From 87b6cde1fbb803c4485ae35e1a3040b79a8f6564 Mon Sep 17 00:00:00 2001
From: Bernhard Schuster <bernhard@ahoi.io>
Date: Wed, 6 Oct 2021 18:25:49 +0200
Subject: [PATCH 001/107] move things around, add filter methods

---
 primitives/src/v1/mod.rs                      |  11 +
 runtime/parachains/src/inclusion.rs           | 162 +++++--------
 runtime/parachains/src/paras_inherent.rs      | 217 +++++++++++++++---
 runtime/parachains/src/runtime_api_impl/v1.rs |   2 +-
 4 files changed, 254 insertions(+), 138 deletions(-)

diff --git a/primitives/src/v1/mod.rs b/primitives/src/v1/mod.rs
index cbea6480efef..3fd2d9dcff82 100644
--- a/primitives/src/v1/mod.rs
+++ b/primitives/src/v1/mod.rs
@@ -533,6 +533,17 @@ impl CandidateCommitments {
 	}
 }
 
+/// A bitfield concering concluded disputes for candidates
+/// associated to the core index equivalent to the bit position.
+#[derive(Default, PartialEq, Eq, Clone, Encode, Decode, RuntimeDebug, TypeInfo)]
+pub struct DisputedBitfield(pub BitVec<bitvec::order::Lsb0, u8>);
+
+impl From<BitVec<bitvec::order::Lsb0, u8>> for DisputedBitfield {
+	fn from(inner: BitVec<bitvec::order::Lsb0, u8>) -> Self {
+		Self(inner)
+	}
+}
+
 /// A bitfield concerning availability of backed candidates.
 #[derive(PartialEq, Eq, Clone, Encode, Decode, RuntimeDebug, TypeInfo)]
 pub struct AvailabilityBitfield(pub BitVec<bitvec::order::Lsb0, u8>);
diff --git a/runtime/parachains/src/inclusion.rs b/runtime/parachains/src/inclusion.rs
index dd865bc8572b..d5e534ecaf52 100644
--- a/runtime/parachains/src/inclusion.rs
+++ b/runtime/parachains/src/inclusion.rs
@@ -26,15 +26,15 @@ use parity_scale_codec::{Decode, Encode};
 use primitives::v1::{
 	AvailabilityBitfield, BackedCandidate, CandidateCommitments, CandidateDescriptor,
 	CandidateHash, CandidateReceipt, CommittedCandidateReceipt, CoreIndex, GroupIndex, Hash,
-	HeadData, Id as ParaId, SigningContext, UncheckedSignedAvailabilityBitfields, ValidatorIndex,
-	ValidityAttestation,
+	HeadData, Id as ParaId, SignedAvailabilityBitfields, SigningContext, ValidatorId,
+	ValidatorIndex, ValidityAttestation,
 };
 use scale_info::TypeInfo;
 use sp_runtime::{
 	traits::{One, Saturating},
 	DispatchError,
 };
-use sp_std::prelude::*;
+use sp_std::{collections::btree_set::BTreeSet, prelude::*};
 
 use crate::{configuration, disputes, dmp, hrmp, paras, scheduler::CoreAssignment, shared, ump};
 
@@ -261,12 +261,10 @@ impl<T: Config> Pallet<T> {
 	/// and cores free.
 	pub(crate) fn process_bitfields(
 		expected_bits: usize,
-		unchecked_bitfields: UncheckedSignedAvailabilityBitfields,
+		checked_bitfields: SignedAvailabilityBitfields,
 		core_lookup: impl Fn(CoreIndex) -> Option<ParaId>,
-	) -> Result<Vec<(CoreIndex, CandidateHash)>, DispatchError> {
-		let validators = shared::Pallet::<T>::active_validator_keys();
-		let session_index = shared::Pallet::<T>::session_index();
-
+		validators: &[ValidatorId],
+	) -> Vec<(CoreIndex, CandidateHash)> {
 		let mut assigned_paras_record = (0..expected_bits)
 			.map(|bit_index| core_lookup(CoreIndex::from(bit_index as u32)))
 			.map(|opt_para_id| {
@@ -274,56 +272,10 @@ impl<T: Config> Pallet<T> {
 			})
 			.collect::<Vec<_>>();
 
-		// do sanity checks on the bitfields:
-		// 1. no more than one bitfield per validator
-		// 2. bitfields are ascending by validator index.
-		// 3. each bitfield has exactly `expected_bits`
-		// 4. signature is valid.
-		let signed_bitfields = {
-			let mut last_index = None;
-
-			let signing_context = SigningContext {
-				parent_hash: <frame_system::Pallet<T>>::parent_hash(),
-				session_index,
-			};
-
-			let mut signed_bitfields = Vec::with_capacity(unchecked_bitfields.len());
-
-			for unchecked_bitfield in unchecked_bitfields {
-				ensure!(
-					unchecked_bitfield.unchecked_payload().0.len() == expected_bits,
-					Error::<T>::WrongBitfieldSize,
-				);
-
-				ensure!(
-					last_index
-						.map_or(true, |last| last < unchecked_bitfield.unchecked_validator_index()),
-					Error::<T>::BitfieldDuplicateOrUnordered,
-				);
-
-				ensure!(
-					(unchecked_bitfield.unchecked_validator_index().0 as usize) < validators.len(),
-					Error::<T>::ValidatorIndexOutOfBounds,
-				);
-
-				let validator_public =
-					&validators[unchecked_bitfield.unchecked_validator_index().0 as usize];
-
-				last_index = Some(unchecked_bitfield.unchecked_validator_index());
-
-				signed_bitfields.push(
-					unchecked_bitfield
-						.try_into_checked(&signing_context, validator_public)
-						.map_err(|_| Error::<T>::InvalidBitfieldSignature)?,
-				);
-			}
-			signed_bitfields
-		};
-
 		let now = <frame_system::Pallet<T>>::block_number();
-		for signed_bitfield in signed_bitfields {
+		for checked_bitfield in checked_bitfields {
 			for (bit_idx, _) in
-				signed_bitfield.payload().0.iter().enumerate().filter(|(_, is_av)| **is_av)
+				checked_bitfield.payload().0.iter().enumerate().filter(|(_, is_av)| **is_av)
 			{
 				let pending_availability = if let Some((_, pending_availability)) =
 					assigned_paras_record[bit_idx].as_mut()
@@ -339,7 +291,7 @@ impl<T: Config> Pallet<T> {
 
 				// defensive check - this is constructed by loading the availability bitfield record,
 				// which is always `Some` if the core is occupied - that's why we're here.
-				let val_idx = signed_bitfield.validator_index().0 as usize;
+				let val_idx = checked_bitfield.validator_index().0 as usize;
 				if let Some(mut bit) =
 					pending_availability.as_mut().and_then(|candidate_pending_availability| {
 						candidate_pending_availability.availability_votes.get_mut(val_idx)
@@ -348,9 +300,9 @@ impl<T: Config> Pallet<T> {
 				}
 			}
 
-			let validator_index = signed_bitfield.validator_index();
+			let validator_index = checked_bitfield.validator_index();
 			let record = AvailabilityBitfieldRecord {
-				bitfield: signed_bitfield.into_payload(),
+				bitfield: checked_bitfield.into_payload(),
 				submitted_at: now,
 			};
 
@@ -398,7 +350,7 @@ impl<T: Config> Pallet<T> {
 			}
 		}
 
-		Ok(freed_cores)
+		freed_cores
 	}
 
 	/// Process candidates that have been backed. Provide the relay storage root, a set of candidates
@@ -822,7 +774,7 @@ impl<T: Config> Pallet<T> {
 	/// Cleans up all paras pending availability that are in the given list of disputed candidates.
 	///
 	/// Returns a vector of cleaned-up core IDs.
-	pub(crate) fn collect_disputed(disputed: Vec<CandidateHash>) -> Vec<CoreIndex> {
+	pub(crate) fn collect_disputed(disputed: &BTreeSet<CandidateHash>) -> Vec<CoreIndex> {
 		let mut cleaned_up_ids = Vec::new();
 		let mut cleaned_up_cores = Vec::new();
 
@@ -1359,12 +1311,14 @@ mod tests {
 					&signing_context,
 				));
 
-				assert!(ParaInclusion::process_bitfields(
-					expected_bits(),
-					vec![signed.into()],
-					&core_lookup,
-				)
-				.is_err());
+				assert_eq!(
+					ParaInclusion::process_bitfields(
+						expected_bits(),
+						vec![signed.into()],
+						&core_lookup,
+					),
+					vec![]
+				);
 			}
 
 			// wrong number of bits: other way around.
@@ -1378,12 +1332,14 @@ mod tests {
 					&signing_context,
 				));
 
-				assert!(ParaInclusion::process_bitfields(
-					expected_bits() + 1,
-					vec![signed.into()],
-					&core_lookup,
-				)
-				.is_err());
+				assert_eq!(
+					ParaInclusion::process_bitfields(
+						expected_bits() + 1,
+						vec![signed.into()],
+						&core_lookup,
+					),
+					vec![]
+				);
 			}
 
 			// duplicate.
@@ -1398,12 +1354,14 @@ mod tests {
 				))
 				.into();
 
-				assert!(ParaInclusion::process_bitfields(
-					expected_bits(),
-					vec![signed.clone(), signed],
-					&core_lookup,
-				)
-				.is_err());
+				assert_eq!(
+					ParaInclusion::process_bitfields(
+						expected_bits(),
+						vec![signed.clone(), signed],
+						&core_lookup,
+					),
+					vec![]
+				);
 			}
 
 			// out of order.
@@ -1427,12 +1385,14 @@ mod tests {
 				))
 				.into();
 
-				assert!(ParaInclusion::process_bitfields(
-					expected_bits(),
-					vec![signed_1, signed_0],
-					&core_lookup,
-				)
-				.is_err());
+				assert_eq!(
+					ParaInclusion::process_bitfields(
+						expected_bits(),
+						vec![signed_1, signed_0],
+						&core_lookup,
+					),
+					vec![]
+				);
 			}
 
 			// non-pending bit set.
@@ -1452,7 +1412,7 @@ mod tests {
 						vec![signed.into()],
 						&core_lookup,
 					),
-					Ok(vec![])
+					vec![]
 				);
 			}
 
@@ -1467,12 +1427,14 @@ mod tests {
 					&signing_context,
 				));
 
-				assert!(ParaInclusion::process_bitfields(
-					expected_bits(),
-					vec![signed.into()],
-					&core_lookup,
-				)
-				.is_ok());
+				assert!(
+					ParaInclusion::process_bitfields(
+						expected_bits(),
+						vec![signed.into()],
+						&core_lookup,
+					)
+					.len() == 1
+				);
 			}
 
 			// bitfield signed with pending bit signed.
@@ -1509,12 +1471,14 @@ mod tests {
 					&signing_context,
 				));
 
-				assert!(ParaInclusion::process_bitfields(
-					expected_bits(),
-					vec![signed.into()],
-					&core_lookup,
-				)
-				.is_ok());
+				assert!(
+					ParaInclusion::process_bitfields(
+						expected_bits(),
+						vec![signed.into()],
+						&core_lookup,
+					)
+					.len() == 1
+				);
 
 				<PendingAvailability<Test>>::remove(chain_a);
 				PendingAvailabilityCommitments::<Test>::remove(chain_a);
@@ -1557,7 +1521,7 @@ mod tests {
 						vec![signed.into()],
 						&core_lookup,
 					),
-					Ok(vec![]),
+					vec![],
 				);
 			}
 		});
diff --git a/runtime/parachains/src/paras_inherent.rs b/runtime/parachains/src/paras_inherent.rs
index c866a077ccb2..94a3e8fc4f58 100644
--- a/runtime/parachains/src/paras_inherent.rs
+++ b/runtime/parachains/src/paras_inherent.rs
@@ -22,22 +22,29 @@
 //! this module.
 
 use crate::{
+	configuration::Config,
 	disputes::DisputesHandler,
 	inclusion,
 	scheduler::{self, FreedReason},
 	shared, ump,
 };
+use bitvec::prelude::BitVec;
 use frame_support::{
 	inherent::{InherentData, InherentIdentifier, MakeFatalError, ProvideInherent},
 	pallet_prelude::*,
 };
 use frame_system::pallet_prelude::*;
 use primitives::v1::{
-	BackedCandidate, InherentData as ParachainsInherentData, ScrapedOnChainVotes,
+	BackedCandidate, CandidateHash, CoreIndex, DisputedBitfield,
+	InherentData as ParachainsInherentData, ScrapedOnChainVotes, SessionIndex,
+	SignedAvailabilityBitfields, SigningContext, UncheckedSignedAvailabilityBitfields, ValidatorId,
 	PARACHAINS_INHERENT_IDENTIFIER,
 };
 use sp_runtime::traits::Header as HeaderT;
-use sp_std::prelude::*;
+use sp_std::{
+	collections::{btree_map::BTreeMap, btree_set::BTreeSet},
+	prelude::*,
+};
 
 pub use pallet::*;
 
@@ -177,9 +184,11 @@ pub mod pallet {
 				Error::<T>::InvalidParentHeader,
 			);
 
+			let expected_bits = <scheduler::Pallet<T>>::availability_cores().len();
+
 			// Handle disputes logic.
 			let current_session = <shared::Pallet<T>>::session_index();
-			{
+			let (disputed_bits, concluded_invalid_disputed_candidates) = {
 				let new_current_dispute_sets: Vec<_> = disputes
 					.iter()
 					.filter(|s| s.session == current_session)
@@ -193,20 +202,38 @@ pub mod pallet {
 					return Ok(Some(MINIMAL_INCLUSION_INHERENT_WEIGHT).into())
 				}
 
-				let mut freed_disputed = if !new_current_dispute_sets.is_empty() {
-					let concluded_invalid_disputes: Vec<_> = new_current_dispute_sets
-						.iter()
-						.filter(|(s, c)| T::DisputesHandler::concluded_invalid(*s, *c))
-						.map(|(_, c)| *c)
-						.collect();
-
-					<inclusion::Pallet<T>>::collect_disputed(concluded_invalid_disputes)
-						.into_iter()
-						.map(|core| (core, FreedReason::Concluded))
-						.collect()
-				} else {
-					Vec::new()
-				};
+				let (mut freed_disputed, concluded_invalid_disputed_candidates) =
+					if !new_current_dispute_sets.is_empty() {
+						let concluded_invalid_disputes = new_current_dispute_sets
+							.iter()
+							.filter(|(session, candidate)| {
+								T::DisputesHandler::concluded_invalid(*session, *candidate)
+							})
+							.map(|(_, candidate)| *candidate)
+							.collect::<BTreeSet<CandidateHash>>();
+
+						let freed_disputed =
+							<inclusion::Pallet<T>>::collect_disputed(&concluded_invalid_disputes)
+								.into_iter()
+								.map(|core| (core, FreedReason::Concluded))
+								.collect();
+						(freed_disputed, concluded_invalid_disputes)
+					} else {
+						(Vec::new(), BTreeSet::new())
+					};
+
+				// create a bit index from the set of core indicies.
+				let mut bitvec = BitVec::with_capacity(expected_bits);
+				if expected_bits > 0 {
+					bitvec.set(expected_bits.saturating_sub(1), false);
+					for (core_idx, _) in &freed_disputed {
+						let core_idx = core_idx.0 as usize;
+						if core_idx < expected_bits {
+							bitvec.set(core_idx, true);
+						}
+					}
+				}
+				let disputed_bitfield = DisputedBitfield::from(bitvec);
 
 				if !freed_disputed.is_empty() {
 					// unstable sort is fine, because core indices are unique
@@ -214,16 +241,29 @@ pub mod pallet {
 					freed_disputed.sort_unstable_by_key(|pair| pair.0); // sort by core index
 					<scheduler::Pallet<T>>::free_cores(freed_disputed);
 				}
+
+				(disputed_bitfield, concluded_invalid_disputed_candidates)
 			};
 
+			let validators = shared::Pallet::<T>::active_validator_keys();
+
+			let checked_bitfields = filter_bitfields::<T>(
+				signed_bitfields,
+				disputed_bits,
+				expected_bits,
+				parent_hash,
+				current_session,
+				&validators,
+			);
+
 			// Process new availability bitfields, yielding any availability cores whose
 			// work has now concluded.
-			let expected_bits = <scheduler::Pallet<T>>::availability_cores().len();
 			let freed_concluded = <inclusion::Pallet<T>>::process_bitfields(
 				expected_bits,
-				signed_bitfields,
+				checked_bitfields,
 				<scheduler::Pallet<T>>::core_para,
-			)?;
+				&validators,
+			);
 
 			// Inform the disputes module of all included candidates.
 			let now = <frame_system::Pallet<T>>::block_number();
@@ -240,32 +280,22 @@ pub mod pallet {
 			};
 
 			// Schedule paras again, given freed cores, and reasons for freeing.
-			let mut freed = freed_concluded
+			let freed = freed_concluded
 				.into_iter()
 				.map(|(c, _hash)| (c, FreedReason::Concluded))
 				.chain(freed_timeout.into_iter().map(|c| (c, FreedReason::TimedOut)))
-				.collect::<Vec<_>>();
+				.collect::<BTreeMap<CoreIndex, FreedReason>>();
 
-			// unstable sort is fine, because core indices are unique.
-			freed.sort_unstable_by_key(|pair| pair.0); // sort by core index
+			let backed_candidates = filter_backed_candidates::<T>(
+				parent_hash,
+				backed_candidates,
+				concluded_invalid_disputed_candidates,
+			);
+			let backed_candidates_len = backed_candidates.len() as Weight;
 
 			<scheduler::Pallet<T>>::clear();
 			<scheduler::Pallet<T>>::schedule(freed, <frame_system::Pallet<T>>::block_number());
 
-			let backed_candidates = limit_backed_candidates::<T>(backed_candidates);
-			let backed_candidates_len = backed_candidates.len() as Weight;
-
-			// Refuse to back any candidates that were disputed and are concluded invalid.
-			for candidate in &backed_candidates {
-				ensure!(
-					!T::DisputesHandler::concluded_invalid(
-						current_session,
-						candidate.candidate.hash(),
-					),
-					Error::<T>::CandidateConcludedInvalid,
-				);
-			}
-
 			// Process backed candidates according to scheduled cores.
 			let parent_storage_root = parent_header.state_root().clone();
 			let inclusion::ProcessedCandidates::<<T::Header as HeaderT>::Hash> {
@@ -304,6 +334,107 @@ pub mod pallet {
 	}
 }
 
+/// Filter bitfields based on freed core indices.
+///
+/// Do sanity checks on the bitfields:
+///
+///  1. no more than one bitfield per validator
+///  2. bitfields are ascending by validator index.
+///  3. each bitfield has exactly `expected_bits`
+///  4. signature is valid
+///  5. remove any disputed core indices
+///
+/// If any of those is not passed, the bitfield is dropped.
+fn filter_bitfields<T: Config>(
+	unchecked_bitfields: UncheckedSignedAvailabilityBitfields,
+	disputed_bits: DisputedBitfield,
+	expected_bits: usize,
+	parent_hash: T::Hash,
+	session_index: SessionIndex,
+	validators: &[ValidatorId],
+) -> SignedAvailabilityBitfields {
+	let mut bitfields = Vec::with_capacity(unchecked_bitfields.len());
+
+	let mut last_index = None;
+
+	if disputed_bits.0.len() != expected_bits {
+		return Default::default()
+	}
+
+	if validators.len() != expected_bits {
+		return Default::default()
+	}
+
+	for unchecked_bitfield in unchecked_bitfields {
+		let signing_context = SigningContext { parent_hash, session_index };
+
+		if unchecked_bitfield.unchecked_payload().0.len() != expected_bits {
+			continue
+		}
+
+		if last_index.map_or(false, |last| last >= unchecked_bitfield.unchecked_validator_index()) {
+			continue
+		}
+		if (unchecked_bitfield.unchecked_validator_index().0 as usize) >= validators.len() {
+			continue
+		}
+
+		let validator_index = unchecked_bitfield.unchecked_validator_index();
+
+		let validator_public = &validators[validator_index.0 as usize];
+
+		let signed_bitfield = if let Ok(signed_bitfield) =
+			unchecked_bitfield.try_into_checked(&signing_context, validator_public)
+		{
+			signed_bitfield
+		} else {
+			continue
+		};
+
+		last_index = Some(validator_index);
+
+		// TODO FIXME do we want to adjust the bitfield or drop it entirely?
+
+		// remove all 1 bits which map to concluded disputes
+		// signed_bitfield.payload.0.iter_mut()
+		// 	.enumerate()
+		// 	.any(|(core_idx, mut bit)| {
+		// 		*bit = if *bit {
+		// 			// ignore those that have matching invalid dispute bits
+		// 			// which always works, since we checked for uniformat bit length
+		// 			// before
+		// 			!disputed_bits.0[core_idx]
+		// 		} else {
+		// 			// bit wasn't set in the first place
+		// 			false
+		// 		};
+		// 	});
+		// TODO at this point the signature won't match, since we modified the payload
+		// TODO double check that the signature is not used at a later point
+		bitfields.push(signed_bitfield)
+	}
+	bitfields
+}
+
+/// Filter candidates based on the concluded disputes.
+fn filter_backed_candidates<T: Config>(
+	relay_parent: T::Hash,
+	mut backed_candidates: Vec<BackedCandidate<T::Hash>>,
+	disputed_candidates: BTreeSet<CandidateHash>,
+) -> Vec<BackedCandidate<T::Hash>> {
+	// Remove any candidates that were concluded invalid.
+	backed_candidates.retain(|backed_candidate| {
+		let candidate_hash = backed_candidate.candidate.hash();
+		!disputed_candidates.contains(&candidate_hash)
+	});
+	// Remove candidates referencing a different relay parent.
+	backed_candidates
+		.retain(|backed_candidate| backed_candidate.descriptor().relay_parent == relay_parent);
+	//
+	// Limit weight, to avoid overweight block.
+	limit_backed_candidates::<T>(backed_candidates)
+}
+
 /// Limit the number of backed candidates processed in order to stay within block weight limits.
 ///
 /// Use a configured assumption about the weight required to process a backed candidate and the
@@ -413,6 +544,16 @@ mod tests {
 		}
 	}
 
+	mod filter_bitfields {
+		use super::*;
+		// TODO
+	}
+
+	mod filter_candidates {
+		use super::*;
+		// TODO
+	}
+
 	mod paras_inherent_weight {
 		use super::*;
 
diff --git a/runtime/parachains/src/runtime_api_impl/v1.rs b/runtime/parachains/src/runtime_api_impl/v1.rs
index 5909ea55290b..1607dcd22528 100644
--- a/runtime/parachains/src/runtime_api_impl/v1.rs
+++ b/runtime/parachains/src/runtime_api_impl/v1.rs
@@ -332,6 +332,6 @@ pub fn validation_code_by_hash<T: paras::Config>(
 }
 
 /// Disputes imported via means of on-chain imports.
-pub fn on_chain_votes<T: paras_inherent::Config>() -> Option<ScrapedOnChainVotes<T::Hash>> {
+pub fn on_chain_votes<T: paras_inherent::pallet::Config>() -> Option<ScrapedOnChainVotes<T::Hash>> {
 	<paras_inherent::Pallet<T>>::on_chain_votes()
 }

From bf1f98b6f7f1fb25bbb7ae01fb72fa11a539bd54 Mon Sep 17 00:00:00 2001
From: Bernhard Schuster <bernhard@ahoi.io>
Date: Fri, 8 Oct 2021 17:33:12 +0200
Subject: [PATCH 002/107] validator keys, modify availability bitfields
 according to disputes

---
 primitives/src/v1/signed.rs              |  5 ++++
 runtime/parachains/src/inclusion.rs      | 20 +++++++++----
 runtime/parachains/src/paras_inherent.rs | 36 +++++++++++-------------
 3 files changed, 35 insertions(+), 26 deletions(-)

diff --git a/primitives/src/v1/signed.rs b/primitives/src/v1/signed.rs
index 612cc1516895..fc2cded195b7 100644
--- a/primitives/src/v1/signed.rs
+++ b/primitives/src/v1/signed.rs
@@ -143,6 +143,11 @@ impl<Payload: EncodeAs<RealPayload>, RealPayload: Encode> Signed<Payload, RealPa
 	}
 }
 
+
+// Payload that passed signature verification, but the content
+// could be normalized/modified _after_ checking.
+// struct Checked<Payload> {}
+
 // We can't bound this on `Payload: Into<RealPayload>` beacuse that conversion consumes
 // the payload, and we don't want that. We can't bound it on `Payload: AsRef<RealPayload>`
 // because there's no blanket impl of `AsRef<T> for T`. In the end, we just invent our
diff --git a/runtime/parachains/src/inclusion.rs b/runtime/parachains/src/inclusion.rs
index d5e534ecaf52..04e6d43f10aa 100644
--- a/runtime/parachains/src/inclusion.rs
+++ b/runtime/parachains/src/inclusion.rs
@@ -261,7 +261,7 @@ impl<T: Config> Pallet<T> {
 	/// and cores free.
 	pub(crate) fn process_bitfields(
 		expected_bits: usize,
-		checked_bitfields: SignedAvailabilityBitfields,
+		checked_bitfields: Vec<(AvailabilityBitfield, ValidatorIndex)>,
 		core_lookup: impl Fn(CoreIndex) -> Option<ParaId>,
 		validators: &[ValidatorId],
 	) -> Vec<(CoreIndex, CandidateHash)> {
@@ -273,9 +273,9 @@ impl<T: Config> Pallet<T> {
 			.collect::<Vec<_>>();
 
 		let now = <frame_system::Pallet<T>>::block_number();
-		for checked_bitfield in checked_bitfields {
+		for (checked_bitfield, validator_index) in checked_bitfields {
 			for (bit_idx, _) in
-				checked_bitfield.payload().0.iter().enumerate().filter(|(_, is_av)| **is_av)
+				checked_bitfield.iter().enumerate().filter(|(_, is_av)| **is_av)
 			{
 				let pending_availability = if let Some((_, pending_availability)) =
 					assigned_paras_record[bit_idx].as_mut()
@@ -291,7 +291,7 @@ impl<T: Config> Pallet<T> {
 
 				// defensive check - this is constructed by loading the availability bitfield record,
 				// which is always `Some` if the core is occupied - that's why we're here.
-				let val_idx = checked_bitfield.validator_index().0 as usize;
+				let validator_index = validator_index.0 as usize;
 				if let Some(mut bit) =
 					pending_availability.as_mut().and_then(|candidate_pending_availability| {
 						candidate_pending_availability.availability_votes.get_mut(val_idx)
@@ -300,9 +300,8 @@ impl<T: Config> Pallet<T> {
 				}
 			}
 
-			let validator_index = checked_bitfield.validator_index();
 			let record = AvailabilityBitfieldRecord {
-				bitfield: checked_bitfield.into_payload(),
+				bitfield: checked_bitfield,
 				submitted_at: now,
 			};
 
@@ -1316,6 +1315,7 @@ mod tests {
 						expected_bits(),
 						vec![signed.into()],
 						&core_lookup,
+						&validator_public,
 					),
 					vec![]
 				);
@@ -1337,6 +1337,7 @@ mod tests {
 						expected_bits() + 1,
 						vec![signed.into()],
 						&core_lookup,
+						&validator_public,
 					),
 					vec![]
 				);
@@ -1359,6 +1360,7 @@ mod tests {
 						expected_bits(),
 						vec![signed.clone(), signed],
 						&core_lookup,
+						&validator_public,
 					),
 					vec![]
 				);
@@ -1390,6 +1392,7 @@ mod tests {
 						expected_bits(),
 						vec![signed_1, signed_0],
 						&core_lookup,
+						&validator_public,
 					),
 					vec![]
 				);
@@ -1411,6 +1414,7 @@ mod tests {
 						expected_bits(),
 						vec![signed.into()],
 						&core_lookup,
+						&validator_public,
 					),
 					vec![]
 				);
@@ -1432,6 +1436,7 @@ mod tests {
 						expected_bits(),
 						vec![signed.into()],
 						&core_lookup,
+						&validator_public,
 					)
 					.len() == 1
 				);
@@ -1476,6 +1481,7 @@ mod tests {
 						expected_bits(),
 						vec![signed.into()],
 						&core_lookup,
+						&validator_public,
 					)
 					.len() == 1
 				);
@@ -1520,6 +1526,7 @@ mod tests {
 						expected_bits(),
 						vec![signed.into()],
 						&core_lookup,
+						&validator_public,
 					),
 					vec![],
 				);
@@ -1662,6 +1669,7 @@ mod tests {
 				expected_bits(),
 				signed_bitfields,
 				&core_lookup,
+				&validator_public,
 			)
 			.is_ok());
 
diff --git a/runtime/parachains/src/paras_inherent.rs b/runtime/parachains/src/paras_inherent.rs
index 94a3e8fc4f58..39328e7bd538 100644
--- a/runtime/parachains/src/paras_inherent.rs
+++ b/runtime/parachains/src/paras_inherent.rs
@@ -352,7 +352,7 @@ fn filter_bitfields<T: Config>(
 	parent_hash: T::Hash,
 	session_index: SessionIndex,
 	validators: &[ValidatorId],
-) -> SignedAvailabilityBitfields {
+) -> Vec<(AvailabilityBitfield, ValidatorIndex)> {
 	let mut bitfields = Vec::with_capacity(unchecked_bitfields.len());
 
 	let mut last_index = None;
@@ -393,25 +393,21 @@ fn filter_bitfields<T: Config>(
 
 		last_index = Some(validator_index);
 
-		// TODO FIXME do we want to adjust the bitfield or drop it entirely?
-
-		// remove all 1 bits which map to concluded disputes
-		// signed_bitfield.payload.0.iter_mut()
-		// 	.enumerate()
-		// 	.any(|(core_idx, mut bit)| {
-		// 		*bit = if *bit {
-		// 			// ignore those that have matching invalid dispute bits
-		// 			// which always works, since we checked for uniformat bit length
-		// 			// before
-		// 			!disputed_bits.0[core_idx]
-		// 		} else {
-		// 			// bit wasn't set in the first place
-		// 			false
-		// 		};
-		// 	});
-		// TODO at this point the signature won't match, since we modified the payload
-		// TODO double check that the signature is not used at a later point
-		bitfields.push(signed_bitfield)
+		let mut checked_bitfield = signed_bitfield.into_payload();
+		checked_bitfield.0.iter_mut()
+			.enumerate()
+			.any(|(core_idx, mut bit)| {
+				*bit = if *bit {
+					// ignore those that have matching invalid dispute bits
+					// which always works, since we checked for uniformat bit length
+					// before
+					!disputed_bits.0[core_idx]
+				} else {
+					// bit wasn't set in the first place
+					false
+				};
+			});
+		bitfields.push((checked_bitfield, validator_index));
 	}
 	bitfields
 }

From 7c2711d59f3b3346d2b5a6894ff73e4d5fc1042d Mon Sep 17 00:00:00 2001
From: Bernhard Schuster <bernhard@ahoi.io>
Date: Wed, 13 Oct 2021 12:58:55 +0200
Subject: [PATCH 003/107] simplify, keep the filter -> sanitize generic for
 both usecases

---
 primitives/src/v1/mod.rs                 |  11 --
 primitives/src/v1/signed.rs              |   5 -
 runtime/parachains/src/inclusion.rs      | 106 +++++-------
 runtime/parachains/src/paras_inherent.rs | 208 ++++++++++++++---------
 4 files changed, 168 insertions(+), 162 deletions(-)

diff --git a/primitives/src/v1/mod.rs b/primitives/src/v1/mod.rs
index 3fd2d9dcff82..cbea6480efef 100644
--- a/primitives/src/v1/mod.rs
+++ b/primitives/src/v1/mod.rs
@@ -533,17 +533,6 @@ impl CandidateCommitments {
 	}
 }
 
-/// A bitfield concering concluded disputes for candidates
-/// associated to the core index equivalent to the bit position.
-#[derive(Default, PartialEq, Eq, Clone, Encode, Decode, RuntimeDebug, TypeInfo)]
-pub struct DisputedBitfield(pub BitVec<bitvec::order::Lsb0, u8>);
-
-impl From<BitVec<bitvec::order::Lsb0, u8>> for DisputedBitfield {
-	fn from(inner: BitVec<bitvec::order::Lsb0, u8>) -> Self {
-		Self(inner)
-	}
-}
-
 /// A bitfield concerning availability of backed candidates.
 #[derive(PartialEq, Eq, Clone, Encode, Decode, RuntimeDebug, TypeInfo)]
 pub struct AvailabilityBitfield(pub BitVec<bitvec::order::Lsb0, u8>);
diff --git a/primitives/src/v1/signed.rs b/primitives/src/v1/signed.rs
index fc2cded195b7..612cc1516895 100644
--- a/primitives/src/v1/signed.rs
+++ b/primitives/src/v1/signed.rs
@@ -143,11 +143,6 @@ impl<Payload: EncodeAs<RealPayload>, RealPayload: Encode> Signed<Payload, RealPa
 	}
 }
 
-
-// Payload that passed signature verification, but the content
-// could be normalized/modified _after_ checking.
-// struct Checked<Payload> {}
-
 // We can't bound this on `Payload: Into<RealPayload>` beacuse that conversion consumes
 // the payload, and we don't want that. We can't bound it on `Payload: AsRef<RealPayload>`
 // because there's no blanket impl of `AsRef<T> for T`. In the end, we just invent our
diff --git a/runtime/parachains/src/inclusion.rs b/runtime/parachains/src/inclusion.rs
index 04e6d43f10aa..e6f12975fe06 100644
--- a/runtime/parachains/src/inclusion.rs
+++ b/runtime/parachains/src/inclusion.rs
@@ -26,7 +26,7 @@ use parity_scale_codec::{Decode, Encode};
 use primitives::v1::{
 	AvailabilityBitfield, BackedCandidate, CandidateCommitments, CandidateDescriptor,
 	CandidateHash, CandidateReceipt, CommittedCandidateReceipt, CoreIndex, GroupIndex, Hash,
-	HeadData, Id as ParaId, SignedAvailabilityBitfields, SigningContext, ValidatorId,
+	HeadData, Id as ParaId, SigningContext, ValidatorId,
 	ValidatorIndex, ValidityAttestation,
 };
 use scale_info::TypeInfo;
@@ -275,7 +275,7 @@ impl<T: Config> Pallet<T> {
 		let now = <frame_system::Pallet<T>>::block_number();
 		for (checked_bitfield, validator_index) in checked_bitfields {
 			for (bit_idx, _) in
-				checked_bitfield.iter().enumerate().filter(|(_, is_av)| **is_av)
+				checked_bitfield.0.iter().enumerate().filter(|(_, is_av)| **is_av)
 			{
 				let pending_availability = if let Some((_, pending_availability)) =
 					assigned_paras_record[bit_idx].as_mut()
@@ -294,7 +294,7 @@ impl<T: Config> Pallet<T> {
 				let validator_index = validator_index.0 as usize;
 				if let Some(mut bit) =
 					pending_availability.as_mut().and_then(|candidate_pending_availability| {
-						candidate_pending_availability.availability_votes.get_mut(val_idx)
+						candidate_pending_availability.availability_votes.get_mut(validator_index)
 					}) {
 					*bit = true;
 				}
@@ -1310,15 +1310,13 @@ mod tests {
 					&signing_context,
 				));
 
-				assert_eq!(
-					ParaInclusion::process_bitfields(
-						expected_bits(),
-						vec![signed.into()],
-						&core_lookup,
-						&validator_public,
-					),
-					vec![]
-				);
+				assert_eq!(ParaInclusion::process_bitfields(
+					expected_bits(),
+					vec![signed.into()],
+					&core_lookup,
+					&validator_public,
+				),
+				Err(Error::WrongBitfieldSize));
 			}
 
 			// wrong number of bits: other way around.
@@ -1332,15 +1330,13 @@ mod tests {
 					&signing_context,
 				));
 
-				assert_eq!(
-					ParaInclusion::process_bitfields(
-						expected_bits() + 1,
-						vec![signed.into()],
-						&core_lookup,
-						&validator_public,
-					),
-					vec![]
-				);
+				assert_eq!(ParaInclusion::process_bitfields(
+					expected_bits() + 1,
+					vec![signed.into()],
+					&core_lookup,
+					&validator_public,
+				),
+				Err(Error::WrongBitfieldSize));
 			}
 
 			// duplicate.
@@ -1355,15 +1351,12 @@ mod tests {
 				))
 				.into();
 
-				assert_eq!(
-					ParaInclusion::process_bitfields(
-						expected_bits(),
-						vec![signed.clone(), signed],
-						&core_lookup,
-						&validator_public,
-					),
-					vec![]
-				);
+				assert_eq!(ParaInclusion::process_bitfields(
+					expected_bits(),
+					vec![signed.clone(), signed],
+					&core_lookup,
+					&validators_public,
+				), Err(Error::BitfieldDuplicateOrUnordered));
 			}
 
 			// out of order.
@@ -1387,15 +1380,13 @@ mod tests {
 				))
 				.into();
 
-				assert_eq!(
-					ParaInclusion::process_bitfields(
-						expected_bits(),
-						vec![signed_1, signed_0],
-						&core_lookup,
-						&validator_public,
-					),
-					vec![]
-				);
+				assert_eq!(ParaInclusion::process_bitfields(
+					expected_bits(),
+					vec![signed_1, signed_0],
+					&core_lookup,
+					&validators_public,
+				),
+				Err(Error::BitfieldDuplicateOrUnordered));
 			}
 
 			// non-pending bit set.
@@ -1414,9 +1405,8 @@ mod tests {
 						expected_bits(),
 						vec![signed.into()],
 						&core_lookup,
-						&validator_public,
 					),
-					vec![]
+					Ok(vec![])
 				);
 			}
 
@@ -1431,15 +1421,12 @@ mod tests {
 					&signing_context,
 				));
 
-				assert!(
-					ParaInclusion::process_bitfields(
-						expected_bits(),
-						vec![signed.into()],
-						&core_lookup,
-						&validator_public,
-					)
-					.len() == 1
-				);
+				assert!(ParaInclusion::process_bitfields(
+					expected_bits(),
+					vec![signed.into()],
+					&core_lookup,
+					&validators_public,
+				).is_ok());
 			}
 
 			// bitfield signed with pending bit signed.
@@ -1476,15 +1463,12 @@ mod tests {
 					&signing_context,
 				));
 
-				assert!(
-					ParaInclusion::process_bitfields(
-						expected_bits(),
-						vec![signed.into()],
-						&core_lookup,
-						&validator_public,
-					)
-					.len() == 1
-				);
+				assert!(ParaInclusion::process_bitfields(
+					expected_bits(),
+					vec![signed.into()],
+					&core_lookup,
+					&validators_public,
+				).is_ok());
 
 				<PendingAvailability<Test>>::remove(chain_a);
 				PendingAvailabilityCommitments::<Test>::remove(chain_a);
@@ -1526,9 +1510,8 @@ mod tests {
 						expected_bits(),
 						vec![signed.into()],
 						&core_lookup,
-						&validator_public,
 					),
-					vec![],
+					Ok(vec![]),
 				);
 			}
 		});
@@ -1669,7 +1652,6 @@ mod tests {
 				expected_bits(),
 				signed_bitfields,
 				&core_lookup,
-				&validator_public,
 			)
 			.is_ok());
 
diff --git a/runtime/parachains/src/paras_inherent.rs b/runtime/parachains/src/paras_inherent.rs
index 39328e7bd538..0aa7d86d044d 100644
--- a/runtime/parachains/src/paras_inherent.rs
+++ b/runtime/parachains/src/paras_inherent.rs
@@ -21,25 +21,22 @@
 //! as it has no initialization logic and its finalization logic depends only on the details of
 //! this module.
 
-use crate::{
-	configuration::Config,
-	disputes::DisputesHandler,
-	inclusion,
-	scheduler::{self, FreedReason},
-	shared, ump,
-};
+use crate::{configuration::Config, disputes::DisputesHandler, inclusion, scheduler::{self, CoreAssignment, FreedReason}, shared, ump};
 use bitvec::prelude::BitVec;
 use frame_support::{
 	inherent::{InherentData, InherentIdentifier, MakeFatalError, ProvideInherent},
 	pallet_prelude::*,
+	fail,
 };
 use frame_system::pallet_prelude::*;
 use primitives::v1::{
-	BackedCandidate, CandidateHash, CoreIndex, DisputedBitfield,
+	AvailabilityBitfield, ValidatorIndex,
+	BackedCandidate, CandidateHash, CoreIndex,
 	InherentData as ParachainsInherentData, ScrapedOnChainVotes, SessionIndex,
-	SignedAvailabilityBitfields, SigningContext, UncheckedSignedAvailabilityBitfields, ValidatorId,
+	SigningContext, UncheckedSignedAvailabilityBitfields, ValidatorId,
 	PARACHAINS_INHERENT_IDENTIFIER,
 };
+use scale_info::TypeInfo;
 use sp_runtime::traits::Header as HeaderT;
 use sp_std::{
 	collections::{btree_map::BTreeMap, btree_set::BTreeSet},
@@ -55,6 +52,17 @@ const INCLUSION_INHERENT_CLAIMED_WEIGHT: Weight = 1_000_000_000;
 // we assume that 75% of an paras inherent's weight is used processing backed candidates
 const MINIMAL_INCLUSION_INHERENT_WEIGHT: Weight = INCLUSION_INHERENT_CLAIMED_WEIGHT / 4;
 
+/// A bitfield concering concluded disputes for candidates
+/// associated to the core index equivalent to the bit position.
+#[derive(Default, PartialEq, Eq, Clone, Encode, Decode, RuntimeDebug, TypeInfo)]
+pub struct DisputedBitfield(pub BitVec<bitvec::order::Lsb0, u8>);
+
+impl From<BitVec<bitvec::order::Lsb0, u8>> for DisputedBitfield {
+	fn from(inner: BitVec<bitvec::order::Lsb0, u8>) -> Self {
+		Self(inner)
+	}
+}
+
 #[frame_support::pallet]
 pub mod pallet {
 	use super::*;
@@ -127,17 +135,17 @@ pub mod pallet {
 			T::DisputesHandler::filter_multi_dispute_data(&mut inherent_data.disputes);
 
 			// Sanity check: session changes can invalidate an inherent, and we _really_ don't want that to happen.
-			// See github.com/paritytech/polkadot/issues/1327
+			// See <https://github.com/paritytech/polkadot/issues/1327>
 			let inherent_data =
 				match Self::enter(frame_system::RawOrigin::None.into(), inherent_data.clone()) {
 					Ok(_) => inherent_data,
 					Err(err) => {
 						log::warn!(
-						target: LOG_TARGET,
-						"dropping signed_bitfields and backed_candidates because they produced \
-						an invalid paras inherent: {:?}",
-						err.error,
-					);
+							target: LOG_TARGET,
+								"dropping signed_bitfields and backed_candidates because they produced \
+							an invalid paras inherent: {:?}",
+							err.error,
+						);
 
 						ParachainsInherentData {
 							bitfields: Vec::new(),
@@ -223,17 +231,19 @@ pub mod pallet {
 					};
 
 				// create a bit index from the set of core indicies.
-				let mut bitvec = BitVec::with_capacity(expected_bits);
-				if expected_bits > 0 {
-					bitvec.set(expected_bits.saturating_sub(1), false);
-					for (core_idx, _) in &freed_disputed {
-						let core_idx = core_idx.0 as usize;
-						if core_idx < expected_bits {
-							bitvec.set(core_idx, true);
+				let disputed_bitfield = {
+					let mut bitvec = BitVec::with_capacity(expected_bits);
+					if expected_bits > 0 {
+						bitvec.set(expected_bits.saturating_sub(1), false);
+						for (core_idx, _) in &freed_disputed {
+							let core_idx = core_idx.0 as usize;
+							if core_idx < expected_bits {
+								bitvec.set(core_idx, true);
+							}
 						}
 					}
-				}
-				let disputed_bitfield = DisputedBitfield::from(bitvec);
+					DisputedBitfield::from(bitvec)
+				};
 
 				if !freed_disputed.is_empty() {
 					// unstable sort is fine, because core indices are unique
@@ -247,14 +257,14 @@ pub mod pallet {
 
 			let validators = shared::Pallet::<T>::active_validator_keys();
 
-			let checked_bitfields = filter_bitfields::<T>(
+			let checked_bitfields = sanitize_bitfields::<T, true>(
 				signed_bitfields,
 				disputed_bits,
 				expected_bits,
 				parent_hash,
 				current_session,
 				&validators,
-			);
+			)?;
 
 			// Process new availability bitfields, yielding any availability cores whose
 			// work has now concluded.
@@ -286,16 +296,19 @@ pub mod pallet {
 				.chain(freed_timeout.into_iter().map(|c| (c, FreedReason::TimedOut)))
 				.collect::<BTreeMap<CoreIndex, FreedReason>>();
 
-			let backed_candidates = filter_backed_candidates::<T>(
+			<scheduler::Pallet<T>>::clear();
+			<scheduler::Pallet<T>>::schedule(freed, <frame_system::Pallet<T>>::block_number());
+
+			let scheduled = <scheduler::Pallet<T>>::scheduled();
+			let backed_candidates = sanitize_backed_candidates::<T, true>(
 				parent_hash,
 				backed_candidates,
 				concluded_invalid_disputed_candidates,
-			);
+				&scheduled[..],
+			)?;
+			let backed_candidates = limit_backed_candidates::<T>(backed_candidates);
 			let backed_candidates_len = backed_candidates.len() as Weight;
 
-			<scheduler::Pallet<T>>::clear();
-			<scheduler::Pallet<T>>::schedule(freed, <frame_system::Pallet<T>>::block_number());
-
 			// Process backed candidates according to scheduled cores.
 			let parent_storage_root = parent_header.state_root().clone();
 			let inclusion::ProcessedCandidates::<<T::Header as HeaderT>::Hash> {
@@ -304,7 +317,7 @@ pub mod pallet {
 			} = <inclusion::Pallet<T>>::process_candidates(
 				parent_storage_root,
 				backed_candidates,
-				<scheduler::Pallet<T>>::scheduled(),
+				scheduled,
 				<scheduler::Pallet<T>>::group_validators,
 			)?;
 
@@ -334,7 +347,18 @@ pub mod pallet {
 	}
 }
 
-/// Filter bitfields based on freed core indices.
+macro_rules! ensure2 {
+	($condition:expr, $err:expr, $action:ident) => {
+		let condition = $condition;
+		if !condition {
+			if $action {
+				ensure!(condition, $err);
+			}
+		}
+	};
+}
+
+/// Filter bitfields based on freed core indices, validity, and other sanity checks.
 ///
 /// Do sanity checks on the bitfields:
 ///
@@ -345,39 +369,43 @@ pub mod pallet {
 ///  5. remove any disputed core indices
 ///
 /// If any of those is not passed, the bitfield is dropped.
-fn filter_bitfields<T: Config>(
+fn sanitize_bitfields<T: Config + crate::inclusion::Config, const EARLY_RETURN: bool>(
 	unchecked_bitfields: UncheckedSignedAvailabilityBitfields,
 	disputed_bits: DisputedBitfield,
 	expected_bits: usize,
 	parent_hash: T::Hash,
 	session_index: SessionIndex,
 	validators: &[ValidatorId],
-) -> Vec<(AvailabilityBitfield, ValidatorIndex)> {
+) -> Result<Vec<(AvailabilityBitfield, ValidatorIndex)>, DispatchError> {
 	let mut bitfields = Vec::with_capacity(unchecked_bitfields.len());
 
 	let mut last_index = None;
 
-	if disputed_bits.0.len() != expected_bits {
-		return Default::default()
-	}
-
-	if validators.len() != expected_bits {
-		return Default::default()
+	if EARLY_RETURN && disputed_bits.0.len() != expected_bits{
+		return Ok(Default::default())
 	}
 
 	for unchecked_bitfield in unchecked_bitfields {
 		let signing_context = SigningContext { parent_hash, session_index };
 
-		if unchecked_bitfield.unchecked_payload().0.len() != expected_bits {
-			continue
-		}
-
-		if last_index.map_or(false, |last| last >= unchecked_bitfield.unchecked_validator_index()) {
-			continue
-		}
-		if (unchecked_bitfield.unchecked_validator_index().0 as usize) >= validators.len() {
-			continue
-		}
+		ensure2!(
+			unchecked_bitfield.unchecked_payload().0.len() == expected_bits,
+			crate::inclusion::pallet::Error::<T>::WrongBitfieldSize,
+			EARLY_RETURN
+		);
+
+		ensure2!(
+			last_index
+				.map_or(true, |last| last < unchecked_bitfield.unchecked_validator_index()),
+				crate::inclusion::pallet::Error::<T>::BitfieldDuplicateOrUnordered,
+			EARLY_RETURN
+		);
+
+		ensure2!(
+			(unchecked_bitfield.unchecked_validator_index().0 as usize) < validators.len(),
+			crate::inclusion::pallet::Error::<T>::ValidatorIndexOutOfBounds,
+			EARLY_RETURN
+		);
 
 		let validator_index = unchecked_bitfield.unchecked_validator_index();
 
@@ -387,48 +415,70 @@ fn filter_bitfields<T: Config>(
 			unchecked_bitfield.try_into_checked(&signing_context, validator_public)
 		{
 			signed_bitfield
+		} else if EARLY_RETURN {
+			fail!(crate::inclusion::pallet::Error::<T>::InvalidBitfieldSignature);
 		} else {
-			continue
+			continue;
 		};
 
 		last_index = Some(validator_index);
 
 		let mut checked_bitfield = signed_bitfield.into_payload();
-		checked_bitfield.0.iter_mut()
-			.enumerate()
-			.any(|(core_idx, mut bit)| {
-				*bit = if *bit {
-					// ignore those that have matching invalid dispute bits
-					// which always works, since we checked for uniformat bit length
-					// before
-					!disputed_bits.0[core_idx]
-				} else {
-					// bit wasn't set in the first place
-					false
-				};
-			});
+
+		// filter the bitfields only
+		if !EARLY_RETURN {
+			checked_bitfield.0.iter_mut()
+				.enumerate()
+				.for_each(|(core_idx, mut bit)| {
+					*bit = if *bit {
+						// ignore those that have matching invalid dispute bits
+						// which always works, since we checked for uniformat bit length
+						// before
+						!disputed_bits.0[core_idx]
+					} else {
+						// bit wasn't set in the first place
+						false
+					};
+				});
+		}
+
 		bitfields.push((checked_bitfield, validator_index));
 	}
-	bitfields
+	Ok(bitfields)
 }
 
-/// Filter candidates based on the concluded disputes.
-fn filter_backed_candidates<T: Config>(
+/// Filter out any candidates, that have a concluded invalid dispute.
+///
+/// `scheduled` follows the same naming scheme as provided in the
+/// guide: Currently `free` but might become `occupied`.
+/// For the filtering here the relevant part is only the current `free`
+/// state.
+fn sanitize_backed_candidates<T: Config + crate::paras_inherent::Config, const EARLY_RETURN: bool>(
 	relay_parent: T::Hash,
 	mut backed_candidates: Vec<BackedCandidate<T::Hash>>,
 	disputed_candidates: BTreeSet<CandidateHash>,
-) -> Vec<BackedCandidate<T::Hash>> {
+	scheduled: &[CoreAssignment],
+) -> Result<Vec<BackedCandidate<T::Hash>>, Error::<T>> {
+	let n = backed_candidates.len();
 	// Remove any candidates that were concluded invalid.
 	backed_candidates.retain(|backed_candidate| {
 		let candidate_hash = backed_candidate.candidate.hash();
 		!disputed_candidates.contains(&candidate_hash)
 	});
-	// Remove candidates referencing a different relay parent.
-	backed_candidates
-		.retain(|backed_candidate| backed_candidate.descriptor().relay_parent == relay_parent);
-	//
+	ensure2!(backed_candidates.len() == n, Error::<T>::CandidateConcludedInvalid, EARLY_RETURN);
+
+	// Assure the backed candidate's `ParaId`'s core is free.
+	// This holds under the assumption that `Scheduler::schedule` is called _before_.
+	// Also checks the candidate references the correct relay parent.
+	backed_candidates.retain(|backed_candidate| {
+		let desc = backed_candidate.descriptor();
+		desc.relay_parent == relay_parent
+		&& scheduled.iter().any(|core| core.para_id == desc.para_id)
+	});
+	ensure2!(backed_candidates.len() == n, Error::<T>::CandidateConcludedInvalid, EARLY_RETURN);
+
 	// Limit weight, to avoid overweight block.
-	limit_backed_candidates::<T>(backed_candidates)
+	Ok(backed_candidates)
 }
 
 /// Limit the number of backed candidates processed in order to stay within block weight limits.
@@ -540,16 +590,6 @@ mod tests {
 		}
 	}
 
-	mod filter_bitfields {
-		use super::*;
-		// TODO
-	}
-
-	mod filter_candidates {
-		use super::*;
-		// TODO
-	}
-
 	mod paras_inherent_weight {
 		use super::*;
 

From 702bceb162555936842b89d488b9fe68012408d1 Mon Sep 17 00:00:00 2001
From: Bernhard Schuster <bernhard@ahoi.io>
Date: Wed, 13 Oct 2021 15:02:24 +0200
Subject: [PATCH 004/107] minor

---
 runtime/parachains/src/inclusion.rs      | 12 +++++++-----
 runtime/parachains/src/mock.rs           |  2 +-
 runtime/parachains/src/paras_inherent.rs |  4 ++--
 3 files changed, 10 insertions(+), 8 deletions(-)

diff --git a/runtime/parachains/src/inclusion.rs b/runtime/parachains/src/inclusion.rs
index e6f12975fe06..d80128399714 100644
--- a/runtime/parachains/src/inclusion.rs
+++ b/runtime/parachains/src/inclusion.rs
@@ -927,8 +927,8 @@ impl<T: Config> CandidateCheckContext<T> {
 #[cfg(test)]
 mod tests {
 	use super::*;
-
 	use crate::{
+		paras_inherent::{sanitize_bitfields, DisputedBitfield},
 		configuration::HostConfiguration,
 		initializer::SessionChangeNotification,
 		mock::{
@@ -1310,13 +1310,15 @@ mod tests {
 					&signing_context,
 				));
 
-				assert_eq!(ParaInclusion::process_bitfields(
-					expected_bits(),
+				assert_eq!(sanitize_bitfields::<Test, true>(
 					vec![signed.into()],
-					&core_lookup,
+					DisputedBitfield::default(),
+					expected_bits(),
+					System::parent_hash(),
+					shared::Pallet::<Test>::session_index(),
 					&validator_public,
 				),
-				Err(Error::WrongBitfieldSize));
+				Err(Error::<T>::WrongBitfieldSize.into()));
 			}
 
 			// wrong number of bits: other way around.
diff --git a/runtime/parachains/src/mock.rs b/runtime/parachains/src/mock.rs
index a5b58cc54b00..f5b7076f41d8 100644
--- a/runtime/parachains/src/mock.rs
+++ b/runtime/parachains/src/mock.rs
@@ -203,7 +203,7 @@ impl crate::inclusion::Config for Test {
 	type RewardValidators = TestRewardValidators;
 }
 
-impl crate::paras_inherent::Config for Test {}
+impl crate::paras_inherent::pallet::Config for Test {}
 
 impl crate::session_info::Config for Test {}
 
diff --git a/runtime/parachains/src/paras_inherent.rs b/runtime/parachains/src/paras_inherent.rs
index 0aa7d86d044d..673f974468df 100644
--- a/runtime/parachains/src/paras_inherent.rs
+++ b/runtime/parachains/src/paras_inherent.rs
@@ -55,7 +55,7 @@ const MINIMAL_INCLUSION_INHERENT_WEIGHT: Weight = INCLUSION_INHERENT_CLAIMED_WEI
 /// A bitfield concering concluded disputes for candidates
 /// associated to the core index equivalent to the bit position.
 #[derive(Default, PartialEq, Eq, Clone, Encode, Decode, RuntimeDebug, TypeInfo)]
-pub struct DisputedBitfield(pub BitVec<bitvec::order::Lsb0, u8>);
+pub(crate) struct DisputedBitfield(pub(crate) BitVec<bitvec::order::Lsb0, u8>);
 
 impl From<BitVec<bitvec::order::Lsb0, u8>> for DisputedBitfield {
 	fn from(inner: BitVec<bitvec::order::Lsb0, u8>) -> Self {
@@ -369,7 +369,7 @@ macro_rules! ensure2 {
 ///  5. remove any disputed core indices
 ///
 /// If any of those is not passed, the bitfield is dropped.
-fn sanitize_bitfields<T: Config + crate::inclusion::Config, const EARLY_RETURN: bool>(
+pub(crate) fn sanitize_bitfields<T: Config + crate::inclusion::Config, const EARLY_RETURN: bool>(
 	unchecked_bitfields: UncheckedSignedAvailabilityBitfields,
 	disputed_bits: DisputedBitfield,
 	expected_bits: usize,

From de29623fdeb36a9aaeb2b34c5366b0c007b992da Mon Sep 17 00:00:00 2001
From: Bernhard Schuster <bernhard@ahoi.io>
Date: Wed, 13 Oct 2021 15:25:34 +0200
Subject: [PATCH 005/107] assure tests still work, reduce changeset

---
 runtime/parachains/src/inclusion.rs      | 158 +++++++++++++----------
 runtime/parachains/src/paras_inherent.rs |  78 ++++++-----
 2 files changed, 128 insertions(+), 108 deletions(-)

diff --git a/runtime/parachains/src/inclusion.rs b/runtime/parachains/src/inclusion.rs
index d80128399714..1226eb853f26 100644
--- a/runtime/parachains/src/inclusion.rs
+++ b/runtime/parachains/src/inclusion.rs
@@ -20,14 +20,20 @@
 //! It is responsible for carrying candidates from being backable to being backed, and then from backed
 //! to included.
 
+use crate::{
+	configuration, disputes, dmp, hrmp, paras,
+	paras_inherent::{sanitize_bitfields, DisputedBitfield},
+	scheduler::CoreAssignment,
+	shared, ump,
+};
 use bitvec::{order::Lsb0 as BitOrderLsb0, vec::BitVec};
 use frame_support::pallet_prelude::*;
 use parity_scale_codec::{Decode, Encode};
 use primitives::v1::{
 	AvailabilityBitfield, BackedCandidate, CandidateCommitments, CandidateDescriptor,
 	CandidateHash, CandidateReceipt, CommittedCandidateReceipt, CoreIndex, GroupIndex, Hash,
-	HeadData, Id as ParaId, SigningContext, ValidatorId,
-	ValidatorIndex, ValidityAttestation,
+	HeadData, Id as ParaId, SigningContext, UncheckedSignedAvailabilityBitfields, ValidatorIndex,
+	ValidityAttestation,
 };
 use scale_info::TypeInfo;
 use sp_runtime::{
@@ -36,8 +42,6 @@ use sp_runtime::{
 };
 use sp_std::{collections::btree_set::BTreeSet, prelude::*};
 
-use crate::{configuration, disputes, dmp, hrmp, paras, scheduler::CoreAssignment, shared, ump};
-
 pub use pallet::*;
 
 /// A bitfield signed by a validator indicating that it is keeping its piece of the erasure-coding
@@ -261,10 +265,23 @@ impl<T: Config> Pallet<T> {
 	/// and cores free.
 	pub(crate) fn process_bitfields(
 		expected_bits: usize,
-		checked_bitfields: Vec<(AvailabilityBitfield, ValidatorIndex)>,
+		signed_bitfields: UncheckedSignedAvailabilityBitfields,
+		disputed_bits: DisputedBitfield,
 		core_lookup: impl Fn(CoreIndex) -> Option<ParaId>,
-		validators: &[ValidatorId],
-	) -> Vec<(CoreIndex, CandidateHash)> {
+	) -> Result<Vec<(CoreIndex, CandidateHash)>, DispatchError> {
+		let validators = shared::Pallet::<T>::active_validator_keys();
+		let session_index = shared::Pallet::<T>::session_index();
+		let parent_hash = frame_system::Pallet::<T>::parent_hash();
+
+		let checked_bitfields = sanitize_bitfields::<T, true>(
+			signed_bitfields,
+			disputed_bits,
+			expected_bits,
+			parent_hash,
+			session_index,
+			&validators[..],
+		)?;
+
 		let mut assigned_paras_record = (0..expected_bits)
 			.map(|bit_index| core_lookup(CoreIndex::from(bit_index as u32)))
 			.map(|opt_para_id| {
@@ -274,9 +291,7 @@ impl<T: Config> Pallet<T> {
 
 		let now = <frame_system::Pallet<T>>::block_number();
 		for (checked_bitfield, validator_index) in checked_bitfields {
-			for (bit_idx, _) in
-				checked_bitfield.0.iter().enumerate().filter(|(_, is_av)| **is_av)
-			{
+			for (bit_idx, _) in checked_bitfield.0.iter().enumerate().filter(|(_, is_av)| **is_av) {
 				let pending_availability = if let Some((_, pending_availability)) =
 					assigned_paras_record[bit_idx].as_mut()
 				{
@@ -300,10 +315,8 @@ impl<T: Config> Pallet<T> {
 				}
 			}
 
-			let record = AvailabilityBitfieldRecord {
-				bitfield: checked_bitfield,
-				submitted_at: now,
-			};
+			let record =
+				AvailabilityBitfieldRecord { bitfield: checked_bitfield, submitted_at: now };
 
 			<AvailabilityBitfields<T>>::insert(&validator_index, record);
 		}
@@ -349,7 +362,7 @@ impl<T: Config> Pallet<T> {
 			}
 		}
 
-		freed_cores
+		Ok(freed_cores)
 	}
 
 	/// Process candidates that have been backed. Provide the relay storage root, a set of candidates
@@ -928,7 +941,6 @@ impl<T: Config> CandidateCheckContext<T> {
 mod tests {
 	use super::*;
 	use crate::{
-		paras_inherent::{sanitize_bitfields, DisputedBitfield},
 		configuration::HostConfiguration,
 		initializer::SessionChangeNotification,
 		mock::{
@@ -936,6 +948,7 @@ mod tests {
 			System, Test,
 		},
 		paras::ParaGenesisArgs,
+		paras_inherent::{sanitize_bitfields, DisputedBitfield},
 		scheduler::AssignmentKind,
 	};
 	use futures::executor::block_on;
@@ -1310,15 +1323,17 @@ mod tests {
 					&signing_context,
 				));
 
-				assert_eq!(sanitize_bitfields::<Test, true>(
-					vec![signed.into()],
-					DisputedBitfield::default(),
-					expected_bits(),
-					System::parent_hash(),
-					shared::Pallet::<Test>::session_index(),
-					&validator_public,
-				),
-				Err(Error::<T>::WrongBitfieldSize.into()));
+				assert_eq!(
+					sanitize_bitfields::<Test, true>(
+						vec![signed.into()],
+						DisputedBitfield::default(),
+						expected_bits(),
+						System::parent_hash(),
+						shared::Pallet::<Test>::session_index(),
+						&validator_public,
+					),
+					Err(Error::<Test>::WrongBitfieldSize.into())
+				);
 			}
 
 			// wrong number of bits: other way around.
@@ -1332,13 +1347,17 @@ mod tests {
 					&signing_context,
 				));
 
-				assert_eq!(ParaInclusion::process_bitfields(
-					expected_bits() + 1,
-					vec![signed.into()],
-					&core_lookup,
-					&validator_public,
-				),
-				Err(Error::WrongBitfieldSize));
+				assert_eq!(
+					sanitize_bitfields::<Test, true>(
+						vec![signed.into()],
+						DisputedBitfield::default(),
+						expected_bits() + 1,
+						System::parent_hash(),
+						shared::Pallet::<Test>::session_index(),
+						&validator_public,
+					),
+					Err(Error::<Test>::WrongBitfieldSize.into())
+				);
 			}
 
 			// duplicate.
@@ -1353,12 +1372,15 @@ mod tests {
 				))
 				.into();
 
-				assert_eq!(ParaInclusion::process_bitfields(
-					expected_bits(),
-					vec![signed.clone(), signed],
-					&core_lookup,
-					&validators_public,
-				), Err(Error::BitfieldDuplicateOrUnordered));
+				assert_eq!(
+					ParaInclusion::process_bitfields(
+						expected_bits(),
+						vec![signed.clone(), signed],
+						DisputedBitfield::default(),
+						&core_lookup,
+					),
+					Err(Error::<Test>::BitfieldDuplicateOrUnordered.into())
+				);
 			}
 
 			// out of order.
@@ -1382,13 +1404,15 @@ mod tests {
 				))
 				.into();
 
-				assert_eq!(ParaInclusion::process_bitfields(
-					expected_bits(),
-					vec![signed_1, signed_0],
-					&core_lookup,
-					&validators_public,
-				),
-				Err(Error::BitfieldDuplicateOrUnordered));
+				assert_eq!(
+					ParaInclusion::process_bitfields(
+						expected_bits(),
+						vec![signed_1, signed_0],
+						DisputedBitfield::default(),
+						&core_lookup,
+					),
+					Err(Error::<Test>::BitfieldDuplicateOrUnordered.into())
+				);
 			}
 
 			// non-pending bit set.
@@ -1402,14 +1426,14 @@ mod tests {
 					bare_bitfield,
 					&signing_context,
 				));
-				assert_eq!(
-					ParaInclusion::process_bitfields(
-						expected_bits(),
-						vec![signed.into()],
-						&core_lookup,
-					),
-					Ok(vec![])
-				);
+
+				assert!(ParaInclusion::process_bitfields(
+					expected_bits(),
+					vec![signed.into()],
+					DisputedBitfield::default(),
+					&core_lookup,
+				)
+				.is_ok());
 			}
 
 			// empty bitfield signed: always OK, but kind of useless.
@@ -1423,12 +1447,13 @@ mod tests {
 					&signing_context,
 				));
 
-				assert!(ParaInclusion::process_bitfields(
+				assert!(!ParaInclusion::process_bitfields(
 					expected_bits(),
 					vec![signed.into()],
+					DisputedBitfield::default(),
 					&core_lookup,
-					&validators_public,
-				).is_ok());
+				)
+				.is_ok());
 			}
 
 			// bitfield signed with pending bit signed.
@@ -1468,9 +1493,10 @@ mod tests {
 				assert!(ParaInclusion::process_bitfields(
 					expected_bits(),
 					vec![signed.into()],
+					DisputedBitfield::default(),
 					&core_lookup,
-					&validators_public,
-				).is_ok());
+				)
+				.is_ok());
 
 				<PendingAvailability<Test>>::remove(chain_a);
 				PendingAvailabilityCommitments::<Test>::remove(chain_a);
@@ -1507,14 +1533,13 @@ mod tests {
 				));
 
 				// no core is freed
-				assert_eq!(
-					ParaInclusion::process_bitfields(
-						expected_bits(),
-						vec![signed.into()],
-						&core_lookup,
-					),
-					Ok(vec![]),
-				);
+				assert!(ParaInclusion::process_bitfields(
+					expected_bits(),
+					vec![signed.into()],
+					DisputedBitfield::default(),
+					&core_lookup,
+				)
+				.is_ok());
 			}
 		});
 	}
@@ -1653,6 +1678,7 @@ mod tests {
 			assert!(ParaInclusion::process_bitfields(
 				expected_bits(),
 				signed_bitfields,
+				DisputedBitfield::default(),
 				&core_lookup,
 			)
 			.is_ok());
diff --git a/runtime/parachains/src/paras_inherent.rs b/runtime/parachains/src/paras_inherent.rs
index 673f974468df..62c0c2f15de9 100644
--- a/runtime/parachains/src/paras_inherent.rs
+++ b/runtime/parachains/src/paras_inherent.rs
@@ -21,19 +21,24 @@
 //! as it has no initialization logic and its finalization logic depends only on the details of
 //! this module.
 
-use crate::{configuration::Config, disputes::DisputesHandler, inclusion, scheduler::{self, CoreAssignment, FreedReason}, shared, ump};
+use crate::{
+	configuration::Config,
+	disputes::DisputesHandler,
+	inclusion,
+	scheduler::{self, CoreAssignment, FreedReason},
+	shared, ump,
+};
 use bitvec::prelude::BitVec;
 use frame_support::{
+	fail,
 	inherent::{InherentData, InherentIdentifier, MakeFatalError, ProvideInherent},
 	pallet_prelude::*,
-	fail,
 };
 use frame_system::pallet_prelude::*;
 use primitives::v1::{
-	AvailabilityBitfield, ValidatorIndex,
-	BackedCandidate, CandidateHash, CoreIndex,
-	InherentData as ParachainsInherentData, ScrapedOnChainVotes, SessionIndex,
-	SigningContext, UncheckedSignedAvailabilityBitfields, ValidatorId,
+	AvailabilityBitfield, BackedCandidate, CandidateHash, CoreIndex,
+	InherentData as ParachainsInherentData, ScrapedOnChainVotes, SessionIndex, SigningContext,
+	UncheckedSignedAvailabilityBitfields, ValidatorId, ValidatorIndex,
 	PARACHAINS_INHERENT_IDENTIFIER,
 };
 use scale_info::TypeInfo;
@@ -255,25 +260,14 @@ pub mod pallet {
 				(disputed_bitfield, concluded_invalid_disputed_candidates)
 			};
 
-			let validators = shared::Pallet::<T>::active_validator_keys();
-
-			let checked_bitfields = sanitize_bitfields::<T, true>(
-				signed_bitfields,
-				disputed_bits,
-				expected_bits,
-				parent_hash,
-				current_session,
-				&validators,
-			)?;
-
 			// Process new availability bitfields, yielding any availability cores whose
 			// work has now concluded.
 			let freed_concluded = <inclusion::Pallet<T>>::process_bitfields(
 				expected_bits,
-				checked_bitfields,
+				signed_bitfields,
+				disputed_bits,
 				<scheduler::Pallet<T>>::core_para,
-				&validators,
-			);
+			)?;
 
 			// Inform the disputes module of all included candidates.
 			let now = <frame_system::Pallet<T>>::block_number();
@@ -381,7 +375,7 @@ pub(crate) fn sanitize_bitfields<T: Config + crate::inclusion::Config, const EAR
 
 	let mut last_index = None;
 
-	if EARLY_RETURN && disputed_bits.0.len() != expected_bits{
+	if EARLY_RETURN && disputed_bits.0.len() != expected_bits {
 		return Ok(Default::default())
 	}
 
@@ -395,9 +389,8 @@ pub(crate) fn sanitize_bitfields<T: Config + crate::inclusion::Config, const EAR
 		);
 
 		ensure2!(
-			last_index
-				.map_or(true, |last| last < unchecked_bitfield.unchecked_validator_index()),
-				crate::inclusion::pallet::Error::<T>::BitfieldDuplicateOrUnordered,
+			last_index.map_or(true, |last| last < unchecked_bitfield.unchecked_validator_index()),
+			crate::inclusion::pallet::Error::<T>::BitfieldDuplicateOrUnordered,
 			EARLY_RETURN
 		);
 
@@ -418,7 +411,7 @@ pub(crate) fn sanitize_bitfields<T: Config + crate::inclusion::Config, const EAR
 		} else if EARLY_RETURN {
 			fail!(crate::inclusion::pallet::Error::<T>::InvalidBitfieldSignature);
 		} else {
-			continue;
+			continue
 		};
 
 		last_index = Some(validator_index);
@@ -427,19 +420,17 @@ pub(crate) fn sanitize_bitfields<T: Config + crate::inclusion::Config, const EAR
 
 		// filter the bitfields only
 		if !EARLY_RETURN {
-			checked_bitfield.0.iter_mut()
-				.enumerate()
-				.for_each(|(core_idx, mut bit)| {
-					*bit = if *bit {
-						// ignore those that have matching invalid dispute bits
-						// which always works, since we checked for uniformat bit length
-						// before
-						!disputed_bits.0[core_idx]
-					} else {
-						// bit wasn't set in the first place
-						false
-					};
-				});
+			checked_bitfield.0.iter_mut().enumerate().for_each(|(core_idx, mut bit)| {
+				*bit = if *bit {
+					// ignore those that have matching invalid dispute bits
+					// which always works, since we checked for uniformat bit length
+					// before
+					!disputed_bits.0[core_idx]
+				} else {
+					// bit wasn't set in the first place
+					false
+				};
+			});
 		}
 
 		bitfields.push((checked_bitfield, validator_index));
@@ -453,12 +444,15 @@ pub(crate) fn sanitize_bitfields<T: Config + crate::inclusion::Config, const EAR
 /// guide: Currently `free` but might become `occupied`.
 /// For the filtering here the relevant part is only the current `free`
 /// state.
-fn sanitize_backed_candidates<T: Config + crate::paras_inherent::Config, const EARLY_RETURN: bool>(
+fn sanitize_backed_candidates<
+	T: Config + crate::paras_inherent::Config,
+	const EARLY_RETURN: bool,
+>(
 	relay_parent: T::Hash,
 	mut backed_candidates: Vec<BackedCandidate<T::Hash>>,
 	disputed_candidates: BTreeSet<CandidateHash>,
 	scheduled: &[CoreAssignment],
-) -> Result<Vec<BackedCandidate<T::Hash>>, Error::<T>> {
+) -> Result<Vec<BackedCandidate<T::Hash>>, Error<T>> {
 	let n = backed_candidates.len();
 	// Remove any candidates that were concluded invalid.
 	backed_candidates.retain(|backed_candidate| {
@@ -472,8 +466,8 @@ fn sanitize_backed_candidates<T: Config + crate::paras_inherent::Config, const E
 	// Also checks the candidate references the correct relay parent.
 	backed_candidates.retain(|backed_candidate| {
 		let desc = backed_candidate.descriptor();
-		desc.relay_parent == relay_parent
-		&& scheduled.iter().any(|core| core.para_id == desc.para_id)
+		desc.relay_parent == relay_parent &&
+			scheduled.iter().any(|core| core.para_id == desc.para_id)
 	});
 	ensure2!(backed_candidates.len() == n, Error::<T>::CandidateConcludedInvalid, EARLY_RETURN);
 

From 98271ae279fac0de357acfe14eb016d2bb55bae8 Mon Sep 17 00:00:00 2001
From: Bernhard Schuster <bernhard@ahoi.io>
Date: Wed, 13 Oct 2021 16:06:30 +0200
Subject: [PATCH 006/107] integration

---
 runtime/parachains/src/inclusion.rs      | 16 +++----
 runtime/parachains/src/paras_inherent.rs | 54 +++++++++++++++++++-----
 2 files changed, 50 insertions(+), 20 deletions(-)

diff --git a/runtime/parachains/src/inclusion.rs b/runtime/parachains/src/inclusion.rs
index 1226eb853f26..81c5615dc254 100644
--- a/runtime/parachains/src/inclusion.rs
+++ b/runtime/parachains/src/inclusion.rs
@@ -1324,13 +1324,11 @@ mod tests {
 				));
 
 				assert_eq!(
-					sanitize_bitfields::<Test, true>(
+					ParaInclusion::process_bitfields(
+						expected_bits(),
 						vec![signed.into()],
 						DisputedBitfield::default(),
-						expected_bits(),
-						System::parent_hash(),
-						shared::Pallet::<Test>::session_index(),
-						&validator_public,
+						&core_lookup,
 					),
 					Err(Error::<Test>::WrongBitfieldSize.into())
 				);
@@ -1348,13 +1346,11 @@ mod tests {
 				));
 
 				assert_eq!(
-					sanitize_bitfields::<Test, true>(
+					ParaInclusion::process_bitfields(
+						expected_bits() + 1,
 						vec![signed.into()],
 						DisputedBitfield::default(),
-						expected_bits() + 1,
-						System::parent_hash(),
-						shared::Pallet::<Test>::session_index(),
-						&validator_public,
+						&core_lookup,
 					),
 					Err(Error::<Test>::WrongBitfieldSize.into())
 				);
diff --git a/runtime/parachains/src/paras_inherent.rs b/runtime/parachains/src/paras_inherent.rs
index 62c0c2f15de9..2fa355c05f09 100644
--- a/runtime/parachains/src/paras_inherent.rs
+++ b/runtime/parachains/src/paras_inherent.rs
@@ -125,20 +125,54 @@ pub mod pallet {
 		const INHERENT_IDENTIFIER: InherentIdentifier = PARACHAINS_INHERENT_IDENTIFIER;
 
 		fn create_inherent(data: &InherentData) -> Option<Self::Call> {
-			let mut inherent_data: ParachainsInherentData<T::Header> =
-				match data.get_data(&Self::INHERENT_IDENTIFIER) {
-					Ok(Some(d)) => d,
-					Ok(None) => return None,
-					Err(_) => {
-						log::warn!(target: LOG_TARGET, "ParachainsInherentData failed to decode");
-
-						return None
-					},
-				};
+			let ParachainsInherentData<T::Header> {
+				mut bitfields,
+				mut backed_candidates,
+				mut disputes,
+				mut parent_header,
+			} = match data.get_data(&Self::INHERENT_IDENTIFIER) {
+				Ok(Some(d)) => d,
+				Ok(None) => return None,
+				Err(_) => {
+					log::warn!(target: LOG_TARGET, "ParachainsInherentData failed to decode");
+
+					return None
+				},
+			};
+
 
 			// filter out any unneeded dispute statements
 			T::DisputesHandler::filter_multi_dispute_data(&mut inherent_data.disputes);
 
+			// sanitize the bitfields and candidates by removing
+			// anything that does not pass a set of checks
+			// will be removed here
+			let validators = shared::Pallet::<T>::active_validator_keys();
+			let parent_hash = <frame_system::Pallet<T>>::parent_hash();
+			let current_session = <shared::Pallet<T>>::session_index();
+
+			let expected_bits = unimplemented!("source?");
+			// FIXME the return value is inapropriate for further usage
+			// TODO move to index return value and do not consume the input
+			let bitfields = sanitize_bitfields::<T, false>(
+				bitfields,
+				DisputedBitfield::default(), // TODO FIXME
+				expected_bits,
+				parent_hash,
+				current_session,
+				&validator_public[..],
+			).ok()?;
+
+			let scheduled = unimplemented!();
+			let disputed_candidates = unimplemented!("");
+			let backed_candidates = sanitize_backed_candidates::<T, false>(
+				parent_hash,
+				backed_candidates,
+				disputed_candidates,
+				&scheduled[..],
+			).ok()?;
+
+
 			// Sanity check: session changes can invalidate an inherent, and we _really_ don't want that to happen.
 			// See <https://github.com/paritytech/polkadot/issues/1327>
 			let inherent_data =

From f7003e839e641705ee7b65da4b1b1fea11b1a668 Mon Sep 17 00:00:00 2001
From: Bernhard Schuster <bernhard@ahoi.io>
Date: Wed, 13 Oct 2021 20:53:57 +0200
Subject: [PATCH 007/107] start entropy passing

---
 Cargo.lock                               | 13 +++++---
 node/core/parachains-inherent/src/lib.rs |  3 +-
 primitives/Cargo.toml                    |  7 ++++-
 primitives/src/v1/mod.rs                 | 40 ++++++++++++++++++++++++
 runtime/parachains/src/paras_inherent.rs | 20 ++++++++----
 5 files changed, 70 insertions(+), 13 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index b3830c482325..798e52cf7a2c 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -68,7 +68,7 @@ version = "0.7.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "43bb833f0bf979d8475d38fbf09ed3b8a55e1885fe93ad3f93239fc6a4f17b98"
 dependencies = [
- "getrandom 0.2.1",
+ "getrandom 0.2.3",
  "once_cell",
  "version_check",
 ]
@@ -2432,9 +2432,9 @@ dependencies = [
 
 [[package]]
 name = "getrandom"
-version = "0.2.1"
+version = "0.2.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4060f4657be78b8e766215b02b18a2e862d83745545de804638e2b545e81aee6"
+checksum = "7fcd999463524c52659517fe2cea98493cfe485d10565e7b0fb07dbba7ad2753"
 dependencies = [
  "cfg-if 1.0.0",
  "libc",
@@ -6624,10 +6624,13 @@ dependencies = [
  "bitvec 0.20.1",
  "frame-system",
  "hex-literal",
+ "lazy_static",
  "parity-scale-codec",
  "parity-util-mem",
  "polkadot-core-primitives",
  "polkadot-parachain",
+ "rand 0.8.4",
+ "rand_chacha 0.3.1",
  "scale-info",
  "serde",
  "sp-api",
@@ -7594,7 +7597,7 @@ version = "0.6.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c026d7df8b298d90ccbbc5190bd04d85e159eaf5576caeacf8741da93ccbd2e5"
 dependencies = [
- "getrandom 0.2.1",
+ "getrandom 0.2.3",
 ]
 
 [[package]]
@@ -7686,7 +7689,7 @@ version = "0.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "528532f3d801c87aec9def2add9ca802fe569e44a544afe633765267840abe64"
 dependencies = [
- "getrandom 0.2.1",
+ "getrandom 0.2.3",
  "redox_syscall 0.2.4",
 ]
 
diff --git a/node/core/parachains-inherent/src/lib.rs b/node/core/parachains-inherent/src/lib.rs
index 87b4f6f247f3..743ffede8ce7 100644
--- a/node/core/parachains-inherent/src/lib.rs
+++ b/node/core/parachains-inherent/src/lib.rs
@@ -28,7 +28,7 @@ use futures::{select, FutureExt};
 use polkadot_node_subsystem::{
 	errors::SubsystemError, messages::ProvisionerMessage, overseer::Handle,
 };
-use polkadot_primitives::v1::{Block, Hash, InherentData as ParachainsInherentData};
+use polkadot_primitives::v1::{Block, SeedEntropy, Hash, InherentData as ParachainsInherentData};
 use sp_blockchain::HeaderBackend;
 use sp_runtime::generic::BlockId;
 use std::time;
@@ -97,6 +97,7 @@ impl ParachainsInherentDataProvider {
 					backed_candidates: Vec::new(),
 					disputes: Vec::new(),
 					parent_header,
+					entropy: SeedEntropy::draw(),
 				}
 			},
 		};
diff --git a/primitives/Cargo.toml b/primitives/Cargo.toml
index 4aa5d00a3a7c..5162667c1de8 100644
--- a/primitives/Cargo.toml
+++ b/primitives/Cargo.toml
@@ -28,7 +28,9 @@ bitvec = { version = "0.20.1", default-features = false, features = ["alloc"] }
 frame-system = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false }
 hex-literal = "0.3.3"
 parity-util-mem = { version = "0.10.0", default-features = false, optional = true }
-
+rand = { version = "0.8.4", optional = true }
+rand_chacha = { version = "0.3.1", optional = true }
+lazy_static = { version = "1.4", optional = true }
 
 [features]
 default = ["std"]
@@ -55,4 +57,7 @@ std = [
 	"polkadot-core-primitives/std",
 	"bitvec/std",
 	"frame-system/std",
+	"rand_chacha",
+	"rand/std",
+	"lazy_static",
 ]
diff --git a/primitives/src/v1/mod.rs b/primitives/src/v1/mod.rs
index cbea6480efef..0d89cb0104f1 100644
--- a/primitives/src/v1/mod.rs
+++ b/primitives/src/v1/mod.rs
@@ -1344,6 +1344,43 @@ pub struct DisputeState<N = BlockNumber> {
 	pub concluded_at: Option<N>,
 }
 
+/// Type abstraction to provide entropy for seeding a `crng`
+#[derive(Clone, Encode, Decode, PartialEq, RuntimeDebug, TypeInfo)]
+pub struct SeedEntropy(pub [u8; 16]);
+
+impl AsRef<[u8]> for SeedEntropy {
+	fn as_ref(&self) -> &[u8] {
+		&self.0[..]
+	}
+}
+
+#[cfg(feature = "std")]
+impl SeedEntropy {
+	/// Collect a random entropy.
+	pub fn draw() -> Self {
+		let mut bytes = [0u8; 16];
+
+		use rand::RngCore;
+
+		#[cfg(not(fuzzing))]
+		{
+
+			let mut rng = rand::thread_rng();
+			rng.fill_bytes(&mut bytes[..]);
+		}
+
+		#[cfg(fuzzing)]
+		{
+			lazy_static::lazy_static!{
+				static ref RNG: rand_chacha::ChaChaRng = rand_chacha::ChaChaRng::from_seed(b"polkadot_on_chain_selection_seed");
+			}
+			RNG.fill_bytes(&mut bytes[..])
+		}
+
+
+		Self(bytes)
+	}
+}
 /// Parachains inherent-data passed into the runtime by a block author
 #[derive(Encode, Decode, Clone, PartialEq, RuntimeDebug, TypeInfo)]
 pub struct InherentData<HDR: HeaderT = Header> {
@@ -1355,6 +1392,9 @@ pub struct InherentData<HDR: HeaderT = Header> {
 	pub disputes: MultiDisputeStatementSet,
 	/// The parent block header. Used for checking state proofs.
 	pub parent_header: HDR,
+	/// Entropy source to initialize a `cprng`, for usage
+	/// in the decimation process of overweight blocks.
+	pub entropy: SeedEntropy,
 }
 
 /// The maximum number of validators `f` which may safely be faulty.
diff --git a/runtime/parachains/src/paras_inherent.rs b/runtime/parachains/src/paras_inherent.rs
index 2fa355c05f09..d26a7d8730c5 100644
--- a/runtime/parachains/src/paras_inherent.rs
+++ b/runtime/parachains/src/paras_inherent.rs
@@ -125,11 +125,12 @@ pub mod pallet {
 		const INHERENT_IDENTIFIER: InherentIdentifier = PARACHAINS_INHERENT_IDENTIFIER;
 
 		fn create_inherent(data: &InherentData) -> Option<Self::Call> {
-			let ParachainsInherentData<T::Header> {
-				mut bitfields,
-				mut backed_candidates,
-				mut disputes,
-				mut parent_header,
+			let ParachainsInherentData::<T::Header> {
+				bitfields,
+				backed_candidates,
+				disputes,
+				parent_header,
+				entropy,
 			} = match data.get_data(&Self::INHERENT_IDENTIFIER) {
 				Ok(Some(d)) => d,
 				Ok(None) => return None,
@@ -142,7 +143,7 @@ pub mod pallet {
 
 
 			// filter out any unneeded dispute statements
-			T::DisputesHandler::filter_multi_dispute_data(&mut inherent_data.disputes);
+			T::DisputesHandler::filter_multi_dispute_data(&mut disputes);
 
 			// sanitize the bitfields and candidates by removing
 			// anything that does not pass a set of checks
@@ -172,6 +173,13 @@ pub mod pallet {
 				&scheduled[..],
 			).ok()?;
 
+			let inherent_data = ParachainsInherentData::<T::Header> {
+				bitfields,
+				backed_candidates,
+				disputes,
+				parent_header,
+				entropy,
+			};
 
 			// Sanity check: session changes can invalidate an inherent, and we _really_ don't want that to happen.
 			// See <https://github.com/paritytech/polkadot/issues/1327>

From eb27dbb7daba7422e07002f7acb232c4ab365bb0 Mon Sep 17 00:00:00 2001
From: Bernhard Schuster <bernhard@ahoi.io>
Date: Thu, 14 Oct 2021 12:03:42 +0200
Subject: [PATCH 008/107] fixins

---
 node/core/parachains-inherent/src/lib.rs |  1 +
 primitives/src/v1/mod.rs                 | 14 +++--
 primitives/src/v1/signed.rs              |  7 +++
 runtime/parachains/Cargo.toml            |  2 +-
 runtime/parachains/src/paras_inherent.rs | 70 +++++++++++++++---------
 5 files changed, 63 insertions(+), 31 deletions(-)

diff --git a/node/core/parachains-inherent/src/lib.rs b/node/core/parachains-inherent/src/lib.rs
index 743ffede8ce7..fddeb158ae0f 100644
--- a/node/core/parachains-inherent/src/lib.rs
+++ b/node/core/parachains-inherent/src/lib.rs
@@ -85,6 +85,7 @@ impl ParachainsInherentDataProvider {
 				bitfields: pd.bitfields.into_iter().map(Into::into).collect(),
 				backed_candidates: pd.backed_candidates,
 				disputes: pd.disputes,
+				seed: pd.entropy,
 				parent_header,
 			},
 			Err(err) => {
diff --git a/primitives/src/v1/mod.rs b/primitives/src/v1/mod.rs
index 0d89cb0104f1..e895cd3fc7ac 100644
--- a/primitives/src/v1/mod.rs
+++ b/primitives/src/v1/mod.rs
@@ -1345,8 +1345,8 @@ pub struct DisputeState<N = BlockNumber> {
 }
 
 /// Type abstraction to provide entropy for seeding a `crng`
-#[derive(Clone, Encode, Decode, PartialEq, RuntimeDebug, TypeInfo)]
-pub struct SeedEntropy(pub [u8; 16]);
+#[derive(Clone, Default, Encode, Decode, PartialEq, RuntimeDebug, TypeInfo)]
+pub struct SeedEntropy(pub [u8; 32]);
 
 impl AsRef<[u8]> for SeedEntropy {
 	fn as_ref(&self) -> &[u8] {
@@ -1354,17 +1354,22 @@ impl AsRef<[u8]> for SeedEntropy {
 	}
 }
 
+impl AsMut<[u8]> for SeedEntropy {
+	fn as_mut(&mut self) -> &mut [u8] {
+		&mut self.0[..]
+	}
+}
+
 #[cfg(feature = "std")]
 impl SeedEntropy {
 	/// Collect a random entropy.
 	pub fn draw() -> Self {
-		let mut bytes = [0u8; 16];
+		let mut bytes = [0u8; 32];
 
 		use rand::RngCore;
 
 		#[cfg(not(fuzzing))]
 		{
-
 			let mut rng = rand::thread_rng();
 			rng.fill_bytes(&mut bytes[..]);
 		}
@@ -1377,7 +1382,6 @@ impl SeedEntropy {
 			RNG.fill_bytes(&mut bytes[..])
 		}
 
-
 		Self(bytes)
 	}
 }
diff --git a/primitives/src/v1/signed.rs b/primitives/src/v1/signed.rs
index 612cc1516895..7edce76cbe72 100644
--- a/primitives/src/v1/signed.rs
+++ b/primitives/src/v1/signed.rs
@@ -41,6 +41,13 @@ use crate::v0::{SigningContext, ValidatorId, ValidatorIndex, ValidatorSignature}
 #[derive(Clone, PartialEq, Eq, RuntimeDebug)]
 pub struct Signed<Payload, RealPayload = Payload>(UncheckedSigned<Payload, RealPayload>);
 
+impl<Payload, RealPayload> Signed<Payload, RealPayload> {
+	/// Convert back to an unchecked type.
+	pub fn into_unchecked(self) -> UncheckedSigned<Payload, RealPayload> {
+		self.0
+	}
+}
+
 /// Unchecked signed data, can be converted to `Signed` by checking the signature.
 #[derive(Clone, PartialEq, Eq, RuntimeDebug, Encode, Decode, TypeInfo)]
 pub struct UncheckedSigned<Payload, RealPayload = Payload> {
diff --git a/runtime/parachains/Cargo.toml b/runtime/parachains/Cargo.toml
index c281269b8682..ddc223c85080 100644
--- a/runtime/parachains/Cargo.toml
+++ b/runtime/parachains/Cargo.toml
@@ -40,7 +40,7 @@ xcm-executor = { package = "xcm-executor", path = "../../xcm/xcm-executor", defa
 primitives = { package = "polkadot-primitives", path = "../../primitives", default-features = false }
 
 rand = { version = "0.8.3", default-features = false }
-rand_chacha = { version = "0.3.1", default-features = false }
+rand_chacha = { version = "0.3.1" }
 
 [dev-dependencies]
 futures = "0.3.17"
diff --git a/runtime/parachains/src/paras_inherent.rs b/runtime/parachains/src/paras_inherent.rs
index d26a7d8730c5..83ad791d654f 100644
--- a/runtime/parachains/src/paras_inherent.rs
+++ b/runtime/parachains/src/paras_inherent.rs
@@ -41,6 +41,7 @@ use primitives::v1::{
 	UncheckedSignedAvailabilityBitfields, ValidatorId, ValidatorIndex,
 	PARACHAINS_INHERENT_IDENTIFIER,
 };
+use rand::Rng;
 use scale_info::TypeInfo;
 use sp_runtime::traits::Header as HeaderT;
 use sp_std::{
@@ -148,13 +149,12 @@ pub mod pallet {
 			// sanitize the bitfields and candidates by removing
 			// anything that does not pass a set of checks
 			// will be removed here
-			let validators = shared::Pallet::<T>::active_validator_keys();
+			let validator_public = shared::Pallet::<T>::active_validator_keys();
 			let parent_hash = <frame_system::Pallet<T>>::parent_hash();
 			let current_session = <shared::Pallet<T>>::session_index();
 
 			let expected_bits = unimplemented!("source?");
-			// FIXME the return value is inapropriate for further usage
-			// TODO move to index return value and do not consume the input
+
 			let bitfields = sanitize_bitfields::<T, false>(
 				bitfields,
 				DisputedBitfield::default(), // TODO FIXME
@@ -164,8 +164,8 @@ pub mod pallet {
 				&validator_public[..],
 			).ok()?;
 
-			let scheduled = unimplemented!();
-			let disputed_candidates = unimplemented!("");
+			let scheduled: Vec<CoreAssignment> = unimplemented!();
+			let disputed_candidates: BTreeSet<CandidateHash> = unimplemented!("");
 			let backed_candidates = sanitize_backed_candidates::<T, false>(
 				parent_hash,
 				backed_candidates,
@@ -394,6 +394,39 @@ macro_rules! ensure2 {
 	};
 }
 
+/// Considers an upper threshold that the candidates must not exceed.
+///
+/// If there is sufficient space, all blocks will be included, otherwise
+/// continues adding blocks, ignoring blocks that exceed the threshold.
+fn pick_random_thresholded_subset(mut candidates: Vec<(CandidateHash, Weight)>, entropy: &EntropySeed, max_weight: Weight) -> (Weight, Vec<CandidateHash>) {
+	if max_weight < candidates.iter().map(|(_, weight)| { weight }).sum() {
+		return candidates.iter().map(|(candidate_hash, _)| candidate_hash).collect()
+	}
+
+	let mut candidates_acc = Vec::with_capacity(candidates.len());
+
+	// TODO use CurrentBlockRandomness as seed here
+	// TODO alt: use the pick index as subject to
+	// TODO obtain an index at the cost of having to impl
+	// TODO the equal probability distribution ourselves
+	let rng = chacha_rand::ChaCha20::from_seed(&entropy);
+	let mut weight_acc: Weight = 0 as _;
+
+	while weight_acc < max_weight || !candidates.empty() {
+		// select a index to try
+		let pick = rng.gen_range(0 .. candidates.len());
+		// remove the candidate from the possible pick set
+		let (picked_candidate, picked_weight) = candidates.swap_remove(pick);
+		// is the candidate small enough to fit
+		if picked_weight + weight_acc <= max_weight {
+			// candidate fits, so pick it and account for its weight
+			candidates_acc.push(picked_candidate);
+			weight_acc += picked_weight;
+		}
+	}
+	(weight_acc, candidates_acc)
+}
+
 /// Filter bitfields based on freed core indices, validity, and other sanity checks.
 ///
 /// Do sanity checks on the bitfields:
@@ -405,6 +438,10 @@ macro_rules! ensure2 {
 ///  5. remove any disputed core indices
 ///
 /// If any of those is not passed, the bitfield is dropped.
+///
+/// While this function technically returns a set of unchecked bitfields,
+/// they were actually checked and filtered to allow using it in both
+/// cases, as `filtering` and `checking` stage.
 pub(crate) fn sanitize_bitfields<T: Config + crate::inclusion::Config, const EARLY_RETURN: bool>(
 	unchecked_bitfields: UncheckedSignedAvailabilityBitfields,
 	disputed_bits: DisputedBitfield,
@@ -412,7 +449,7 @@ pub(crate) fn sanitize_bitfields<T: Config + crate::inclusion::Config, const EAR
 	parent_hash: T::Hash,
 	session_index: SessionIndex,
 	validators: &[ValidatorId],
-) -> Result<Vec<(AvailabilityBitfield, ValidatorIndex)>, DispatchError> {
+) -> Result<UncheckedSignedAvailabilityBitfields, DispatchError> {
 	let mut bitfields = Vec::with_capacity(unchecked_bitfields.len());
 
 	let mut last_index = None;
@@ -456,26 +493,9 @@ pub(crate) fn sanitize_bitfields<T: Config + crate::inclusion::Config, const EAR
 			continue
 		};
 
-		last_index = Some(validator_index);
+		bitfields.push(signed_bitfield.into_unchecked());
 
-		let mut checked_bitfield = signed_bitfield.into_payload();
-
-		// filter the bitfields only
-		if !EARLY_RETURN {
-			checked_bitfield.0.iter_mut().enumerate().for_each(|(core_idx, mut bit)| {
-				*bit = if *bit {
-					// ignore those that have matching invalid dispute bits
-					// which always works, since we checked for uniformat bit length
-					// before
-					!disputed_bits.0[core_idx]
-				} else {
-					// bit wasn't set in the first place
-					false
-				};
-			});
-		}
-
-		bitfields.push((checked_bitfield, validator_index));
+		last_index = Some(validator_index);
 	}
 	Ok(bitfields)
 }

From 0e5d2ff023e9319f7c78a48be1dece3f056dae17 Mon Sep 17 00:00:00 2001
From: Bernhard Schuster <bernhard@ahoi.io>
Date: Thu, 14 Oct 2021 16:30:11 +0200
Subject: [PATCH 009/107] compile, 1 failing test

---
 Cargo.lock                               |   1 +
 node/core/parachains-inherent/src/lib.rs |   2 +-
 primitives/src/v1/mod.rs                 |   8 +-
 runtime/parachains/Cargo.toml            |   2 +-
 runtime/parachains/src/inclusion.rs      |  75 ++++++++------
 runtime/parachains/src/paras_inherent.rs | 126 +++++++++++++++++------
 6 files changed, 148 insertions(+), 66 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 798e52cf7a2c..5df55a2415e2 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -6815,6 +6815,7 @@ dependencies = [
 name = "polkadot-runtime-parachains"
 version = "0.9.11"
 dependencies = [
+ "assert_matches",
  "bitflags",
  "bitvec 0.20.1",
  "derive_more",
diff --git a/node/core/parachains-inherent/src/lib.rs b/node/core/parachains-inherent/src/lib.rs
index fddeb158ae0f..2c4a23eebc4f 100644
--- a/node/core/parachains-inherent/src/lib.rs
+++ b/node/core/parachains-inherent/src/lib.rs
@@ -28,7 +28,7 @@ use futures::{select, FutureExt};
 use polkadot_node_subsystem::{
 	errors::SubsystemError, messages::ProvisionerMessage, overseer::Handle,
 };
-use polkadot_primitives::v1::{Block, SeedEntropy, Hash, InherentData as ParachainsInherentData};
+use polkadot_primitives::v1::{Block, Hash, InherentData as ParachainsInherentData, SeedEntropy};
 use sp_blockchain::HeaderBackend;
 use sp_runtime::generic::BlockId;
 use std::time;
diff --git a/primitives/src/v1/mod.rs b/primitives/src/v1/mod.rs
index e895cd3fc7ac..ec2a0ffbd706 100644
--- a/primitives/src/v1/mod.rs
+++ b/primitives/src/v1/mod.rs
@@ -1360,6 +1360,12 @@ impl AsMut<[u8]> for SeedEntropy {
 	}
 }
 
+impl Into<[u8; 32]> for SeedEntropy {
+	fn into(self) -> [u8; 32] {
+		self.0
+	}
+}
+
 #[cfg(feature = "std")]
 impl SeedEntropy {
 	/// Collect a random entropy.
@@ -1376,7 +1382,7 @@ impl SeedEntropy {
 
 		#[cfg(fuzzing)]
 		{
-			lazy_static::lazy_static!{
+			lazy_static::lazy_static! {
 				static ref RNG: rand_chacha::ChaChaRng = rand_chacha::ChaChaRng::from_seed(b"polkadot_on_chain_selection_seed");
 			}
 			RNG.fill_bytes(&mut bytes[..])
diff --git a/runtime/parachains/Cargo.toml b/runtime/parachains/Cargo.toml
index ddc223c85080..fae566e7dedd 100644
--- a/runtime/parachains/Cargo.toml
+++ b/runtime/parachains/Cargo.toml
@@ -48,7 +48,7 @@ hex-literal = "0.3.3"
 keyring = { package = "sp-keyring", git = "https://github.com/paritytech/substrate", branch = "master" }
 frame-support-test = { git = "https://github.com/paritytech/substrate", branch = "master" }
 sc-keystore = { git = "https://github.com/paritytech/substrate", branch = "master" }
-
+assert_matches = "1"
 
 [features]
 default = ["std"]
diff --git a/runtime/parachains/src/inclusion.rs b/runtime/parachains/src/inclusion.rs
index 81c5615dc254..bc5aa1a09698 100644
--- a/runtime/parachains/src/inclusion.rs
+++ b/runtime/parachains/src/inclusion.rs
@@ -290,7 +290,13 @@ impl<T: Config> Pallet<T> {
 			.collect::<Vec<_>>();
 
 		let now = <frame_system::Pallet<T>>::block_number();
-		for (checked_bitfield, validator_index) in checked_bitfields {
+		for (checked_bitfield, validator_index) in
+			checked_bitfields.into_iter().map(|signed_bitfield| {
+				// extracting unchecked data, since it's checked in `fn sanitize_bitfields` already.
+				let validator_idx = signed_bitfield.unchecked_validator_index();
+				let checked_bitfield = signed_bitfield.unchecked_into_payload();
+				(checked_bitfield, validator_idx)
+			}) {
 			for (bit_idx, _) in checked_bitfield.0.iter().enumerate().filter(|(_, is_av)| **is_av) {
 				let pending_availability = if let Some((_, pending_availability)) =
 					assigned_paras_record[bit_idx].as_mut()
@@ -948,9 +954,10 @@ mod tests {
 			System, Test,
 		},
 		paras::ParaGenesisArgs,
-		paras_inherent::{sanitize_bitfields, DisputedBitfield},
+		paras_inherent::DisputedBitfield,
 		scheduler::AssignmentKind,
 	};
+	use assert_matches::assert_matches;
 	use futures::executor::block_on;
 	use keyring::Sr25519Keyring;
 	use primitives::{
@@ -1311,7 +1318,7 @@ mod tests {
 				_ => panic!("out of bounds for testing"),
 			};
 
-			// wrong number of bits.
+			// too many bits in bitfield
 			{
 				let mut bare_bitfield = default_bitfield();
 				bare_bitfield.0.push(false);
@@ -1327,14 +1334,14 @@ mod tests {
 					ParaInclusion::process_bitfields(
 						expected_bits(),
 						vec![signed.into()],
-						DisputedBitfield::default(),
+						DisputedBitfield::zeros(expected_bits()),
 						&core_lookup,
 					),
 					Err(Error::<Test>::WrongBitfieldSize.into())
 				);
 			}
 
-			// wrong number of bits: other way around.
+			// not enough bits
 			{
 				let bare_bitfield = default_bitfield();
 				let signed = block_on(sign_bitfield(
@@ -1349,7 +1356,7 @@ mod tests {
 					ParaInclusion::process_bitfields(
 						expected_bits() + 1,
 						vec![signed.into()],
-						DisputedBitfield::default(),
+						DisputedBitfield::zeros(expected_bits()),
 						&core_lookup,
 					),
 					Err(Error::<Test>::WrongBitfieldSize.into())
@@ -1372,7 +1379,7 @@ mod tests {
 					ParaInclusion::process_bitfields(
 						expected_bits(),
 						vec![signed.clone(), signed],
-						DisputedBitfield::default(),
+						DisputedBitfield::zeros(expected_bits()),
 						&core_lookup,
 					),
 					Err(Error::<Test>::BitfieldDuplicateOrUnordered.into())
@@ -1404,7 +1411,7 @@ mod tests {
 					ParaInclusion::process_bitfields(
 						expected_bits(),
 						vec![signed_1, signed_0],
-						DisputedBitfield::default(),
+						DisputedBitfield::zeros(expected_bits()),
 						&core_lookup,
 					),
 					Err(Error::<Test>::BitfieldDuplicateOrUnordered.into())
@@ -1426,7 +1433,7 @@ mod tests {
 				assert!(ParaInclusion::process_bitfields(
 					expected_bits(),
 					vec![signed.into()],
-					DisputedBitfield::default(),
+					DisputedBitfield::zeros(expected_bits()),
 					&core_lookup,
 				)
 				.is_ok());
@@ -1443,13 +1450,15 @@ mod tests {
 					&signing_context,
 				));
 
-				assert!(!ParaInclusion::process_bitfields(
-					expected_bits(),
-					vec![signed.into()],
-					DisputedBitfield::default(),
-					&core_lookup,
-				)
-				.is_ok());
+				assert_matches!(
+					ParaInclusion::process_bitfields(
+						expected_bits(),
+						vec![signed.into()],
+						DisputedBitfield::zeros(expected_bits()),
+						&core_lookup,
+					),
+					Ok(_)
+				);
 			}
 
 			// bitfield signed with pending bit signed.
@@ -1486,13 +1495,15 @@ mod tests {
 					&signing_context,
 				));
 
-				assert!(ParaInclusion::process_bitfields(
-					expected_bits(),
-					vec![signed.into()],
-					DisputedBitfield::default(),
-					&core_lookup,
-				)
-				.is_ok());
+				assert_matches!(
+					ParaInclusion::process_bitfields(
+						expected_bits(),
+						vec![signed.into()],
+						DisputedBitfield::zeros(expected_bits()),
+						&core_lookup,
+					),
+					Ok(_)
+				);
 
 				<PendingAvailability<Test>>::remove(chain_a);
 				PendingAvailabilityCommitments::<Test>::remove(chain_a);
@@ -1529,13 +1540,15 @@ mod tests {
 				));
 
 				// no core is freed
-				assert!(ParaInclusion::process_bitfields(
-					expected_bits(),
-					vec![signed.into()],
-					DisputedBitfield::default(),
-					&core_lookup,
-				)
-				.is_ok());
+				assert_matches!(
+					ParaInclusion::process_bitfields(
+						expected_bits(),
+						vec![signed.into()],
+						DisputedBitfield::zeros(expected_bits()),
+						&core_lookup,
+					),
+					Ok(_)
+				);
 			}
 		});
 	}
@@ -1674,7 +1687,7 @@ mod tests {
 			assert!(ParaInclusion::process_bitfields(
 				expected_bits(),
 				signed_bitfields,
-				DisputedBitfield::default(),
+				DisputedBitfield::zeros(expected_bits()),
 				&core_lookup,
 			)
 			.is_ok());
diff --git a/runtime/parachains/src/paras_inherent.rs b/runtime/parachains/src/paras_inherent.rs
index 83ad791d654f..2cbbeb071cb8 100644
--- a/runtime/parachains/src/paras_inherent.rs
+++ b/runtime/parachains/src/paras_inherent.rs
@@ -36,10 +36,9 @@ use frame_support::{
 };
 use frame_system::pallet_prelude::*;
 use primitives::v1::{
-	AvailabilityBitfield, BackedCandidate, CandidateHash, CoreIndex,
-	InherentData as ParachainsInherentData, ScrapedOnChainVotes, SessionIndex, SigningContext,
-	UncheckedSignedAvailabilityBitfields, ValidatorId, ValidatorIndex,
-	PARACHAINS_INHERENT_IDENTIFIER,
+	BackedCandidate, CandidateHash, CoreIndex, InherentData as ParachainsInherentData,
+	ScrapedOnChainVotes, SeedEntropy, SessionIndex, SigningContext,
+	UncheckedSignedAvailabilityBitfields, ValidatorId, PARACHAINS_INHERENT_IDENTIFIER,
 };
 use rand::Rng;
 use scale_info::TypeInfo;
@@ -69,6 +68,12 @@ impl From<BitVec<bitvec::order::Lsb0, u8>> for DisputedBitfield {
 	}
 }
 
+impl DisputedBitfield {
+	pub fn zeros(n: usize) -> Self {
+		Self::from(BitVec::<bitvec::order::Lsb0, u8>::repeat(false, n))
+	}
+}
+
 #[frame_support::pallet]
 pub mod pallet {
 	use super::*;
@@ -129,7 +134,7 @@ pub mod pallet {
 			let ParachainsInherentData::<T::Header> {
 				bitfields,
 				backed_candidates,
-				disputes,
+				mut disputes,
 				parent_header,
 				entropy,
 			} = match data.get_data(&Self::INHERENT_IDENTIFIER) {
@@ -142,36 +147,56 @@ pub mod pallet {
 				},
 			};
 
+			let parent_hash = <frame_system::Pallet<T>>::parent_hash();
+			let current_session = <shared::Pallet<T>>::session_index();
 
 			// filter out any unneeded dispute statements
 			T::DisputesHandler::filter_multi_dispute_data(&mut disputes);
 
+			let concluded_invalid_disputes = disputes
+				.iter()
+				.filter(|s| s.session == current_session)
+				.map(|s| (s.session, s.candidate_hash))
+				.filter(|(session, candidate)| {
+					T::DisputesHandler::concluded_invalid(*session, *candidate)
+				})
+				.map(|(_session, candidate)| candidate)
+				.collect::<BTreeSet<CandidateHash>>();
+
 			// sanitize the bitfields and candidates by removing
 			// anything that does not pass a set of checks
 			// will be removed here
 			let validator_public = shared::Pallet::<T>::active_validator_keys();
-			let parent_hash = <frame_system::Pallet<T>>::parent_hash();
-			let current_session = <shared::Pallet<T>>::session_index();
 
-			let expected_bits = unimplemented!("source?");
+			let expected_bits = <scheduler::Pallet<T>>::availability_cores().len();
 
 			let bitfields = sanitize_bitfields::<T, false>(
 				bitfields,
-				DisputedBitfield::default(), // TODO FIXME
+				DisputedBitfield::zeros(expected_bits), // TODO FIXME
 				expected_bits,
 				parent_hash,
 				current_session,
 				&validator_public[..],
-			).ok()?;
+			)
+			.ok()?;
 
-			let scheduled: Vec<CoreAssignment> = unimplemented!();
-			let disputed_candidates: BTreeSet<CandidateHash> = unimplemented!("");
+			let scheduled: Vec<CoreAssignment> = <scheduler::Pallet<T>>::scheduled();
 			let backed_candidates = sanitize_backed_candidates::<T, false>(
 				parent_hash,
 				backed_candidates,
-				disputed_candidates,
+				concluded_invalid_disputes,
 				&scheduled[..],
-			).ok()?;
+			)
+			.ok()?;
+
+			// TODO this cannot be correct
+			// TODO filter bitfields as well
+			let remaining_weight = MINIMAL_INCLUSION_INHERENT_WEIGHT;
+			let (_backed_candidates_weight, backed_candidates) = pick_random_thresholded_subset::<T>(
+				backed_candidates,
+				entropy.clone(),
+				remaining_weight,
+			);
 
 			let inherent_data = ParachainsInherentData::<T::Header> {
 				bitfields,
@@ -187,7 +212,7 @@ pub mod pallet {
 				match Self::enter(frame_system::RawOrigin::None.into(), inherent_data.clone()) {
 					Ok(_) => inherent_data,
 					Err(err) => {
-						log::warn!(
+						log::error!(
 							target: LOG_TARGET,
 								"dropping signed_bitfields and backed_candidates because they produced \
 							an invalid paras inherent: {:?}",
@@ -199,6 +224,7 @@ pub mod pallet {
 							backed_candidates: Vec::new(),
 							disputes: Vec::new(),
 							parent_header: inherent_data.parent_header,
+							entropy: inherent_data.entropy,
 						}
 					},
 				};
@@ -227,6 +253,7 @@ pub mod pallet {
 				backed_candidates,
 				parent_header,
 				disputes,
+				entropy: _entropy,
 			} = data;
 
 			ensure_none(origin)?;
@@ -243,7 +270,7 @@ pub mod pallet {
 
 			// Handle disputes logic.
 			let current_session = <shared::Pallet<T>>::session_index();
-			let (disputed_bits, concluded_invalid_disputed_candidates) = {
+			let (disputed_bitfield, concluded_invalid_disputed_candidates) = {
 				let new_current_dispute_sets: Vec<_> = disputes
 					.iter()
 					.filter(|s| s.session == current_session)
@@ -307,7 +334,7 @@ pub mod pallet {
 			let freed_concluded = <inclusion::Pallet<T>>::process_bitfields(
 				expected_bits,
 				signed_bitfields,
-				disputed_bits,
+				disputed_bitfield,
 				<scheduler::Pallet<T>>::core_para,
 			)?;
 
@@ -384,23 +411,47 @@ pub mod pallet {
 }
 
 macro_rules! ensure2 {
-	($condition:expr, $err:expr, $action:ident) => {
+	($condition:expr, $err:expr, $action:ident $(, $alt:expr)? $(,)?) => {
 		let condition = $condition;
 		if !condition {
 			if $action {
 				ensure!(condition, $err);
+			} else {
+				$($alt)?
 			}
 		}
 	};
 }
 
+/// Calculate the weight of a single backed candidate.
+fn backed_candidate_weight<T: Config>(backed_candidate: &BackedCandidate<<T>::Hash>) -> Weight {
+	// FIXME
+	const CODE_UPGRADE_WEIGHT: Weight = 10 as Weight;
+	const DISPUTE_PER_STATEMENT_WEIGHT: Weight = 1 as Weight;
+
+	backed_candidate.validity_votes.len() as Weight * DISPUTE_PER_STATEMENT_WEIGHT +
+		if backed_candidate.candidate.commitments.new_validation_code.is_some() {
+			CODE_UPGRADE_WEIGHT
+		} else {
+			0 as Weight
+		}
+}
+
 /// Considers an upper threshold that the candidates must not exceed.
 ///
 /// If there is sufficient space, all blocks will be included, otherwise
 /// continues adding blocks, ignoring blocks that exceed the threshold.
-fn pick_random_thresholded_subset(mut candidates: Vec<(CandidateHash, Weight)>, entropy: &EntropySeed, max_weight: Weight) -> (Weight, Vec<CandidateHash>) {
-	if max_weight < candidates.iter().map(|(_, weight)| { weight }).sum() {
-		return candidates.iter().map(|(candidate_hash, _)| candidate_hash).collect()
+fn pick_random_thresholded_subset<T: Config>(
+	mut candidates: Vec<BackedCandidate<<T>::Hash>>,
+	entropy: SeedEntropy,
+	max_weight: Weight,
+) -> (Weight, Vec<BackedCandidate<<T as frame_system::Config>::Hash>>) {
+	let total = candidates
+		.iter()
+		.map(|backed_candidate| backed_candidate_weight::<T>(backed_candidate))
+		.sum();
+	if max_weight < total {
+		return (total, candidates)
 	}
 
 	let mut candidates_acc = Vec::with_capacity(candidates.len());
@@ -409,14 +460,18 @@ fn pick_random_thresholded_subset(mut candidates: Vec<(CandidateHash, Weight)>,
 	// TODO alt: use the pick index as subject to
 	// TODO obtain an index at the cost of having to impl
 	// TODO the equal probability distribution ourselves
-	let rng = chacha_rand::ChaCha20::from_seed(&entropy);
+	use rand_chacha::rand_core::SeedableRng;
+	let mut rng = rand_chacha::ChaChaRng::from_seed(entropy.into());
 	let mut weight_acc: Weight = 0 as _;
 
-	while weight_acc < max_weight || !candidates.empty() {
+	while weight_acc < max_weight || !candidates.is_empty() {
 		// select a index to try
-		let pick = rng.gen_range(0 .. candidates.len());
+		let pick = rng.gen_range(0..candidates.len());
 		// remove the candidate from the possible pick set
-		let (picked_candidate, picked_weight) = candidates.swap_remove(pick);
+		let picked_candidate = candidates.swap_remove(pick);
+
+		let picked_weight = backed_candidate_weight::<T>(&picked_candidate);
+
 		// is the candidate small enough to fit
 		if picked_weight + weight_acc <= max_weight {
 			// candidate fits, so pick it and account for its weight
@@ -454,9 +509,11 @@ pub(crate) fn sanitize_bitfields<T: Config + crate::inclusion::Config, const EAR
 
 	let mut last_index = None;
 
-	if EARLY_RETURN && disputed_bits.0.len() != expected_bits {
-		return Ok(Default::default())
-	}
+	ensure2!(
+		disputed_bits.0.len() == expected_bits,
+		crate::inclusion::pallet::Error::<T>::WrongBitfieldSize,
+		EARLY_RETURN
+	);
 
 	for unchecked_bitfield in unchecked_bitfields {
 		let signing_context = SigningContext { parent_hash, session_index };
@@ -464,19 +521,22 @@ pub(crate) fn sanitize_bitfields<T: Config + crate::inclusion::Config, const EAR
 		ensure2!(
 			unchecked_bitfield.unchecked_payload().0.len() == expected_bits,
 			crate::inclusion::pallet::Error::<T>::WrongBitfieldSize,
-			EARLY_RETURN
+			EARLY_RETURN,
+			continue
 		);
 
 		ensure2!(
 			last_index.map_or(true, |last| last < unchecked_bitfield.unchecked_validator_index()),
 			crate::inclusion::pallet::Error::<T>::BitfieldDuplicateOrUnordered,
-			EARLY_RETURN
+			EARLY_RETURN,
+			continue
 		);
 
 		ensure2!(
 			(unchecked_bitfield.unchecked_validator_index().0 as usize) < validators.len(),
 			crate::inclusion::pallet::Error::<T>::ValidatorIndexOutOfBounds,
-			EARLY_RETURN
+			EARLY_RETURN,
+			continue
 		);
 
 		let validator_index = unchecked_bitfield.unchecked_validator_index();
@@ -696,6 +756,7 @@ mod tests {
 						backed_candidates,
 						disputes: Vec::new(),
 						parent_header: default_header(),
+						entropy: Default::default(),
 					},
 				}
 				.dispatch_bypass_filter(None.into())
@@ -746,6 +807,7 @@ mod tests {
 						backed_candidates,
 						disputes: Vec::new(),
 						parent_header: header,
+						entropy: Default::default(),
 					},
 				}
 				.dispatch_bypass_filter(None.into())
@@ -754,7 +816,7 @@ mod tests {
 				// we don't directly check the block's weight post-call. Instead, we check that the
 				// call has returned the appropriate post-dispatch weight for refund, and trust
 				// Substrate to do the right thing with that information.
-				assert_eq!(post_info.actual_weight.unwrap(), expected_weight);
+				assert_eq!(post_info.actual_weight, Some(expected_weight));
 			});
 		}
 	}

From ceaefae0dd12f30faa3c9115d3faabd04fed9ab0 Mon Sep 17 00:00:00 2001
From: Bernhard Schuster <bernhard@ahoi.io>
Date: Fri, 15 Oct 2021 12:45:06 +0200
Subject: [PATCH 010/107] filter with coverage

---
 primitives/src/v1/mod.rs                 |  2 +
 runtime/parachains/src/paras_inherent.rs | 94 +++++++++++++++++++-----
 2 files changed, 79 insertions(+), 17 deletions(-)

diff --git a/primitives/src/v1/mod.rs b/primitives/src/v1/mod.rs
index ec2a0ffbd706..ced970185714 100644
--- a/primitives/src/v1/mod.rs
+++ b/primitives/src/v1/mod.rs
@@ -534,6 +534,8 @@ impl CandidateCommitments {
 }
 
 /// A bitfield concerning availability of backed candidates.
+///
+/// Every bit refers to an availability core index.
 #[derive(PartialEq, Eq, Clone, Encode, Decode, RuntimeDebug, TypeInfo)]
 pub struct AvailabilityBitfield(pub BitVec<bitvec::order::Lsb0, u8>);
 
diff --git a/runtime/parachains/src/paras_inherent.rs b/runtime/parachains/src/paras_inherent.rs
index 2cbbeb071cb8..bcc763e6efbe 100644
--- a/runtime/parachains/src/paras_inherent.rs
+++ b/runtime/parachains/src/paras_inherent.rs
@@ -28,18 +28,14 @@ use crate::{
 	scheduler::{self, CoreAssignment, FreedReason},
 	shared, ump,
 };
-use bitvec::prelude::BitVec;
+use bitvec::{bitvec, order::Lsb0, prelude::BitVec};
 use frame_support::{
 	fail,
 	inherent::{InherentData, InherentIdentifier, MakeFatalError, ProvideInherent},
 	pallet_prelude::*,
 };
 use frame_system::pallet_prelude::*;
-use primitives::v1::{
-	BackedCandidate, CandidateHash, CoreIndex, InherentData as ParachainsInherentData,
-	ScrapedOnChainVotes, SeedEntropy, SessionIndex, SigningContext,
-	UncheckedSignedAvailabilityBitfields, ValidatorId, PARACHAINS_INHERENT_IDENTIFIER,
-};
+use primitives::v1::{AvailabilityBitfield, BackedCandidate, CandidateHash, CoreIndex, InherentData as ParachainsInherentData, PARACHAINS_INHERENT_IDENTIFIER, ScrapedOnChainVotes, SeedEntropy, SessionIndex, SigningContext, UncheckedSignedAvailabilityBitfields, ValidatorId};
 use rand::Rng;
 use scale_info::TypeInfo;
 use sp_runtime::traits::Header as HeaderT;
@@ -189,13 +185,13 @@ pub mod pallet {
 			)
 			.ok()?;
 
-			// TODO this cannot be correct
-			// TODO filter bitfields as well
 			let remaining_weight = MINIMAL_INCLUSION_INHERENT_WEIGHT;
-			let (_backed_candidates_weight, backed_candidates) = pick_random_thresholded_subset::<T>(
+			let (_backed_candidates_weight, backed_candidates) = apply_weight_limit::<T>(
+				expected_bits,
 				backed_candidates,
 				entropy.clone(),
 				remaining_weight,
+				<scheduler::Pallet<T>>::core_para,
 			);
 
 			let inherent_data = ParachainsInherentData::<T::Header> {
@@ -426,8 +422,8 @@ macro_rules! ensure2 {
 /// Calculate the weight of a single backed candidate.
 fn backed_candidate_weight<T: Config>(backed_candidate: &BackedCandidate<<T>::Hash>) -> Weight {
 	// FIXME
-	const CODE_UPGRADE_WEIGHT: Weight = 10 as Weight;
-	const DISPUTE_PER_STATEMENT_WEIGHT: Weight = 1 as Weight;
+	const CODE_UPGRADE_WEIGHT: Weight = 10_000 as Weight;
+	const DISPUTE_PER_STATEMENT_WEIGHT: Weight = 1_000 as Weight;
 
 	backed_candidate.validity_votes.len() as Weight * DISPUTE_PER_STATEMENT_WEIGHT +
 		if backed_candidate.candidate.commitments.new_validation_code.is_some() {
@@ -437,21 +433,37 @@ fn backed_candidate_weight<T: Config>(backed_candidate: &BackedCandidate<<T>::Ha
 		}
 }
 
+/// Calculate the weight of a individual bitfield.
+fn bitfield_weight<T: Config>(bitfield: &AvailabilityBitfield) -> Weight {
+	7_000 as Weight
+}
+
 /// Considers an upper threshold that the candidates must not exceed.
 ///
 /// If there is sufficient space, all blocks will be included, otherwise
 /// continues adding blocks, ignoring blocks that exceed the threshold.
-fn pick_random_thresholded_subset<T: Config>(
+///
+/// Since there is the relation of `backed candidate <-> occupied core <-> bitfield`
+/// this is used to pick the candidate, but also include all relevant
+/// bitfields.
+fn apply_weight_limit<T: Config>(
+	expected_bits: usize,
 	mut candidates: Vec<BackedCandidate<<T>::Hash>>,
+	mut bitfields: UncheckedSignedAvailabilityBitfields,
 	entropy: SeedEntropy,
 	max_weight: Weight,
+	core_lookup: impl Fn(CoreIndex) -> Option<ParaId>,
 ) -> (Weight, Vec<BackedCandidate<<T as frame_system::Config>::Hash>>) {
-	let total = candidates
+	let mut total = candidates
 		.iter()
 		.map(|backed_candidate| backed_candidate_weight::<T>(backed_candidate))
 		.sum();
+	total += bitfields.iter()
+		.map(|bitfield| bitfield_weight::<T>(bitfield))
+		.sum();
+
 	if max_weight < total {
-		return (total, candidates)
+		return (total, candidates, bitfields)
 	}
 
 	let mut candidates_acc = Vec::with_capacity(candidates.len());
@@ -464,6 +476,24 @@ fn pick_random_thresholded_subset<T: Config>(
 	let mut rng = rand_chacha::ChaChaRng::from_seed(entropy.into());
 	let mut weight_acc: Weight = 0 as _;
 
+	// a bitfield to determine which bitfields to include
+	let bitfields_to_include_coverage = bitvec![Lsb0, u8; false; bitfields.len()];
+
+	// create a mapping of `CandidateHash` to `CoreIndex`
+	let mut candidates_core_index: BTreeMap<CandidateHash, CoreIndex> = (0..expected_bits)
+		.map(|bit_index| core_lookup(CoreIndex::from(bit_index as u32)))
+		.filter_map(|opt_para_id| {
+			opt_para_id
+		})
+		.map(|para_id| {
+			PendingAvailability::<T>::get(&para_id);
+		})
+		.map(|cpa: CandidatePendingAvailability<T> | {
+			(cpa.candidate, cpa.core_index)
+		})
+		.collect();
+
+
 	while weight_acc < max_weight || !candidates.is_empty() {
 		// select a index to try
 		let pick = rng.gen_range(0..candidates.len());
@@ -472,14 +502,44 @@ fn pick_random_thresholded_subset<T: Config>(
 
 		let picked_weight = backed_candidate_weight::<T>(&picked_candidate);
 
+		// collect all bitfields that reference the candidate
+		// (or: the availability core index that is responsible for the candidate)
+		let bitfields_weight = 0 as Weight;
+		let covered_bitfields = bitvec![Lsb0, u8; false; bitfields.len()];
+
+		for (i, bitfield) in bitfields
+			.iter()
+			.enumerate() {
+
+			// lookup the core index that is responsible for the candidate
+			if let Some(core_index) = candidates_core_index.get(&picked_candidate) {
+				if bitfield[core_index] {
+					// avoid duplicate accounting if it was already included before
+					if bitfields_to_include_coverage[i] {
+						// mark the `i`-th bit
+						covered_bitfields.set(i, true);
+						// account for the added weight of the bitfield
+						bitfields_weight += bitfield_weight(bitfield.unchecked_payload());
+					}
+				}
+			}
+		}
+
 		// is the candidate small enough to fit
-		if picked_weight + weight_acc <= max_weight {
+		let prospective_weight = picked_weight + bitfields_weight + weight_acc;
+		if prospective_weight <= max_weight {
 			// candidate fits, so pick it and account for its weight
 			candidates_acc.push(picked_candidate);
-			weight_acc += picked_weight;
+			weight_acc = prospective_weight;
+			bitfields_to_include_coverage |= bitfields_weight;
 		}
 	}
-	(weight_acc, candidates_acc)
+
+	let bitfields_acc = bitfields.into_iter().iter().filter(|(i, _)| {
+		bitfields_to_include_coverage[i]
+	}).collect::<Vec<_>>();
+
+	(weight_acc, candidates_acc, bitfields_acc)
 }
 
 /// Filter bitfields based on freed core indices, validity, and other sanity checks.

From 2f8385f942045906f9344785ba68c604bb009bb4 Mon Sep 17 00:00:00 2001
From: Bernhard Schuster <bernhard@ahoi.io>
Date: Fri, 15 Oct 2021 17:16:21 +0200
Subject: [PATCH 011/107] fixins

---
 node/core/parachains-inherent/src/lib.rs |  5 ++-
 runtime/parachains/src/paras_inherent.rs | 54 ++++++++++++------------
 2 files changed, 30 insertions(+), 29 deletions(-)

diff --git a/node/core/parachains-inherent/src/lib.rs b/node/core/parachains-inherent/src/lib.rs
index 2c4a23eebc4f..c1101e9f49e9 100644
--- a/node/core/parachains-inherent/src/lib.rs
+++ b/node/core/parachains-inherent/src/lib.rs
@@ -80,12 +80,13 @@ impl ParachainsInherentDataProvider {
 			_ = timeout => Err(Error::Timeout),
 		};
 
+		let entropy = SeedEntropy::draw();
 		let inherent_data = match res {
 			Ok(pd) => ParachainsInherentData {
 				bitfields: pd.bitfields.into_iter().map(Into::into).collect(),
 				backed_candidates: pd.backed_candidates,
 				disputes: pd.disputes,
-				seed: pd.entropy,
+				entropy,
 				parent_header,
 			},
 			Err(err) => {
@@ -98,7 +99,7 @@ impl ParachainsInherentDataProvider {
 					backed_candidates: Vec::new(),
 					disputes: Vec::new(),
 					parent_header,
-					entropy: SeedEntropy::draw(),
+					entropy,
 				}
 			},
 		};
diff --git a/runtime/parachains/src/paras_inherent.rs b/runtime/parachains/src/paras_inherent.rs
index bcc763e6efbe..a3230ddf7110 100644
--- a/runtime/parachains/src/paras_inherent.rs
+++ b/runtime/parachains/src/paras_inherent.rs
@@ -25,8 +25,9 @@ use crate::{
 	configuration::Config,
 	disputes::DisputesHandler,
 	inclusion,
+	inclusion::{CandidatePendingAvailability, PendingAvailability},
 	scheduler::{self, CoreAssignment, FreedReason},
-	shared, ump,
+	shared, ump, ParaId,
 };
 use bitvec::{bitvec, order::Lsb0, prelude::BitVec};
 use frame_support::{
@@ -35,7 +36,12 @@ use frame_support::{
 	pallet_prelude::*,
 };
 use frame_system::pallet_prelude::*;
-use primitives::v1::{AvailabilityBitfield, BackedCandidate, CandidateHash, CoreIndex, InherentData as ParachainsInherentData, PARACHAINS_INHERENT_IDENTIFIER, ScrapedOnChainVotes, SeedEntropy, SessionIndex, SigningContext, UncheckedSignedAvailabilityBitfields, ValidatorId};
+use primitives::v1::{
+	AvailabilityBitfield, BackedCandidate, CandidateHash, CoreIndex,
+	InherentData as ParachainsInherentData, ScrapedOnChainVotes, SeedEntropy, SessionIndex,
+	SigningContext, UncheckedSignedAvailabilityBitfields, ValidatorId,
+	PARACHAINS_INHERENT_IDENTIFIER,
+};
 use rand::Rng;
 use scale_info::TypeInfo;
 use sp_runtime::traits::Header as HeaderT;
@@ -186,9 +192,10 @@ pub mod pallet {
 			.ok()?;
 
 			let remaining_weight = MINIMAL_INCLUSION_INHERENT_WEIGHT;
-			let (_backed_candidates_weight, backed_candidates) = apply_weight_limit::<T>(
+			let (_backed_candidates_weight, backed_candidates, bitfields) = apply_weight_limit::<T, _>(
 				expected_bits,
 				backed_candidates,
+				bitfields,
 				entropy.clone(),
 				remaining_weight,
 				<scheduler::Pallet<T>>::core_para,
@@ -446,21 +453,19 @@ fn bitfield_weight<T: Config>(bitfield: &AvailabilityBitfield) -> Weight {
 /// Since there is the relation of `backed candidate <-> occupied core <-> bitfield`
 /// this is used to pick the candidate, but also include all relevant
 /// bitfields.
-fn apply_weight_limit<T: Config>(
+fn apply_weight_limit<T: Config, F: Fn(CoreIndex) -> Option<ParaId>>(
 	expected_bits: usize,
 	mut candidates: Vec<BackedCandidate<<T>::Hash>>,
 	mut bitfields: UncheckedSignedAvailabilityBitfields,
 	entropy: SeedEntropy,
 	max_weight: Weight,
-	core_lookup: impl Fn(CoreIndex) -> Option<ParaId>,
-) -> (Weight, Vec<BackedCandidate<<T as frame_system::Config>::Hash>>) {
+	core_lookup: F,
+) -> (Weight, Vec<BackedCandidate<<T>::Hash>>, UncheckedSignedAvailabilityBitfields) {
 	let mut total = candidates
 		.iter()
 		.map(|backed_candidate| backed_candidate_weight::<T>(backed_candidate))
 		.sum();
-	total += bitfields.iter()
-		.map(|bitfield| bitfield_weight::<T>(bitfield))
-		.sum();
+	total += bitfields.iter().map(|bitfield| bitfield_weight::<T>(bitfield.unchecked_payload())).sum();
 
 	if max_weight < total {
 		return (total, candidates, bitfields)
@@ -477,23 +482,18 @@ fn apply_weight_limit<T: Config>(
 	let mut weight_acc: Weight = 0 as _;
 
 	// a bitfield to determine which bitfields to include
-	let bitfields_to_include_coverage = bitvec![Lsb0, u8; false; bitfields.len()];
+	let bitfields_to_include_coverage = BitVec::<Lsb0, u8>::repeat(false, expected_bits);
 
 	// create a mapping of `CandidateHash` to `CoreIndex`
 	let mut candidates_core_index: BTreeMap<CandidateHash, CoreIndex> = (0..expected_bits)
 		.map(|bit_index| core_lookup(CoreIndex::from(bit_index as u32)))
-		.filter_map(|opt_para_id| {
-			opt_para_id
-		})
+		.filter_map(|opt_para_id| opt_para_id)
 		.map(|para_id| {
 			PendingAvailability::<T>::get(&para_id);
 		})
-		.map(|cpa: CandidatePendingAvailability<T> | {
-			(cpa.candidate, cpa.core_index)
-		})
+		.map(|cpa: CandidatePendingAvailability<T,<T>::BlockNumber>| (cpa.candidate_hash(), cpa.core_occupied()))
 		.collect();
 
-
 	while weight_acc < max_weight || !candidates.is_empty() {
 		// select a index to try
 		let pick = rng.gen_range(0..candidates.len());
@@ -505,15 +505,12 @@ fn apply_weight_limit<T: Config>(
 		// collect all bitfields that reference the candidate
 		// (or: the availability core index that is responsible for the candidate)
 		let bitfields_weight = 0 as Weight;
-		let covered_bitfields = bitvec![Lsb0, u8; false; bitfields.len()];
-
-		for (i, bitfield) in bitfields
-			.iter()
-			.enumerate() {
+		let covered_bitfields = BitVec::<Lsb0, u8>::repeat(false, expected_bits);
 
+		for (i, bitfield) in bitfields.iter().enumerate() {
 			// lookup the core index that is responsible for the candidate
-			if let Some(core_index) = candidates_core_index.get(&picked_candidate) {
-				if bitfield[core_index] {
+			if let Some(core_index) = candidates_core_index.get(&picked_candidate.hash()) {
+				if bitfield.unchecked_payload().0[core_index.0 as _] {
 					// avoid duplicate accounting if it was already included before
 					if bitfields_to_include_coverage[i] {
 						// mark the `i`-th bit
@@ -535,9 +532,12 @@ fn apply_weight_limit<T: Config>(
 		}
 	}
 
-	let bitfields_acc = bitfields.into_iter().iter().filter(|(i, _)| {
-		bitfields_to_include_coverage[i]
-	}).collect::<Vec<_>>();
+	let bitfields_acc = bitfields
+		.into_iter()
+		.enumerate()
+		.filter(|(i, _)| bitfields_to_include_coverage[*i])
+		.map(|(_, bitfield)| bitfield)
+		.collect::<Vec<_>>();
 
 	(weight_acc, candidates_acc, bitfields_acc)
 }

From c169ae0005db4529fd707f3be819bf23cc807a13 Mon Sep 17 00:00:00 2001
From: Bernhard Schuster <bernhard@ahoi.io>
Date: Mon, 18 Oct 2021 11:00:04 +0200
Subject: [PATCH 012/107] Update runtime/parachains/src/paras_inherent.rs

Co-authored-by: Robert Habermeier <rphmeier@gmail.com>
---
 runtime/parachains/src/paras_inherent.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/parachains/src/paras_inherent.rs b/runtime/parachains/src/paras_inherent.rs
index a3230ddf7110..b1ebd43ce67d 100644
--- a/runtime/parachains/src/paras_inherent.rs
+++ b/runtime/parachains/src/paras_inherent.rs
@@ -217,7 +217,7 @@ pub mod pallet {
 					Err(err) => {
 						log::error!(
 							target: LOG_TARGET,
-								"dropping signed_bitfields and backed_candidates because they produced \
+								"dropping paras inherent data because they produced \
 							an invalid paras inherent: {:?}",
 							err.error,
 						);

From 623a28364f0ae43541bcbe0029650c336dd0d4a0 Mon Sep 17 00:00:00 2001
From: Bernhard Schuster <bernhard@ahoi.io>
Date: Mon, 18 Oct 2021 10:13:12 +0200
Subject: [PATCH 013/107] slip of the pen

---
 runtime/parachains/Cargo.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/parachains/Cargo.toml b/runtime/parachains/Cargo.toml
index fae566e7dedd..5708dccf115e 100644
--- a/runtime/parachains/Cargo.toml
+++ b/runtime/parachains/Cargo.toml
@@ -40,7 +40,7 @@ xcm-executor = { package = "xcm-executor", path = "../../xcm/xcm-executor", defa
 primitives = { package = "polkadot-primitives", path = "../../primitives", default-features = false }
 
 rand = { version = "0.8.3", default-features = false }
-rand_chacha = { version = "0.3.1" }
+rand_chacha = { version = "0.3.1", default-features = false }
 
 [dev-dependencies]
 futures = "0.3.17"

From 9e1cb252a394cec6272585a25122ec0406a818e7 Mon Sep 17 00:00:00 2001
From: Bernhard Schuster <bernhard@ahoi.io>
Date: Mon, 18 Oct 2021 10:13:30 +0200
Subject: [PATCH 014/107] improve test cases

---
 runtime/parachains/src/inclusion.rs | 97 +++++++++++++++--------------
 1 file changed, 51 insertions(+), 46 deletions(-)

diff --git a/runtime/parachains/src/inclusion.rs b/runtime/parachains/src/inclusion.rs
index bc5aa1a09698..c8334cba0759 100644
--- a/runtime/parachains/src/inclusion.rs
+++ b/runtime/parachains/src/inclusion.rs
@@ -968,6 +968,7 @@ mod tests {
 			UncheckedSignedAvailabilityBitfield, ValidationCode, ValidatorId, ValidityAttestation,
 		},
 	};
+	use frame_support::assert_noop;
 	use sc_keystore::LocalKeystore;
 	use sp_keystore::{SyncCryptoStore, SyncCryptoStorePtr};
 	use std::sync::Arc;
@@ -1330,14 +1331,14 @@ mod tests {
 					&signing_context,
 				));
 
-				assert_eq!(
+				assert_noop!(
 					ParaInclusion::process_bitfields(
 						expected_bits(),
 						vec![signed.into()],
 						DisputedBitfield::zeros(expected_bits()),
 						&core_lookup,
 					),
-					Err(Error::<Test>::WrongBitfieldSize.into())
+					Error::<Test>::WrongBitfieldSize.into()
 				);
 			}
 
@@ -1352,14 +1353,14 @@ mod tests {
 					&signing_context,
 				));
 
-				assert_eq!(
+				assert_noop!(
 					ParaInclusion::process_bitfields(
 						expected_bits() + 1,
 						vec![signed.into()],
 						DisputedBitfield::zeros(expected_bits()),
 						&core_lookup,
 					),
-					Err(Error::<Test>::WrongBitfieldSize.into())
+					Error::<Test>::WrongBitfieldSize.into()
 				);
 			}
 
@@ -1375,14 +1376,14 @@ mod tests {
 				))
 				.into();
 
-				assert_eq!(
+				assert_noop!(
 					ParaInclusion::process_bitfields(
 						expected_bits(),
 						vec![signed.clone(), signed],
 						DisputedBitfield::zeros(expected_bits()),
 						&core_lookup,
 					),
-					Err(Error::<Test>::BitfieldDuplicateOrUnordered.into())
+					Error::<Test>::BitfieldDuplicateOrUnordered.into()
 				);
 			}
 
@@ -1407,14 +1408,14 @@ mod tests {
 				))
 				.into();
 
-				assert_eq!(
+				assert_noop!(
 					ParaInclusion::process_bitfields(
 						expected_bits(),
 						vec![signed_1, signed_0],
 						DisputedBitfield::zeros(expected_bits()),
 						&core_lookup,
 					),
-					Err(Error::<Test>::BitfieldDuplicateOrUnordered.into())
+					Error::<Test>::BitfieldDuplicateOrUnordered.into()
 				);
 			}
 
@@ -1430,13 +1431,15 @@ mod tests {
 					&signing_context,
 				));
 
-				assert!(ParaInclusion::process_bitfields(
-					expected_bits(),
-					vec![signed.into()],
-					DisputedBitfield::zeros(expected_bits()),
-					&core_lookup,
-				)
-				.is_ok());
+				assert_matches!(
+					ParaInclusion::process_bitfields(
+						expected_bits(),
+						vec![signed.into()],
+						DisputedBitfield::zeros(expected_bits()),
+						&core_lookup,
+					),
+					Ok(_)
+				);
 			}
 
 			// empty bitfield signed: always OK, but kind of useless.
@@ -1540,14 +1543,14 @@ mod tests {
 				));
 
 				// no core is freed
-				assert_matches!(
+				assert_eq!(
 					ParaInclusion::process_bitfields(
 						expected_bits(),
 						vec![signed.into()],
 						DisputedBitfield::zeros(expected_bits()),
 						&core_lookup,
 					),
-					Ok(_)
+					Ok(vec![])
 				);
 			}
 		});
@@ -1684,13 +1687,15 @@ mod tests {
 				})
 				.collect();
 
-			assert!(ParaInclusion::process_bitfields(
-				expected_bits(),
-				signed_bitfields,
-				DisputedBitfield::zeros(expected_bits()),
-				&core_lookup,
-			)
-			.is_ok());
+			assert_matches!(
+				ParaInclusion::process_bitfields(
+					expected_bits(),
+					signed_bitfields,
+					DisputedBitfield::zeros(expected_bits()),
+					&core_lookup,
+				),
+				Ok(_)
+			);
 
 			// chain A had 4 signing off, which is >= threshold.
 			// chain B has 3 signing off, which is < threshold.
@@ -1824,14 +1829,14 @@ mod tests {
 					BackingKind::Threshold,
 				));
 
-				assert_eq!(
+				assert_noop!(
 					ParaInclusion::process_candidates(
 						Default::default(),
 						vec![backed],
 						vec![chain_b_assignment.clone()],
 						&group_validators,
 					),
-					Err(Error::<Test>::UnscheduledCandidate.into()),
+					Error::<Test>::UnscheduledCandidate.into(),
 				);
 			}
 
@@ -1879,14 +1884,14 @@ mod tests {
 				));
 
 				// out-of-order manifests as unscheduled.
-				assert_eq!(
+				assert_noop!(
 					ParaInclusion::process_candidates(
 						Default::default(),
 						vec![backed_b, backed_a],
 						vec![chain_a_assignment.clone(), chain_b_assignment.clone()],
 						&group_validators,
 					),
-					Err(Error::<Test>::UnscheduledCandidate.into()),
+					Error::<Test>::UnscheduledCandidate.into(),
 				);
 			}
 
@@ -1912,14 +1917,14 @@ mod tests {
 					BackingKind::Lacking,
 				));
 
-				assert_eq!(
+				assert_noop!(
 					ParaInclusion::process_candidates(
 						Default::default(),
 						vec![backed],
 						vec![chain_a_assignment.clone()],
 						&group_validators,
 					),
-					Err(Error::<Test>::InsufficientBacking.into()),
+					Error::<Test>::InsufficientBacking.into(),
 				);
 			}
 
@@ -1947,14 +1952,14 @@ mod tests {
 					BackingKind::Threshold,
 				));
 
-				assert_eq!(
+				assert_noop!(
 					ParaInclusion::process_candidates(
 						Default::default(),
 						vec![backed],
 						vec![chain_a_assignment.clone()],
 						&group_validators,
 					),
-					Err(Error::<Test>::CandidateNotInParentContext.into()),
+					Error::<Test>::CandidateNotInParentContext.into(),
 				);
 			}
 
@@ -1982,7 +1987,7 @@ mod tests {
 					BackingKind::Threshold,
 				));
 
-				assert_eq!(
+				assert_noop!(
 					ParaInclusion::process_candidates(
 						Default::default(),
 						vec![backed],
@@ -1993,7 +1998,7 @@ mod tests {
 						],
 						&group_validators,
 					),
-					Err(Error::<Test>::WrongCollator.into()),
+					Error::<Test>::WrongCollator.into(),
 				);
 			}
 
@@ -2024,14 +2029,14 @@ mod tests {
 					BackingKind::Threshold,
 				));
 
-				assert_eq!(
+				assert_noop!(
 					ParaInclusion::process_candidates(
 						Default::default(),
 						vec![backed],
 						vec![thread_a_assignment.clone()],
 						&group_validators,
 					),
-					Err(Error::<Test>::NotCollatorSigned.into()),
+					Error::<Test>::NotCollatorSigned.into(),
 				);
 			}
 
@@ -2074,14 +2079,14 @@ mod tests {
 				);
 				<PendingAvailabilityCommitments<Test>>::insert(&chain_a, candidate.commitments);
 
-				assert_eq!(
+				assert_noop!(
 					ParaInclusion::process_candidates(
 						Default::default(),
 						vec![backed],
 						vec![chain_a_assignment.clone()],
 						&group_validators,
 					),
-					Err(Error::<Test>::CandidateScheduledBeforeParaFree.into()),
+					Error::<Test>::CandidateScheduledBeforeParaFree.into(),
 				);
 
 				<PendingAvailability<Test>>::remove(&chain_a);
@@ -2117,14 +2122,14 @@ mod tests {
 					BackingKind::Threshold,
 				));
 
-				assert_eq!(
+				assert_noop!(
 					ParaInclusion::process_candidates(
 						Default::default(),
 						vec![backed],
 						vec![chain_a_assignment.clone()],
 						&group_validators,
 					),
-					Err(Error::<Test>::CandidateScheduledBeforeParaFree.into()),
+					Error::<Test>::CandidateScheduledBeforeParaFree.into(),
 				);
 
 				<PendingAvailabilityCommitments<Test>>::remove(&chain_a);
@@ -2168,14 +2173,14 @@ mod tests {
 					assert_eq!(Paras::last_code_upgrade(chain_a, true), Some(expected_at));
 				}
 
-				assert_eq!(
+				assert_noop!(
 					ParaInclusion::process_candidates(
 						Default::default(),
 						vec![backed],
 						vec![chain_a_assignment.clone()],
 						&group_validators,
 					),
-					Err(Error::<Test>::PrematureCodeUpgrade.into()),
+					Error::<Test>::PrematureCodeUpgrade.into(),
 				);
 			}
 
@@ -2237,14 +2242,14 @@ mod tests {
 					BackingKind::Threshold,
 				));
 
-				assert_eq!(
+				assert_noop!(
 					ParaInclusion::process_candidates(
 						Default::default(),
 						vec![backed],
 						vec![chain_a_assignment.clone()],
 						&group_validators,
 					),
-					Err(Error::<Test>::InvalidValidationCodeHash.into()),
+					Error::<Test>::InvalidValidationCodeHash.into(),
 				);
 			}
 
@@ -2272,14 +2277,14 @@ mod tests {
 					BackingKind::Threshold,
 				));
 
-				assert_eq!(
+				assert_noop!(
 					ParaInclusion::process_candidates(
 						Default::default(),
 						vec![backed],
 						vec![chain_a_assignment.clone()],
 						&group_validators,
 					),
-					Err(Error::<Test>::ParaHeadMismatch.into()),
+					Error::<Test>::ParaHeadMismatch.into(),
 				);
 			}
 		});

From e0f4b9e5095b739b297b44e13218a55a73531948 Mon Sep 17 00:00:00 2001
From: Bernhard Schuster <bernhard@ahoi.io>
Date: Mon, 18 Oct 2021 10:13:50 +0200
Subject: [PATCH 015/107] misc

---
 runtime/parachains/src/paras_inherent.rs | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/runtime/parachains/src/paras_inherent.rs b/runtime/parachains/src/paras_inherent.rs
index b1ebd43ce67d..87832406107d 100644
--- a/runtime/parachains/src/paras_inherent.rs
+++ b/runtime/parachains/src/paras_inherent.rs
@@ -29,7 +29,7 @@ use crate::{
 	scheduler::{self, CoreAssignment, FreedReason},
 	shared, ump, ParaId,
 };
-use bitvec::{bitvec, order::Lsb0, prelude::BitVec};
+use bitvec::{order::Lsb0, prelude::BitVec};
 use frame_support::{
 	fail,
 	inherent::{InherentData, InherentIdentifier, MakeFatalError, ProvideInherent},
@@ -71,6 +71,7 @@ impl From<BitVec<bitvec::order::Lsb0, u8>> for DisputedBitfield {
 }
 
 impl DisputedBitfield {
+	/// Create a new bitfield, where each bit is set to `false`.
 	pub fn zeros(n: usize) -> Self {
 		Self::from(BitVec::<bitvec::order::Lsb0, u8>::repeat(false, n))
 	}
@@ -489,9 +490,11 @@ fn apply_weight_limit<T: Config, F: Fn(CoreIndex) -> Option<ParaId>>(
 		.map(|bit_index| core_lookup(CoreIndex::from(bit_index as u32)))
 		.filter_map(|opt_para_id| opt_para_id)
 		.map(|para_id| {
-			PendingAvailability::<T>::get(&para_id);
+			PendingAvailability::<T>::get(&para_id)
+		})
+		.map(|cpa: CandidatePendingAvailability<T,<T>::BlockNumber>| {
+			(cpa.candidate_hash(), cpa.core_occupied())
 		})
-		.map(|cpa: CandidatePendingAvailability<T,<T>::BlockNumber>| (cpa.candidate_hash(), cpa.core_occupied()))
 		.collect();
 
 	while weight_acc < max_weight || !candidates.is_empty() {

From 9ec43fa246f17221ce3aa41e40b8d93f01f1077f Mon Sep 17 00:00:00 2001
From: Bernhard Schuster <bernhard@ahoi.io>
Date: Mon, 18 Oct 2021 10:13:57 +0200
Subject: [PATCH 016/107] fix

---
 runtime/parachains/src/paras_inherent.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/parachains/src/paras_inherent.rs b/runtime/parachains/src/paras_inherent.rs
index 87832406107d..3fe32e10c482 100644
--- a/runtime/parachains/src/paras_inherent.rs
+++ b/runtime/parachains/src/paras_inherent.rs
@@ -531,7 +531,7 @@ fn apply_weight_limit<T: Config, F: Fn(CoreIndex) -> Option<ParaId>>(
 			// candidate fits, so pick it and account for its weight
 			candidates_acc.push(picked_candidate);
 			weight_acc = prospective_weight;
-			bitfields_to_include_coverage |= bitfields_weight;
+			bitfields_to_include_coverage |= covered_bitfields;
 		}
 	}
 

From 8bd9af6e34c31caaacd6e864181ac3ae9d2a986f Mon Sep 17 00:00:00 2001
From: Bernhard Schuster <bernhard@ahoi.io>
Date: Mon, 18 Oct 2021 11:24:15 +0200
Subject: [PATCH 017/107] fixins

---
 runtime/parachains/src/disputes.rs       |  1 +
 runtime/parachains/src/paras_inherent.rs | 64 ++++++++++++++++--------
 2 files changed, 45 insertions(+), 20 deletions(-)

diff --git a/runtime/parachains/src/disputes.rs b/runtime/parachains/src/disputes.rs
index 1ec5515667fb..f816a514b8c9 100644
--- a/runtime/parachains/src/disputes.rs
+++ b/runtime/parachains/src/disputes.rs
@@ -738,6 +738,7 @@ impl<T: Config> Pallet<T> {
 		Ok(fresh)
 	}
 
+	/// Removes all duplicate disputes.
 	fn filter_multi_dispute_data(statement_sets: &mut MultiDisputeStatementSet) {
 		let config = <configuration::Pallet<T>>::config();
 
diff --git a/runtime/parachains/src/paras_inherent.rs b/runtime/parachains/src/paras_inherent.rs
index 3fe32e10c482..51fbf4e9a1c7 100644
--- a/runtime/parachains/src/paras_inherent.rs
+++ b/runtime/parachains/src/paras_inherent.rs
@@ -59,7 +59,7 @@ const INCLUSION_INHERENT_CLAIMED_WEIGHT: Weight = 1_000_000_000;
 // we assume that 75% of an paras inherent's weight is used processing backed candidates
 const MINIMAL_INCLUSION_INHERENT_WEIGHT: Weight = INCLUSION_INHERENT_CLAIMED_WEIGHT / 4;
 
-/// A bitfield concering concluded disputes for candidates
+/// A bitfield concerning concluded disputes for candidates
 /// associated to the core index equivalent to the bit position.
 #[derive(Default, PartialEq, Eq, Clone, Encode, Decode, RuntimeDebug, TypeInfo)]
 pub(crate) struct DisputedBitfield(pub(crate) BitVec<bitvec::order::Lsb0, u8>);
@@ -134,6 +134,8 @@ pub mod pallet {
 		const INHERENT_IDENTIFIER: InherentIdentifier = PARACHAINS_INHERENT_IDENTIFIER;
 
 		fn create_inherent(data: &InherentData) -> Option<Self::Call> {
+			let parent_hash = <frame_system::Pallet<T>>::parent_hash();
+
 			let ParachainsInherentData::<T::Header> {
 				bitfields,
 				backed_candidates,
@@ -146,11 +148,21 @@ pub mod pallet {
 				Err(_) => {
 					log::warn!(target: LOG_TARGET, "ParachainsInherentData failed to decode");
 
-					return None
+					let parent_header = unimplemented!();
+					ParachainsInherentData {
+						// this is ok, since there are no backed candidates
+						// to pick from in case of overweight.
+						entropy: SeedEntropy::default(),
+
+						parent_header,
+
+						bitfields: Default::default(),
+						backed_candidates: Default::default(),
+						disputes: Default::default(),
+					}
 				},
 			};
 
-			let parent_hash = <frame_system::Pallet<T>>::parent_hash();
 			let current_session = <shared::Pallet<T>>::session_index();
 
 			// filter out any unneeded dispute statements
@@ -192,6 +204,8 @@ pub mod pallet {
 			)
 			.ok()?;
 
+			// XXX @Lldenaurois
+			// FIXME these weights are garbage
 			let remaining_weight = MINIMAL_INCLUSION_INHERENT_WEIGHT;
 			let (_backed_candidates_weight, backed_candidates, bitfields) = apply_weight_limit::<T, _>(
 				expected_bits,
@@ -429,7 +443,8 @@ macro_rules! ensure2 {
 
 /// Calculate the weight of a single backed candidate.
 fn backed_candidate_weight<T: Config>(backed_candidate: &BackedCandidate<<T>::Hash>) -> Weight {
-	// FIXME
+	// XXX @Lldenaurois
+	// FIXME these weights are garbage
 	const CODE_UPGRADE_WEIGHT: Weight = 10_000 as Weight;
 	const DISPUTE_PER_STATEMENT_WEIGHT: Weight = 1_000 as Weight;
 
@@ -443,6 +458,8 @@ fn backed_candidate_weight<T: Config>(backed_candidate: &BackedCandidate<<T>::Ha
 
 /// Calculate the weight of a individual bitfield.
 fn bitfield_weight<T: Config>(bitfield: &AvailabilityBitfield) -> Weight {
+	// XXX @Lldenaurois
+	// FIXME these weights are garbage
 	7_000 as Weight
 }
 
@@ -454,7 +471,7 @@ fn bitfield_weight<T: Config>(bitfield: &AvailabilityBitfield) -> Weight {
 /// Since there is the relation of `backed candidate <-> occupied core <-> bitfield`
 /// this is used to pick the candidate, but also include all relevant
 /// bitfields.
-fn apply_weight_limit<T: Config, F: Fn(CoreIndex) -> Option<ParaId>>(
+fn apply_weight_limit<T: Config + inclusion::Config, F: Fn(CoreIndex) -> Option<ParaId>>(
 	expected_bits: usize,
 	mut candidates: Vec<BackedCandidate<<T>::Hash>>,
 	mut bitfields: UncheckedSignedAvailabilityBitfields,
@@ -466,7 +483,7 @@ fn apply_weight_limit<T: Config, F: Fn(CoreIndex) -> Option<ParaId>>(
 		.iter()
 		.map(|backed_candidate| backed_candidate_weight::<T>(backed_candidate))
 		.sum();
-	total += bitfields.iter().map(|bitfield| bitfield_weight::<T>(bitfield.unchecked_payload())).sum();
+	total += bitfields.iter().map(|bitfield| bitfield_weight::<T>(bitfield.unchecked_payload())).sum::<Weight>();
 
 	if max_weight < total {
 		return (total, candidates, bitfields)
@@ -489,10 +506,10 @@ fn apply_weight_limit<T: Config, F: Fn(CoreIndex) -> Option<ParaId>>(
 	let mut candidates_core_index: BTreeMap<CandidateHash, CoreIndex> = (0..expected_bits)
 		.map(|bit_index| core_lookup(CoreIndex::from(bit_index as u32)))
 		.filter_map(|opt_para_id| opt_para_id)
-		.map(|para_id| {
+		.filter_map(|para_id| {
 			PendingAvailability::<T>::get(&para_id)
 		})
-		.map(|cpa: CandidatePendingAvailability<T,<T>::BlockNumber>| {
+		.map(|cpa: CandidatePendingAvailability<<T>::Hash,<T>::BlockNumber>| {
 			(cpa.candidate_hash(), cpa.core_occupied())
 		})
 		.collect();
@@ -513,13 +530,13 @@ fn apply_weight_limit<T: Config, F: Fn(CoreIndex) -> Option<ParaId>>(
 		for (i, bitfield) in bitfields.iter().enumerate() {
 			// lookup the core index that is responsible for the candidate
 			if let Some(core_index) = candidates_core_index.get(&picked_candidate.hash()) {
-				if bitfield.unchecked_payload().0[core_index.0 as _] {
+				if bitfield.unchecked_payload().0[core_index.0 as usize] {
 					// avoid duplicate accounting if it was already included before
 					if bitfields_to_include_coverage[i] {
 						// mark the `i`-th bit
 						covered_bitfields.set(i, true);
 						// account for the added weight of the bitfield
-						bitfields_weight += bitfield_weight(bitfield.unchecked_payload());
+						bitfields_weight += bitfield_weight::<T>(bitfield.unchecked_payload());
 					}
 				}
 			}
@@ -560,6 +577,11 @@ fn apply_weight_limit<T: Config, F: Fn(CoreIndex) -> Option<ParaId>>(
 /// While this function technically returns a set of unchecked bitfields,
 /// they were actually checked and filtered to allow using it in both
 /// cases, as `filtering` and `checking` stage.
+///
+/// `EARLY_RETURN` determines the behavior.
+/// `false` assures that all inputs are filtered, and invalid ones are filtered out.
+/// It also skips signature verification.
+/// `true` returns an `Err(_)` on the first check failing.
 pub(crate) fn sanitize_bitfields<T: Config + crate::inclusion::Config, const EARLY_RETURN: bool>(
 	unchecked_bitfields: UncheckedSignedAvailabilityBitfields,
 	disputed_bits: DisputedBitfield,
@@ -606,17 +628,19 @@ pub(crate) fn sanitize_bitfields<T: Config + crate::inclusion::Config, const EAR
 
 		let validator_public = &validators[validator_index.0 as usize];
 
-		let signed_bitfield = if let Ok(signed_bitfield) =
-			unchecked_bitfield.try_into_checked(&signing_context, validator_public)
-		{
-			signed_bitfield
-		} else if EARLY_RETURN {
-			fail!(crate::inclusion::pallet::Error::<T>::InvalidBitfieldSignature);
+		// only check the signatures when returning early
+		if EARLY_RETURN {
+			let signed_bitfield = if let Ok(signed_bitfield) =
+				unchecked_bitfield.try_into_checked(&signing_context, validator_public)
+			{
+				signed_bitfield
+			} else {
+				fail!(crate::inclusion::pallet::Error::<T>::InvalidBitfieldSignature);
+			};
+			bitfields.push(signed_bitfield.into_unchecked());
 		} else {
-			continue
-		};
-
-		bitfields.push(signed_bitfield.into_unchecked());
+			bitfields.push(unchecked_bitfield);
+		}
 
 		last_index = Some(validator_index);
 	}

From 2c232512d2797210d444acde454d45f7c6973908 Mon Sep 17 00:00:00 2001
From: Bernhard Schuster <bernhard@ahoi.io>
Date: Mon, 18 Oct 2021 11:31:44 +0200
Subject: [PATCH 018/107] test avoid extra into() calls in assert_noop!

---
 runtime/parachains/src/inclusion.rs      | 28 ++++++++++++------------
 runtime/parachains/src/paras_inherent.rs | 14 +-----------
 2 files changed, 15 insertions(+), 27 deletions(-)

diff --git a/runtime/parachains/src/inclusion.rs b/runtime/parachains/src/inclusion.rs
index c8334cba0759..9bf22add323f 100644
--- a/runtime/parachains/src/inclusion.rs
+++ b/runtime/parachains/src/inclusion.rs
@@ -1338,7 +1338,7 @@ mod tests {
 						DisputedBitfield::zeros(expected_bits()),
 						&core_lookup,
 					),
-					Error::<Test>::WrongBitfieldSize.into()
+					Error::<Test>::WrongBitfieldSize
 				);
 			}
 
@@ -1360,7 +1360,7 @@ mod tests {
 						DisputedBitfield::zeros(expected_bits()),
 						&core_lookup,
 					),
-					Error::<Test>::WrongBitfieldSize.into()
+					Error::<Test>::WrongBitfieldSize
 				);
 			}
 
@@ -1383,7 +1383,7 @@ mod tests {
 						DisputedBitfield::zeros(expected_bits()),
 						&core_lookup,
 					),
-					Error::<Test>::BitfieldDuplicateOrUnordered.into()
+					Error::<Test>::BitfieldDuplicateOrUnordered
 				);
 			}
 
@@ -1415,7 +1415,7 @@ mod tests {
 						DisputedBitfield::zeros(expected_bits()),
 						&core_lookup,
 					),
-					Error::<Test>::BitfieldDuplicateOrUnordered.into()
+					Error::<Test>::BitfieldDuplicateOrUnordered
 				);
 			}
 
@@ -1836,7 +1836,7 @@ mod tests {
 						vec![chain_b_assignment.clone()],
 						&group_validators,
 					),
-					Error::<Test>::UnscheduledCandidate.into(),
+					Error::<Test>::UnscheduledCandidate
 				);
 			}
 
@@ -1891,7 +1891,7 @@ mod tests {
 						vec![chain_a_assignment.clone(), chain_b_assignment.clone()],
 						&group_validators,
 					),
-					Error::<Test>::UnscheduledCandidate.into(),
+					Error::<Test>::UnscheduledCandidate
 				);
 			}
 
@@ -1924,7 +1924,7 @@ mod tests {
 						vec![chain_a_assignment.clone()],
 						&group_validators,
 					),
-					Error::<Test>::InsufficientBacking.into(),
+					Error::<Test>::InsufficientBacking
 				);
 			}
 
@@ -1959,7 +1959,7 @@ mod tests {
 						vec![chain_a_assignment.clone()],
 						&group_validators,
 					),
-					Error::<Test>::CandidateNotInParentContext.into(),
+					Error::<Test>::CandidateNotInParentContext
 				);
 			}
 
@@ -2036,7 +2036,7 @@ mod tests {
 						vec![thread_a_assignment.clone()],
 						&group_validators,
 					),
-					Error::<Test>::NotCollatorSigned.into(),
+					Error::<Test>::NotCollatorSigned
 				);
 			}
 
@@ -2086,7 +2086,7 @@ mod tests {
 						vec![chain_a_assignment.clone()],
 						&group_validators,
 					),
-					Error::<Test>::CandidateScheduledBeforeParaFree.into(),
+					Error::<Test>::CandidateScheduledBeforeParaFree
 				);
 
 				<PendingAvailability<Test>>::remove(&chain_a);
@@ -2129,7 +2129,7 @@ mod tests {
 						vec![chain_a_assignment.clone()],
 						&group_validators,
 					),
-					Error::<Test>::CandidateScheduledBeforeParaFree.into(),
+					Error::<Test>::CandidateScheduledBeforeParaFree
 				);
 
 				<PendingAvailabilityCommitments<Test>>::remove(&chain_a);
@@ -2180,7 +2180,7 @@ mod tests {
 						vec![chain_a_assignment.clone()],
 						&group_validators,
 					),
-					Error::<Test>::PrematureCodeUpgrade.into(),
+					Error::<Test>::PrematureCodeUpgrade
 				);
 			}
 
@@ -2249,7 +2249,7 @@ mod tests {
 						vec![chain_a_assignment.clone()],
 						&group_validators,
 					),
-					Error::<Test>::InvalidValidationCodeHash.into(),
+					Error::<Test>::InvalidValidationCodeHash
 				);
 			}
 
@@ -2284,7 +2284,7 @@ mod tests {
 						vec![chain_a_assignment.clone()],
 						&group_validators,
 					),
-					Error::<Test>::ParaHeadMismatch.into(),
+					Error::<Test>::ParaHeadMismatch
 				);
 			}
 		});
diff --git a/runtime/parachains/src/paras_inherent.rs b/runtime/parachains/src/paras_inherent.rs
index 51fbf4e9a1c7..3efd3d10b3a3 100644
--- a/runtime/parachains/src/paras_inherent.rs
+++ b/runtime/parachains/src/paras_inherent.rs
@@ -147,19 +147,7 @@ pub mod pallet {
 				Ok(None) => return None,
 				Err(_) => {
 					log::warn!(target: LOG_TARGET, "ParachainsInherentData failed to decode");
-
-					let parent_header = unimplemented!();
-					ParachainsInherentData {
-						// this is ok, since there are no backed candidates
-						// to pick from in case of overweight.
-						entropy: SeedEntropy::default(),
-
-						parent_header,
-
-						bitfields: Default::default(),
-						backed_candidates: Default::default(),
-						disputes: Default::default(),
-					}
+					return None
 				},
 			};
 

From b815172c82647693a47c4f5244fca31e8c62f280 Mon Sep 17 00:00:00 2001
From: Bernhard Schuster <bernhard@ahoi.io>
Date: Mon, 18 Oct 2021 11:39:29 +0200
Subject: [PATCH 019/107] chores

---
 runtime/parachains/src/inclusion.rs      |  2 +-
 runtime/parachains/src/paras_inherent.rs | 34 +++++++++++++-----------
 2 files changed, 19 insertions(+), 17 deletions(-)

diff --git a/runtime/parachains/src/inclusion.rs b/runtime/parachains/src/inclusion.rs
index 9bf22add323f..91c37b9a53fd 100644
--- a/runtime/parachains/src/inclusion.rs
+++ b/runtime/parachains/src/inclusion.rs
@@ -958,6 +958,7 @@ mod tests {
 		scheduler::AssignmentKind,
 	};
 	use assert_matches::assert_matches;
+	use frame_support::assert_noop;
 	use futures::executor::block_on;
 	use keyring::Sr25519Keyring;
 	use primitives::{
@@ -968,7 +969,6 @@ mod tests {
 			UncheckedSignedAvailabilityBitfield, ValidationCode, ValidatorId, ValidityAttestation,
 		},
 	};
-	use frame_support::assert_noop;
 	use sc_keystore::LocalKeystore;
 	use sp_keystore::{SyncCryptoStore, SyncCryptoStorePtr};
 	use std::sync::Arc;
diff --git a/runtime/parachains/src/paras_inherent.rs b/runtime/parachains/src/paras_inherent.rs
index 3efd3d10b3a3..e81880c73c44 100644
--- a/runtime/parachains/src/paras_inherent.rs
+++ b/runtime/parachains/src/paras_inherent.rs
@@ -181,7 +181,7 @@ pub mod pallet {
 				current_session,
 				&validator_public[..],
 			)
-			.ok()?;
+			.ok()?; // by convention, when called with `EARLY_RETURN=false`, will always return `Ok()`
 
 			let scheduled: Vec<CoreAssignment> = <scheduler::Pallet<T>>::scheduled();
 			let backed_candidates = sanitize_backed_candidates::<T, false>(
@@ -190,19 +190,20 @@ pub mod pallet {
 				concluded_invalid_disputes,
 				&scheduled[..],
 			)
-			.ok()?;
+			.ok()?; // by convention, when called with `EARLY_RETURN=false`, will always return `Ok()`
 
 			// XXX @Lldenaurois
 			// FIXME these weights are garbage
 			let remaining_weight = MINIMAL_INCLUSION_INHERENT_WEIGHT;
-			let (_backed_candidates_weight, backed_candidates, bitfields) = apply_weight_limit::<T, _>(
-				expected_bits,
-				backed_candidates,
-				bitfields,
-				entropy.clone(),
-				remaining_weight,
-				<scheduler::Pallet<T>>::core_para,
-			);
+			let (_backed_candidates_weight, backed_candidates, bitfields) =
+				apply_weight_limit::<T, _>(
+					expected_bits,
+					backed_candidates,
+					bitfields,
+					entropy.clone(),
+					remaining_weight,
+					<scheduler::Pallet<T>>::core_para,
+				);
 
 			let inherent_data = ParachainsInherentData::<T::Header> {
 				bitfields,
@@ -220,7 +221,7 @@ pub mod pallet {
 					Err(err) => {
 						log::error!(
 							target: LOG_TARGET,
-								"dropping paras inherent data because they produced \
+							"dropping paras inherent data because they produced \
 							an invalid paras inherent: {:?}",
 							err.error,
 						);
@@ -471,7 +472,10 @@ fn apply_weight_limit<T: Config + inclusion::Config, F: Fn(CoreIndex) -> Option<
 		.iter()
 		.map(|backed_candidate| backed_candidate_weight::<T>(backed_candidate))
 		.sum();
-	total += bitfields.iter().map(|bitfield| bitfield_weight::<T>(bitfield.unchecked_payload())).sum::<Weight>();
+	total += bitfields
+		.iter()
+		.map(|bitfield| bitfield_weight::<T>(bitfield.unchecked_payload()))
+		.sum::<Weight>();
 
 	if max_weight < total {
 		return (total, candidates, bitfields)
@@ -494,10 +498,8 @@ fn apply_weight_limit<T: Config + inclusion::Config, F: Fn(CoreIndex) -> Option<
 	let mut candidates_core_index: BTreeMap<CandidateHash, CoreIndex> = (0..expected_bits)
 		.map(|bit_index| core_lookup(CoreIndex::from(bit_index as u32)))
 		.filter_map(|opt_para_id| opt_para_id)
-		.filter_map(|para_id| {
-			PendingAvailability::<T>::get(&para_id)
-		})
-		.map(|cpa: CandidatePendingAvailability<<T>::Hash,<T>::BlockNumber>| {
+		.filter_map(|para_id| PendingAvailability::<T>::get(&para_id))
+		.map(|cpa: CandidatePendingAvailability<<T>::Hash, <T>::BlockNumber>| {
 			(cpa.candidate_hash(), cpa.core_occupied())
 		})
 		.collect();

From 903382ba4813f7fe7065db13b34701c2fa9d7bc4 Mon Sep 17 00:00:00 2001
From: Bernhard Schuster <bernhard@ahoi.io>
Date: Mon, 18 Oct 2021 13:18:58 +0200
Subject: [PATCH 020/107] ff

---
 runtime/parachains/src/paras_inherent.rs | 39 +++++++++++++++++-------
 1 file changed, 28 insertions(+), 11 deletions(-)

diff --git a/runtime/parachains/src/paras_inherent.rs b/runtime/parachains/src/paras_inherent.rs
index e81880c73c44..71815e2148f6 100644
--- a/runtime/parachains/src/paras_inherent.rs
+++ b/runtime/parachains/src/paras_inherent.rs
@@ -156,12 +156,17 @@ pub mod pallet {
 			// filter out any unneeded dispute statements
 			T::DisputesHandler::filter_multi_dispute_data(&mut disputes);
 
+			// `with_transaction`
+			// T::DisputesHandler::provide_multi_dispute_data(disputes);
+
 			let concluded_invalid_disputes = disputes
 				.iter()
-				.filter(|s| s.session == current_session)
-				.map(|s| (s.session, s.candidate_hash))
+				.filter(|dss| dss.session == current_session)
+				.map(|dss| (dss.session, dss.candidate_hash))
 				.filter(|(session, candidate)| {
-					T::DisputesHandler::concluded_invalid(*session, *candidate)
+					// newly concluded votes are not accounted for _yet_
+					// as such we need to explicitly check for them
+					<T>::DisputesHandler::concluded_invalid(*session, *candidate)
 				})
 				.map(|(_session, candidate)| candidate)
 				.collect::<BTreeSet<CandidateHash>>();
@@ -188,6 +193,7 @@ pub mod pallet {
 				parent_hash,
 				backed_candidates,
 				concluded_invalid_disputes,
+				current_session,
 				&scheduled[..],
 			)
 			.ok()?; // by convention, when called with `EARLY_RETURN=false`, will always return `Ok()`
@@ -374,8 +380,17 @@ pub mod pallet {
 				parent_hash,
 				backed_candidates,
 				concluded_invalid_disputed_candidates,
+				current_session,
 				&scheduled[..],
-			)?;
+			).unwrap_or_else(|err| {
+				log::error!(
+					target: LOG_TARGET,
+					"dropping all backed candidates due to sanitization error: {:?}",
+					err,
+				);
+				Vec::new()
+			});
+
 			let backed_candidates = limit_backed_candidates::<T>(backed_candidates);
 			let backed_candidates_len = backed_candidates.len() as Weight;
 
@@ -446,7 +461,7 @@ fn backed_candidate_weight<T: Config>(backed_candidate: &BackedCandidate<<T>::Ha
 }
 
 /// Calculate the weight of a individual bitfield.
-fn bitfield_weight<T: Config>(bitfield: &AvailabilityBitfield) -> Weight {
+fn bitfield_weight<T: Config>(_bitfield: &AvailabilityBitfield) -> Weight {
 	// XXX @Lldenaurois
 	// FIXME these weights are garbage
 	7_000 as Weight
@@ -463,7 +478,7 @@ fn bitfield_weight<T: Config>(bitfield: &AvailabilityBitfield) -> Weight {
 fn apply_weight_limit<T: Config + inclusion::Config, F: Fn(CoreIndex) -> Option<ParaId>>(
 	expected_bits: usize,
 	mut candidates: Vec<BackedCandidate<<T>::Hash>>,
-	mut bitfields: UncheckedSignedAvailabilityBitfields,
+	bitfields: UncheckedSignedAvailabilityBitfields,
 	entropy: SeedEntropy,
 	max_weight: Weight,
 	core_lookup: F,
@@ -492,10 +507,10 @@ fn apply_weight_limit<T: Config + inclusion::Config, F: Fn(CoreIndex) -> Option<
 	let mut weight_acc: Weight = 0 as _;
 
 	// a bitfield to determine which bitfields to include
-	let bitfields_to_include_coverage = BitVec::<Lsb0, u8>::repeat(false, expected_bits);
+	let mut bitfields_to_include_coverage = BitVec::<Lsb0, u8>::repeat(false, expected_bits);
 
 	// create a mapping of `CandidateHash` to `CoreIndex`
-	let mut candidates_core_index: BTreeMap<CandidateHash, CoreIndex> = (0..expected_bits)
+	let candidates_core_index: BTreeMap<CandidateHash, CoreIndex> = (0..expected_bits)
 		.map(|bit_index| core_lookup(CoreIndex::from(bit_index as u32)))
 		.filter_map(|opt_para_id| opt_para_id)
 		.filter_map(|para_id| PendingAvailability::<T>::get(&para_id))
@@ -514,8 +529,8 @@ fn apply_weight_limit<T: Config + inclusion::Config, F: Fn(CoreIndex) -> Option<
 
 		// collect all bitfields that reference the candidate
 		// (or: the availability core index that is responsible for the candidate)
-		let bitfields_weight = 0 as Weight;
-		let covered_bitfields = BitVec::<Lsb0, u8>::repeat(false, expected_bits);
+		let mut bitfields_weight = 0 as Weight;
+		let mut covered_bitfields = BitVec::<Lsb0, u8>::repeat(false, expected_bits);
 
 		for (i, bitfield) in bitfields.iter().enumerate() {
 			// lookup the core index that is responsible for the candidate
@@ -644,12 +659,13 @@ pub(crate) fn sanitize_bitfields<T: Config + crate::inclusion::Config, const EAR
 /// For the filtering here the relevant part is only the current `free`
 /// state.
 fn sanitize_backed_candidates<
-	T: Config + crate::paras_inherent::Config,
+	T: Config + crate::inclusion::Config + crate::paras_inherent::Config,
 	const EARLY_RETURN: bool,
 >(
 	relay_parent: T::Hash,
 	mut backed_candidates: Vec<BackedCandidate<T::Hash>>,
 	disputed_candidates: BTreeSet<CandidateHash>,
+	session_index: SessionIndex,
 	scheduled: &[CoreAssignment],
 ) -> Result<Vec<BackedCandidate<T::Hash>>, Error<T>> {
 	let n = backed_candidates.len();
@@ -657,6 +673,7 @@ fn sanitize_backed_candidates<
 	backed_candidates.retain(|backed_candidate| {
 		let candidate_hash = backed_candidate.candidate.hash();
 		!disputed_candidates.contains(&candidate_hash)
+		|| T::DisputesHandler::concluded_invalid(session_index, candidate_hash)
 	});
 	ensure2!(backed_candidates.len() == n, Error::<T>::CandidateConcludedInvalid, EARLY_RETURN);
 

From 75b48eb840cd27c73ff6f9cc6169c87645af733e Mon Sep 17 00:00:00 2001
From: Bernhard Schuster <bernhard@ahoi.io>
Date: Mon, 18 Oct 2021 13:55:09 +0200
Subject: [PATCH 021/107] test fixup superfluous into call

---
 runtime/parachains/src/inclusion.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/parachains/src/inclusion.rs b/runtime/parachains/src/inclusion.rs
index 91c37b9a53fd..0a7d57463db7 100644
--- a/runtime/parachains/src/inclusion.rs
+++ b/runtime/parachains/src/inclusion.rs
@@ -1998,7 +1998,7 @@ mod tests {
 						],
 						&group_validators,
 					),
-					Error::<Test>::WrongCollator.into(),
+					Error::<Test>::WrongCollator,
 				);
 			}
 

From 1c2b4de6579a0299744a4b02ea76e77468eaf52d Mon Sep 17 00:00:00 2001
From: Bernhard Schuster <bernhard@ahoi.io>
Date: Mon, 18 Oct 2021 13:55:35 +0200
Subject: [PATCH 022/107] chore: pfmt

---
 runtime/parachains/src/paras_inherent.rs | 17 ++++++++++-------
 1 file changed, 10 insertions(+), 7 deletions(-)

diff --git a/runtime/parachains/src/paras_inherent.rs b/runtime/parachains/src/paras_inherent.rs
index 71815e2148f6..23b07e6569b7 100644
--- a/runtime/parachains/src/paras_inherent.rs
+++ b/runtime/parachains/src/paras_inherent.rs
@@ -22,10 +22,9 @@
 //! this module.
 
 use crate::{
-	configuration::Config,
+	configuration,
 	disputes::DisputesHandler,
-	inclusion,
-	inclusion::{CandidatePendingAvailability, PendingAvailability},
+	inclusion::{self, CandidatePendingAvailability, PendingAvailability},
 	scheduler::{self, CoreAssignment, FreedReason},
 	shared, ump, ParaId,
 };
@@ -382,7 +381,8 @@ pub mod pallet {
 				concluded_invalid_disputed_candidates,
 				current_session,
 				&scheduled[..],
-			).unwrap_or_else(|err| {
+			)
+			.unwrap_or_else(|err| {
 				log::error!(
 					target: LOG_TARGET,
 					"dropping all backed candidates due to sanitization error: {:?}",
@@ -587,7 +587,10 @@ fn apply_weight_limit<T: Config + inclusion::Config, F: Fn(CoreIndex) -> Option<
 /// `false` assures that all inputs are filtered, and invalid ones are filtered out.
 /// It also skips signature verification.
 /// `true` returns an `Err(_)` on the first check failing.
-pub(crate) fn sanitize_bitfields<T: Config + crate::inclusion::Config, const EARLY_RETURN: bool>(
+pub(crate) fn sanitize_bitfields<
+	T: configuration::Config + crate::inclusion::Config,
+	const EARLY_RETURN: bool,
+>(
 	unchecked_bitfields: UncheckedSignedAvailabilityBitfields,
 	disputed_bits: DisputedBitfield,
 	expected_bits: usize,
@@ -672,8 +675,8 @@ fn sanitize_backed_candidates<
 	// Remove any candidates that were concluded invalid.
 	backed_candidates.retain(|backed_candidate| {
 		let candidate_hash = backed_candidate.candidate.hash();
-		!disputed_candidates.contains(&candidate_hash)
-		|| T::DisputesHandler::concluded_invalid(session_index, candidate_hash)
+		!disputed_candidates.contains(&candidate_hash) ||
+			T::DisputesHandler::concluded_invalid(session_index, candidate_hash)
 	});
 	ensure2!(backed_candidates.len() == n, Error::<T>::CandidateConcludedInvalid, EARLY_RETURN);
 

From c38771fde298ab1c44ae4a602fb3aa235b30d68f Mon Sep 17 00:00:00 2001
From: Bernhard Schuster <bernhard@ahoi.io>
Date: Mon, 18 Oct 2021 18:12:58 +0200
Subject: [PATCH 023/107] improve apply_block_weight_limit to try to maximize
 the number of sufficiently backed

blocks and add extra bitfields in a round-robin fashion
---
 runtime/parachains/src/paras_inherent.rs | 76 +++++++++++++++++++++---
 1 file changed, 69 insertions(+), 7 deletions(-)

diff --git a/runtime/parachains/src/paras_inherent.rs b/runtime/parachains/src/paras_inherent.rs
index 23b07e6569b7..163f0846101a 100644
--- a/runtime/parachains/src/paras_inherent.rs
+++ b/runtime/parachains/src/paras_inherent.rs
@@ -36,6 +36,7 @@ use frame_support::{
 };
 use frame_system::pallet_prelude::*;
 use primitives::v1::{
+	supermajority_threshold,
 	AvailabilityBitfield, BackedCandidate, CandidateHash, CoreIndex,
 	InherentData as ParachainsInherentData, ScrapedOnChainVotes, SeedEntropy, SessionIndex,
 	SigningContext, UncheckedSignedAvailabilityBitfields, ValidatorId,
@@ -199,7 +200,7 @@ pub mod pallet {
 
 			// XXX @Lldenaurois
 			// FIXME these weights are garbage
-			let remaining_weight = MINIMAL_INCLUSION_INHERENT_WEIGHT;
+			let remaining_weight = <T as frame_system::Config>::BlockWeights::get().max_block;
 			let (_backed_candidates_weight, backed_candidates, bitfields) =
 				apply_weight_limit::<T, _>(
 					expected_bits,
@@ -208,6 +209,7 @@ pub mod pallet {
 					entropy.clone(),
 					remaining_weight,
 					<scheduler::Pallet<T>>::core_para,
+					supermajority_threshold(validator_public.len())
 				);
 
 			let inherent_data = ParachainsInherentData::<T::Header> {
@@ -482,6 +484,7 @@ fn apply_weight_limit<T: Config + inclusion::Config, F: Fn(CoreIndex) -> Option<
 	entropy: SeedEntropy,
 	max_weight: Weight,
 	core_lookup: F,
+	validator_supermajority_availability_threshold: usize,
 ) -> (Weight, Vec<BackedCandidate<<T>::Hash>>, UncheckedSignedAvailabilityBitfields) {
 	let mut total = candidates
 		.iter()
@@ -506,7 +509,8 @@ fn apply_weight_limit<T: Config + inclusion::Config, F: Fn(CoreIndex) -> Option<
 	let mut rng = rand_chacha::ChaChaRng::from_seed(entropy.into());
 	let mut weight_acc: Weight = 0 as _;
 
-	// a bitfield to determine which bitfields to include
+	// a bitfield to determine which availablity bitfields to include
+	// essentially a `Vec<bool>` for our purposes
 	let mut bitfields_to_include_coverage = BitVec::<Lsb0, u8>::repeat(false, expected_bits);
 
 	// create a mapping of `CandidateHash` to `CoreIndex`
@@ -519,25 +523,43 @@ fn apply_weight_limit<T: Config + inclusion::Config, F: Fn(CoreIndex) -> Option<
 		})
 		.collect();
 
-	while weight_acc < max_weight || !candidates.is_empty() {
+	// Keep track the coverage for each candidate, and avoid
+	// including excessive coverage that is not required since the `f+1` threshold is already reached
+	let mut sufficiency_count = BTreeMap::<CandidateHash, usize>::new();
+
+	// the candidates that have remaining bitfields which
+	// could be used to maximize the block utilization
+	let mut superfluous = Vec::new();
+	'next: while weight_acc < max_weight || !candidates.is_empty() {
 		// select a index to try
 		let pick = rng.gen_range(0..candidates.len());
 		// remove the candidate from the possible pick set
 		let picked_candidate = candidates.swap_remove(pick);
 
 		let picked_weight = backed_candidate_weight::<T>(&picked_candidate);
-
 		// collect all bitfields that reference the candidate
 		// (or: the availability core index that is responsible for the candidate)
 		let mut bitfields_weight = 0 as Weight;
 		let mut covered_bitfields = BitVec::<Lsb0, u8>::repeat(false, expected_bits);
 
 		for (i, bitfield) in bitfields.iter().enumerate() {
+
+			// check if there are already sufficient bitfields included to pass the `f+1` threshold
+			if *sufficiency_count.entry(picked_candidate.hash())
+				.and_modify(|x| { *x += 1; } )
+				.or_insert(1_usize) > validator_supermajority_availability_threshold {
+				// supermajority of bitfields reached, don't worry about extra bitfields
+				// they are used lazily to fill remaining space
+				superfluous.push(picked_candidate);
+				continue 'next;
+			}
+
 			// lookup the core index that is responsible for the candidate
 			if let Some(core_index) = candidates_core_index.get(&picked_candidate.hash()) {
+				// check the corresponding bit is `true`
 				if bitfield.unchecked_payload().0[core_index.0 as usize] {
 					// avoid duplicate accounting if it was already included before
-					if bitfields_to_include_coverage[i] {
+					if !bitfields_to_include_coverage[i] {
 						// mark the `i`-th bit
 						covered_bitfields.set(i, true);
 						// account for the added weight of the bitfield
@@ -557,6 +579,44 @@ fn apply_weight_limit<T: Config + inclusion::Config, F: Fn(CoreIndex) -> Option<
 		}
 	}
 
+
+	// If there is any remaining weight, try to fit in some
+	// additional bitfields for candidates that are included with
+	// corresponding bitfields that were not that are not included yet.
+	// see if any further bitfields that cover included candidates.
+	'outer: for (i, bitfield) in bitfields
+		.iter()
+		.enumerate()
+	{
+		// avoid duplicate accounting if it was already included before
+		if bitfields_to_include_coverage[i] {
+			continue 'outer;
+		}
+		// Processing in order here is ok, it's round robbin with an initially
+		// shuffeled ordering.
+		for picked_candidate in superfluous.iter() {
+			if weight_acc >= max_weight {
+				break 'outer;
+			}
+
+			// lookup the core index that is responsible for the candidate
+			if let Some(core_index) = candidates_core_index.get(&picked_candidate.hash()) {
+				// check the corresponding core index represented by the bit offset is `true`
+				if bitfield.unchecked_payload().0[core_index.0 as usize] {
+					// mark the `i`-th bit, we assured in the outer loop
+					// it was not set
+					bitfields_to_include_coverage.set(i, true);
+					// account for the added weight of the bitfield
+					weight_acc += bitfield_weight::<T>(bitfield.unchecked_payload());
+					// if a bitfield is included once, the inner loop is done
+					continue 'outer;
+				}
+			}
+		}
+	}
+
+	// actually collect all unchecked signed availablity bitfields based
+	// on the accumulative bitfield that was created above
 	let bitfields_acc = bitfields
 		.into_iter()
 		.enumerate()
@@ -675,8 +735,8 @@ fn sanitize_backed_candidates<
 	// Remove any candidates that were concluded invalid.
 	backed_candidates.retain(|backed_candidate| {
 		let candidate_hash = backed_candidate.candidate.hash();
-		!disputed_candidates.contains(&candidate_hash) ||
-			T::DisputesHandler::concluded_invalid(session_index, candidate_hash)
+		!disputed_candidates.contains(&candidate_hash) &&
+			!T::DisputesHandler::concluded_invalid(session_index, candidate_hash)
 	});
 	ensure2!(backed_candidates.len() == n, Error::<T>::CandidateConcludedInvalid, EARLY_RETURN);
 
@@ -846,6 +906,8 @@ mod tests {
 				let used_block_weight = max_block_weight / 2;
 				System::set_block_consumed_resources(used_block_weight, 0);
 
+				// TODO add scheduled cores
+
 				// execute the paras inherent
 				let post_info = Call::<Test>::enter {
 					data: ParachainsInherentData {

From 29083565436e7f6d9f504ce41442d491ec7ef6ad Mon Sep 17 00:00:00 2001
From: Bernhard Schuster <bernhard@ahoi.io>
Date: Mon, 18 Oct 2021 18:14:42 +0200
Subject: [PATCH 024/107] new code treats the lack of backed candidates as ok

---
 runtime/parachains/src/paras_inherent.rs | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/runtime/parachains/src/paras_inherent.rs b/runtime/parachains/src/paras_inherent.rs
index 163f0846101a..30f4b7cd2dbf 100644
--- a/runtime/parachains/src/paras_inherent.rs
+++ b/runtime/parachains/src/paras_inherent.rs
@@ -919,8 +919,7 @@ mod tests {
 					},
 				}
 				.dispatch_bypass_filter(None.into())
-				.unwrap_err()
-				.post_info;
+				.unwrap();
 
 				// we don't directly check the block's weight post-call. Instead, we check that the
 				// call has returned the appropriate post-dispatch weight for refund, and trust

From 82617f20f366904ff31d25fcc62e4bc1ae59dac5 Mon Sep 17 00:00:00 2001
From: Bernhard Schuster <bernhard@ahoi.io>
Date: Mon, 18 Oct 2021 19:40:59 +0200
Subject: [PATCH 025/107] Use vrf based entropy

---
 Cargo.lock                               | 14 +++----
 node/core/parachains-inherent/src/lib.rs |  5 ++-
 primitives/Cargo.toml                    |  7 +---
 primitives/src/v1/mod.rs                 | 50 ------------------------
 runtime/parachains/Cargo.toml            |  1 +
 runtime/parachains/src/paras_inherent.rs | 46 +++++++++++-----------
 6 files changed, 35 insertions(+), 88 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 5df55a2415e2..2ae87ca28f1d 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -68,7 +68,7 @@ version = "0.7.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "43bb833f0bf979d8475d38fbf09ed3b8a55e1885fe93ad3f93239fc6a4f17b98"
 dependencies = [
- "getrandom 0.2.3",
+ "getrandom 0.2.1",
  "once_cell",
  "version_check",
 ]
@@ -2432,9 +2432,9 @@ dependencies = [
 
 [[package]]
 name = "getrandom"
-version = "0.2.3"
+version = "0.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7fcd999463524c52659517fe2cea98493cfe485d10565e7b0fb07dbba7ad2753"
+checksum = "4060f4657be78b8e766215b02b18a2e862d83745545de804638e2b545e81aee6"
 dependencies = [
  "cfg-if 1.0.0",
  "libc",
@@ -6624,13 +6624,10 @@ dependencies = [
  "bitvec 0.20.1",
  "frame-system",
  "hex-literal",
- "lazy_static",
  "parity-scale-codec",
  "parity-util-mem",
  "polkadot-core-primitives",
  "polkadot-parachain",
- "rand 0.8.4",
- "rand_chacha 0.3.1",
  "scale-info",
  "serde",
  "sp-api",
@@ -6828,6 +6825,7 @@ dependencies = [
  "log",
  "pallet-authority-discovery",
  "pallet-authorship",
+ "pallet-babe",
  "pallet-balances",
  "pallet-session",
  "pallet-staking",
@@ -7598,7 +7596,7 @@ version = "0.6.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c026d7df8b298d90ccbbc5190bd04d85e159eaf5576caeacf8741da93ccbd2e5"
 dependencies = [
- "getrandom 0.2.3",
+ "getrandom 0.2.1",
 ]
 
 [[package]]
@@ -7690,7 +7688,7 @@ version = "0.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "528532f3d801c87aec9def2add9ca802fe569e44a544afe633765267840abe64"
 dependencies = [
- "getrandom 0.2.3",
+ "getrandom 0.2.1",
  "redox_syscall 0.2.4",
 ]
 
diff --git a/node/core/parachains-inherent/src/lib.rs b/node/core/parachains-inherent/src/lib.rs
index c1101e9f49e9..b123da195506 100644
--- a/node/core/parachains-inherent/src/lib.rs
+++ b/node/core/parachains-inherent/src/lib.rs
@@ -28,7 +28,7 @@ use futures::{select, FutureExt};
 use polkadot_node_subsystem::{
 	errors::SubsystemError, messages::ProvisionerMessage, overseer::Handle,
 };
-use polkadot_primitives::v1::{Block, Hash, InherentData as ParachainsInherentData, SeedEntropy};
+use polkadot_primitives::v1::{Block, Hash, InherentData as ParachainsInherentData};
 use sp_blockchain::HeaderBackend;
 use sp_runtime::generic::BlockId;
 use std::time;
@@ -99,7 +99,10 @@ impl ParachainsInherentDataProvider {
 					backed_candidates: Vec::new(),
 					disputes: Vec::new(),
 					parent_header,
+<<<<<<< HEAD
 					entropy,
+=======
+>>>>>>> parent of f7003e839e (start entropy passing)
 				}
 			},
 		};
diff --git a/primitives/Cargo.toml b/primitives/Cargo.toml
index 5162667c1de8..4aa5d00a3a7c 100644
--- a/primitives/Cargo.toml
+++ b/primitives/Cargo.toml
@@ -28,9 +28,7 @@ bitvec = { version = "0.20.1", default-features = false, features = ["alloc"] }
 frame-system = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false }
 hex-literal = "0.3.3"
 parity-util-mem = { version = "0.10.0", default-features = false, optional = true }
-rand = { version = "0.8.4", optional = true }
-rand_chacha = { version = "0.3.1", optional = true }
-lazy_static = { version = "1.4", optional = true }
+
 
 [features]
 default = ["std"]
@@ -57,7 +55,4 @@ std = [
 	"polkadot-core-primitives/std",
 	"bitvec/std",
 	"frame-system/std",
-	"rand_chacha",
-	"rand/std",
-	"lazy_static",
 ]
diff --git a/primitives/src/v1/mod.rs b/primitives/src/v1/mod.rs
index ced970185714..56ba018c5fd5 100644
--- a/primitives/src/v1/mod.rs
+++ b/primitives/src/v1/mod.rs
@@ -1346,53 +1346,6 @@ pub struct DisputeState<N = BlockNumber> {
 	pub concluded_at: Option<N>,
 }
 
-/// Type abstraction to provide entropy for seeding a `crng`
-#[derive(Clone, Default, Encode, Decode, PartialEq, RuntimeDebug, TypeInfo)]
-pub struct SeedEntropy(pub [u8; 32]);
-
-impl AsRef<[u8]> for SeedEntropy {
-	fn as_ref(&self) -> &[u8] {
-		&self.0[..]
-	}
-}
-
-impl AsMut<[u8]> for SeedEntropy {
-	fn as_mut(&mut self) -> &mut [u8] {
-		&mut self.0[..]
-	}
-}
-
-impl Into<[u8; 32]> for SeedEntropy {
-	fn into(self) -> [u8; 32] {
-		self.0
-	}
-}
-
-#[cfg(feature = "std")]
-impl SeedEntropy {
-	/// Collect a random entropy.
-	pub fn draw() -> Self {
-		let mut bytes = [0u8; 32];
-
-		use rand::RngCore;
-
-		#[cfg(not(fuzzing))]
-		{
-			let mut rng = rand::thread_rng();
-			rng.fill_bytes(&mut bytes[..]);
-		}
-
-		#[cfg(fuzzing)]
-		{
-			lazy_static::lazy_static! {
-				static ref RNG: rand_chacha::ChaChaRng = rand_chacha::ChaChaRng::from_seed(b"polkadot_on_chain_selection_seed");
-			}
-			RNG.fill_bytes(&mut bytes[..])
-		}
-
-		Self(bytes)
-	}
-}
 /// Parachains inherent-data passed into the runtime by a block author
 #[derive(Encode, Decode, Clone, PartialEq, RuntimeDebug, TypeInfo)]
 pub struct InherentData<HDR: HeaderT = Header> {
@@ -1404,9 +1357,6 @@ pub struct InherentData<HDR: HeaderT = Header> {
 	pub disputes: MultiDisputeStatementSet,
 	/// The parent block header. Used for checking state proofs.
 	pub parent_header: HDR,
-	/// Entropy source to initialize a `cprng`, for usage
-	/// in the decimation process of overweight blocks.
-	pub entropy: SeedEntropy,
 }
 
 /// The maximum number of validators `f` which may safely be faulty.
diff --git a/runtime/parachains/Cargo.toml b/runtime/parachains/Cargo.toml
index 5708dccf115e..293d4c34d575 100644
--- a/runtime/parachains/Cargo.toml
+++ b/runtime/parachains/Cargo.toml
@@ -27,6 +27,7 @@ sp-keystore = { git = "https://github.com/paritytech/substrate", branch = "maste
 pallet-authority-discovery = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false }
 pallet-authorship = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false }
 pallet-balances = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false }
+pallet-babe = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false }
 pallet-session = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false }
 pallet-staking = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false }
 pallet-timestamp = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false }
diff --git a/runtime/parachains/src/paras_inherent.rs b/runtime/parachains/src/paras_inherent.rs
index 30f4b7cd2dbf..529e68c8dd6f 100644
--- a/runtime/parachains/src/paras_inherent.rs
+++ b/runtime/parachains/src/paras_inherent.rs
@@ -32,13 +32,14 @@ use bitvec::{order::Lsb0, prelude::BitVec};
 use frame_support::{
 	fail,
 	inherent::{InherentData, InherentIdentifier, MakeFatalError, ProvideInherent},
+	traits::Randomness,
 	pallet_prelude::*,
 };
 use frame_system::pallet_prelude::*;
+use pallet_babe::CurrentBlockRandomness;
 use primitives::v1::{
-	supermajority_threshold,
-	AvailabilityBitfield, BackedCandidate, CandidateHash, CoreIndex,
-	InherentData as ParachainsInherentData, ScrapedOnChainVotes, SeedEntropy, SessionIndex,
+	supermajority_threshold, AvailabilityBitfield, BackedCandidate, CandidateHash, CoreIndex,
+	InherentData as ParachainsInherentData, ScrapedOnChainVotes, SessionIndex,
 	SigningContext, UncheckedSignedAvailabilityBitfields, ValidatorId,
 	PARACHAINS_INHERENT_IDENTIFIER,
 };
@@ -141,7 +142,6 @@ pub mod pallet {
 				backed_candidates,
 				mut disputes,
 				parent_header,
-				entropy,
 			} = match data.get_data(&Self::INHERENT_IDENTIFIER) {
 				Ok(Some(d)) => d,
 				Ok(None) => return None,
@@ -198,6 +198,9 @@ pub mod pallet {
 			)
 			.ok()?; // by convention, when called with `EARLY_RETURN=false`, will always return `Ok()`
 
+
+			let entropy = CurrentBlockRandomness::random(&b"candidate-pick"[..]).0;
+
 			// XXX @Lldenaurois
 			// FIXME these weights are garbage
 			let remaining_weight = <T as frame_system::Config>::BlockWeights::get().max_block;
@@ -206,10 +209,10 @@ pub mod pallet {
 					expected_bits,
 					backed_candidates,
 					bitfields,
-					entropy.clone(),
+					entropy,
 					remaining_weight,
 					<scheduler::Pallet<T>>::core_para,
-					supermajority_threshold(validator_public.len())
+					supermajority_threshold(validator_public.len()),
 				);
 
 			let inherent_data = ParachainsInherentData::<T::Header> {
@@ -217,7 +220,6 @@ pub mod pallet {
 				backed_candidates,
 				disputes,
 				parent_header,
-				entropy,
 			};
 
 			// Sanity check: session changes can invalidate an inherent, and we _really_ don't want that to happen.
@@ -238,7 +240,6 @@ pub mod pallet {
 							backed_candidates: Vec::new(),
 							disputes: Vec::new(),
 							parent_header: inherent_data.parent_header,
-							entropy: inherent_data.entropy,
 						}
 					},
 				};
@@ -267,7 +268,6 @@ pub mod pallet {
 				backed_candidates,
 				parent_header,
 				disputes,
-				entropy: _entropy,
 			} = data;
 
 			ensure_none(origin)?;
@@ -481,7 +481,7 @@ fn apply_weight_limit<T: Config + inclusion::Config, F: Fn(CoreIndex) -> Option<
 	expected_bits: usize,
 	mut candidates: Vec<BackedCandidate<<T>::Hash>>,
 	bitfields: UncheckedSignedAvailabilityBitfields,
-	entropy: SeedEntropy,
+	entropy: [u8; 32],
 	max_weight: Weight,
 	core_lookup: F,
 	validator_supermajority_availability_threshold: usize,
@@ -543,15 +543,19 @@ fn apply_weight_limit<T: Config + inclusion::Config, F: Fn(CoreIndex) -> Option<
 		let mut covered_bitfields = BitVec::<Lsb0, u8>::repeat(false, expected_bits);
 
 		for (i, bitfield) in bitfields.iter().enumerate() {
-
 			// check if there are already sufficient bitfields included to pass the `f+1` threshold
-			if *sufficiency_count.entry(picked_candidate.hash())
-				.and_modify(|x| { *x += 1; } )
-				.or_insert(1_usize) > validator_supermajority_availability_threshold {
+			if *sufficiency_count
+				.entry(picked_candidate.hash())
+				.and_modify(|x| {
+					*x += 1;
+				})
+				.or_insert(1_usize) >
+				validator_supermajority_availability_threshold
+			{
 				// supermajority of bitfields reached, don't worry about extra bitfields
 				// they are used lazily to fill remaining space
 				superfluous.push(picked_candidate);
-				continue 'next;
+				continue 'next
 			}
 
 			// lookup the core index that is responsible for the candidate
@@ -579,24 +583,20 @@ fn apply_weight_limit<T: Config + inclusion::Config, F: Fn(CoreIndex) -> Option<
 		}
 	}
 
-
 	// If there is any remaining weight, try to fit in some
 	// additional bitfields for candidates that are included with
 	// corresponding bitfields that were not that are not included yet.
 	// see if any further bitfields that cover included candidates.
-	'outer: for (i, bitfield) in bitfields
-		.iter()
-		.enumerate()
-	{
+	'outer: for (i, bitfield) in bitfields.iter().enumerate() {
 		// avoid duplicate accounting if it was already included before
 		if bitfields_to_include_coverage[i] {
-			continue 'outer;
+			continue 'outer
 		}
 		// Processing in order here is ok, it's round robbin with an initially
 		// shuffeled ordering.
 		for picked_candidate in superfluous.iter() {
 			if weight_acc >= max_weight {
-				break 'outer;
+				break 'outer
 			}
 
 			// lookup the core index that is responsible for the candidate
@@ -609,7 +609,7 @@ fn apply_weight_limit<T: Config + inclusion::Config, F: Fn(CoreIndex) -> Option<
 					// account for the added weight of the bitfield
 					weight_acc += bitfield_weight::<T>(bitfield.unchecked_payload());
 					// if a bitfield is included once, the inner loop is done
-					continue 'outer;
+					continue 'outer
 				}
 			}
 		}

From 1b236a51a1221c3d720a5d788dea72317bd05ad3 Mon Sep 17 00:00:00 2001
From: Bernhard Schuster <bernhard@ahoi.io>
Date: Tue, 19 Oct 2021 11:33:29 +0200
Subject: [PATCH 026/107] fixup vrf random

---
 node/core/parachains-inherent/src/lib.rs |  4 ----
 runtime/parachains/src/paras_inherent.rs | 20 +++++++++++++-------
 2 files changed, 13 insertions(+), 11 deletions(-)

diff --git a/node/core/parachains-inherent/src/lib.rs b/node/core/parachains-inherent/src/lib.rs
index b123da195506..9f95404fff8e 100644
--- a/node/core/parachains-inherent/src/lib.rs
+++ b/node/core/parachains-inherent/src/lib.rs
@@ -99,10 +99,6 @@ impl ParachainsInherentDataProvider {
 					backed_candidates: Vec::new(),
 					disputes: Vec::new(),
 					parent_header,
-<<<<<<< HEAD
-					entropy,
-=======
->>>>>>> parent of f7003e839e (start entropy passing)
 				}
 			},
 		};
diff --git a/runtime/parachains/src/paras_inherent.rs b/runtime/parachains/src/paras_inherent.rs
index 529e68c8dd6f..9fc574e95675 100644
--- a/runtime/parachains/src/paras_inherent.rs
+++ b/runtime/parachains/src/paras_inherent.rs
@@ -32,16 +32,15 @@ use bitvec::{order::Lsb0, prelude::BitVec};
 use frame_support::{
 	fail,
 	inherent::{InherentData, InherentIdentifier, MakeFatalError, ProvideInherent},
-	traits::Randomness,
 	pallet_prelude::*,
+	traits::Randomness,
 };
 use frame_system::pallet_prelude::*;
 use pallet_babe::CurrentBlockRandomness;
 use primitives::v1::{
 	supermajority_threshold, AvailabilityBitfield, BackedCandidate, CandidateHash, CoreIndex,
-	InherentData as ParachainsInherentData, ScrapedOnChainVotes, SessionIndex,
-	SigningContext, UncheckedSignedAvailabilityBitfields, ValidatorId,
-	PARACHAINS_INHERENT_IDENTIFIER,
+	InherentData as ParachainsInherentData, ScrapedOnChainVotes, SessionIndex, SigningContext,
+	UncheckedSignedAvailabilityBitfields, ValidatorId, PARACHAINS_INHERENT_IDENTIFIER,
 };
 use rand::Rng;
 use scale_info::TypeInfo;
@@ -88,7 +87,7 @@ pub mod pallet {
 
 	#[pallet::config]
 	#[pallet::disable_frame_system_supertrait_check]
-	pub trait Config: inclusion::Config + scheduler::Config {}
+	pub trait Config: inclusion::Config + scheduler::Config + pallet_babe::Config {}
 
 	#[pallet::error]
 	pub enum Error<T> {
@@ -198,8 +197,15 @@ pub mod pallet {
 			)
 			.ok()?; // by convention, when called with `EARLY_RETURN=false`, will always return `Ok()`
 
-
-			let entropy = CurrentBlockRandomness::random(&b"candidate-pick"[..]).0;
+			let entropy = {
+				const CANDIDATE_SEED_SUBJECT: [u8; 32] = *b"candidate-seed-selection-subject";
+				let vrf_random = CurrentBlockRandomness::<T>::random(&CANDIDATE_SEED_SUBJECT[..]).0;
+				let mut entropy: [u8; 32] = CANDIDATE_SEED_SUBJECT.clone();
+				if let Some(vrf_random) = vrf_random {
+					entropy.as_mut().copy_from_slice(vrf_random.as_ref());
+				}
+				entropy
+			};
 
 			// XXX @Lldenaurois
 			// FIXME these weights are garbage

From 56d2752960c73b8a1391ab63a9503da3a1b716cd Mon Sep 17 00:00:00 2001
From: Bernhard Schuster <bernhard@ahoi.io>
Date: Tue, 19 Oct 2021 11:35:52 +0200
Subject: [PATCH 027/107] add warn

---
 runtime/parachains/src/paras_inherent.rs | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/runtime/parachains/src/paras_inherent.rs b/runtime/parachains/src/paras_inherent.rs
index 9fc574e95675..d9264267f265 100644
--- a/runtime/parachains/src/paras_inherent.rs
+++ b/runtime/parachains/src/paras_inherent.rs
@@ -203,6 +203,11 @@ pub mod pallet {
 				let mut entropy: [u8; 32] = CANDIDATE_SEED_SUBJECT.clone();
 				if let Some(vrf_random) = vrf_random {
 					entropy.as_mut().copy_from_slice(vrf_random.as_ref());
+				} else {
+					// in case there is no vrf randomness present, we utilize the relay parent
+					// as seed, it's better than a static value.
+					log::warn!(target: LOG_TARGET, "CurrentBlockRandomness did not provide entropy");
+					entropy.as_mut().copy_from_slice(relay_parent.as_ref());
 				}
 				entropy
 			};

From e11c985f4f217bb17008aa30c47a2a12e2fb5cb2 Mon Sep 17 00:00:00 2001
From: Bernhard Schuster <bernhard@ahoi.io>
Date: Tue, 19 Oct 2021 11:42:23 +0200
Subject: [PATCH 028/107] slip of the pen

---
 runtime/parachains/src/paras_inherent.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/parachains/src/paras_inherent.rs b/runtime/parachains/src/paras_inherent.rs
index d9264267f265..f695954875fd 100644
--- a/runtime/parachains/src/paras_inherent.rs
+++ b/runtime/parachains/src/paras_inherent.rs
@@ -207,7 +207,7 @@ pub mod pallet {
 					// in case there is no vrf randomness present, we utilize the relay parent
 					// as seed, it's better than a static value.
 					log::warn!(target: LOG_TARGET, "CurrentBlockRandomness did not provide entropy");
-					entropy.as_mut().copy_from_slice(relay_parent.as_ref());
+					entropy.as_mut().copy_from_slice(parent_hash.as_ref());
 				}
 				entropy
 			};

From 4e13c1b9945897dee04432a91135f1c113eba276 Mon Sep 17 00:00:00 2001
From: Bernhard Schuster <bernhard@ahoi.io>
Date: Tue, 19 Oct 2021 15:01:54 +0200
Subject: [PATCH 029/107] fixup

---
 node/core/parachains-inherent/src/lib.rs | 2 --
 1 file changed, 2 deletions(-)

diff --git a/node/core/parachains-inherent/src/lib.rs b/node/core/parachains-inherent/src/lib.rs
index 9f95404fff8e..87b4f6f247f3 100644
--- a/node/core/parachains-inherent/src/lib.rs
+++ b/node/core/parachains-inherent/src/lib.rs
@@ -80,13 +80,11 @@ impl ParachainsInherentDataProvider {
 			_ = timeout => Err(Error::Timeout),
 		};
 
-		let entropy = SeedEntropy::draw();
 		let inherent_data = match res {
 			Ok(pd) => ParachainsInherentData {
 				bitfields: pd.bitfields.into_iter().map(Into::into).collect(),
 				backed_candidates: pd.backed_candidates,
 				disputes: pd.disputes,
-				entropy,
 				parent_header,
 			},
 			Err(err) => {

From 7d00562a3f4a4e19d9234f24f86ed5be68e91603 Mon Sep 17 00:00:00 2001
From: Bernhard Schuster <bernhard@ahoi.io>
Date: Tue, 19 Oct 2021 16:14:51 +0200
Subject: [PATCH 030/107] assure ordering

---
 runtime/parachains/src/paras_inherent.rs | 29 ++++++++++++++++++------
 1 file changed, 22 insertions(+), 7 deletions(-)

diff --git a/runtime/parachains/src/paras_inherent.rs b/runtime/parachains/src/paras_inherent.rs
index f695954875fd..40707be9debc 100644
--- a/runtime/parachains/src/paras_inherent.rs
+++ b/runtime/parachains/src/paras_inherent.rs
@@ -499,7 +499,9 @@ fn apply_weight_limit<T: Config + inclusion::Config, F: Fn(CoreIndex) -> Option<
 ) -> (Weight, Vec<BackedCandidate<<T>::Hash>>, UncheckedSignedAvailabilityBitfields) {
 	let mut total = candidates
 		.iter()
-		.map(|backed_candidate| backed_candidate_weight::<T>(backed_candidate))
+		.map(|backed_candidate| {
+			backed_candidate_weight::<T>(backed_candidate)
+		})
 		.sum();
 	total += bitfields
 		.iter()
@@ -510,6 +512,9 @@ fn apply_weight_limit<T: Config + inclusion::Config, F: Fn(CoreIndex) -> Option<
 		return (total, candidates, bitfields)
 	}
 
+	// add the original index _before_ randomly picking
+	let candidates = candidates.into_iter().enumerate().collect::<Vec<_>>();
+
 	let mut candidates_acc = Vec::with_capacity(candidates.len());
 
 	// TODO use CurrentBlockRandomness as seed here
@@ -535,7 +540,8 @@ fn apply_weight_limit<T: Config + inclusion::Config, F: Fn(CoreIndex) -> Option<
 		.collect();
 
 	// Keep track the coverage for each candidate, and avoid
-	// including excessive coverage that is not required since the `f+1` threshold is already reached
+	// including excessive coverage that is not required since the
+	// `2f + 1` threshold is already reached.
 	let mut sufficiency_count = BTreeMap::<CandidateHash, usize>::new();
 
 	// the candidates that have remaining bitfields which
@@ -545,7 +551,7 @@ fn apply_weight_limit<T: Config + inclusion::Config, F: Fn(CoreIndex) -> Option<
 		// select a index to try
 		let pick = rng.gen_range(0..candidates.len());
 		// remove the candidate from the possible pick set
-		let picked_candidate = candidates.swap_remove(pick);
+		let (original_vec_idx, picked_candidate) = candidates.swap_remove(pick);
 
 		let picked_weight = backed_candidate_weight::<T>(&picked_candidate);
 		// collect all bitfields that reference the candidate
@@ -554,7 +560,7 @@ fn apply_weight_limit<T: Config + inclusion::Config, F: Fn(CoreIndex) -> Option<
 		let mut covered_bitfields = BitVec::<Lsb0, u8>::repeat(false, expected_bits);
 
 		for (i, bitfield) in bitfields.iter().enumerate() {
-			// check if there are already sufficient bitfields included to pass the `f+1` threshold
+			// check if there are already sufficient bitfields included to pass the `2f + 1` threshold
 			if *sufficiency_count
 				.entry(picked_candidate.hash())
 				.and_modify(|x| {
@@ -565,7 +571,7 @@ fn apply_weight_limit<T: Config + inclusion::Config, F: Fn(CoreIndex) -> Option<
 			{
 				// supermajority of bitfields reached, don't worry about extra bitfields
 				// they are used lazily to fill remaining space
-				superfluous.push(picked_candidate);
+				superfluous.push((original_vec_idx, picked_candidate));
 				continue 'next
 			}
 
@@ -588,7 +594,7 @@ fn apply_weight_limit<T: Config + inclusion::Config, F: Fn(CoreIndex) -> Option<
 		let prospective_weight = picked_weight + bitfields_weight + weight_acc;
 		if prospective_weight <= max_weight {
 			// candidate fits, so pick it and account for its weight
-			candidates_acc.push(picked_candidate);
+			candidates_acc.push((original_vec_idx, picked_candidate));
 			weight_acc = prospective_weight;
 			bitfields_to_include_coverage |= covered_bitfields;
 		}
@@ -605,7 +611,7 @@ fn apply_weight_limit<T: Config + inclusion::Config, F: Fn(CoreIndex) -> Option<
 		}
 		// Processing in order here is ok, it's round robbin with an initially
 		// shuffeled ordering.
-		for picked_candidate in superfluous.iter() {
+		for (_original_vec_idx, picked_candidate) in superfluous.iter() {
 			if weight_acc >= max_weight {
 				break 'outer
 			}
@@ -635,6 +641,15 @@ fn apply_weight_limit<T: Config + inclusion::Config, F: Fn(CoreIndex) -> Option<
 		.map(|(_, bitfield)| bitfield)
 		.collect::<Vec<_>>();
 
+	candidates_acc.sort_by(|(x_idx, _), (y_idx, _)| {
+		x_idx.cmp(&y_idx)
+	});
+
+	// drop the original indices _after_ sorting
+	let candidates_acc = candidates_acc.into_iter().map(|(_original_vec_idx, backed_candidate)| {
+		backed_candidate
+	}).collect::<Vec<_>>();
+
 	(weight_acc, candidates_acc, bitfields_acc)
 }
 

From 4e06e8d682a4ba900673333fb8b6664157656ddb Mon Sep 17 00:00:00 2001
From: Bernhard Schuster <bernhard@ahoi.io>
Date: Tue, 19 Oct 2021 17:52:35 +0200
Subject: [PATCH 031/107] rethink apply_weights

---
 runtime/parachains/src/paras_inherent.rs | 258 ++++++++---------------
 1 file changed, 89 insertions(+), 169 deletions(-)

diff --git a/runtime/parachains/src/paras_inherent.rs b/runtime/parachains/src/paras_inherent.rs
index 40707be9debc..55e735d7df18 100644
--- a/runtime/parachains/src/paras_inherent.rs
+++ b/runtime/parachains/src/paras_inherent.rs
@@ -24,11 +24,11 @@
 use crate::{
 	configuration,
 	disputes::DisputesHandler,
-	inclusion::{self, CandidatePendingAvailability, PendingAvailability},
+	inclusion,
 	scheduler::{self, CoreAssignment, FreedReason},
-	shared, ump, ParaId,
+	shared, ump,
 };
-use bitvec::{order::Lsb0, prelude::BitVec};
+use bitvec::prelude::BitVec;
 use frame_support::{
 	fail,
 	inherent::{InherentData, InherentIdentifier, MakeFatalError, ProvideInherent},
@@ -38,8 +38,8 @@ use frame_support::{
 use frame_system::pallet_prelude::*;
 use pallet_babe::CurrentBlockRandomness;
 use primitives::v1::{
-	supermajority_threshold, AvailabilityBitfield, BackedCandidate, CandidateHash, CoreIndex,
-	InherentData as ParachainsInherentData, ScrapedOnChainVotes, SessionIndex, SigningContext,
+	BackedCandidate, CandidateHash, CoreIndex, InherentData as ParachainsInherentData,
+	ScrapedOnChainVotes, SessionIndex, SigningContext, UncheckedSignedAvailabilityBitfield,
 	UncheckedSignedAvailabilityBitfields, ValidatorId, PARACHAINS_INHERENT_IDENTIFIER,
 };
 use rand::Rng;
@@ -155,8 +155,16 @@ pub mod pallet {
 			// filter out any unneeded dispute statements
 			T::DisputesHandler::filter_multi_dispute_data(&mut disputes);
 
-			// `with_transaction`
-			// T::DisputesHandler::provide_multi_dispute_data(disputes);
+			let fresh_disputes = frame_support::storage::with_transaction(|| {
+				frame_support::storage::TransactionOutcome::Rollback(
+					T::DisputesHandler::provide_multi_dispute_data(disputes.clone()),
+				)
+			})
+			.map_err(|e| {
+				log::warn!(target: LOG_TARGET, "MultiDisputesData failed to load: {:?}", e);
+				e
+			})
+			.unwrap_or_default();
 
 			let concluded_invalid_disputes = disputes
 				.iter()
@@ -165,7 +173,8 @@ pub mod pallet {
 				.filter(|(session, candidate)| {
 					// newly concluded votes are not accounted for _yet_
 					// as such we need to explicitly check for them
-					<T>::DisputesHandler::concluded_invalid(*session, *candidate)
+					!fresh_disputes.contains(&(*session, *candidate)) &&
+						!<T>::DisputesHandler::concluded_invalid(*session, *candidate)
 				})
 				.map(|(_session, candidate)| candidate)
 				.collect::<BTreeSet<CandidateHash>>();
@@ -206,7 +215,10 @@ pub mod pallet {
 				} else {
 					// in case there is no vrf randomness present, we utilize the relay parent
 					// as seed, it's better than a static value.
-					log::warn!(target: LOG_TARGET, "CurrentBlockRandomness did not provide entropy");
+					log::warn!(
+						target: LOG_TARGET,
+						"CurrentBlockRandomness did not provide entropy"
+					);
 					entropy.as_mut().copy_from_slice(parent_hash.as_ref());
 				}
 				entropy
@@ -216,15 +228,7 @@ pub mod pallet {
 			// FIXME these weights are garbage
 			let remaining_weight = <T as frame_system::Config>::BlockWeights::get().max_block;
 			let (_backed_candidates_weight, backed_candidates, bitfields) =
-				apply_weight_limit::<T, _>(
-					expected_bits,
-					backed_candidates,
-					bitfields,
-					entropy,
-					remaining_weight,
-					<scheduler::Pallet<T>>::core_para,
-					supermajority_threshold(validator_public.len()),
-				);
+				apply_weight_limit::<T>(backed_candidates, bitfields, entropy, remaining_weight);
 
 			let inherent_data = ParachainsInherentData::<T::Header> {
 				bitfields,
@@ -474,7 +478,7 @@ fn backed_candidate_weight<T: Config>(backed_candidate: &BackedCandidate<<T>::Ha
 }
 
 /// Calculate the weight of a individual bitfield.
-fn bitfield_weight<T: Config>(_bitfield: &AvailabilityBitfield) -> Weight {
+fn bitfield_weight<T: Config>(_bitfield: &UncheckedSignedAvailabilityBitfield) -> Weight {
 	// XXX @Lldenaurois
 	// FIXME these weights are garbage
 	7_000 as Weight
@@ -482,175 +486,92 @@ fn bitfield_weight<T: Config>(_bitfield: &AvailabilityBitfield) -> Weight {
 
 /// Considers an upper threshold that the candidates must not exceed.
 ///
-/// If there is sufficient space, all blocks will be included, otherwise
-/// continues adding blocks, ignoring blocks that exceed the threshold.
+/// If there is sufficient space, all bitfields and candidates will be included.
 ///
-/// Since there is the relation of `backed candidate <-> occupied core <-> bitfield`
-/// this is used to pick the candidate, but also include all relevant
-/// bitfields.
-fn apply_weight_limit<T: Config + inclusion::Config, F: Fn(CoreIndex) -> Option<ParaId>>(
-	expected_bits: usize,
-	mut candidates: Vec<BackedCandidate<<T>::Hash>>,
+/// Otherwise tries to include all bitfields, and fills in the remaining weight with candidates.
+///
+/// If even the bitfields are too large to fit into the `max_weight` limit, bitfields are randomly
+/// picked and _no_ candidates will be included.
+fn apply_weight_limit<T: Config + inclusion::Config>(
+	candidates: Vec<BackedCandidate<<T>::Hash>>,
 	bitfields: UncheckedSignedAvailabilityBitfields,
 	entropy: [u8; 32],
 	max_weight: Weight,
-	core_lookup: F,
-	validator_supermajority_availability_threshold: usize,
 ) -> (Weight, Vec<BackedCandidate<<T>::Hash>>, UncheckedSignedAvailabilityBitfields) {
-	let mut total = candidates
-		.iter()
-		.map(|backed_candidate| {
-			backed_candidate_weight::<T>(backed_candidate)
-		})
-		.sum();
-	total += bitfields
+	let total_bitfields_weight =
+		bitfields.iter().map(|bitfield| bitfield_weight::<T>(bitfield)).sum::<Weight>();
+
+	let total_candidates_weight = candidates
 		.iter()
-		.map(|bitfield| bitfield_weight::<T>(bitfield.unchecked_payload()))
+		.map(|backed_candidate| backed_candidate_weight::<T>(backed_candidate))
 		.sum::<Weight>();
 
+	let total = total_bitfields_weight + total_candidates_weight;
+
+	// everything fits into the block
 	if max_weight < total {
 		return (total, candidates, bitfields)
 	}
 
-	// add the original index _before_ randomly picking
-	let candidates = candidates.into_iter().enumerate().collect::<Vec<_>>();
-
-	let mut candidates_acc = Vec::with_capacity(candidates.len());
-
-	// TODO use CurrentBlockRandomness as seed here
-	// TODO alt: use the pick index as subject to
-	// TODO obtain an index at the cost of having to impl
-	// TODO the equal probability distribution ourselves
 	use rand_chacha::rand_core::SeedableRng;
 	let mut rng = rand_chacha::ChaChaRng::from_seed(entropy.into());
-	let mut weight_acc: Weight = 0 as _;
-
-	// a bitfield to determine which availablity bitfields to include
-	// essentially a `Vec<bool>` for our purposes
-	let mut bitfields_to_include_coverage = BitVec::<Lsb0, u8>::repeat(false, expected_bits);
-
-	// create a mapping of `CandidateHash` to `CoreIndex`
-	let candidates_core_index: BTreeMap<CandidateHash, CoreIndex> = (0..expected_bits)
-		.map(|bit_index| core_lookup(CoreIndex::from(bit_index as u32)))
-		.filter_map(|opt_para_id| opt_para_id)
-		.filter_map(|para_id| PendingAvailability::<T>::get(&para_id))
-		.map(|cpa: CandidatePendingAvailability<<T>::Hash, <T>::BlockNumber>| {
-			(cpa.candidate_hash(), cpa.core_occupied())
-		})
-		.collect();
-
-	// Keep track the coverage for each candidate, and avoid
-	// including excessive coverage that is not required since the
-	// `2f + 1` threshold is already reached.
-	let mut sufficiency_count = BTreeMap::<CandidateHash, usize>::new();
-
-	// the candidates that have remaining bitfields which
-	// could be used to maximize the block utilization
-	let mut superfluous = Vec::new();
-	'next: while weight_acc < max_weight || !candidates.is_empty() {
-		// select a index to try
-		let pick = rng.gen_range(0..candidates.len());
-		// remove the candidate from the possible pick set
-		let (original_vec_idx, picked_candidate) = candidates.swap_remove(pick);
-
-		let picked_weight = backed_candidate_weight::<T>(&picked_candidate);
-		// collect all bitfields that reference the candidate
-		// (or: the availability core index that is responsible for the candidate)
-		let mut bitfields_weight = 0 as Weight;
-		let mut covered_bitfields = BitVec::<Lsb0, u8>::repeat(false, expected_bits);
-
-		for (i, bitfield) in bitfields.iter().enumerate() {
-			// check if there are already sufficient bitfields included to pass the `2f + 1` threshold
-			if *sufficiency_count
-				.entry(picked_candidate.hash())
-				.and_modify(|x| {
-					*x += 1;
-				})
-				.or_insert(1_usize) >
-				validator_supermajority_availability_threshold
-			{
-				// supermajority of bitfields reached, don't worry about extra bitfields
-				// they are used lazily to fill remaining space
-				superfluous.push((original_vec_idx, picked_candidate));
-				continue 'next
-			}
 
-			// lookup the core index that is responsible for the candidate
-			if let Some(core_index) = candidates_core_index.get(&picked_candidate.hash()) {
-				// check the corresponding bit is `true`
-				if bitfield.unchecked_payload().0[core_index.0 as usize] {
-					// avoid duplicate accounting if it was already included before
-					if !bitfields_to_include_coverage[i] {
-						// mark the `i`-th bit
-						covered_bitfields.set(i, true);
-						// account for the added weight of the bitfield
-						bitfields_weight += bitfield_weight::<T>(bitfield.unchecked_payload());
-					}
-				}
-			}
-		}
-
-		// is the candidate small enough to fit
-		let prospective_weight = picked_weight + bitfields_weight + weight_acc;
-		if prospective_weight <= max_weight {
-			// candidate fits, so pick it and account for its weight
-			candidates_acc.push((original_vec_idx, picked_candidate));
-			weight_acc = prospective_weight;
-			bitfields_to_include_coverage |= covered_bitfields;
+	fn random_sel<X, F: Fn(&X) -> Weight>(
+		rng: &mut rand_chacha::ChaChaRng,
+		selectables: &[X],
+		weight_fn: F,
+		weight_limit: Weight,
+	) -> (Weight, Vec<usize>) {
+		let mut indices = (0..selectables.len()).into_iter().collect::<Vec<_>>();
+		let mut picked_indices = Vec::with_capacity(selectables.len().saturating_sub(1));
+
+		let mut weight_acc = 0 as Weight;
+		while weight_acc < weight_limit || !selectables.is_empty() {
+			// randomly pick an index
+			let pick = rng.gen_range(0..indices.len());
+			// remove the index from the available set of indices
+			let idx = indices.swap_remove(pick);
+
+			let item = &selectables[idx];
+
+			picked_indices.push(idx);
+			weight_acc = weight_fn(item);
 		}
+		// sorting indices, so the ordering is retained
+		// unstable sorting is fine, since there are no duplicates
+		picked_indices.sort_unstable();
+		(weight_acc, picked_indices)
 	}
 
-	// If there is any remaining weight, try to fit in some
-	// additional bitfields for candidates that are included with
-	// corresponding bitfields that were not that are not included yet.
-	// see if any further bitfields that cover included candidates.
-	'outer: for (i, bitfield) in bitfields.iter().enumerate() {
-		// avoid duplicate accounting if it was already included before
-		if bitfields_to_include_coverage[i] {
-			continue 'outer
-		}
-		// Processing in order here is ok, it's round robbin with an initially
-		// shuffeled ordering.
-		for (_original_vec_idx, picked_candidate) in superfluous.iter() {
-			if weight_acc >= max_weight {
-				break 'outer
-			}
-
-			// lookup the core index that is responsible for the candidate
-			if let Some(core_index) = candidates_core_index.get(&picked_candidate.hash()) {
-				// check the corresponding core index represented by the bit offset is `true`
-				if bitfield.unchecked_payload().0[core_index.0 as usize] {
-					// mark the `i`-th bit, we assured in the outer loop
-					// it was not set
-					bitfields_to_include_coverage.set(i, true);
-					// account for the added weight of the bitfield
-					weight_acc += bitfield_weight::<T>(bitfield.unchecked_payload());
-					// if a bitfield is included once, the inner loop is done
-					continue 'outer
-				}
-			}
-		}
+	// There is weight remaining to be consumed by a subset of candidates
+	// which are going to be picked now.
+	if let Some(remaining_weight) = max_weight.checked_sub(total_bitfields_weight) {
+		let (acc_candidate_weight, indices) = random_sel::<BackedCandidate<<T>::Hash>, _>(
+			&mut rng,
+			&candidates[..],
+			backed_candidate_weight::<T>,
+			remaining_weight,
+		);
+		let candidates =
+			indices.into_iter().map(move |idx| candidates[idx].clone()).collect::<Vec<_>>();
+		// pick all bitfields, and
+		// fill the remaining space with candidates
+		let total = acc_candidate_weight + total_bitfields_weight;
+		return (total, candidates, bitfields)
 	}
 
-	// actually collect all unchecked signed availablity bitfields based
-	// on the accumulative bitfield that was created above
-	let bitfields_acc = bitfields
-		.into_iter()
-		.enumerate()
-		.filter(|(i, _)| bitfields_to_include_coverage[*i])
-		.map(|(_, bitfield)| bitfield)
-		.collect::<Vec<_>>();
-
-	candidates_acc.sort_by(|(x_idx, _), (y_idx, _)| {
-		x_idx.cmp(&y_idx)
-	});
-
-	// drop the original indices _after_ sorting
-	let candidates_acc = candidates_acc.into_iter().map(|(_original_vec_idx, backed_candidate)| {
-		backed_candidate
-	}).collect::<Vec<_>>();
-
-	(weight_acc, candidates_acc, bitfields_acc)
+	// insufficient space for even the bitfields alone, so only try to fit as many of those
+	// into the block and skip the candidates entirely
+	let (total, indices) = random_sel::<UncheckedSignedAvailabilityBitfield, _>(
+		&mut rng,
+		&bitfields[..],
+		bitfield_weight::<T>,
+		max_weight,
+	);
+	let bitfields = indices.into_iter().map(move |idx| bitfields[idx].clone()).collect::<Vec<_>>();
+	// pick all bitfields, and
+	// fill the remaining space with candidates
+	(total, candidates, bitfields)
 }
 
 /// Filter bitfields based on freed core indices, validity, and other sanity checks.
@@ -991,7 +912,6 @@ mod tests {
 						backed_candidates,
 						disputes: Vec::new(),
 						parent_header: header,
-						entropy: Default::default(),
 					},
 				}
 				.dispatch_bypass_filter(None.into())

From 383711d05d91fbc83cfcaf03870f9e06a83208e3 Mon Sep 17 00:00:00 2001
From: Bernhard Schuster <bernhard@ahoi.io>
Date: Tue, 19 Oct 2021 17:59:31 +0200
Subject: [PATCH 032/107] mock

---
 runtime/parachains/src/mock.rs           | 65 ++++++++++++++++++++++--
 runtime/parachains/src/paras_inherent.rs |  1 -
 2 files changed, 62 insertions(+), 4 deletions(-)

diff --git a/runtime/parachains/src/mock.rs b/runtime/parachains/src/mock.rs
index f5b7076f41d8..09dc4b4787c7 100644
--- a/runtime/parachains/src/mock.rs
+++ b/runtime/parachains/src/mock.rs
@@ -22,15 +22,28 @@ use crate::{
 	ump::{self, MessageId, UmpSink},
 	ParaId,
 };
-use frame_support::{parameter_types, traits::GenesisBuild, weights::Weight};
+
+use frame_support::{
+	construct_runtime, parameter_types,
+	traits::{
+		Contains, Everything, GenesisBuild, InstanceFilter, KeyOwnerProofSystem, LockIdentifier,
+		Nothing, OnRuntimeUpgrade,
+	},
+	weights::Weight,
+	PalletId, RuntimeDebug,
+};
 use frame_support_test::TestRandomness;
 use parity_scale_codec::Decode;
 use primitives::v1::{
-	AuthorityDiscoveryId, Balance, BlockNumber, Header, SessionIndex, UpwardMessage, ValidatorIndex,
+	AuthorityDiscoveryId, Balance, BlockNumber, Header, Moment, SessionIndex, UpwardMessage,
+	ValidatorIndex,
 };
 use sp_core::H256;
 use sp_io::TestExternalities;
-use sp_runtime::traits::{BlakeTwo256, IdentityLookup};
+use sp_runtime::{
+	traits::{BlakeTwo256, IdentityLookup},
+	KeyTypeId,
+};
 use std::{cell::RefCell, collections::HashMap};
 
 type UncheckedExtrinsic = frame_system::mocking::MockUncheckedExtrinsic<Test>;
@@ -109,6 +122,52 @@ impl pallet_balances::Config for Test {
 	type WeightInfo = ();
 }
 
+parameter_types! {
+	pub const EpochDuration: u64 = 10;
+	pub const ExpectedBlockTime: Moment = 6_000;
+	pub const ReportLongevity: u64 = 10;
+	pub const MaxAuthorities: u32 = 100_000;
+}
+
+impl pallet_babe::Config for Test {
+	type EpochDuration = EpochDuration;
+	type ExpectedBlockTime = ExpectedBlockTime;
+
+	// session module is the trigger
+	type EpochChangeTrigger = pallet_babe::ExternalTrigger;
+
+	type DisabledValidators = ();
+
+	type KeyOwnerProof = <Self::KeyOwnerProofSystem as KeyOwnerProofSystem<(
+		KeyTypeId,
+		pallet_babe::AuthorityId,
+	)>>::Proof;
+
+	type KeyOwnerIdentification = <Self::KeyOwnerProofSystem as KeyOwnerProofSystem<(
+		KeyTypeId,
+		pallet_babe::AuthorityId,
+	)>>::IdentificationTuple;
+
+	type KeyOwnerProofSystem = ();
+
+	type HandleEquivocation = ();
+
+	type WeightInfo = ();
+
+	type MaxAuthorities = MaxAuthorities;
+}
+
+parameter_types! {
+	pub const MinimumPeriod: Moment = 6_000 / 2;
+}
+
+impl pallet_timestamp::Config for Test {
+	type Moment = Moment;
+	type OnTimestampSet = ();
+	type MinimumPeriod = MinimumPeriod;
+	type WeightInfo = ();
+}
+
 impl crate::initializer::Config for Test {
 	type Randomness = TestRandomness<Self>;
 	type ForceOrigin = frame_system::EnsureRoot<u64>;
diff --git a/runtime/parachains/src/paras_inherent.rs b/runtime/parachains/src/paras_inherent.rs
index 55e735d7df18..1f48e8bfda64 100644
--- a/runtime/parachains/src/paras_inherent.rs
+++ b/runtime/parachains/src/paras_inherent.rs
@@ -862,7 +862,6 @@ mod tests {
 						backed_candidates,
 						disputes: Vec::new(),
 						parent_header: default_header(),
-						entropy: Default::default(),
 					},
 				}
 				.dispatch_bypass_filter(None.into())

From bcd70ebc8d5feeee9a6cba73f52f8933db566a88 Mon Sep 17 00:00:00 2001
From: Bernhard Schuster <bernhard@ahoi.io>
Date: Wed, 20 Oct 2021 14:45:46 +0200
Subject: [PATCH 033/107] use a closure as predicate check

---
 runtime/parachains/src/inclusion.rs      |  4 +-
 runtime/parachains/src/mock.rs           |  6 +--
 runtime/parachains/src/paras_inherent.rs | 66 ++++++++++++------------
 3 files changed, 37 insertions(+), 39 deletions(-)

diff --git a/runtime/parachains/src/inclusion.rs b/runtime/parachains/src/inclusion.rs
index 0a7d57463db7..3fc0f78dbca9 100644
--- a/runtime/parachains/src/inclusion.rs
+++ b/runtime/parachains/src/inclusion.rs
@@ -266,7 +266,7 @@ impl<T: Config> Pallet<T> {
 	pub(crate) fn process_bitfields(
 		expected_bits: usize,
 		signed_bitfields: UncheckedSignedAvailabilityBitfields,
-		disputed_bits: DisputedBitfield,
+		disputed_bitfield: DisputedBitfield,
 		core_lookup: impl Fn(CoreIndex) -> Option<ParaId>,
 	) -> Result<Vec<(CoreIndex, CandidateHash)>, DispatchError> {
 		let validators = shared::Pallet::<T>::active_validator_keys();
@@ -275,7 +275,7 @@ impl<T: Config> Pallet<T> {
 
 		let checked_bitfields = sanitize_bitfields::<T, true>(
 			signed_bitfields,
-			disputed_bits,
+			disputed_bitfield,
 			expected_bits,
 			parent_hash,
 			session_index,
diff --git a/runtime/parachains/src/mock.rs b/runtime/parachains/src/mock.rs
index 09dc4b4787c7..56e63863de58 100644
--- a/runtime/parachains/src/mock.rs
+++ b/runtime/parachains/src/mock.rs
@@ -24,13 +24,11 @@ use crate::{
 };
 
 use frame_support::{
-	construct_runtime, parameter_types,
+	parameter_types,
 	traits::{
-		Contains, Everything, GenesisBuild, InstanceFilter, KeyOwnerProofSystem, LockIdentifier,
-		Nothing, OnRuntimeUpgrade,
+		GenesisBuild, KeyOwnerProofSystem,
 	},
 	weights::Weight,
-	PalletId, RuntimeDebug,
 };
 use frame_support_test::TestRandomness;
 use parity_scale_codec::Decode;
diff --git a/runtime/parachains/src/paras_inherent.rs b/runtime/parachains/src/paras_inherent.rs
index 1f48e8bfda64..ae427b832896 100644
--- a/runtime/parachains/src/paras_inherent.rs
+++ b/runtime/parachains/src/paras_inherent.rs
@@ -155,29 +155,29 @@ pub mod pallet {
 			// filter out any unneeded dispute statements
 			T::DisputesHandler::filter_multi_dispute_data(&mut disputes);
 
-			let fresh_disputes = frame_support::storage::with_transaction(|| {
+			let concluded_invalid_disputes = frame_support::storage::with_transaction(|| {
+				// we don't care about fresh or not disputes
+				// this writes them to storage, so let's query it via those means
+				// if this fails for whatever reason, that's ok
+				let _ = T::DisputesHandler::provide_multi_dispute_data(disputes.clone())
+					.map_err(|e| {
+						log::warn!(target: LOG_TARGET, "MultiDisputesData failed to update: {:?}", e);
+						e
+					});
+				let concluded_invalid_disputes = disputes
+					.iter()
+					.filter(|dss| dss.session == current_session)
+					.map(|dss| (dss.session, dss.candidate_hash))
+					.filter(|(session, candidate)| {
+						<T>::DisputesHandler::concluded_invalid(*session, *candidate)
+					})
+					.map(|(_session, candidate)| candidate)
+					.collect::<BTreeSet<CandidateHash>>();
+
 				frame_support::storage::TransactionOutcome::Rollback(
-					T::DisputesHandler::provide_multi_dispute_data(disputes.clone()),
+					concluded_invalid_disputes
 				)
-			})
-			.map_err(|e| {
-				log::warn!(target: LOG_TARGET, "MultiDisputesData failed to load: {:?}", e);
-				e
-			})
-			.unwrap_or_default();
-
-			let concluded_invalid_disputes = disputes
-				.iter()
-				.filter(|dss| dss.session == current_session)
-				.map(|dss| (dss.session, dss.candidate_hash))
-				.filter(|(session, candidate)| {
-					// newly concluded votes are not accounted for _yet_
-					// as such we need to explicitly check for them
-					!fresh_disputes.contains(&(*session, *candidate)) &&
-						!<T>::DisputesHandler::concluded_invalid(*session, *candidate)
-				})
-				.map(|(_session, candidate)| candidate)
-				.collect::<BTreeSet<CandidateHash>>();
+			});
 
 			// sanitize the bitfields and candidates by removing
 			// anything that does not pass a set of checks
@@ -185,6 +185,8 @@ pub mod pallet {
 			let validator_public = shared::Pallet::<T>::active_validator_keys();
 
 			let expected_bits = <scheduler::Pallet<T>>::availability_cores().len();
+			let scheduled: Vec<CoreAssignment> = <scheduler::Pallet<T>>::scheduled();
+
 
 			let bitfields = sanitize_bitfields::<T, false>(
 				bitfields,
@@ -196,12 +198,10 @@ pub mod pallet {
 			)
 			.ok()?; // by convention, when called with `EARLY_RETURN=false`, will always return `Ok()`
 
-			let scheduled: Vec<CoreAssignment> = <scheduler::Pallet<T>>::scheduled();
-			let backed_candidates = sanitize_backed_candidates::<T, false>(
+			let backed_candidates = sanitize_backed_candidates::<T,_, false>(
 				parent_hash,
 				backed_candidates,
-				concluded_invalid_disputes,
-				current_session,
+				move |candidate_hash: CandidateHash| -> bool { <T>::DisputesHandler::concluded_invalid(current_session, candidate_hash) },
 				&scheduled[..],
 			)
 			.ok()?; // by convention, when called with `EARLY_RETURN=false`, will always return `Ok()`
@@ -392,11 +392,10 @@ pub mod pallet {
 			<scheduler::Pallet<T>>::schedule(freed, <frame_system::Pallet<T>>::block_number());
 
 			let scheduled = <scheduler::Pallet<T>>::scheduled();
-			let backed_candidates = sanitize_backed_candidates::<T, true>(
+			let backed_candidates = sanitize_backed_candidates::<T, _, true>(
 				parent_hash,
 				backed_candidates,
-				concluded_invalid_disputed_candidates,
-				current_session,
+				move |candidate_hash: CandidateHash| -> bool { <T>::DisputesHandler::concluded_invalid(current_session, candidate_hash) },
 				&scheduled[..],
 			)
 			.unwrap_or_else(|err| {
@@ -668,22 +667,23 @@ pub(crate) fn sanitize_bitfields<
 /// guide: Currently `free` but might become `occupied`.
 /// For the filtering here the relevant part is only the current `free`
 /// state.
+///
+/// `candidate_has_concluded_invalid_dispute` must return `true` if the candidate
+/// is disputed, false otherwise
 fn sanitize_backed_candidates<
 	T: Config + crate::inclusion::Config + crate::paras_inherent::Config,
+	F: Fn(CandidateHash) -> bool,
 	const EARLY_RETURN: bool,
 >(
 	relay_parent: T::Hash,
 	mut backed_candidates: Vec<BackedCandidate<T::Hash>>,
-	disputed_candidates: BTreeSet<CandidateHash>,
-	session_index: SessionIndex,
+	candidate_has_concluded_invalid_dispute: F,
 	scheduled: &[CoreAssignment],
 ) -> Result<Vec<BackedCandidate<T::Hash>>, Error<T>> {
 	let n = backed_candidates.len();
 	// Remove any candidates that were concluded invalid.
 	backed_candidates.retain(|backed_candidate| {
-		let candidate_hash = backed_candidate.candidate.hash();
-		!disputed_candidates.contains(&candidate_hash) &&
-			!T::DisputesHandler::concluded_invalid(session_index, candidate_hash)
+		!candidate_has_concluded_invalid_dispute(backed_candidate.candidate.hash())
 	});
 	ensure2!(backed_candidates.len() == n, Error::<T>::CandidateConcludedInvalid, EARLY_RETURN);
 

From 84cbec3d41e7c92dd6e134c68db3fba4ae3dcbbe Mon Sep 17 00:00:00 2001
From: Bernhard Schuster <bernhard@ahoi.io>
Date: Wed, 20 Oct 2021 15:52:20 +0200
Subject: [PATCH 034/107] extract and use DisputedBitfield

---
 runtime/parachains/src/inclusion.rs      |  4 ++
 runtime/parachains/src/paras_inherent.rs | 86 ++++++++++++++----------
 2 files changed, 54 insertions(+), 36 deletions(-)

diff --git a/runtime/parachains/src/inclusion.rs b/runtime/parachains/src/inclusion.rs
index 3fc0f78dbca9..43682692cc96 100644
--- a/runtime/parachains/src/inclusion.rs
+++ b/runtime/parachains/src/inclusion.rs
@@ -216,6 +216,10 @@ pub mod pallet {
 		/// The `para_head` hash in the candidate descriptor doesn't match the hash of the actual para head in the
 		/// commitments.
 		ParaHeadMismatch,
+		/// A bitfield that references a freed core,
+		/// either intentionally or as part of a concluded
+		/// invalid dispute.
+		BitfieldReferencesFreedCore,
 	}
 
 	/// The latest bitfield for each validator, referred to by their index in the validator set.
diff --git a/runtime/parachains/src/paras_inherent.rs b/runtime/parachains/src/paras_inherent.rs
index ae427b832896..d860534ac9ec 100644
--- a/runtime/parachains/src/paras_inherent.rs
+++ b/runtime/parachains/src/paras_inherent.rs
@@ -151,11 +151,15 @@ pub mod pallet {
 			};
 
 			let current_session = <shared::Pallet<T>>::session_index();
+			let expected_bits = <scheduler::Pallet<T>>::availability_cores().len();
+			let validator_public = shared::Pallet::<T>::active_validator_keys();
 
 			// filter out any unneeded dispute statements
 			T::DisputesHandler::filter_multi_dispute_data(&mut disputes);
 
-			let concluded_invalid_disputes = frame_support::storage::with_transaction(|| {
+			let scheduled: Vec<CoreAssignment> = <scheduler::Pallet<T>>::scheduled();
+
+			let (freed_cores, concluded_invalid_disputes) = frame_support::storage::with_transaction(|| {
 				// we don't care about fresh or not disputes
 				// this writes them to storage, so let's query it via those means
 				// if this fails for whatever reason, that's ok
@@ -164,6 +168,7 @@ pub mod pallet {
 						log::warn!(target: LOG_TARGET, "MultiDisputesData failed to update: {:?}", e);
 						e
 					});
+				// all concluded invalid disputes, including the current block's votes
 				let concluded_invalid_disputes = disputes
 					.iter()
 					.filter(|dss| dss.session == current_session)
@@ -174,23 +179,18 @@ pub mod pallet {
 					.map(|(_session, candidate)| candidate)
 					.collect::<BTreeSet<CandidateHash>>();
 
+				let freed_cores = <inclusion::Pallet<T>>::collect_disputed(&concluded_invalid_disputes);
+
 				frame_support::storage::TransactionOutcome::Rollback(
-					concluded_invalid_disputes
+					(freed_cores, concluded_invalid_disputes)
 				)
 			});
 
-			// sanitize the bitfields and candidates by removing
-			// anything that does not pass a set of checks
-			// will be removed here
-			let validator_public = shared::Pallet::<T>::active_validator_keys();
-
-			let expected_bits = <scheduler::Pallet<T>>::availability_cores().len();
-			let scheduled: Vec<CoreAssignment> = <scheduler::Pallet<T>>::scheduled();
-
+			let disputed_bitfield = create_disputed_bitfield(expected_bits, freed_cores.iter());
 
 			let bitfields = sanitize_bitfields::<T, false>(
 				bitfields,
-				DisputedBitfield::zeros(expected_bits), // TODO FIXME
+				disputed_bitfield,
 				expected_bits,
 				parent_hash,
 				current_session,
@@ -201,7 +201,7 @@ pub mod pallet {
 			let backed_candidates = sanitize_backed_candidates::<T,_, false>(
 				parent_hash,
 				backed_candidates,
-				move |candidate_hash: CandidateHash| -> bool { <T>::DisputesHandler::concluded_invalid(current_session, candidate_hash) },
+				move |candidate_hash: CandidateHash| -> bool { concluded_invalid_disputes.contains(&candidate_hash) },
 				&scheduled[..],
 			)
 			.ok()?; // by convention, when called with `EARLY_RETURN=false`, will always return `Ok()`
@@ -299,7 +299,7 @@ pub mod pallet {
 
 			// Handle disputes logic.
 			let current_session = <shared::Pallet<T>>::session_index();
-			let (disputed_bitfield, concluded_invalid_disputed_candidates) = {
+			let disputed_bitfield = {
 				let new_current_dispute_sets: Vec<_> = disputes
 					.iter()
 					.filter(|s| s.session == current_session)
@@ -313,7 +313,7 @@ pub mod pallet {
 					return Ok(Some(MINIMAL_INCLUSION_INHERENT_WEIGHT).into())
 				}
 
-				let (mut freed_disputed, concluded_invalid_disputed_candidates) =
+				let mut freed_disputed =
 					if !new_current_dispute_sets.is_empty() {
 						let concluded_invalid_disputes = new_current_dispute_sets
 							.iter()
@@ -328,25 +328,18 @@ pub mod pallet {
 								.into_iter()
 								.map(|core| (core, FreedReason::Concluded))
 								.collect();
-						(freed_disputed, concluded_invalid_disputes)
+						freed_disputed
 					} else {
-						(Vec::new(), BTreeSet::new())
+						Vec::new()
 					};
 
-				// create a bit index from the set of core indicies.
-				let disputed_bitfield = {
-					let mut bitvec = BitVec::with_capacity(expected_bits);
-					if expected_bits > 0 {
-						bitvec.set(expected_bits.saturating_sub(1), false);
-						for (core_idx, _) in &freed_disputed {
-							let core_idx = core_idx.0 as usize;
-							if core_idx < expected_bits {
-								bitvec.set(core_idx, true);
-							}
-						}
-					}
-					DisputedBitfield::from(bitvec)
-				};
+				// create a bit index from the set of core indicies
+				// where each index corresponds to a core index
+				// that was freed due to a dispute
+				let disputed_bitfield = create_disputed_bitfield(
+					expected_bits,
+					freed_disputed.iter().map(|(core_index, _)| core_index)
+				);
 
 				if !freed_disputed.is_empty() {
 					// unstable sort is fine, because core indices are unique
@@ -355,7 +348,7 @@ pub mod pallet {
 					<scheduler::Pallet<T>>::free_cores(freed_disputed);
 				}
 
-				(disputed_bitfield, concluded_invalid_disputed_candidates)
+				disputed_bitfield
 			};
 
 			// Process new availability bitfields, yielding any availability cores whose
@@ -461,6 +454,22 @@ macro_rules! ensure2 {
 	};
 }
 
+/// Derive a bitfield from dispute
+pub(super) fn create_disputed_bitfield<'a, I: 'a + IntoIterator<Item=&'a CoreIndex>>(expected_bits: usize, freed_cores: I) -> DisputedBitfield
+{
+	let mut bitvec = BitVec::with_capacity(expected_bits);
+	if expected_bits > 0 {
+		bitvec.set(expected_bits.saturating_sub(1), false);
+		for core_idx in freed_cores {
+			let core_idx = core_idx.0 as usize;
+			if core_idx < expected_bits {
+				bitvec.set(core_idx, true);
+			}
+		}
+	}
+	DisputedBitfield::from(bitvec)
+}
+
 /// Calculate the weight of a single backed candidate.
 fn backed_candidate_weight<T: Config>(backed_candidate: &BackedCandidate<<T>::Hash>) -> Weight {
 	// XXX @Lldenaurois
@@ -598,7 +607,7 @@ pub(crate) fn sanitize_bitfields<
 	const EARLY_RETURN: bool,
 >(
 	unchecked_bitfields: UncheckedSignedAvailabilityBitfields,
-	disputed_bits: DisputedBitfield,
+	disputed_bitfield: DisputedBitfield,
 	expected_bits: usize,
 	parent_hash: T::Hash,
 	session_index: SessionIndex,
@@ -609,11 +618,12 @@ pub(crate) fn sanitize_bitfields<
 	let mut last_index = None;
 
 	ensure2!(
-		disputed_bits.0.len() == expected_bits,
+		disputed_bitfield.0.len() == expected_bits,
 		crate::inclusion::pallet::Error::<T>::WrongBitfieldSize,
 		EARLY_RETURN
 	);
 
+	let all_zeros = BitVec::<bitvec::order::Lsb0, u8>::repeat(false, expected_bits);
 	for unchecked_bitfield in unchecked_bitfields {
 		let signing_context = SigningContext { parent_hash, session_index };
 
@@ -624,6 +634,12 @@ pub(crate) fn sanitize_bitfields<
 			continue
 		);
 
+		ensure2!(unchecked_bitfield.unchecked_payload().0.clone() & disputed_bitfield.0.clone() != all_zeros,
+			crate::inclusion::pallet::Error::<T>::BitfieldReferencesFreedCore,
+			EARLY_RETURN,
+			continue
+		);
+
 		ensure2!(
 			last_index.map_or(true, |last| last < unchecked_bitfield.unchecked_validator_index()),
 			crate::inclusion::pallet::Error::<T>::BitfieldDuplicateOrUnordered,
@@ -850,11 +866,9 @@ mod tests {
 				// we've used half the block weight; there's plenty of margin
 				let max_block_weight =
 					<Test as frame_system::Config>::BlockWeights::get().max_block;
-				let used_block_weight = max_block_weight / 2;
+				let used_block_weight = dbg!(max_block_weight / 2);
 				System::set_block_consumed_resources(used_block_weight, 0);
 
-				// TODO add scheduled cores
-
 				// execute the paras inherent
 				let post_info = Call::<Test>::enter {
 					data: ParachainsInherentData {

From f347b3fa08d9bffac809f685b31aa99bfeb40449 Mon Sep 17 00:00:00 2001
From: Bernhard Schuster <bernhard@ahoi.io>
Date: Wed, 20 Oct 2021 15:53:28 +0200
Subject: [PATCH 035/107] chore: simplify

---
 runtime/parachains/src/paras_inherent.rs | 13 +++++--------
 1 file changed, 5 insertions(+), 8 deletions(-)

diff --git a/runtime/parachains/src/paras_inherent.rs b/runtime/parachains/src/paras_inherent.rs
index d860534ac9ec..c5b213780527 100644
--- a/runtime/parachains/src/paras_inherent.rs
+++ b/runtime/parachains/src/paras_inherent.rs
@@ -457,14 +457,11 @@ macro_rules! ensure2 {
 /// Derive a bitfield from dispute
 pub(super) fn create_disputed_bitfield<'a, I: 'a + IntoIterator<Item=&'a CoreIndex>>(expected_bits: usize, freed_cores: I) -> DisputedBitfield
 {
-	let mut bitvec = BitVec::with_capacity(expected_bits);
-	if expected_bits > 0 {
-		bitvec.set(expected_bits.saturating_sub(1), false);
-		for core_idx in freed_cores {
-			let core_idx = core_idx.0 as usize;
-			if core_idx < expected_bits {
-				bitvec.set(core_idx, true);
-			}
+	let mut bitvec = BitVec::repeat(false, expected_bits);
+	for core_idx in freed_cores {
+		let core_idx = core_idx.0 as usize;
+		if core_idx < expected_bits {
+			bitvec.set(core_idx, true);
 		}
 	}
 	DisputedBitfield::from(bitvec)

From 0df5cf4decd3a9388351fd4b5e1d09a354afa5b0 Mon Sep 17 00:00:00 2001
From: Bernhard Schuster <bernhard@ahoi.io>
Date: Wed, 20 Oct 2021 15:53:47 +0200
Subject: [PATCH 036/107] remove stray dbg

---
 runtime/parachains/src/paras_inherent.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/parachains/src/paras_inherent.rs b/runtime/parachains/src/paras_inherent.rs
index c5b213780527..50dc9ecdade9 100644
--- a/runtime/parachains/src/paras_inherent.rs
+++ b/runtime/parachains/src/paras_inherent.rs
@@ -863,7 +863,7 @@ mod tests {
 				// we've used half the block weight; there's plenty of margin
 				let max_block_weight =
 					<Test as frame_system::Config>::BlockWeights::get().max_block;
-				let used_block_weight = dbg!(max_block_weight / 2);
+				let used_block_weight = max_block_weight / 2;
 				System::set_block_consumed_resources(used_block_weight, 0);
 
 				// execute the paras inherent

From 376d158d54fad2c20a86cc34ba73da133bc0d6b2 Mon Sep 17 00:00:00 2001
From: Bernhard Schuster <bernhard@ahoi.io>
Date: Wed, 20 Oct 2021 16:02:54 +0200
Subject: [PATCH 037/107] chore: fmt

---
 runtime/parachains/src/mock.rs           |   4 +-
 runtime/parachains/src/paras_inherent.rs | 118 +++++++++++++----------
 2 files changed, 67 insertions(+), 55 deletions(-)

diff --git a/runtime/parachains/src/mock.rs b/runtime/parachains/src/mock.rs
index 56e63863de58..5ba8e046aff0 100644
--- a/runtime/parachains/src/mock.rs
+++ b/runtime/parachains/src/mock.rs
@@ -25,9 +25,7 @@ use crate::{
 
 use frame_support::{
 	parameter_types,
-	traits::{
-		GenesisBuild, KeyOwnerProofSystem,
-	},
+	traits::{GenesisBuild, KeyOwnerProofSystem},
 	weights::Weight,
 };
 use frame_support_test::TestRandomness;
diff --git a/runtime/parachains/src/paras_inherent.rs b/runtime/parachains/src/paras_inherent.rs
index 50dc9ecdade9..b4f1e51a2af7 100644
--- a/runtime/parachains/src/paras_inherent.rs
+++ b/runtime/parachains/src/paras_inherent.rs
@@ -159,32 +159,39 @@ pub mod pallet {
 
 			let scheduled: Vec<CoreAssignment> = <scheduler::Pallet<T>>::scheduled();
 
-			let (freed_cores, concluded_invalid_disputes) = frame_support::storage::with_transaction(|| {
-				// we don't care about fresh or not disputes
-				// this writes them to storage, so let's query it via those means
-				// if this fails for whatever reason, that's ok
-				let _ = T::DisputesHandler::provide_multi_dispute_data(disputes.clone())
-					.map_err(|e| {
-						log::warn!(target: LOG_TARGET, "MultiDisputesData failed to update: {:?}", e);
-						e
-					});
-				// all concluded invalid disputes, including the current block's votes
-				let concluded_invalid_disputes = disputes
-					.iter()
-					.filter(|dss| dss.session == current_session)
-					.map(|dss| (dss.session, dss.candidate_hash))
-					.filter(|(session, candidate)| {
-						<T>::DisputesHandler::concluded_invalid(*session, *candidate)
-					})
-					.map(|(_session, candidate)| candidate)
-					.collect::<BTreeSet<CandidateHash>>();
-
-				let freed_cores = <inclusion::Pallet<T>>::collect_disputed(&concluded_invalid_disputes);
-
-				frame_support::storage::TransactionOutcome::Rollback(
-					(freed_cores, concluded_invalid_disputes)
-				)
-			});
+			let (freed_cores, concluded_invalid_disputes) =
+				frame_support::storage::with_transaction(|| {
+					// we don't care about fresh or not disputes
+					// this writes them to storage, so let's query it via those means
+					// if this fails for whatever reason, that's ok
+					let _ = T::DisputesHandler::provide_multi_dispute_data(disputes.clone())
+						.map_err(|e| {
+							log::warn!(
+								target: LOG_TARGET,
+								"MultiDisputesData failed to update: {:?}",
+								e
+							);
+							e
+						});
+					// all concluded invalid disputes, including the current block's votes
+					let concluded_invalid_disputes = disputes
+						.iter()
+						.filter(|dss| dss.session == current_session)
+						.map(|dss| (dss.session, dss.candidate_hash))
+						.filter(|(session, candidate)| {
+							<T>::DisputesHandler::concluded_invalid(*session, *candidate)
+						})
+						.map(|(_session, candidate)| candidate)
+						.collect::<BTreeSet<CandidateHash>>();
+
+					let freed_cores =
+						<inclusion::Pallet<T>>::collect_disputed(&concluded_invalid_disputes);
+
+					frame_support::storage::TransactionOutcome::Rollback((
+						freed_cores,
+						concluded_invalid_disputes,
+					))
+				});
 
 			let disputed_bitfield = create_disputed_bitfield(expected_bits, freed_cores.iter());
 
@@ -198,10 +205,12 @@ pub mod pallet {
 			)
 			.ok()?; // by convention, when called with `EARLY_RETURN=false`, will always return `Ok()`
 
-			let backed_candidates = sanitize_backed_candidates::<T,_, false>(
+			let backed_candidates = sanitize_backed_candidates::<T, _, false>(
 				parent_hash,
 				backed_candidates,
-				move |candidate_hash: CandidateHash| -> bool { concluded_invalid_disputes.contains(&candidate_hash) },
+				move |candidate_hash: CandidateHash| -> bool {
+					concluded_invalid_disputes.contains(&candidate_hash)
+				},
 				&scheduled[..],
 			)
 			.ok()?; // by convention, when called with `EARLY_RETURN=false`, will always return `Ok()`
@@ -313,32 +322,31 @@ pub mod pallet {
 					return Ok(Some(MINIMAL_INCLUSION_INHERENT_WEIGHT).into())
 				}
 
-				let mut freed_disputed =
-					if !new_current_dispute_sets.is_empty() {
-						let concluded_invalid_disputes = new_current_dispute_sets
-							.iter()
-							.filter(|(session, candidate)| {
-								T::DisputesHandler::concluded_invalid(*session, *candidate)
-							})
-							.map(|(_, candidate)| *candidate)
-							.collect::<BTreeSet<CandidateHash>>();
-
-						let freed_disputed =
-							<inclusion::Pallet<T>>::collect_disputed(&concluded_invalid_disputes)
-								.into_iter()
-								.map(|core| (core, FreedReason::Concluded))
-								.collect();
-						freed_disputed
-					} else {
-						Vec::new()
-					};
+				let mut freed_disputed = if !new_current_dispute_sets.is_empty() {
+					let concluded_invalid_disputes = new_current_dispute_sets
+						.iter()
+						.filter(|(session, candidate)| {
+							T::DisputesHandler::concluded_invalid(*session, *candidate)
+						})
+						.map(|(_, candidate)| *candidate)
+						.collect::<BTreeSet<CandidateHash>>();
+
+					let freed_disputed =
+						<inclusion::Pallet<T>>::collect_disputed(&concluded_invalid_disputes)
+							.into_iter()
+							.map(|core| (core, FreedReason::Concluded))
+							.collect();
+					freed_disputed
+				} else {
+					Vec::new()
+				};
 
 				// create a bit index from the set of core indicies
 				// where each index corresponds to a core index
 				// that was freed due to a dispute
 				let disputed_bitfield = create_disputed_bitfield(
 					expected_bits,
-					freed_disputed.iter().map(|(core_index, _)| core_index)
+					freed_disputed.iter().map(|(core_index, _)| core_index),
 				);
 
 				if !freed_disputed.is_empty() {
@@ -388,7 +396,9 @@ pub mod pallet {
 			let backed_candidates = sanitize_backed_candidates::<T, _, true>(
 				parent_hash,
 				backed_candidates,
-				move |candidate_hash: CandidateHash| -> bool { <T>::DisputesHandler::concluded_invalid(current_session, candidate_hash) },
+				move |candidate_hash: CandidateHash| -> bool {
+					<T>::DisputesHandler::concluded_invalid(current_session, candidate_hash)
+				},
 				&scheduled[..],
 			)
 			.unwrap_or_else(|err| {
@@ -455,8 +465,10 @@ macro_rules! ensure2 {
 }
 
 /// Derive a bitfield from dispute
-pub(super) fn create_disputed_bitfield<'a, I: 'a + IntoIterator<Item=&'a CoreIndex>>(expected_bits: usize, freed_cores: I) -> DisputedBitfield
-{
+pub(super) fn create_disputed_bitfield<'a, I: 'a + IntoIterator<Item = &'a CoreIndex>>(
+	expected_bits: usize,
+	freed_cores: I,
+) -> DisputedBitfield {
 	let mut bitvec = BitVec::repeat(false, expected_bits);
 	for core_idx in freed_cores {
 		let core_idx = core_idx.0 as usize;
@@ -631,7 +643,9 @@ pub(crate) fn sanitize_bitfields<
 			continue
 		);
 
-		ensure2!(unchecked_bitfield.unchecked_payload().0.clone() & disputed_bitfield.0.clone() != all_zeros,
+		ensure2!(
+			unchecked_bitfield.unchecked_payload().0.clone() & disputed_bitfield.0.clone() !=
+				all_zeros,
 			crate::inclusion::pallet::Error::<T>::BitfieldReferencesFreedCore,
 			EARLY_RETURN,
 			continue

From 2d2f8a8e3fa011452fe38d56663dc4dc9a3ddb03 Mon Sep 17 00:00:00 2001
From: Bernhard Schuster <bernhard@ahoi.io>
Date: Wed, 20 Oct 2021 17:54:13 +0200
Subject: [PATCH 038/107] address feedback

---
 runtime/parachains/src/paras_inherent.rs | 24 ++++++++++++++++++++----
 1 file changed, 20 insertions(+), 4 deletions(-)

diff --git a/runtime/parachains/src/paras_inherent.rs b/runtime/parachains/src/paras_inherent.rs
index b4f1e51a2af7..38e828b785ce 100644
--- a/runtime/parachains/src/paras_inherent.rs
+++ b/runtime/parachains/src/paras_inherent.rs
@@ -173,8 +173,9 @@ pub mod pallet {
 							);
 							e
 						});
-					// all concluded invalid disputes, including the current block's votes
-					let concluded_invalid_disputes = disputes
+
+					// current concluded invalid disputes, only including the current block's votes
+					let current_concluded_invalid_disputes = disputes
 						.iter()
 						.filter(|dss| dss.session == current_session)
 						.map(|dss| (dss.session, dss.candidate_hash))
@@ -184,8 +185,19 @@ pub mod pallet {
 						.map(|(_session, candidate)| candidate)
 						.collect::<BTreeSet<CandidateHash>>();
 
-					let freed_cores =
-						<inclusion::Pallet<T>>::collect_disputed(&concluded_invalid_disputes);
+					let freed_cores = <inclusion::Pallet<T>>::collect_disputed(
+						&current_concluded_invalid_disputes,
+					);
+
+					// all concluded invalid disputes, that are relevant for the set of candidates
+					// the inherent provided
+					let concluded_invalid_disputes = backed_candidates
+						.iter()
+						.map(|backed_candidate| backed_candidate.hash())
+						.filter(|candidate| {
+							<T>::DisputesHandler::concluded_invalid(current_session, candidate)
+						})
+						.collect::<BTreeSet<CandidateHash>>();
 
 					frame_support::storage::TransactionOutcome::Rollback((
 						freed_cores,
@@ -248,6 +260,10 @@ pub mod pallet {
 
 			// Sanity check: session changes can invalidate an inherent, and we _really_ don't want that to happen.
 			// See <https://github.com/paritytech/polkadot/issues/1327>
+
+			// Calling `Self::enter` here is a safe-guard, to avoid any discrepancy
+			// between on-chain logic and the logic that is executed
+			// at the block producer off-chain (this code path).
 			let inherent_data =
 				match Self::enter(frame_system::RawOrigin::None.into(), inherent_data.clone()) {
 					Ok(_) => inherent_data,

From a1a567c49c078ac92e2c6199e9fe066bd517bd5f Mon Sep 17 00:00:00 2001
From: Bernhard Schuster <bernhard@ahoi.io>
Date: Wed, 20 Oct 2021 19:52:06 +0200
Subject: [PATCH 039/107] fix test, halfway there

---
 runtime/parachains/src/paras_inherent.rs | 60 ++++++++++++++++++------
 1 file changed, 46 insertions(+), 14 deletions(-)

diff --git a/runtime/parachains/src/paras_inherent.rs b/runtime/parachains/src/paras_inherent.rs
index 38e828b785ce..abe851c38132 100644
--- a/runtime/parachains/src/paras_inherent.rs
+++ b/runtime/parachains/src/paras_inherent.rs
@@ -70,6 +70,7 @@ impl From<BitVec<bitvec::order::Lsb0, u8>> for DisputedBitfield {
 	}
 }
 
+#[cfg(test)]
 impl DisputedBitfield {
 	/// Create a new bitfield, where each bit is set to `false`.
 	pub fn zeros(n: usize) -> Self {
@@ -195,7 +196,7 @@ pub mod pallet {
 						.iter()
 						.map(|backed_candidate| backed_candidate.hash())
 						.filter(|candidate| {
-							<T>::DisputesHandler::concluded_invalid(current_session, candidate)
+							<T>::DisputesHandler::concluded_invalid(current_session, *candidate)
 						})
 						.collect::<BTreeSet<CandidateHash>>();
 
@@ -496,18 +497,19 @@ pub(super) fn create_disputed_bitfield<'a, I: 'a + IntoIterator<Item = &'a CoreI
 }
 
 /// Calculate the weight of a single backed candidate.
-fn backed_candidate_weight<T: Config>(backed_candidate: &BackedCandidate<<T>::Hash>) -> Weight {
+fn backed_candidate_weight<T: Config>(_backed_candidate: &BackedCandidate<<T>::Hash>) -> Weight {
 	// XXX @Lldenaurois
 	// FIXME these weights are garbage
-	const CODE_UPGRADE_WEIGHT: Weight = 10_000 as Weight;
-	const DISPUTE_PER_STATEMENT_WEIGHT: Weight = 1_000 as Weight;
-
-	backed_candidate.validity_votes.len() as Weight * DISPUTE_PER_STATEMENT_WEIGHT +
-		if backed_candidate.candidate.commitments.new_validation_code.is_some() {
-			CODE_UPGRADE_WEIGHT
-		} else {
-			0 as Weight
-		}
+	// const CODE_UPGRADE_WEIGHT: Weight = 10_000 as Weight;
+	// const DISPUTE_PER_STATEMENT_WEIGHT: Weight = 1_000 as Weight;
+
+	// backed_candidate.validity_votes.len() as Weight * DISPUTE_PER_STATEMENT_WEIGHT +
+	// 	if backed_candidate.candidate.commitments.new_validation_code.is_some() {
+	// 		CODE_UPGRADE_WEIGHT
+	// 	} else {
+	// 		0 as Weight
+	// 	}
+	BACKED_CANDIDATE_WEIGHT
 }
 
 /// Calculate the weight of a individual bitfield.
@@ -857,7 +859,7 @@ mod tests {
 		use super::*;
 
 		use crate::mock::{new_test_ext, MockGenesisConfig, System, Test};
-		use primitives::v1::Header;
+		use primitives::v1::{Hash, Header, GroupIndex, Id as ParaId};
 
 		use frame_support::traits::UnfilteredDispatchable;
 
@@ -884,7 +886,13 @@ mod tests {
 				// number of bitfields doesn't affect the paras inherent weight, so we can mock it with an empty one
 				let signed_bitfields = Vec::new();
 				// backed candidates must not be empty, so we can demonstrate that the weight has not changed
-				let backed_candidates = vec![BackedCandidate::default(); 10];
+				let backed_candidates = std::iter::repeat_with(|| BackedCandidate::default())
+					.take(10)
+					.map(|mut x| {
+						x.candidate.descriptor.relay_parent = header.hash();
+						x
+					})
+					.collect::<Vec<_>>();
 
 				// the expected weight can always be computed by this formula
 				let expected_weight = MINIMAL_INCLUSION_INHERENT_WEIGHT +
@@ -896,13 +904,37 @@ mod tests {
 				let used_block_weight = max_block_weight / 2;
 				System::set_block_consumed_resources(used_block_weight, 0);
 
+				let chain_a = ParaId::from(1);
+				let chain_b = ParaId::from(2);
+
+				let chain_a_assignment = CoreAssignment {
+					core: CoreIndex::from(0),
+					para_id: chain_a,
+					kind: crate::scheduler::AssignmentKind::Parachain,
+					group_idx: GroupIndex::from(0),
+				};
+
+				let chain_b_assignment = CoreAssignment {
+					core: CoreIndex::from(1),
+					para_id: chain_b,
+					kind: crate::scheduler::AssignmentKind::Parachain,
+					group_idx: GroupIndex::from(1),
+				};
+               // TODO scheduled in ::enter is still empty11
+				crate::scheduler::AvailabilityCores::<Test>::set(vec![None, None]);
+				crate::paras::Parachains::<Test>::set(vec![chain_a, chain_b]);
+				<crate::scheduler::Pallet<Test>>::schedule(vec![(CoreIndex(0), FreedReason::Concluded), (CoreIndex(1), FreedReason::Concluded)], <frame_system::Pallet<Test>>::block_number());
+
+				let scheduled = vec![chain_a_assignment, chain_b_assignment];
+				crate::scheduler::Scheduled::<Test>::set(scheduled);
+
 				// execute the paras inherent
 				let post_info = Call::<Test>::enter {
 					data: ParachainsInherentData {
 						bitfields: signed_bitfields,
 						backed_candidates,
 						disputes: Vec::new(),
-						parent_header: default_header(),
+						parent_header: header,
 					},
 				}
 				.dispatch_bypass_filter(None.into())

From 64a759af6a6285730033e65cf4dcf14b74fe1182 Mon Sep 17 00:00:00 2001
From: Bernhard Schuster <bernhard@ahoi.io>
Date: Thu, 21 Oct 2021 18:03:12 +0200
Subject: [PATCH 040/107] stage1

---
 runtime/parachains/src/inclusion.rs      |  56 +++++------
 runtime/parachains/src/paras_inherent.rs | 121 +++++++++++++++++++----
 2 files changed, 128 insertions(+), 49 deletions(-)

diff --git a/runtime/parachains/src/inclusion.rs b/runtime/parachains/src/inclusion.rs
index 43682692cc96..6988686fe636 100644
--- a/runtime/parachains/src/inclusion.rs
+++ b/runtime/parachains/src/inclusion.rs
@@ -948,7 +948,7 @@ impl<T: Config> CandidateCheckContext<T> {
 }
 
 #[cfg(test)]
-mod tests {
+pub(crate) mod tests {
 	use super::*;
 	use crate::{
 		configuration::HostConfiguration,
@@ -984,7 +984,7 @@ mod tests {
 		config
 	}
 
-	fn genesis_config(paras: Vec<(ParaId, bool)>) -> MockGenesisConfig {
+	pub(crate) fn genesis_config(paras: Vec<(ParaId, bool)>) -> MockGenesisConfig {
 		MockGenesisConfig {
 			paras: paras::GenesisConfig {
 				paras: paras
@@ -1011,14 +1011,14 @@ mod tests {
 	}
 
 	#[derive(Debug, Clone, Copy, PartialEq)]
-	enum BackingKind {
+	pub(crate) enum BackingKind {
 		#[allow(unused)]
 		Unanimous,
 		Threshold,
 		Lacking,
 	}
 
-	fn collator_sign_candidate(
+	pub(crate) fn collator_sign_candidate(
 		collator: Sr25519Keyring,
 		candidate: &mut CommittedCandidateReceipt,
 	) {
@@ -1036,7 +1036,7 @@ mod tests {
 		assert!(candidate.descriptor().check_collator_signature().is_ok());
 	}
 
-	async fn back_candidate(
+	pub(crate) async fn back_candidate(
 		candidate: CommittedCandidateReceipt,
 		validators: &[Sr25519Keyring],
 		group: &[ValidatorIndex],
@@ -1078,11 +1078,6 @@ mod tests {
 
 		let backed = BackedCandidate { candidate, validity_votes, validator_indices };
 
-		let should_pass = match kind {
-			BackingKind::Unanimous | BackingKind::Threshold => true,
-			BackingKind::Lacking => false,
-		};
-
 		let successfully_backed =
 			primitives::v1::check_candidate_backing(&backed, signing_context, group.len(), |i| {
 				Some(validators[group[i].0 as usize].public().into())
@@ -1091,16 +1086,15 @@ mod tests {
 			.unwrap_or(0) * 2 >
 				group.len();
 
-		if should_pass {
-			assert!(successfully_backed);
-		} else {
-			assert!(!successfully_backed);
-		}
+		match kind {
+			BackingKind::Unanimous | BackingKind::Threshold => assert!(successfully_backed),
+			BackingKind::Lacking => assert!(!successfully_backed),
+		};
 
 		backed
 	}
 
-	fn run_to_block(
+	pub(crate) fn run_to_block(
 		to: BlockNumber,
 		new_session: impl Fn(BlockNumber) -> Option<SessionChangeNotification<BlockNumber>>,
 	) {
@@ -1133,7 +1127,7 @@ mod tests {
 		}
 	}
 
-	fn expected_bits() -> usize {
+	pub(crate) fn expected_bits() -> usize {
 		Paras::parachains().len() + Configuration::config().parathread_cores as usize
 	}
 
@@ -1157,11 +1151,11 @@ mod tests {
 		b
 	}
 
-	fn validator_pubkeys(val_ids: &[Sr25519Keyring]) -> Vec<ValidatorId> {
+	pub(crate) fn validator_pubkeys(val_ids: &[Sr25519Keyring]) -> Vec<ValidatorId> {
 		val_ids.iter().map(|v| v.public().into()).collect()
 	}
 
-	async fn sign_bitfield(
+	pub(crate) async fn sign_bitfield(
 		keystore: &SyncCryptoStorePtr,
 		key: &Sr25519Keyring,
 		validator_index: ValidatorIndex,
@@ -1181,20 +1175,20 @@ mod tests {
 	}
 
 	#[derive(Default)]
-	struct TestCandidateBuilder {
-		para_id: ParaId,
-		head_data: HeadData,
-		para_head_hash: Option<Hash>,
-		pov_hash: Hash,
-		relay_parent: Hash,
-		persisted_validation_data_hash: Hash,
-		new_validation_code: Option<ValidationCode>,
-		validation_code: ValidationCode,
-		hrmp_watermark: BlockNumber,
+	pub(crate) struct TestCandidateBuilder {
+		pub(crate) para_id: ParaId,
+		pub(crate) head_data: HeadData,
+		pub(crate) para_head_hash: Option<Hash>,
+		pub(crate) pov_hash: Hash,
+		pub(crate) relay_parent: Hash,
+		pub(crate) persisted_validation_data_hash: Hash,
+		pub(crate) new_validation_code: Option<ValidationCode>,
+		pub(crate) validation_code: ValidationCode,
+		pub(crate) hrmp_watermark: BlockNumber,
 	}
 
 	impl TestCandidateBuilder {
-		fn build(self) -> CommittedCandidateReceipt {
+		pub(crate) fn build(self) -> CommittedCandidateReceipt {
 			CommittedCandidateReceipt {
 				descriptor: CandidateDescriptor {
 					para_id: self.para_id,
@@ -1215,7 +1209,7 @@ mod tests {
 		}
 	}
 
-	fn make_vdata_hash(para_id: ParaId) -> Option<Hash> {
+	pub(crate) fn make_vdata_hash(para_id: ParaId) -> Option<Hash> {
 		let relay_parent_number = <frame_system::Pallet<Test>>::block_number() - 1;
 		let persisted_validation_data = crate::util::make_persisted_validation_data::<Test>(
 			para_id,
diff --git a/runtime/parachains/src/paras_inherent.rs b/runtime/parachains/src/paras_inherent.rs
index abe851c38132..0a4bd6e5db9d 100644
--- a/runtime/parachains/src/paras_inherent.rs
+++ b/runtime/parachains/src/paras_inherent.rs
@@ -857,11 +857,24 @@ mod tests {
 
 	mod paras_inherent_weight {
 		use super::*;
+		use crate::inclusion::tests::*;
 
-		use crate::mock::{new_test_ext, MockGenesisConfig, System, Test};
-		use primitives::v1::{Hash, Header, GroupIndex, Id as ParaId};
+		use primitives::v1::{CoreOccupied, GroupIndex, Hash, Header, Id as ParaId, ValidatorIndex};
 
 		use frame_support::traits::UnfilteredDispatchable;
+		use crate::{
+			mock::{
+				new_test_ext, MockGenesisConfig,
+				System, Test,
+			},
+		};
+		use futures::executor::block_on;
+		use primitives::{
+			v0::PARACHAIN_KEY_TYPE_ID,
+		};
+		use sc_keystore::LocalKeystore;
+		use sp_keystore::{SyncCryptoStore, SyncCryptoStorePtr};
+		use std::sync::Arc;
 
 		fn default_header() -> Header {
 			Header {
@@ -873,24 +886,87 @@ mod tests {
 			}
 		}
 
+
 		/// We expect the weight of the paras inherent not to change when no truncation occurs:
 		/// its weight is dynamically computed from the size of the backed candidates list, and is
 		/// already incorporated into the current block weight when it is selected by the provisioner.
 		#[test]
 		fn weight_does_not_change_on_happy_path() {
-			new_test_ext(MockGenesisConfig::default()).execute_with(|| {
+
+			let chain_a = ParaId::from(1);
+			let chain_b = ParaId::from(2);
+			let chains = vec![chain_a, chain_b];
+
+			new_test_ext(genesis_config(vec![(chain_a, true), (chain_b, true)])).execute_with(|| {
 				let header = default_header();
 				System::set_block_number(1);
 				System::set_parent_hash(header.hash());
 
+				let keystore: SyncCryptoStorePtr = Arc::new(LocalKeystore::in_memory());
+				let validators = vec![
+					keyring::Sr25519Keyring::Alice,
+					keyring::Sr25519Keyring::Bob,
+					keyring::Sr25519Keyring::Charlie,
+					keyring::Sr25519Keyring::Dave,
+					keyring::Sr25519Keyring::Ferdie,
+				];
+				for validator in validators.iter() {
+					SyncCryptoStore::sr25519_generate_new(
+						&*keystore,
+						PARACHAIN_KEY_TYPE_ID,
+						Some(&validator.to_seed()),
+					)
+					.unwrap();
+				}
+				let validator_public = validator_pubkeys(&validators);
+
+				shared::Pallet::<Test>::set_active_validators_ascending(validator_public.clone());
+
+				let group_validators = |group_index: GroupIndex| {
+					match group_index {
+						group_index if group_index == GroupIndex::from(0) => vec![0, 1],
+						group_index if group_index == GroupIndex::from(1) => vec![2, 3],
+						x => panic!("Group index out of bounds for 2 parachains and 1 parathread core {}", x.0),
+					}
+					.into_iter().map(ValidatorIndex).collect::<Vec<_>>()
+				};
+
+				let signing_context =
+					SigningContext { parent_hash: System::parent_hash(), session_index: 5 };
+
 				// number of bitfields doesn't affect the paras inherent weight, so we can mock it with an empty one
 				let signed_bitfields = Vec::new();
 				// backed candidates must not be empty, so we can demonstrate that the weight has not changed
-				let backed_candidates = std::iter::repeat_with(|| BackedCandidate::default())
-					.take(10)
-					.map(|mut x| {
-						x.candidate.descriptor.relay_parent = header.hash();
-						x
+				crate::paras::Parachains::<Test>::set(chains.clone());
+				dbg!(crate::paras::Parachains::<Test>::get());
+
+				let backed_candidates = chains.iter().cloned().enumerate()
+					.map(|(idx, para_id)| {
+						let mut candidate = TestCandidateBuilder {
+							para_id,
+							relay_parent: header.hash(),
+							pov_hash: Hash::repeat_byte(1_u8 + idx as u8),
+							persisted_validation_data_hash: make_vdata_hash(para_id).unwrap(),
+							hrmp_watermark: 0,
+							..Default::default()
+						}
+						.build();
+
+						collator_sign_candidate(
+							keyring::Sr25519Keyring::One,
+							&mut candidate,
+						);
+
+						let backed_candidate = block_on(back_candidate(
+							candidate,
+							&validators,
+							group_validators(GroupIndex::from(((1 + idx) % chains.len()) as u32)).as_slice(),
+							&keystore,
+							&signing_context,
+							BackingKind::Threshold,
+						));
+						backed_candidate
+
 					})
 					.collect::<Vec<_>>();
 
@@ -901,32 +977,41 @@ mod tests {
 				// we've used half the block weight; there's plenty of margin
 				let max_block_weight =
 					<Test as frame_system::Config>::BlockWeights::get().max_block;
+
+				dbg!(backed_candidates
+					.iter()
+					.map(backed_candidate_weight::<Test>)
+					.sum::<Weight>());
+				dbg!(MINIMAL_INCLUSION_INHERENT_WEIGHT);
+
 				let used_block_weight = max_block_weight / 2;
 				System::set_block_consumed_resources(used_block_weight, 0);
 
-				let chain_a = ParaId::from(1);
-				let chain_b = ParaId::from(2);
-
+				// before these are used, schedule is called
+				// and the assignments change ag
 				let chain_a_assignment = CoreAssignment {
 					core: CoreIndex::from(0),
 					para_id: chain_a,
 					kind: crate::scheduler::AssignmentKind::Parachain,
-					group_idx: GroupIndex::from(0),
+					group_idx: GroupIndex::from(1),
 				};
 
 				let chain_b_assignment = CoreAssignment {
 					core: CoreIndex::from(1),
 					para_id: chain_b,
 					kind: crate::scheduler::AssignmentKind::Parachain,
-					group_idx: GroupIndex::from(1),
+					group_idx: GroupIndex::from(0),
 				};
-               // TODO scheduled in ::enter is still empty11
-				crate::scheduler::AvailabilityCores::<Test>::set(vec![None, None]);
-				crate::paras::Parachains::<Test>::set(vec![chain_a, chain_b]);
-				<crate::scheduler::Pallet<Test>>::schedule(vec![(CoreIndex(0), FreedReason::Concluded), (CoreIndex(1), FreedReason::Concluded)], <frame_system::Pallet<Test>>::block_number());
+				crate::scheduler::ValidatorGroups::<Test>::set(vec![
+					group_validators(GroupIndex::from(0)),
+					group_validators(GroupIndex::from(1)),
+				]);
 
+				crate::scheduler::AvailabilityCores::<Test>::set(vec![None, None]);
+				// crate::scheduler::AvailabilityCores::<Test>::set(vec![Some(CoreOccupied::Parachain), Some(CoreOccupied::Parachain)]);
 				let scheduled = vec![chain_a_assignment, chain_b_assignment];
 				crate::scheduler::Scheduled::<Test>::set(scheduled);
+				dbg!(<crate::scheduler::Pallet<Test>>::scheduled());
 
 				// execute the paras inherent
 				let post_info = Call::<Test>::enter {
@@ -947,7 +1032,7 @@ mod tests {
 				// In this case, the weight system can update the actual weight with the same amount,
 				// or return `None` to indicate that the pre-computed weight should not change.
 				// Either option is acceptable for our purposes.
-				if let Some(actual_weight) = post_info.actual_weight {
+				if let Some(actual_weight) = dbg!(post_info).actual_weight {
 					assert_eq!(actual_weight, expected_weight);
 				}
 			});

From 98fb6d41866c00037c0a0b0dfce3997b99bdc4ae Mon Sep 17 00:00:00 2001
From: Bernhard Schuster <bernhard@ahoi.io>
Date: Thu, 21 Oct 2021 18:03:34 +0200
Subject: [PATCH 041/107] dbg stuff

---
 primitives/src/v1/mod.rs                 | 1 +
 runtime/parachains/src/inclusion.rs      | 3 ++-
 runtime/parachains/src/paras_inherent.rs | 6 +++---
 3 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/primitives/src/v1/mod.rs b/primitives/src/v1/mod.rs
index 56ba018c5fd5..c9e9138fccf2 100644
--- a/primitives/src/v1/mod.rs
+++ b/primitives/src/v1/mod.rs
@@ -632,6 +632,7 @@ pub fn check_candidate_backing<H: AsRef<[u8]> + Clone + Encode>(
 		let payload = attestation.signed_payload(hash.clone(), signing_context);
 		let sig = attestation.signature();
 
+		println!("Verify candidate {:?} with val[group_idx={}]", &hash, val_in_group_idx);
 		if sig.verify(&payload[..], &validator_id) {
 			signed += 1;
 		} else {
diff --git a/runtime/parachains/src/inclusion.rs b/runtime/parachains/src/inclusion.rs
index 6988686fe636..88c92e55e920 100644
--- a/runtime/parachains/src/inclusion.rs
+++ b/runtime/parachains/src/inclusion.rs
@@ -540,7 +540,7 @@ impl<T: Config> Pallet<T> {
 								|idx| {
 									group_vals
 										.get(idx)
-										.and_then(|i| validators.get(i.0 as usize))
+										.and_then(|i| validators.get(dbg!(i.0) as usize))
 										.map(|v| v.clone())
 								},
 							);
@@ -1058,6 +1058,7 @@ pub(crate) mod tests {
 
 		for (idx_in_group, val_idx) in group.iter().enumerate().take(signing) {
 			let key: Sr25519Keyring = validators[val_idx.0 as usize];
+			println!("Signing candidate {:?} with val[{}]", candidate.hash(), val_idx.0);
 			*validator_indices.get_mut(idx_in_group).unwrap() = true;
 
 			let signature = SignedStatement::sign(
diff --git a/runtime/parachains/src/paras_inherent.rs b/runtime/parachains/src/paras_inherent.rs
index 0a4bd6e5db9d..b626014b38fb 100644
--- a/runtime/parachains/src/paras_inherent.rs
+++ b/runtime/parachains/src/paras_inherent.rs
@@ -409,7 +409,7 @@ pub mod pallet {
 			<scheduler::Pallet<T>>::clear();
 			<scheduler::Pallet<T>>::schedule(freed, <frame_system::Pallet<T>>::block_number());
 
-			let scheduled = <scheduler::Pallet<T>>::scheduled();
+			let scheduled = dbg!(<scheduler::Pallet<T>>::scheduled());
 			let backed_candidates = sanitize_backed_candidates::<T, _, true>(
 				parent_hash,
 				backed_candidates,
@@ -737,8 +737,8 @@ fn sanitize_backed_candidates<
 	// Also checks the candidate references the correct relay parent.
 	backed_candidates.retain(|backed_candidate| {
 		let desc = backed_candidate.descriptor();
-		desc.relay_parent == relay_parent &&
-			scheduled.iter().any(|core| core.para_id == desc.para_id)
+		dbg!(desc.relay_parent) == dbg!(relay_parent) &&
+			dbg!(scheduled.iter().any(|core| dbg!(core.para_id) == dbg!(desc.para_id)))
 	});
 	ensure2!(backed_candidates.len() == n, Error::<T>::CandidateConcludedInvalid, EARLY_RETURN);
 

From 208506903d19a0879934a2b7df68f4c24d718196 Mon Sep 17 00:00:00 2001
From: Bernhard Schuster <bernhard@ahoi.io>
Date: Thu, 21 Oct 2021 22:18:11 +0200
Subject: [PATCH 042/107] make group selection align

---
 runtime/parachains/src/inclusion.rs      |  6 +--
 runtime/parachains/src/paras.rs          |  2 +-
 runtime/parachains/src/paras_inherent.rs | 66 +++++++++++++++---------
 3 files changed, 45 insertions(+), 29 deletions(-)

diff --git a/runtime/parachains/src/inclusion.rs b/runtime/parachains/src/inclusion.rs
index 88c92e55e920..abae734c0faf 100644
--- a/runtime/parachains/src/inclusion.rs
+++ b/runtime/parachains/src/inclusion.rs
@@ -537,10 +537,10 @@ impl<T: Config> Pallet<T> {
 								&backed_candidate,
 								&signing_context,
 								group_vals.len(),
-								|idx| {
+								|intra_group_vi| {
 									group_vals
-										.get(idx)
-										.and_then(|i| validators.get(dbg!(i.0) as usize))
+										.get(intra_group_vi)
+										.and_then(|vi| validators.get(vi.0 as usize))
 										.map(|v| v.clone())
 								},
 							);
diff --git a/runtime/parachains/src/paras.rs b/runtime/parachains/src/paras.rs
index 4582e3e99c44..e9bc009800a0 100644
--- a/runtime/parachains/src/paras.rs
+++ b/runtime/parachains/src/paras.rs
@@ -350,7 +350,7 @@ pub mod pallet {
 	/// All parachains. Ordered ascending by `ParaId`. Parathreads are not included.
 	#[pallet::storage]
 	#[pallet::getter(fn parachains)]
-	pub(super) type Parachains<T: Config> = StorageValue<_, Vec<ParaId>, ValueQuery>;
+	pub(crate) type Parachains<T: Config> = StorageValue<_, Vec<ParaId>, ValueQuery>;
 
 	/// The current lifecycle of a all known Para IDs.
 	#[pallet::storage]
diff --git a/runtime/parachains/src/paras_inherent.rs b/runtime/parachains/src/paras_inherent.rs
index b626014b38fb..3a501dc9cf66 100644
--- a/runtime/parachains/src/paras_inherent.rs
+++ b/runtime/parachains/src/paras_inherent.rs
@@ -737,8 +737,8 @@ fn sanitize_backed_candidates<
 	// Also checks the candidate references the correct relay parent.
 	backed_candidates.retain(|backed_candidate| {
 		let desc = backed_candidate.descriptor();
-		dbg!(desc.relay_parent) == dbg!(relay_parent) &&
-			dbg!(scheduled.iter().any(|core| dbg!(core.para_id) == dbg!(desc.para_id)))
+		desc.relay_parent == relay_parent &&
+			scheduled.iter().any(|core| core.para_id == desc.para_id)
 	});
 	ensure2!(backed_candidates.len() == n, Error::<T>::CandidateConcludedInvalid, EARLY_RETURN);
 
@@ -859,7 +859,7 @@ mod tests {
 		use super::*;
 		use crate::inclusion::tests::*;
 
-		use primitives::v1::{CoreOccupied, GroupIndex, Hash, Header, Id as ParaId, ValidatorIndex};
+		use primitives::v1::{GroupIndex, Hash, Header, Id as ParaId, ValidatorIndex};
 
 		use frame_support::traits::UnfilteredDispatchable;
 		use crate::{
@@ -940,6 +940,22 @@ mod tests {
 				crate::paras::Parachains::<Test>::set(chains.clone());
 				dbg!(crate::paras::Parachains::<Test>::get());
 
+
+				crate::scheduler::ValidatorGroups::<Test>::set(vec![
+					group_validators(GroupIndex::from(0)),
+					group_validators(GroupIndex::from(1)),
+				]);
+
+				crate::scheduler::AvailabilityCores::<Test>::set(vec![None, None]);
+
+				let core_a = CoreIndex::from(0);
+				let grp_idx_a = crate::scheduler::Pallet::<Test>::group_assigned_to_core(core_a, System::block_number()).unwrap();
+
+				let core_b = CoreIndex::from(1);
+				let grp_idx_b = crate::scheduler::Pallet::<Test>::group_assigned_to_core(core_b, System::block_number()).unwrap();
+
+				let grp_indices = vec![grp_idx_a.clone(), grp_idx_b.clone()];
+
 				let backed_candidates = chains.iter().cloned().enumerate()
 					.map(|(idx, para_id)| {
 						let mut candidate = TestCandidateBuilder {
@@ -957,10 +973,14 @@ mod tests {
 							&mut candidate,
 						);
 
+						let group_to_use_for_signing = grp_indices[idx].clone();
+						let backing_validators =
+							group_validators(group_to_use_for_signing)
+						;
 						let backed_candidate = block_on(back_candidate(
 							candidate,
 							&validators,
-							group_validators(GroupIndex::from(((1 + idx) % chains.len()) as u32)).as_slice(),
+							backing_validators.as_slice(),
 							&keystore,
 							&signing_context,
 							BackingKind::Threshold,
@@ -989,29 +1009,25 @@ mod tests {
 
 				// before these are used, schedule is called
 				// and the assignments change ag
-				let chain_a_assignment = CoreAssignment {
-					core: CoreIndex::from(0),
-					para_id: chain_a,
-					kind: crate::scheduler::AssignmentKind::Parachain,
-					group_idx: GroupIndex::from(1),
-				};
 
-				let chain_b_assignment = CoreAssignment {
-					core: CoreIndex::from(1),
-					para_id: chain_b,
-					kind: crate::scheduler::AssignmentKind::Parachain,
-					group_idx: GroupIndex::from(0),
-				};
-				crate::scheduler::ValidatorGroups::<Test>::set(vec![
-					group_validators(GroupIndex::from(0)),
-					group_validators(GroupIndex::from(1)),
-				]);
+				// let chain_a_assignment = CoreAssignment {
+				// 	core: core_a,
+				// 	para_id: chain_a,
+				// 	kind: crate::scheduler::AssignmentKind::Parachain,
+				// 	group_idx: grp_idx_a,
+				// };
 
-				crate::scheduler::AvailabilityCores::<Test>::set(vec![None, None]);
-				// crate::scheduler::AvailabilityCores::<Test>::set(vec![Some(CoreOccupied::Parachain), Some(CoreOccupied::Parachain)]);
-				let scheduled = vec![chain_a_assignment, chain_b_assignment];
-				crate::scheduler::Scheduled::<Test>::set(scheduled);
-				dbg!(<crate::scheduler::Pallet<Test>>::scheduled());
+				// let chain_b_assignment = CoreAssignment {
+				// 	core: core_b,
+				// 	para_id: chain_b,
+				// 	kind: crate::scheduler::AssignmentKind::Parachain,
+				// 	group_idx: grp_idx_b,
+				// };
+
+
+				// let scheduled = vec![chain_a_assignment, chain_b_assignment];
+				// crate::scheduler::Scheduled::<Test>::set(scheduled);
+				// dbg!(<crate::scheduler::Pallet<Test>>::scheduled());
 
 				// execute the paras inherent
 				let post_info = Call::<Test>::enter {

From 2795af0d426ec9583955e099171fcb5458b6b3d6 Mon Sep 17 00:00:00 2001
From: Bernhard Schuster <bernhard@ahoi.io>
Date: Fri, 22 Oct 2021 09:49:41 +0200
Subject: [PATCH 043/107] fix session index

---
 runtime/parachains/src/paras_inherent.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/runtime/parachains/src/paras_inherent.rs b/runtime/parachains/src/paras_inherent.rs
index 3a501dc9cf66..a82b4e079c4a 100644
--- a/runtime/parachains/src/paras_inherent.rs
+++ b/runtime/parachains/src/paras_inherent.rs
@@ -651,8 +651,8 @@ pub(crate) fn sanitize_bitfields<
 	);
 
 	let all_zeros = BitVec::<bitvec::order::Lsb0, u8>::repeat(false, expected_bits);
+	let signing_context = SigningContext { parent_hash, session_index };
 	for unchecked_bitfield in unchecked_bitfields {
-		let signing_context = SigningContext { parent_hash, session_index };
 
 		ensure2!(
 			unchecked_bitfield.unchecked_payload().0.len() == expected_bits,
@@ -932,7 +932,7 @@ mod tests {
 				};
 
 				let signing_context =
-					SigningContext { parent_hash: System::parent_hash(), session_index: 5 };
+					SigningContext { parent_hash: System::parent_hash(), session_index: current_session };
 
 				// number of bitfields doesn't affect the paras inherent weight, so we can mock it with an empty one
 				let signed_bitfields = Vec::new();

From 3271fd314347ee96a5cd3dbbcaafec2df8005345 Mon Sep 17 00:00:00 2001
From: Bernhard Schuster <bernhard@ahoi.io>
Date: Fri, 22 Oct 2021 11:27:07 +0200
Subject: [PATCH 044/107] fix wrongly returned candidates

---
 runtime/parachains/src/paras_inherent.rs | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/runtime/parachains/src/paras_inherent.rs b/runtime/parachains/src/paras_inherent.rs
index a82b4e079c4a..1896e2d4abc1 100644
--- a/runtime/parachains/src/paras_inherent.rs
+++ b/runtime/parachains/src/paras_inherent.rs
@@ -604,9 +604,7 @@ fn apply_weight_limit<T: Config + inclusion::Config>(
 		max_weight,
 	);
 	let bitfields = indices.into_iter().map(move |idx| bitfields[idx].clone()).collect::<Vec<_>>();
-	// pick all bitfields, and
-	// fill the remaining space with candidates
-	(total, candidates, bitfields)
+	(total, vec![], bitfields)
 }
 
 /// Filter bitfields based on freed core indices, validity, and other sanity checks.

From 4375b83df07f7dfc6785b456d31ccb6973fccf14 Mon Sep 17 00:00:00 2001
From: Bernhard Schuster <bernhard@ahoi.io>
Date: Fri, 22 Oct 2021 11:49:43 +0200
Subject: [PATCH 045/107] cleanup

---
 runtime/parachains/src/paras_inherent.rs | 38 ++++--------------------
 runtime/parachains/src/shared.rs         |  2 +-
 2 files changed, 7 insertions(+), 33 deletions(-)

diff --git a/runtime/parachains/src/paras_inherent.rs b/runtime/parachains/src/paras_inherent.rs
index 1896e2d4abc1..339bb9d47b02 100644
--- a/runtime/parachains/src/paras_inherent.rs
+++ b/runtime/parachains/src/paras_inherent.rs
@@ -899,14 +899,14 @@ mod tests {
 				let header = default_header();
 				System::set_block_number(1);
 				System::set_parent_hash(header.hash());
-
+				let session_index = SessionIndex::from(0_u32);
+				crate::shared::CurrentSessionIndex::<Test>::set(session_index);
 				let keystore: SyncCryptoStorePtr = Arc::new(LocalKeystore::in_memory());
 				let validators = vec![
 					keyring::Sr25519Keyring::Alice,
 					keyring::Sr25519Keyring::Bob,
 					keyring::Sr25519Keyring::Charlie,
 					keyring::Sr25519Keyring::Dave,
-					keyring::Sr25519Keyring::Ferdie,
 				];
 				for validator in validators.iter() {
 					SyncCryptoStore::sr25519_generate_new(
@@ -930,13 +930,12 @@ mod tests {
 				};
 
 				let signing_context =
-					SigningContext { parent_hash: System::parent_hash(), session_index: current_session };
+					SigningContext { parent_hash: System::parent_hash(), session_index };
 
 				// number of bitfields doesn't affect the paras inherent weight, so we can mock it with an empty one
 				let signed_bitfields = Vec::new();
 				// backed candidates must not be empty, so we can demonstrate that the weight has not changed
 				crate::paras::Parachains::<Test>::set(chains.clone());
-				dbg!(crate::paras::Parachains::<Test>::get());
 
 
 				crate::scheduler::ValidatorGroups::<Test>::set(vec![
@@ -996,36 +995,11 @@ mod tests {
 				let max_block_weight =
 					<Test as frame_system::Config>::BlockWeights::get().max_block;
 
-				dbg!(backed_candidates
-					.iter()
-					.map(backed_candidate_weight::<Test>)
-					.sum::<Weight>());
-				dbg!(MINIMAL_INCLUSION_INHERENT_WEIGHT);
-
 				let used_block_weight = max_block_weight / 2;
 				System::set_block_consumed_resources(used_block_weight, 0);
 
-				// before these are used, schedule is called
-				// and the assignments change ag
-
-				// let chain_a_assignment = CoreAssignment {
-				// 	core: core_a,
-				// 	para_id: chain_a,
-				// 	kind: crate::scheduler::AssignmentKind::Parachain,
-				// 	group_idx: grp_idx_a,
-				// };
-
-				// let chain_b_assignment = CoreAssignment {
-				// 	core: core_b,
-				// 	para_id: chain_b,
-				// 	kind: crate::scheduler::AssignmentKind::Parachain,
-				// 	group_idx: grp_idx_b,
-				// };
-
-
-				// let scheduled = vec![chain_a_assignment, chain_b_assignment];
-				// crate::scheduler::Scheduled::<Test>::set(scheduled);
-				// dbg!(<crate::scheduler::Pallet<Test>>::scheduled());
+				// no need to schedule anything, `schedule(..)` is called
+				// form the `fn` under test.
 
 				// execute the paras inherent
 				let post_info = Call::<Test>::enter {
@@ -1046,7 +1020,7 @@ mod tests {
 				// In this case, the weight system can update the actual weight with the same amount,
 				// or return `None` to indicate that the pre-computed weight should not change.
 				// Either option is acceptable for our purposes.
-				if let Some(actual_weight) = dbg!(post_info).actual_weight {
+				if let Some(actual_weight) = post_info.actual_weight {
 					assert_eq!(actual_weight, expected_weight);
 				}
 			});
diff --git a/runtime/parachains/src/shared.rs b/runtime/parachains/src/shared.rs
index 9b5aa5c59629..28242e1a7637 100644
--- a/runtime/parachains/src/shared.rs
+++ b/runtime/parachains/src/shared.rs
@@ -49,7 +49,7 @@ pub mod pallet {
 	/// The current session index.
 	#[pallet::storage]
 	#[pallet::getter(fn session_index)]
-	pub(super) type CurrentSessionIndex<T: Config> = StorageValue<_, SessionIndex, ValueQuery>;
+	pub(crate) type CurrentSessionIndex<T: Config> = StorageValue<_, SessionIndex, ValueQuery>;
 
 	/// All the validators actively participating in parachain consensus.
 	/// Indices are into the broader validator set.

From cbed7031b607003b852d79cebf7e1daf8b98fae7 Mon Sep 17 00:00:00 2001
From: Bernhard Schuster <bernhard@ahoi.io>
Date: Fri, 22 Oct 2021 12:18:21 +0200
Subject: [PATCH 046/107] chore fmt

---
 primitives/src/v1/mod.rs                      |   1 -
 .../src/runtime/inclusion.md                  |   3 +
 runtime/parachains/src/inclusion.rs           |   1 -
 runtime/parachains/src/paras_inherent.rs      | 331 +++++++++++-------
 4 files changed, 205 insertions(+), 131 deletions(-)

diff --git a/primitives/src/v1/mod.rs b/primitives/src/v1/mod.rs
index c9e9138fccf2..56ba018c5fd5 100644
--- a/primitives/src/v1/mod.rs
+++ b/primitives/src/v1/mod.rs
@@ -632,7 +632,6 @@ pub fn check_candidate_backing<H: AsRef<[u8]> + Clone + Encode>(
 		let payload = attestation.signed_payload(hash.clone(), signing_context);
 		let sig = attestation.signature();
 
-		println!("Verify candidate {:?} with val[group_idx={}]", &hash, val_in_group_idx);
 		if sig.verify(&payload[..], &validator_id) {
 			signed += 1;
 		} else {
diff --git a/roadmap/implementers-guide/src/runtime/inclusion.md b/roadmap/implementers-guide/src/runtime/inclusion.md
index c800abd8f7a7..bab3ae455c60 100644
--- a/roadmap/implementers-guide/src/runtime/inclusion.md
+++ b/roadmap/implementers-guide/src/runtime/inclusion.md
@@ -78,6 +78,9 @@ All failed checks should lead to an unrecoverable error making the block invalid
   1. call `Hrmp::prune_hrmp` with the para id of the candiate and the candidate's `hrmp_watermark`.
   1. call `Hrmp::queue_outbound_hrmp` with the para id of the candidate and the list of horizontal messages taken from the commitment,
   1. Call `Paras::note_new_head` using the `HeadData` from the receipt and `relay_parent_number`.
+
+* `fn sanitize_bitfields`
+
 * `collect_pending`:
 
   ```rust
diff --git a/runtime/parachains/src/inclusion.rs b/runtime/parachains/src/inclusion.rs
index abae734c0faf..2ca0c62af075 100644
--- a/runtime/parachains/src/inclusion.rs
+++ b/runtime/parachains/src/inclusion.rs
@@ -1058,7 +1058,6 @@ pub(crate) mod tests {
 
 		for (idx_in_group, val_idx) in group.iter().enumerate().take(signing) {
 			let key: Sr25519Keyring = validators[val_idx.0 as usize];
-			println!("Signing candidate {:?} with val[{}]", candidate.hash(), val_idx.0);
 			*validator_indices.get_mut(idx_in_group).unwrap() = true;
 
 			let signature = SignedStatement::sign(
diff --git a/runtime/parachains/src/paras_inherent.rs b/runtime/parachains/src/paras_inherent.rs
index 339bb9d47b02..62e8e26ff624 100644
--- a/runtime/parachains/src/paras_inherent.rs
+++ b/runtime/parachains/src/paras_inherent.rs
@@ -409,7 +409,7 @@ pub mod pallet {
 			<scheduler::Pallet<T>>::clear();
 			<scheduler::Pallet<T>>::schedule(freed, <frame_system::Pallet<T>>::block_number());
 
-			let scheduled = dbg!(<scheduler::Pallet<T>>::scheduled());
+			let scheduled = <scheduler::Pallet<T>>::scheduled();
 			let backed_candidates = sanitize_backed_candidates::<T, _, true>(
 				parent_hash,
 				backed_candidates,
@@ -651,7 +651,6 @@ pub(crate) fn sanitize_bitfields<
 	let all_zeros = BitVec::<bitvec::order::Lsb0, u8>::repeat(false, expected_bits);
 	let signing_context = SigningContext { parent_hash, session_index };
 	for unchecked_bitfield in unchecked_bitfields {
-
 		ensure2!(
 			unchecked_bitfield.unchecked_payload().0.len() == expected_bits,
 			crate::inclusion::pallet::Error::<T>::WrongBitfieldSize,
@@ -853,177 +852,251 @@ mod tests {
 		}
 	}
 
-	mod paras_inherent_weight {
+	fn default_header() -> primitives::v1::Header {
+		primitives::v1::Header {
+			parent_hash: Default::default(),
+			number: 0,
+			state_root: Default::default(),
+			extrinsics_root: Default::default(),
+			digest: Default::default(),
+		}
+	}
+
+	mod sanitizers {
 		use super::*;
-		use crate::inclusion::tests::*;
+		use assert_matches::assert_matches;
 
 		use primitives::v1::{GroupIndex, Hash, Header, Id as ParaId, ValidatorIndex};
 
+		use crate::mock::{new_test_ext, MockGenesisConfig, System, Test};
 		use frame_support::traits::UnfilteredDispatchable;
-		use crate::{
-			mock::{
-				new_test_ext, MockGenesisConfig,
-				System, Test,
-			},
-		};
 		use futures::executor::block_on;
-		use primitives::{
-			v0::PARACHAIN_KEY_TYPE_ID,
-		};
+		use primitives::v0::PARACHAIN_KEY_TYPE_ID;
 		use sc_keystore::LocalKeystore;
 		use sp_keystore::{SyncCryptoStore, SyncCryptoStorePtr};
 		use std::sync::Arc;
 
-		fn default_header() -> Header {
-			Header {
-				parent_hash: Default::default(),
-				number: 0,
-				state_root: Default::default(),
-				extrinsics_root: Default::default(),
-				digest: Default::default(),
+		fn validator_pubkeys(val_ids: &[keyring::Sr25519Keyring]) -> Vec<ValidatorId> {
+			val_ids.iter().map(|v| v.public().into()).collect()
+		}
+
+		#[test]
+		fn bitfields() {
+			let header = default_header();
+			let parent_hash = header.hash();
+			// 2 cores means two bits
+			let expected_bits = 2;
+			let unchecked_bitfields = vec![];
+			let disputed_bitfield = DisputedBitfield::zeros(expected_bits);
+			let session_index = SessionIndex::from(0_u32);
+
+			let keystore: SyncCryptoStorePtr = Arc::new(LocalKeystore::in_memory());
+			let validators = vec![
+				keyring::Sr25519Keyring::Alice,
+				keyring::Sr25519Keyring::Bob,
+				keyring::Sr25519Keyring::Charlie,
+				keyring::Sr25519Keyring::Dave,
+			];
+			for validator in validators.iter() {
+				SyncCryptoStore::sr25519_generate_new(
+					&*keystore,
+					PARACHAIN_KEY_TYPE_ID,
+					Some(&validator.to_seed()),
+				)
+				.unwrap();
+			}
+			let validator_public = validator_pubkeys(&validators);
+
+			{
+				assert_matches!(
+					sanitize_bitfields::<Test, true>(
+						unchecked_bitfields.clone(),
+						disputed_bitfield.clone(),
+						expected_bits,
+						parent_hash,
+						session_index,
+						&validator_public[..]
+					),
+					Ok(_)
+				);
+				assert_eq!(
+					sanitize_bitfields::<Test, false>(
+						unchecked_bitfields.clone(),
+						disputed_bitfield.clone(),
+						expected_bits,
+						parent_hash,
+						session_index,
+						&validator_public[..]
+					),
+					Ok(unchecked_bitfields.clone())
+				);
 			}
 		}
 
+		#[test]
+		fn candidates() {}
+	}
+
+	mod paras_inherent_weight {
+		use super::*;
+		use crate::inclusion::tests::*;
+
+		use primitives::v1::{GroupIndex, Hash, Id as ParaId, ValidatorIndex};
+
+		use crate::mock::{new_test_ext, MockGenesisConfig, System, Test};
+		use frame_support::traits::UnfilteredDispatchable;
+		use futures::executor::block_on;
+		use primitives::v0::PARACHAIN_KEY_TYPE_ID;
+		use sc_keystore::LocalKeystore;
+		use sp_keystore::{SyncCryptoStore, SyncCryptoStorePtr};
+		use std::sync::Arc;
 
 		/// We expect the weight of the paras inherent not to change when no truncation occurs:
 		/// its weight is dynamically computed from the size of the backed candidates list, and is
 		/// already incorporated into the current block weight when it is selected by the provisioner.
 		#[test]
 		fn weight_does_not_change_on_happy_path() {
-
 			let chain_a = ParaId::from(1);
 			let chain_b = ParaId::from(2);
 			let chains = vec![chain_a, chain_b];
 
-			new_test_ext(genesis_config(vec![(chain_a, true), (chain_b, true)])).execute_with(|| {
-				let header = default_header();
-				System::set_block_number(1);
-				System::set_parent_hash(header.hash());
-				let session_index = SessionIndex::from(0_u32);
-				crate::shared::CurrentSessionIndex::<Test>::set(session_index);
-				let keystore: SyncCryptoStorePtr = Arc::new(LocalKeystore::in_memory());
-				let validators = vec![
-					keyring::Sr25519Keyring::Alice,
-					keyring::Sr25519Keyring::Bob,
-					keyring::Sr25519Keyring::Charlie,
-					keyring::Sr25519Keyring::Dave,
-				];
-				for validator in validators.iter() {
-					SyncCryptoStore::sr25519_generate_new(
-						&*keystore,
-						PARACHAIN_KEY_TYPE_ID,
-						Some(&validator.to_seed()),
-					)
-					.unwrap();
-				}
-				let validator_public = validator_pubkeys(&validators);
+			new_test_ext(genesis_config(vec![(chain_a, true), (chain_b, true)])).execute_with(
+				|| {
+					let header = default_header();
+					System::set_block_number(1);
+					System::set_parent_hash(header.hash());
+					let session_index = SessionIndex::from(0_u32);
+					crate::shared::CurrentSessionIndex::<Test>::set(session_index);
+					let keystore: SyncCryptoStorePtr = Arc::new(LocalKeystore::in_memory());
+					let validators = vec![
+						keyring::Sr25519Keyring::Alice,
+						keyring::Sr25519Keyring::Bob,
+						keyring::Sr25519Keyring::Charlie,
+						keyring::Sr25519Keyring::Dave,
+					];
+					for validator in validators.iter() {
+						SyncCryptoStore::sr25519_generate_new(
+							&*keystore,
+							PARACHAIN_KEY_TYPE_ID,
+							Some(&validator.to_seed()),
+						)
+						.unwrap();
+					}
+					let validator_public = validator_pubkeys(&validators);
 
-				shared::Pallet::<Test>::set_active_validators_ascending(validator_public.clone());
+					shared::Pallet::<Test>::set_active_validators_ascending(
+						validator_public.clone(),
+					);
 
-				let group_validators = |group_index: GroupIndex| {
-					match group_index {
+					let group_validators = |group_index: GroupIndex| {
+						match group_index {
 						group_index if group_index == GroupIndex::from(0) => vec![0, 1],
 						group_index if group_index == GroupIndex::from(1) => vec![2, 3],
 						x => panic!("Group index out of bounds for 2 parachains and 1 parathread core {}", x.0),
 					}
 					.into_iter().map(ValidatorIndex).collect::<Vec<_>>()
-				};
+					};
 
-				let signing_context =
-					SigningContext { parent_hash: System::parent_hash(), session_index };
+					let signing_context =
+						SigningContext { parent_hash: System::parent_hash(), session_index };
 
-				// number of bitfields doesn't affect the paras inherent weight, so we can mock it with an empty one
-				let signed_bitfields = Vec::new();
-				// backed candidates must not be empty, so we can demonstrate that the weight has not changed
-				crate::paras::Parachains::<Test>::set(chains.clone());
+					// number of bitfields doesn't affect the paras inherent weight, so we can mock it with an empty one
+					let signed_bitfields = Vec::new();
+					// backed candidates must not be empty, so we can demonstrate that the weight has not changed
+					crate::paras::Parachains::<Test>::set(chains.clone());
 
+					crate::scheduler::ValidatorGroups::<Test>::set(vec![
+						group_validators(GroupIndex::from(0)),
+						group_validators(GroupIndex::from(1)),
+					]);
 
-				crate::scheduler::ValidatorGroups::<Test>::set(vec![
-					group_validators(GroupIndex::from(0)),
-					group_validators(GroupIndex::from(1)),
-				]);
+					crate::scheduler::AvailabilityCores::<Test>::set(vec![None, None]);
 
-				crate::scheduler::AvailabilityCores::<Test>::set(vec![None, None]);
-
-				let core_a = CoreIndex::from(0);
-				let grp_idx_a = crate::scheduler::Pallet::<Test>::group_assigned_to_core(core_a, System::block_number()).unwrap();
+					let core_a = CoreIndex::from(0);
+					let grp_idx_a = crate::scheduler::Pallet::<Test>::group_assigned_to_core(
+						core_a,
+						System::block_number(),
+					)
+					.unwrap();
 
-				let core_b = CoreIndex::from(1);
-				let grp_idx_b = crate::scheduler::Pallet::<Test>::group_assigned_to_core(core_b, System::block_number()).unwrap();
+					let core_b = CoreIndex::from(1);
+					let grp_idx_b = crate::scheduler::Pallet::<Test>::group_assigned_to_core(
+						core_b,
+						System::block_number(),
+					)
+					.unwrap();
 
-				let grp_indices = vec![grp_idx_a.clone(), grp_idx_b.clone()];
+					let grp_indices = vec![grp_idx_a.clone(), grp_idx_b.clone()];
 
-				let backed_candidates = chains.iter().cloned().enumerate()
-					.map(|(idx, para_id)| {
-						let mut candidate = TestCandidateBuilder {
-							para_id,
-							relay_parent: header.hash(),
-							pov_hash: Hash::repeat_byte(1_u8 + idx as u8),
-							persisted_validation_data_hash: make_vdata_hash(para_id).unwrap(),
-							hrmp_watermark: 0,
-							..Default::default()
-						}
-						.build();
+					let backed_candidates = chains
+						.iter()
+						.cloned()
+						.enumerate()
+						.map(|(idx, para_id)| {
+							let mut candidate = TestCandidateBuilder {
+								para_id,
+								relay_parent: header.hash(),
+								pov_hash: Hash::repeat_byte(1_u8 + idx as u8),
+								persisted_validation_data_hash: make_vdata_hash(para_id).unwrap(),
+								hrmp_watermark: 0,
+								..Default::default()
+							}
+							.build();
+
+							collator_sign_candidate(keyring::Sr25519Keyring::One, &mut candidate);
+
+							let group_to_use_for_signing = grp_indices[idx].clone();
+							let backing_validators = group_validators(group_to_use_for_signing);
+							let backed_candidate = block_on(back_candidate(
+								candidate,
+								&validators,
+								backing_validators.as_slice(),
+								&keystore,
+								&signing_context,
+								BackingKind::Threshold,
+							));
+							backed_candidate
+						})
+						.collect::<Vec<_>>();
 
-						collator_sign_candidate(
-							keyring::Sr25519Keyring::One,
-							&mut candidate,
-						);
+					// the expected weight can always be computed by this formula
+					let expected_weight = MINIMAL_INCLUSION_INHERENT_WEIGHT +
+						(backed_candidates.len() as Weight * BACKED_CANDIDATE_WEIGHT);
 
-						let group_to_use_for_signing = grp_indices[idx].clone();
-						let backing_validators =
-							group_validators(group_to_use_for_signing)
-						;
-						let backed_candidate = block_on(back_candidate(
-							candidate,
-							&validators,
-							backing_validators.as_slice(),
-							&keystore,
-							&signing_context,
-							BackingKind::Threshold,
-						));
-						backed_candidate
-
-					})
-					.collect::<Vec<_>>();
-
-				// the expected weight can always be computed by this formula
-				let expected_weight = MINIMAL_INCLUSION_INHERENT_WEIGHT +
-					(backed_candidates.len() as Weight * BACKED_CANDIDATE_WEIGHT);
-
-				// we've used half the block weight; there's plenty of margin
-				let max_block_weight =
-					<Test as frame_system::Config>::BlockWeights::get().max_block;
+					// we've used half the block weight; there's plenty of margin
+					let max_block_weight =
+						<Test as frame_system::Config>::BlockWeights::get().max_block;
 
-				let used_block_weight = max_block_weight / 2;
-				System::set_block_consumed_resources(used_block_weight, 0);
+					let used_block_weight = max_block_weight / 2;
+					System::set_block_consumed_resources(used_block_weight, 0);
 
-				// no need to schedule anything, `schedule(..)` is called
-				// form the `fn` under test.
+					// no need to schedule anything, `schedule(..)` is called
+					// form the `fn` under test.
 
-				// execute the paras inherent
-				let post_info = Call::<Test>::enter {
-					data: ParachainsInherentData {
-						bitfields: signed_bitfields,
-						backed_candidates,
-						disputes: Vec::new(),
-						parent_header: header,
-					},
-				}
-				.dispatch_bypass_filter(None.into())
-				.unwrap();
+					// execute the paras inherent
+					let post_info = Call::<Test>::enter {
+						data: ParachainsInherentData {
+							bitfields: signed_bitfields,
+							backed_candidates,
+							disputes: Vec::new(),
+							parent_header: header,
+						},
+					}
+					.dispatch_bypass_filter(None.into())
+					.unwrap();
 
-				// we don't directly check the block's weight post-call. Instead, we check that the
-				// call has returned the appropriate post-dispatch weight for refund, and trust
-				// Substrate to do the right thing with that information.
-				//
-				// In this case, the weight system can update the actual weight with the same amount,
-				// or return `None` to indicate that the pre-computed weight should not change.
-				// Either option is acceptable for our purposes.
-				if let Some(actual_weight) = post_info.actual_weight {
-					assert_eq!(actual_weight, expected_weight);
-				}
-			});
+					// we don't directly check the block's weight post-call. Instead, we check that the
+					// call has returned the appropriate post-dispatch weight for refund, and trust
+					// Substrate to do the right thing with that information.
+					//
+					// In this case, the weight system can update the actual weight with the same amount,
+					// or return `None` to indicate that the pre-computed weight should not change.
+					// Either option is acceptable for our purposes.
+					if let Some(actual_weight) = post_info.actual_weight {
+						assert_eq!(actual_weight, expected_weight);
+					}
+				},
+			);
 		}
 
 		/// We expect the weight of the paras inherent to change when truncation occurs: its

From 88a7ee12ff5f53415f155767cabb4aa9f49442d4 Mon Sep 17 00:00:00 2001
From: Bernhard Schuster <bernhard@ahoi.io>
Date: Sun, 24 Oct 2021 09:52:12 +0200
Subject: [PATCH 047/107] fix ensure check

---
 runtime/parachains/src/paras_inherent.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/parachains/src/paras_inherent.rs b/runtime/parachains/src/paras_inherent.rs
index 62e8e26ff624..60a4531aca80 100644
--- a/runtime/parachains/src/paras_inherent.rs
+++ b/runtime/parachains/src/paras_inherent.rs
@@ -659,7 +659,7 @@ pub(crate) fn sanitize_bitfields<
 		);
 
 		ensure2!(
-			unchecked_bitfield.unchecked_payload().0.clone() & disputed_bitfield.0.clone() !=
+			unchecked_bitfield.unchecked_payload().0.clone() & disputed_bitfield.0.clone() ==
 				all_zeros,
 			crate::inclusion::pallet::Error::<T>::BitfieldReferencesFreedCore,
 			EARLY_RETURN,

From a7332e0e094a4bd0e1d4ecece383e8cc327d8b32 Mon Sep 17 00:00:00 2001
From: Bernhard Schuster <bernhard@ahoi.io>
Date: Sun, 24 Oct 2021 09:52:39 +0200
Subject: [PATCH 048/107] make good case test work

---
 runtime/parachains/src/paras_inherent.rs | 38 ++++++++++++++++++++----
 1 file changed, 33 insertions(+), 5 deletions(-)

diff --git a/runtime/parachains/src/paras_inherent.rs b/runtime/parachains/src/paras_inherent.rs
index 60a4531aca80..3c187669d907 100644
--- a/runtime/parachains/src/paras_inherent.rs
+++ b/runtime/parachains/src/paras_inherent.rs
@@ -866,7 +866,11 @@ mod tests {
 		use super::*;
 		use assert_matches::assert_matches;
 
-		use primitives::v1::{GroupIndex, Hash, Header, Id as ParaId, ValidatorIndex};
+		use bitvec::order::Lsb0;
+		use primitives::v1::{
+			AvailabilityBitfield, GroupIndex, Hash, Header, Id as ParaId,
+			SignedAvailabilityBitfield, ValidatorIndex,
+		};
 
 		use crate::mock::{new_test_ext, MockGenesisConfig, System, Test};
 		use frame_support::traits::UnfilteredDispatchable;
@@ -886,11 +890,12 @@ mod tests {
 			let parent_hash = header.hash();
 			// 2 cores means two bits
 			let expected_bits = 2;
-			let unchecked_bitfields = vec![];
-			let disputed_bitfield = DisputedBitfield::zeros(expected_bits);
 			let session_index = SessionIndex::from(0_u32);
 
-			let keystore: SyncCryptoStorePtr = Arc::new(LocalKeystore::in_memory());
+			let crypto_store = LocalKeystore::in_memory();
+			let crypto_store = Arc::new(crypto_store) as SyncCryptoStorePtr;
+			let signing_context = SigningContext { parent_hash, session_index };
+
 			let validators = vec![
 				keyring::Sr25519Keyring::Alice,
 				keyring::Sr25519Keyring::Bob,
@@ -899,7 +904,7 @@ mod tests {
 			];
 			for validator in validators.iter() {
 				SyncCryptoStore::sr25519_generate_new(
-					&*keystore,
+					&*crypto_store,
 					PARACHAIN_KEY_TYPE_ID,
 					Some(&validator.to_seed()),
 				)
@@ -907,6 +912,29 @@ mod tests {
 			}
 			let validator_public = validator_pubkeys(&validators);
 
+			let unchecked_bitfields = [
+				BitVec::<Lsb0, u8>::repeat(true, expected_bits),
+				BitVec::<Lsb0, u8>::repeat(true, expected_bits),
+			]
+			.iter()
+			.enumerate()
+			.map(|(vi, ab)| {
+				let validator_index = ValidatorIndex::from(vi as u32);
+				block_on(SignedAvailabilityBitfield::sign(
+					&crypto_store,
+					AvailabilityBitfield::from(ab.clone()),
+					&signing_context,
+					validator_index,
+					&validator_public[vi],
+				))
+				.unwrap()
+				.unwrap()
+				.into_unchecked()
+			})
+			.collect::<Vec<_>>();
+
+			let disputed_bitfield = DisputedBitfield::zeros(expected_bits);
+
 			{
 				assert_matches!(
 					sanitize_bitfields::<Test, true>(

From 849af66ec85b22154d858729e2b09db2d4232e8c Mon Sep 17 00:00:00 2001
From: Bernhard Schuster <bernhard@ahoi.io>
Date: Sun, 24 Oct 2021 10:52:15 +0200
Subject: [PATCH 049/107] more tests for bitfields

---
 runtime/parachains/src/paras_inherent.rs | 150 +++++++++++++++++++++--
 1 file changed, 143 insertions(+), 7 deletions(-)

diff --git a/runtime/parachains/src/paras_inherent.rs b/runtime/parachains/src/paras_inherent.rs
index 3c187669d907..b238bfe22273 100644
--- a/runtime/parachains/src/paras_inherent.rs
+++ b/runtime/parachains/src/paras_inherent.rs
@@ -155,7 +155,6 @@ pub mod pallet {
 			let expected_bits = <scheduler::Pallet<T>>::availability_cores().len();
 			let validator_public = shared::Pallet::<T>::active_validator_keys();
 
-			// filter out any unneeded dispute statements
 			T::DisputesHandler::filter_multi_dispute_data(&mut disputes);
 
 			let scheduled: Vec<CoreAssignment> = <scheduler::Pallet<T>>::scheduled();
@@ -915,6 +914,11 @@ mod tests {
 			let unchecked_bitfields = [
 				BitVec::<Lsb0, u8>::repeat(true, expected_bits),
 				BitVec::<Lsb0, u8>::repeat(true, expected_bits),
+				{
+					let mut bv = BitVec::<Lsb0, u8>::repeat(false, expected_bits);
+					bv.set(expected_bits - 1, true);
+					bv
+				},
 			]
 			.iter()
 			.enumerate()
@@ -959,10 +963,142 @@ mod tests {
 					Ok(unchecked_bitfields.clone())
 				);
 			}
+
+			// disputed bitfield is non-zero
+			{
+				let mut disputed_bitfield = DisputedBitfield::zeros(expected_bits);
+				// pretend the first core was freed by either a malicious validator
+				// or by resolved dispute
+				disputed_bitfield.0.set(0, true);
+
+				assert_eq!(
+					sanitize_bitfields::<Test, true>(
+						unchecked_bitfields.clone(),
+						disputed_bitfield.clone(),
+						expected_bits,
+						parent_hash,
+						session_index,
+						&validator_public[..]
+					),
+					Err(inclusion::Error::<Test>::BitfieldReferencesFreedCore.into())
+				);
+				assert_matches!(
+					sanitize_bitfields::<Test, false>(
+						unchecked_bitfields.clone(),
+						disputed_bitfield.clone(),
+						expected_bits,
+						parent_hash,
+						session_index,
+						&validator_public[..]
+					),
+					Ok(unchecked_bitfields) => {
+						assert_eq!(unchecked_bitfields.len(), 1);
+					}
+				);
+			}
+
+			// bitfield size mismatch
+			{
+				assert_eq!(
+					sanitize_bitfields::<Test, true>(
+						unchecked_bitfields.clone(),
+						disputed_bitfield.clone(),
+						expected_bits + 1,
+						parent_hash,
+						session_index,
+						&validator_public[..]
+					),
+					Err(inclusion::Error::<Test>::WrongBitfieldSize.into())
+				);
+				assert_matches!(
+					sanitize_bitfields::<Test, false>(
+						unchecked_bitfields.clone(),
+						disputed_bitfield.clone(),
+						expected_bits + 1,
+						parent_hash,
+						session_index,
+						&validator_public[..]
+					),
+					Ok(unchecked_bitfields) => {
+						assert!(unchecked_bitfields.is_empty());
+					}
+				);
+			}
+
+			// remove the last validator
+			{
+				let shortened = validator_public.len() - 2;
+				assert_eq!(
+					sanitize_bitfields::<Test, true>(
+						unchecked_bitfields.clone(),
+						disputed_bitfield.clone(),
+						expected_bits,
+						parent_hash,
+						session_index,
+						&validator_public[..shortened]
+					),
+					Err(inclusion::Error::<Test>::ValidatorIndexOutOfBounds.into())
+				);
+				assert_matches!(
+					sanitize_bitfields::<Test, false>(
+						unchecked_bitfields.clone(),
+						disputed_bitfield.clone(),
+						expected_bits,
+						parent_hash,
+						session_index,
+						&validator_public[..shortened]
+					),
+					Ok(unchecked_bitfields_filtered) => {
+						assert_eq!(
+							&unchecked_bitfields_filtered[..],
+							&unchecked_bitfields[..shortened]
+						);
+					}
+				);
+			}
+
+			// switch ordering of bitfields, bail
+			{
+				let mut unchecked_bitfields = unchecked_bitfields.clone();
+				let x = unchecked_bitfields.swap_remove(0);
+				unchecked_bitfields.push(x);
+				assert_eq!(
+					sanitize_bitfields::<Test, true>(
+						unchecked_bitfields.clone(),
+						disputed_bitfield.clone(),
+						expected_bits,
+						parent_hash,
+						session_index,
+						&validator_public[..]
+					),
+					Err(inclusion::Error::<Test>::BitfieldDuplicateOrUnordered.into())
+				);
+				assert_matches!(
+					sanitize_bitfields::<Test, false>(
+						unchecked_bitfields.clone(),
+						disputed_bitfield.clone(),
+						expected_bits,
+						parent_hash,
+						session_index,
+						&validator_public[..]
+					),
+					Ok(unchecked_bitfields_filtered) => {
+						// the last element is out of order
+						// hence sanitization will have removed only that
+						// one
+						assert_eq!(
+							&unchecked_bitfields[..(unchecked_bitfields.len()-2)],
+							&unchecked_bitfields_filtered[..]
+							);
+					}
+				);
+			}
 		}
 
 		#[test]
-		fn candidates() {}
+		fn candidates() {
+			todo!("check sanitize_candidates here!");
+		}
 	}
 
 	mod paras_inherent_weight {
@@ -1018,11 +1154,11 @@ mod tests {
 
 					let group_validators = |group_index: GroupIndex| {
 						match group_index {
-						group_index if group_index == GroupIndex::from(0) => vec![0, 1],
-						group_index if group_index == GroupIndex::from(1) => vec![2, 3],
-						x => panic!("Group index out of bounds for 2 parachains and 1 parathread core {}", x.0),
-					}
-					.into_iter().map(ValidatorIndex).collect::<Vec<_>>()
+							group_index if group_index == GroupIndex::from(0) => vec![0, 1],
+							group_index if group_index == GroupIndex::from(1) => vec![2, 3],
+							x => panic!("Group index out of bounds for 2 parachains and 1 parathread core {}", x.0),
+						}
+						.into_iter().map(ValidatorIndex).collect::<Vec<_>>()
 					};
 
 					let signing_context =

From bfa1f1a240f0b00feed192ce3dbc88c2336ac051 Mon Sep 17 00:00:00 2001
From: Bernhard Schuster <bernhard@ahoi.io>
Date: Fri, 29 Oct 2021 18:48:35 +0200
Subject: [PATCH 050/107] create sanitize_backed_candidates

---
 runtime/parachains/src/paras_inherent.rs | 199 ++++++++++++++++++++++-
 1 file changed, 194 insertions(+), 5 deletions(-)

diff --git a/runtime/parachains/src/paras_inherent.rs b/runtime/parachains/src/paras_inherent.rs
index b238bfe22273..e10c0593b8e2 100644
--- a/runtime/parachains/src/paras_inherent.rs
+++ b/runtime/parachains/src/paras_inherent.rs
@@ -865,15 +865,18 @@ mod tests {
 		use super::*;
 		use assert_matches::assert_matches;
 
+		use crate::inclusion::tests::{
+			back_candidate, collator_sign_candidate, BackingKind, TestCandidateBuilder,
+		};
 		use bitvec::order::Lsb0;
 		use primitives::v1::{
-			AvailabilityBitfield, GroupIndex, Hash, Header, Id as ParaId,
-			SignedAvailabilityBitfield, ValidatorIndex,
+			AvailabilityBitfield, GroupIndex, Hash, Id as ParaId, SignedAvailabilityBitfield,
+			ValidatorIndex,
 		};
 
-		use crate::mock::{new_test_ext, MockGenesisConfig, System, Test};
-		use frame_support::traits::UnfilteredDispatchable;
+		use crate::mock::Test;
 		use futures::executor::block_on;
+		use keyring::Sr25519Keyring;
 		use primitives::v0::PARACHAIN_KEY_TYPE_ID;
 		use sc_keystore::LocalKeystore;
 		use sp_keystore::{SyncCryptoStore, SyncCryptoStorePtr};
@@ -1097,7 +1100,193 @@ mod tests {
 
 		#[test]
 		fn candidates() {
-			todo!("check sanitize_candidates here!");
+			const RELAY_PARENT_NUM: u32 = 3;
+
+			let header = default_header();
+			let relay_parent = header.hash();
+			// 2 cores means two bits
+			let session_index = SessionIndex::from(0_u32);
+
+			let keystore = LocalKeystore::in_memory();
+			let keystore = Arc::new(keystore) as SyncCryptoStorePtr;
+			let signing_context = SigningContext { parent_hash: relay_parent, session_index };
+
+			let validators = vec![
+				keyring::Sr25519Keyring::Alice,
+				keyring::Sr25519Keyring::Bob,
+				keyring::Sr25519Keyring::Charlie,
+				keyring::Sr25519Keyring::Dave,
+			];
+			for validator in validators.iter() {
+				SyncCryptoStore::sr25519_generate_new(
+					&*keystore,
+					PARACHAIN_KEY_TYPE_ID,
+					Some(&validator.to_seed()),
+				)
+				.unwrap();
+			}
+
+			let has_concluded_invalid = |_candidate: CandidateHash| -> bool { false };
+
+			let scheduled = (1_usize..4)
+				.into_iter()
+				.map(|idx| {
+					let ca = CoreAssignment {
+						kind: scheduler::AssignmentKind::Parachain,
+						group_idx: GroupIndex::from(idx as u32),
+						para_id: ParaId::from(idx as u32),
+						core: CoreIndex::from(idx as u32),
+					};
+					ca
+				})
+				.collect::<Vec<_>>();
+			let scheduled = &scheduled[..];
+
+			let group_validators = |group_index: GroupIndex| {
+				match group_index {
+					group_index if group_index == GroupIndex::from(0) => Some(vec![0, 1]),
+					group_index if group_index == GroupIndex::from(1) => Some(vec![2, 3]),
+					group_index if group_index == GroupIndex::from(2) => Some(vec![4]),
+					_ => panic!("Group index out of bounds for 2 parachains and 1 parathread core"),
+				}
+				.map(|m| m.into_iter().map(ValidatorIndex).collect::<Vec<_>>())
+			};
+
+			let backed_candidates = (1_usize..4)
+				.into_iter()
+				.map(|idx| {
+					let mut candidate = TestCandidateBuilder {
+						para_id: ParaId::from(idx),
+						relay_parent,
+						pov_hash: Hash::repeat_byte(idx as u8),
+						persisted_validation_data_hash: [42u8; 32].into(),
+						hrmp_watermark: RELAY_PARENT_NUM,
+						..Default::default()
+					}
+					.build();
+
+					collator_sign_candidate(Sr25519Keyring::One, &mut candidate);
+
+					let backed = block_on(back_candidate(
+						candidate,
+						&validators,
+						group_validators(GroupIndex::from(0)).unwrap().as_ref(),
+						&keystore,
+						&signing_context,
+						BackingKind::Threshold,
+					));
+					backed
+				})
+				.collect::<Vec<_>>();
+
+			{
+				assert_matches!(
+					sanitize_backed_candidates::<Test, _, true>(
+						relay_parent,
+						backed_candidates.clone(),
+						has_concluded_invalid,
+						scheduled
+					),
+					Ok(sanitized_backed_candidates) => {
+						assert_eq!(backed_candidates, sanitized_backed_candidates);
+					}
+				);
+				assert_matches!(
+					sanitize_backed_candidates::<Test, _, false>(
+						relay_parent,
+						backed_candidates.clone(),
+						has_concluded_invalid,
+						scheduled
+					),
+					Ok(sanitized_backed_candidates) => {
+						assert_eq!(backed_candidates, sanitized_backed_candidates);
+					}
+				);
+			}
+
+			// nothing is scheduled, so no paraids match,
+			// hence no backed candidate makes it through
+			{
+				assert_matches!(
+					sanitize_backed_candidates::<Test, _, true>(
+						relay_parent,
+						backed_candidates.clone(),
+						has_concluded_invalid,
+						&[][..]
+					),
+					Err(Error::<Test>::CandidateConcludedInvalid)
+				);
+				assert_matches!(
+					sanitize_backed_candidates::<Test, _, false>(
+						relay_parent,
+						backed_candidates.clone(),
+						has_concluded_invalid,
+						&[][..]
+					),
+					Ok(sanitized_backed_candidates) => {
+						assert_eq!(0, sanitized_backed_candidates.len());
+					}
+				);
+			}
+
+			// relay parent mismatch
+			{
+				let relay_parent = Hash::repeat_byte(0xFA);
+				assert_matches!(
+					sanitize_backed_candidates::<Test, _, true>(
+						relay_parent,
+						backed_candidates.clone(),
+						has_concluded_invalid,
+						scheduled
+					),
+					Err(Error::<Test>::CandidateConcludedInvalid)
+				);
+				assert_matches!(
+					sanitize_backed_candidates::<Test, _, false>(
+						relay_parent,
+						backed_candidates.clone(),
+						has_concluded_invalid,
+						scheduled
+					),
+					Ok(sanitized_backed_candidates) => {
+						assert_eq!(0, sanitized_backed_candidates.len());
+					}
+				);
+			}
+
+			// relay parent mismatch
+			{
+				let set = {
+					let mut set = std::collections::HashSet::new();
+					for (idx, backed_candidate) in backed_candidates.iter().enumerate() {
+						if idx & 0x01 == 0 {
+							set.insert(backed_candidate.hash().clone());
+						}
+					}
+					set
+				};
+				let has_concluded_invalid = |candidate: CandidateHash| set.contains(&candidate);
+				assert_matches!(
+					sanitize_backed_candidates::<Test, _, true>(
+						relay_parent,
+						backed_candidates.clone(),
+						has_concluded_invalid,
+						scheduled
+					),
+					Err(Error::<Test>::CandidateConcludedInvalid)
+				);
+				assert_matches!(
+					sanitize_backed_candidates::<Test, _, false>(
+						relay_parent,
+						backed_candidates.clone(),
+						has_concluded_invalid,
+						scheduled
+					),
+					Ok(sanitized_backed_candidates) => {
+						assert_eq!(0, sanitized_backed_candidates.len() >> 1);
+					}
+				);
+			}
 		}
 	}
 

From c2826a79c88505fe782e443e3142732e399d07e8 Mon Sep 17 00:00:00 2001
From: Bernhard Schuster <bernhard@ahoi.io>
Date: Fri, 29 Oct 2021 21:24:50 +0200
Subject: [PATCH 051/107] fixup tests

---
 runtime/parachains/src/paras_inherent.rs | 25 ++++++++++++------------
 1 file changed, 13 insertions(+), 12 deletions(-)

diff --git a/runtime/parachains/src/paras_inherent.rs b/runtime/parachains/src/paras_inherent.rs
index e10c0593b8e2..dfafee95e5b1 100644
--- a/runtime/parachains/src/paras_inherent.rs
+++ b/runtime/parachains/src/paras_inherent.rs
@@ -1104,7 +1104,6 @@ mod tests {
 
 			let header = default_header();
 			let relay_parent = header.hash();
-			// 2 cores means two bits
 			let session_index = SessionIndex::from(0_u32);
 
 			let keystore = LocalKeystore::in_memory();
@@ -1128,13 +1127,13 @@ mod tests {
 
 			let has_concluded_invalid = |_candidate: CandidateHash| -> bool { false };
 
-			let scheduled = (1_usize..4)
+			let scheduled = (0_usize..2)
 				.into_iter()
 				.map(|idx| {
 					let ca = CoreAssignment {
 						kind: scheduler::AssignmentKind::Parachain,
 						group_idx: GroupIndex::from(idx as u32),
-						para_id: ParaId::from(idx as u32),
+						para_id: ParaId::from(1_u32 + idx as u32),
 						core: CoreIndex::from(idx as u32),
 					};
 					ca
@@ -1146,19 +1145,19 @@ mod tests {
 				match group_index {
 					group_index if group_index == GroupIndex::from(0) => Some(vec![0, 1]),
 					group_index if group_index == GroupIndex::from(1) => Some(vec![2, 3]),
-					group_index if group_index == GroupIndex::from(2) => Some(vec![4]),
 					_ => panic!("Group index out of bounds for 2 parachains and 1 parathread core"),
 				}
 				.map(|m| m.into_iter().map(ValidatorIndex).collect::<Vec<_>>())
 			};
 
-			let backed_candidates = (1_usize..4)
+			let backed_candidates = (0_usize..2)
 				.into_iter()
-				.map(|idx| {
+				.map(|idx0| {
+					let idx1 = idx0 + 1;
 					let mut candidate = TestCandidateBuilder {
-						para_id: ParaId::from(idx),
+						para_id: ParaId::from(idx1),
 						relay_parent,
-						pov_hash: Hash::repeat_byte(idx as u8),
+						pov_hash: Hash::repeat_byte(idx1 as u8),
 						persisted_validation_data_hash: [42u8; 32].into(),
 						hrmp_watermark: RELAY_PARENT_NUM,
 						..Default::default()
@@ -1170,7 +1169,7 @@ mod tests {
 					let backed = block_on(back_candidate(
 						candidate,
 						&validators,
-						group_validators(GroupIndex::from(0)).unwrap().as_ref(),
+						group_validators(GroupIndex::from(idx0 as u32)).unwrap().as_ref(),
 						&keystore,
 						&signing_context,
 						BackingKind::Threshold,
@@ -1207,12 +1206,13 @@ mod tests {
 			// nothing is scheduled, so no paraids match,
 			// hence no backed candidate makes it through
 			{
+				let scheduled = &[][..];
 				assert_matches!(
 					sanitize_backed_candidates::<Test, _, true>(
 						relay_parent,
 						backed_candidates.clone(),
 						has_concluded_invalid,
-						&[][..]
+						scheduled
 					),
 					Err(Error::<Test>::CandidateConcludedInvalid)
 				);
@@ -1221,7 +1221,7 @@ mod tests {
 						relay_parent,
 						backed_candidates.clone(),
 						has_concluded_invalid,
-						&[][..]
+						scheduled
 					),
 					Ok(sanitized_backed_candidates) => {
 						assert_eq!(0, sanitized_backed_candidates.len());
@@ -1256,6 +1256,7 @@ mod tests {
 
 			// relay parent mismatch
 			{
+				// mark every second one as concluded invalid
 				let set = {
 					let mut set = std::collections::HashSet::new();
 					for (idx, backed_candidate) in backed_candidates.iter().enumerate() {
@@ -1283,7 +1284,7 @@ mod tests {
 						scheduled
 					),
 					Ok(sanitized_backed_candidates) => {
-						assert_eq!(0, sanitized_backed_candidates.len() >> 1);
+						assert_eq!(0, sanitized_backed_candidates.len() / 2);
 					}
 				);
 			}

From d2f03f4b4702f6c8a2c64bc534235f86b14ee1a1 Mon Sep 17 00:00:00 2001
From: Bernhard Schuster <bernhard@ahoi.io>
Date: Sun, 31 Oct 2021 17:41:56 +0100
Subject: [PATCH 052/107] update guide

---
 .../src/runtime/inclusion.md                  | 31 +++++++++++++++++--
 .../src/runtime/parainherent.md               | 19 ++++++++++++
 2 files changed, 47 insertions(+), 3 deletions(-)

diff --git a/roadmap/implementers-guide/src/runtime/inclusion.md b/roadmap/implementers-guide/src/runtime/inclusion.md
index bab3ae455c60..0d7c1b3523b1 100644
--- a/roadmap/implementers-guide/src/runtime/inclusion.md
+++ b/roadmap/implementers-guide/src/runtime/inclusion.md
@@ -45,13 +45,38 @@ PendingAvailabilityCommitments: map ParaId => CandidateCommitments;
 All failed checks should lead to an unrecoverable error making the block invalid.
 
 * `process_bitfields(expected_bits, Bitfields, core_lookup: Fn(CoreIndex) -> Option<ParaId>)`:
-  1. check that there is at most 1 bitfield per validator and that the number of bits in each bitfield is equal to `expected_bits`.
-  1. check that there are no duplicates
-  1. check all validator signatures.
+  1. call `sanitize_bitfields<true>` and use the sanitized `signed_bitfields` from now on.
+  1. call `sanitize_backed_candidates<true>` and use the sanitized `backed_candidates` from now on.
   1. apply each bit of bitfield to the corresponding pending candidate. looking up parathread cores using the `core_lookup`. Disregard bitfields that have a `1` bit for any free cores.
   1. For each applied bit of each availability-bitfield, set the bit for the validator in the `CandidatePendingAvailability`'s `availability_votes` bitfield. Track all candidates that now have >2/3 of bits set in their `availability_votes`. These candidates are now available and can be enacted.
   1. For all now-available candidates, invoke the `enact_candidate` routine with the candidate and relay-parent number.
   1. Return a list of `(CoreIndex, CandidateHash)` from freed cores consisting of the cores where candidates have become available.
+* `sanitize_bitfields<const EARLY_RETURN>(
+    usab: UncheckedSignedAvailabilityBitfields,
+    db: DisputedBitfield,
+    expected_bits: usize,
+    parent_hash: Hash,
+    session_index: SessionIndex,
+    validators: &[ValidatorId]
+  )`:
+  1. if `EARLY_RETURN` is `true`, return an error when encountering a bitfield that does not pass the checks, if `false`, drop the bitfield from the set that will be returned.
+  1. check that there is at most 1 bitfield per validator and that the number of bits in each bitfield is equal to `expected_bits`.
+  1. check that there are no duplicates
+  1. check that the validator bit index is not out of bounds
+  1. check all validator signatures, iff `EARLY_RETURN=true`, since in the other case, checking is supposed to be done before the call
+  1. check that there are no bits set that reference a disputed candidate
+
+* `sanitize_backed_candidates<const EARLY_RETURN: bool>(
+	relay_parent: Hash,
+	mut backed_candidates: Vec<BackedCandidate>,
+	candidate_has_concluded_invalid_dispute: Fn(CandidateHash) -> bool,
+	scheduled: &[CoreAssignment],
+)`
+  1. if `EARLY_RETURN` is `true`, return an error when encountering a backed candidates that does not pass the checks, if `false`, drop the backed candidates from the set that will be returned.
+  1. check all backed candidates have no concluded invalid dispute by means of the provided closure `candidate_has_concluded_invalid_dispute`
+  1. check all backed candidates have the matchin `relay_parent`
+  1. check all backed candidates paraid was scheduled by means of the provided `scheduled` parameter
+
 * `process_candidates(parent_storage_root, BackedCandidates, scheduled: Vec<CoreAssignment>, group_validators: Fn(GroupIndex) -> Option<Vec<ValidatorIndex>>)`:
   1. check that each candidate corresponds to a scheduled core and that they are ordered in the same order the cores appear in assignments in `scheduled`.
   1. check that `scheduled` is sorted ascending by `CoreIndex`, without duplicates.
diff --git a/roadmap/implementers-guide/src/runtime/parainherent.md b/roadmap/implementers-guide/src/runtime/parainherent.md
index cc5e209362e9..82804cf1cd95 100644
--- a/roadmap/implementers-guide/src/runtime/parainherent.md
+++ b/roadmap/implementers-guide/src/runtime/parainherent.md
@@ -48,3 +48,22 @@ OnChainVotes: Option<ScrapedOnChainVotes>,
     1. Call `Scheduler::occupied` using the `occupied` core indices of the returned  above, first sorting the list of assigned core indices.
     1. Call the `Ump::process_pending_upward_messages` routine to execute all messages in upward dispatch queues.
     1. If all of the above succeeds, set `Included` to `Some(())`.
+
+
+* `create_inherent`: This entry-point accepts one parameter: `InherentData`.
+  1. Unpack `InherentData` into its parts, `bitfields`, `backed_candidates`, `disputes` and the `parent_header`.
+  1. Hash the `parent_header` and make sure that it corresponds to the block hash of the parent (tracked by the `frame_system` FRAME module),
+  1. Invoke `Disputes::filter_multi_dispute_data` to remove duplicates et al from `disputes`.
+  1. Run the following within a  `with_transaction` closure to avoid side effects:
+    1. Invoke `Disputes::provide_multi_dispute_data`.
+    1. Collect the newly concluded disputes as `current_concluded_invalid_disputes`.
+    1. If there are any concluded disputes from the current session, invoke `Inclusion::collect_disputed` with the newly disputed candidates. Annotate each returned core with `FreedReason::Concluded`, sort them, and invoke `Scheduler::free_cores` with them.
+    1. Collect the concluded invalid disputes in the current session as `conlcuded_invalid_disputes`.
+    1. Return `TransactionOutcome::Rollback(freed_cores, concluded_invalid_disputes)`.
+  1. Call `sanitize_bitfields<false>` and only use the sanitized set of bitfields afterwards.
+  1. Call `sanitize_backed_candidates<false>`.
+  1. Collect entropy based on `CurrentBlockRandomness::random`.
+  1. Call `apply_weight_limit` to utilize the block as well as possible, with a randomized heuristic.
+  1. Re-create a `InherentData` from the sanitized and weight bounded information.
+  1. Call `Self::enter` which now should not error anymore, but we do this call anyways for now.
+  1. Return `Call::enter { .. }`.
\ No newline at end of file

From 209e267c00df145695dfb1c36b524c11b4f1f6e3 Mon Sep 17 00:00:00 2001
From: Bernhard Schuster <bernhard@ahoi.io>
Date: Sun, 31 Oct 2021 17:42:47 +0100
Subject: [PATCH 053/107] add check referenced in the guide

---
 runtime/parachains/src/paras_inherent.rs | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/runtime/parachains/src/paras_inherent.rs b/runtime/parachains/src/paras_inherent.rs
index dfafee95e5b1..03b885309822 100644
--- a/runtime/parachains/src/paras_inherent.rs
+++ b/runtime/parachains/src/paras_inherent.rs
@@ -151,6 +151,11 @@ pub mod pallet {
 				},
 			};
 
+			if parent_hash != parent_header.hash() {
+				log::warn!(target: LOG_TARGET, "ParachainsInherentData references a different parent header hash than frame");
+				return None
+			}
+
 			let current_session = <shared::Pallet<T>>::session_index();
 			let expected_bits = <scheduler::Pallet<T>>::availability_cores().len();
 			let validator_public = shared::Pallet::<T>::active_validator_keys();

From 03e77ad228535880b008772f1df97766c8bbf572 Mon Sep 17 00:00:00 2001
From: Bernhard Schuster <bernhard@ahoi.io>
Date: Sun, 31 Oct 2021 17:59:53 +0100
Subject: [PATCH 054/107] improve weights code

---
 runtime/parachains/src/paras_inherent.rs | 92 +++++++++++++++---------
 1 file changed, 58 insertions(+), 34 deletions(-)

diff --git a/runtime/parachains/src/paras_inherent.rs b/runtime/parachains/src/paras_inherent.rs
index 03b885309822..5d6840fe00fe 100644
--- a/runtime/parachains/src/paras_inherent.rs
+++ b/runtime/parachains/src/paras_inherent.rs
@@ -54,11 +54,58 @@ pub use pallet::*;
 
 const LOG_TARGET: &str = "runtime::inclusion-inherent";
 // In the future, we should benchmark these consts; these are all untested assumptions for now.
-const BACKED_CANDIDATE_WEIGHT: Weight = 100_000;
 const INCLUSION_INHERENT_CLAIMED_WEIGHT: Weight = 1_000_000_000;
 // we assume that 75% of an paras inherent's weight is used processing backed candidates
 const MINIMAL_INCLUSION_INHERENT_WEIGHT: Weight = INCLUSION_INHERENT_CLAIMED_WEIGHT / 4;
 
+/// Weights
+///
+/// TODO replaced this with the true weights
+/// TODO as part of <https://github.com/paritytech/polkadot/pull/4044>
+/// TODO The current ones are _preliminary_ copies of those.
+fn paras_inherent_total_weight<T: Config>(
+	n_backed_candidates: u32,
+	n_bitfields: u32,
+	n_disputes: u32,
+) -> Weight {
+	let weights = T::DbWeight::get();
+	// static
+	(10_901_789_000 as Weight)
+		// backed candidates
+		.saturating_add((424_633_000 as Weight).saturating_mul(n_backed_candidates as Weight))
+		.saturating_add(weights.reads(58 as Weight))
+		.saturating_add(weights.reads((8 as Weight).saturating_mul(n_backed_candidates as Weight)))
+		.saturating_add(weights.writes(212 as Weight))
+		.saturating_add(weights.writes((2 as Weight).saturating_mul(n_backed_candidates as Weight)))
+		// disputes
+		.saturating_add((103_899_000 as Weight).saturating_mul(n_disputes as Weight))
+		.saturating_add(weights.reads(61 as Weight))
+		.saturating_add(weights.reads((1 as Weight).saturating_mul(n_disputes as Weight)))
+		.saturating_add(weights.writes(212 as Weight))
+		.saturating_add(weights.writes((2 as Weight).saturating_mul(n_disputes as Weight)))
+		// bitfields
+		.saturating_add((10_000_000 as Weight).saturating_mul(n_disputes as Weight))
+		.saturating_add(weights.reads(10 as Weight))
+		.saturating_add(weights.reads((20 as Weight).saturating_mul(n_bitfields as Weight)))
+		.saturating_add(weights.writes(10 as Weight))
+		.saturating_add(weights.writes((20 as Weight).saturating_mul(n_bitfields as Weight)))
+}
+
+/// Extract the static weight.
+fn static_weight<T: Config>() -> Weight {
+	paras_inherent_total_weight::<T>(0, 0, 0)
+}
+
+/// Extract the weight that is added _per_ bitfield.
+fn bitfield_weight<T: Config>() -> Weight {
+	paras_inherent_total_weight::<T>(0, 1, 0) - static_weight::<T>()
+}
+
+/// Extract the weight that is adder _per_ backed candidate.
+fn backed_candidate_weight<T: Config>() -> Weight {
+	paras_inherent_total_weight::<T>(1, 0, 0) - static_weight::<T>()
+}
+
 /// A bitfield concerning concluded disputes for candidates
 /// associated to the core index equivalent to the bit position.
 #[derive(Default, PartialEq, Eq, Clone, Encode, Decode, RuntimeDebug, TypeInfo)]
@@ -250,9 +297,9 @@ pub mod pallet {
 				entropy
 			};
 
-			// XXX @Lldenaurois
-			// FIXME these weights are garbage
-			let remaining_weight = <T as frame_system::Config>::BlockWeights::get().max_block;
+			let remaining_weight = <T as frame_system::Config>::BlockWeights::get()
+				.max_block
+				.saturating_sub(static_weight());
 			let (_backed_candidates_weight, backed_candidates, bitfields) =
 				apply_weight_limit::<T>(backed_candidates, bitfields, entropy, remaining_weight);
 
@@ -301,7 +348,11 @@ pub mod pallet {
 	impl<T: Config> Pallet<T> {
 		/// Enter the paras inherent. This will process bitfields and backed candidates.
 		#[pallet::weight((
-			MINIMAL_INCLUSION_INHERENT_WEIGHT + data.backed_candidates.len() as Weight * BACKED_CANDIDATE_WEIGHT,
+			paras_inherent_total_weight(
+				data.backed_candidates.len(),
+				data.bitfields.len(),
+				data.disputes.len(),
+			)
 			DispatchClass::Mandatory,
 		))]
 		pub fn enter(
@@ -500,29 +551,6 @@ pub(super) fn create_disputed_bitfield<'a, I: 'a + IntoIterator<Item = &'a CoreI
 	DisputedBitfield::from(bitvec)
 }
 
-/// Calculate the weight of a single backed candidate.
-fn backed_candidate_weight<T: Config>(_backed_candidate: &BackedCandidate<<T>::Hash>) -> Weight {
-	// XXX @Lldenaurois
-	// FIXME these weights are garbage
-	// const CODE_UPGRADE_WEIGHT: Weight = 10_000 as Weight;
-	// const DISPUTE_PER_STATEMENT_WEIGHT: Weight = 1_000 as Weight;
-
-	// backed_candidate.validity_votes.len() as Weight * DISPUTE_PER_STATEMENT_WEIGHT +
-	// 	if backed_candidate.candidate.commitments.new_validation_code.is_some() {
-	// 		CODE_UPGRADE_WEIGHT
-	// 	} else {
-	// 		0 as Weight
-	// 	}
-	BACKED_CANDIDATE_WEIGHT
-}
-
-/// Calculate the weight of a individual bitfield.
-fn bitfield_weight<T: Config>(_bitfield: &UncheckedSignedAvailabilityBitfield) -> Weight {
-	// XXX @Lldenaurois
-	// FIXME these weights are garbage
-	7_000 as Weight
-}
-
 /// Considers an upper threshold that the candidates must not exceed.
 ///
 /// If there is sufficient space, all bitfields and candidates will be included.
@@ -537,13 +565,9 @@ fn apply_weight_limit<T: Config + inclusion::Config>(
 	entropy: [u8; 32],
 	max_weight: Weight,
 ) -> (Weight, Vec<BackedCandidate<<T>::Hash>>, UncheckedSignedAvailabilityBitfields) {
-	let total_bitfields_weight =
-		bitfields.iter().map(|bitfield| bitfield_weight::<T>(bitfield)).sum::<Weight>();
+	let total_bitfields_weight = bitfields.len().saturating_mul(bitfield_weight::<T>());
 
-	let total_candidates_weight = candidates
-		.iter()
-		.map(|backed_candidate| backed_candidate_weight::<T>(backed_candidate))
-		.sum::<Weight>();
+	let total_candidates_weight = candidates.len().saturating_mul(backed_candidate_weight::<T>());
 
 	let total = total_bitfields_weight + total_candidates_weight;
 

From 3b89d79d02223865341b03e960bfa3510ab2a06d Mon Sep 17 00:00:00 2001
From: Bernhard Schuster <bernhard@ahoi.io>
Date: Sun, 31 Oct 2021 18:00:10 +0100
Subject: [PATCH 055/107] fmt

---
 runtime/parachains/src/paras_inherent.rs | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/runtime/parachains/src/paras_inherent.rs b/runtime/parachains/src/paras_inherent.rs
index 5d6840fe00fe..4cd932ba0931 100644
--- a/runtime/parachains/src/paras_inherent.rs
+++ b/runtime/parachains/src/paras_inherent.rs
@@ -199,7 +199,10 @@ pub mod pallet {
 			};
 
 			if parent_hash != parent_header.hash() {
-				log::warn!(target: LOG_TARGET, "ParachainsInherentData references a different parent header hash than frame");
+				log::warn!(
+					target: LOG_TARGET,
+					"ParachainsInherentData references a different parent header hash than frame"
+				);
 				return None
 			}
 
@@ -310,7 +313,8 @@ pub mod pallet {
 				parent_header,
 			};
 
-			// Sanity check: session changes can invalidate an inherent, and we _really_ don't want that to happen.
+			// Sanity check: session changes can invalidate an inherent,
+			// and we _really_ don't want that to happen.
 			// See <https://github.com/paritytech/polkadot/issues/1327>
 
 			// Calling `Self::enter` here is a safe-guard, to avoid any discrepancy

From 20249dd4cab3e58f7a0010f9b4f591bc6a35cf21 Mon Sep 17 00:00:00 2001
From: Bernhard Schuster <bernhard@ahoi.io>
Date: Sun, 31 Oct 2021 19:14:24 +0100
Subject: [PATCH 056/107] fixins

---
 runtime/parachains/src/paras_inherent.rs | 41 ++++++++++++------------
 1 file changed, 21 insertions(+), 20 deletions(-)

diff --git a/runtime/parachains/src/paras_inherent.rs b/runtime/parachains/src/paras_inherent.rs
index 4cd932ba0931..fb3582c80b12 100644
--- a/runtime/parachains/src/paras_inherent.rs
+++ b/runtime/parachains/src/paras_inherent.rs
@@ -176,7 +176,7 @@ pub mod pallet {
 	}
 
 	#[pallet::inherent]
-	impl<T: Config> ProvideInherent for Pallet<T> {
+	impl<T: Config + crate::configuration::Config> ProvideInherent for Pallet<T> {
 		type Call = Call<T>;
 		type Error = MakeFatalError<()>;
 		const INHERENT_IDENTIFIER: InherentIdentifier = PARACHAINS_INHERENT_IDENTIFIER;
@@ -302,7 +302,7 @@ pub mod pallet {
 
 			let remaining_weight = <T as frame_system::Config>::BlockWeights::get()
 				.max_block
-				.saturating_sub(static_weight());
+				.saturating_sub(static_weight::<T>());
 			let (_backed_candidates_weight, backed_candidates, bitfields) =
 				apply_weight_limit::<T>(backed_candidates, bitfields, entropy, remaining_weight);
 
@@ -352,11 +352,11 @@ pub mod pallet {
 	impl<T: Config> Pallet<T> {
 		/// Enter the paras inherent. This will process bitfields and backed candidates.
 		#[pallet::weight((
-			paras_inherent_total_weight(
-				data.backed_candidates.len(),
-				data.bitfields.len(),
-				data.disputes.len(),
-			)
+			paras_inherent_total_weight::<T>(
+				data.backed_candidates.len() as u32,
+				data.bitfields.len() as u32,
+				data.disputes.len() as u32,
+			),
 			DispatchClass::Mandatory,
 		))]
 		pub fn enter(
@@ -369,6 +369,8 @@ pub mod pallet {
 				parent_header,
 				disputes,
 			} = data;
+			let disputes_len = disputes.len() as u32;
+			let bitfields_len = signed_bitfields.len() as u32;
 
 			ensure_none(origin)?;
 			ensure!(!Included::<T>::exists(), Error::<T>::TooManyInclusionInherents);
@@ -487,7 +489,7 @@ pub mod pallet {
 			});
 
 			let backed_candidates = limit_backed_candidates::<T>(backed_candidates);
-			let backed_candidates_len = backed_candidates.len() as Weight;
+			let backed_candidates_len = backed_candidates.len() as u32;
 
 			// Process backed candidates according to scheduled cores.
 			let parent_storage_root = parent_header.state_root().clone();
@@ -518,10 +520,11 @@ pub mod pallet {
 			// And track that we've finished processing the inherent for this block.
 			Included::<T>::set(Some(()));
 
-			Ok(Some(
-				MINIMAL_INCLUSION_INHERENT_WEIGHT +
-					(backed_candidates_len * BACKED_CANDIDATE_WEIGHT),
-			)
+			Ok(Some(paras_inherent_total_weight::<T>(
+				backed_candidates_len,
+				bitfields_len,
+				disputes_len,
+			))
 			.into())
 		}
 	}
@@ -569,9 +572,10 @@ fn apply_weight_limit<T: Config + inclusion::Config>(
 	entropy: [u8; 32],
 	max_weight: Weight,
 ) -> (Weight, Vec<BackedCandidate<<T>::Hash>>, UncheckedSignedAvailabilityBitfields) {
-	let total_bitfields_weight = bitfields.len().saturating_mul(bitfield_weight::<T>());
+	let total_bitfields_weight = (bitfields.len() as Weight).saturating_mul(bitfield_weight::<T>());
 
-	let total_candidates_weight = candidates.len().saturating_mul(backed_candidate_weight::<T>());
+	let total_candidates_weight =
+		(candidates.len() as Weight).saturating_mul(backed_candidate_weight::<T>());
 
 	let total = total_bitfields_weight + total_candidates_weight;
 
@@ -616,7 +620,7 @@ fn apply_weight_limit<T: Config + inclusion::Config>(
 		let (acc_candidate_weight, indices) = random_sel::<BackedCandidate<<T>::Hash>, _>(
 			&mut rng,
 			&candidates[..],
-			backed_candidate_weight::<T>,
+			|_| backed_candidate_weight::<T>(),
 			remaining_weight,
 		);
 		let candidates =
@@ -632,7 +636,7 @@ fn apply_weight_limit<T: Config + inclusion::Config>(
 	let (total, indices) = random_sel::<UncheckedSignedAvailabilityBitfield, _>(
 		&mut rng,
 		&bitfields[..],
-		bitfield_weight::<T>,
+		|_| bitfield_weight::<T>(),
 		max_weight,
 	);
 	let bitfields = indices.into_iter().map(move |idx| bitfields[idx].clone()).collect::<Vec<_>>();
@@ -659,10 +663,7 @@ fn apply_weight_limit<T: Config + inclusion::Config>(
 /// `false` assures that all inputs are filtered, and invalid ones are filtered out.
 /// It also skips signature verification.
 /// `true` returns an `Err(_)` on the first check failing.
-pub(crate) fn sanitize_bitfields<
-	T: configuration::Config + crate::inclusion::Config,
-	const EARLY_RETURN: bool,
->(
+pub(crate) fn sanitize_bitfields<T: Config + crate::inclusion::Config, const EARLY_RETURN: bool>(
 	unchecked_bitfields: UncheckedSignedAvailabilityBitfields,
 	disputed_bitfield: DisputedBitfield,
 	expected_bits: usize,

From 57e8daa21a092a9cb0a582e7125f05bd2aa221ee Mon Sep 17 00:00:00 2001
From: Bernhard Schuster <bernhard@ahoi.io>
Date: Mon, 1 Nov 2021 01:21:06 +0100
Subject: [PATCH 057/107] Update
 roadmap/implementers-guide/src/runtime/inclusion.md

Co-authored-by: Zeke Mostov <32168567+emostov@users.noreply.github.com>
---
 roadmap/implementers-guide/src/runtime/inclusion.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/roadmap/implementers-guide/src/runtime/inclusion.md b/roadmap/implementers-guide/src/runtime/inclusion.md
index 0d7c1b3523b1..52e09fd60f9f 100644
--- a/roadmap/implementers-guide/src/runtime/inclusion.md
+++ b/roadmap/implementers-guide/src/runtime/inclusion.md
@@ -74,7 +74,7 @@ All failed checks should lead to an unrecoverable error making the block invalid
 )`
   1. if `EARLY_RETURN` is `true`, return an error when encountering a backed candidates that does not pass the checks, if `false`, drop the backed candidates from the set that will be returned.
   1. check all backed candidates have no concluded invalid dispute by means of the provided closure `candidate_has_concluded_invalid_dispute`
-  1. check all backed candidates have the matchin `relay_parent`
+  1. check all backed candidates have the matching `relay_parent`
   1. check all backed candidates paraid was scheduled by means of the provided `scheduled` parameter
 
 * `process_candidates(parent_storage_root, BackedCandidates, scheduled: Vec<CoreAssignment>, group_validators: Fn(GroupIndex) -> Option<Vec<ValidatorIndex>>)`:

From 352872b509f22da9639393f24a0872b428cd9e91 Mon Sep 17 00:00:00 2001
From: Bernhard Schuster <bernhard@ahoi.io>
Date: Mon, 1 Nov 2021 01:34:09 +0100
Subject: [PATCH 058/107] compiling + address review

---
 runtime/parachains/src/mock.rs                |  2 +-
 runtime/parachains/src/paras_inherent.rs      | 19 +++++++++----------
 runtime/parachains/src/runtime_api_impl/v1.rs |  2 +-
 3 files changed, 11 insertions(+), 12 deletions(-)

diff --git a/runtime/parachains/src/mock.rs b/runtime/parachains/src/mock.rs
index 5ba8e046aff0..df5f4a8aa285 100644
--- a/runtime/parachains/src/mock.rs
+++ b/runtime/parachains/src/mock.rs
@@ -258,7 +258,7 @@ impl crate::inclusion::Config for Test {
 	type RewardValidators = TestRewardValidators;
 }
 
-impl crate::paras_inherent::pallet::Config for Test {}
+impl crate::paras_inherent::Config for Test {}
 
 impl crate::session_info::Config for Test {}
 
diff --git a/runtime/parachains/src/paras_inherent.rs b/runtime/parachains/src/paras_inherent.rs
index fb3582c80b12..4e852706960e 100644
--- a/runtime/parachains/src/paras_inherent.rs
+++ b/runtime/parachains/src/paras_inherent.rs
@@ -22,7 +22,6 @@
 //! this module.
 
 use crate::{
-	configuration,
 	disputes::DisputesHandler,
 	inclusion,
 	scheduler::{self, CoreAssignment, FreedReason},
@@ -176,7 +175,7 @@ pub mod pallet {
 	}
 
 	#[pallet::inherent]
-	impl<T: Config + crate::configuration::Config> ProvideInherent for Pallet<T> {
+	impl<T: Config> ProvideInherent for Pallet<T> {
 		type Call = Call<T>;
 		type Error = MakeFatalError<()>;
 		const INHERENT_IDENTIFIER: InherentIdentifier = PARACHAINS_INHERENT_IDENTIFIER;
@@ -663,7 +662,7 @@ fn apply_weight_limit<T: Config + inclusion::Config>(
 /// `false` assures that all inputs are filtered, and invalid ones are filtered out.
 /// It also skips signature verification.
 /// `true` returns an `Err(_)` on the first check failing.
-pub(crate) fn sanitize_bitfields<T: Config + crate::inclusion::Config, const EARLY_RETURN: bool>(
+pub(crate) fn sanitize_bitfields<T: crate::inclusion::Config, const EARLY_RETURN: bool>(
 	unchecked_bitfields: UncheckedSignedAvailabilityBitfields,
 	disputed_bitfield: DisputedBitfield,
 	expected_bits: usize,
@@ -677,7 +676,7 @@ pub(crate) fn sanitize_bitfields<T: Config + crate::inclusion::Config, const EAR
 
 	ensure2!(
 		disputed_bitfield.0.len() == expected_bits,
-		crate::inclusion::pallet::Error::<T>::WrongBitfieldSize,
+		crate::inclusion::Error::<T>::WrongBitfieldSize,
 		EARLY_RETURN
 	);
 
@@ -686,7 +685,7 @@ pub(crate) fn sanitize_bitfields<T: Config + crate::inclusion::Config, const EAR
 	for unchecked_bitfield in unchecked_bitfields {
 		ensure2!(
 			unchecked_bitfield.unchecked_payload().0.len() == expected_bits,
-			crate::inclusion::pallet::Error::<T>::WrongBitfieldSize,
+			crate::inclusion::Error::<T>::WrongBitfieldSize,
 			EARLY_RETURN,
 			continue
 		);
@@ -694,21 +693,21 @@ pub(crate) fn sanitize_bitfields<T: Config + crate::inclusion::Config, const EAR
 		ensure2!(
 			unchecked_bitfield.unchecked_payload().0.clone() & disputed_bitfield.0.clone() ==
 				all_zeros,
-			crate::inclusion::pallet::Error::<T>::BitfieldReferencesFreedCore,
+			crate::inclusion::Error::<T>::BitfieldReferencesFreedCore,
 			EARLY_RETURN,
 			continue
 		);
 
 		ensure2!(
 			last_index.map_or(true, |last| last < unchecked_bitfield.unchecked_validator_index()),
-			crate::inclusion::pallet::Error::<T>::BitfieldDuplicateOrUnordered,
+			crate::inclusion::Error::<T>::BitfieldDuplicateOrUnordered,
 			EARLY_RETURN,
 			continue
 		);
 
 		ensure2!(
 			(unchecked_bitfield.unchecked_validator_index().0 as usize) < validators.len(),
-			crate::inclusion::pallet::Error::<T>::ValidatorIndexOutOfBounds,
+			crate::inclusion::Error::<T>::ValidatorIndexOutOfBounds,
 			EARLY_RETURN,
 			continue
 		);
@@ -724,7 +723,7 @@ pub(crate) fn sanitize_bitfields<T: Config + crate::inclusion::Config, const EAR
 			{
 				signed_bitfield
 			} else {
-				fail!(crate::inclusion::pallet::Error::<T>::InvalidBitfieldSignature);
+				fail!(crate::inclusion::Error::<T>::InvalidBitfieldSignature);
 			};
 			bitfields.push(signed_bitfield.into_unchecked());
 		} else {
@@ -746,7 +745,7 @@ pub(crate) fn sanitize_bitfields<T: Config + crate::inclusion::Config, const EAR
 /// `candidate_has_concluded_invalid_dispute` must return `true` if the candidate
 /// is disputed, false otherwise
 fn sanitize_backed_candidates<
-	T: Config + crate::inclusion::Config + crate::paras_inherent::Config,
+	T: crate::inclusion::Config,
 	F: Fn(CandidateHash) -> bool,
 	const EARLY_RETURN: bool,
 >(
diff --git a/runtime/parachains/src/runtime_api_impl/v1.rs b/runtime/parachains/src/runtime_api_impl/v1.rs
index 1607dcd22528..5909ea55290b 100644
--- a/runtime/parachains/src/runtime_api_impl/v1.rs
+++ b/runtime/parachains/src/runtime_api_impl/v1.rs
@@ -332,6 +332,6 @@ pub fn validation_code_by_hash<T: paras::Config>(
 }
 
 /// Disputes imported via means of on-chain imports.
-pub fn on_chain_votes<T: paras_inherent::pallet::Config>() -> Option<ScrapedOnChainVotes<T::Hash>> {
+pub fn on_chain_votes<T: paras_inherent::Config>() -> Option<ScrapedOnChainVotes<T::Hash>> {
 	<paras_inherent::Pallet<T>>::on_chain_votes()
 }

From c592b4889d900dceff87f877f9e7bf4f6b7589cb Mon Sep 17 00:00:00 2001
From: Bernhard Schuster <bernhard@ahoi.io>
Date: Mon, 1 Nov 2021 08:51:35 +0100
Subject: [PATCH 059/107] add comments

---
 runtime/parachains/src/paras_inherent.rs | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/runtime/parachains/src/paras_inherent.rs b/runtime/parachains/src/paras_inherent.rs
index 4e852706960e..2f2107a3fe1c 100644
--- a/runtime/parachains/src/paras_inherent.rs
+++ b/runtime/parachains/src/paras_inherent.rs
@@ -529,6 +529,9 @@ pub mod pallet {
 	}
 }
 
+/// Assures the `$condition` is `true`, and raises
+/// an error if `$action` is `true`.
+/// If `$action` is `false`, executes `$alt` if present.
 macro_rules! ensure2 {
 	($condition:expr, $err:expr, $action:ident $(, $alt:expr)? $(,)?) => {
 		let condition = $condition;

From 7dd4773081a231890ff4469ee465e8c283edd7cc Mon Sep 17 00:00:00 2001
From: Bernhard Schuster <bernhard@ahoi.io>
Date: Mon, 1 Nov 2021 09:37:25 +0100
Subject: [PATCH 060/107] fix weight calc

---
 runtime/parachains/src/paras_inherent.rs | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/runtime/parachains/src/paras_inherent.rs b/runtime/parachains/src/paras_inherent.rs
index 2f2107a3fe1c..796ebecc2d29 100644
--- a/runtime/parachains/src/paras_inherent.rs
+++ b/runtime/parachains/src/paras_inherent.rs
@@ -83,7 +83,7 @@ fn paras_inherent_total_weight<T: Config>(
 		.saturating_add(weights.writes(212 as Weight))
 		.saturating_add(weights.writes((2 as Weight).saturating_mul(n_disputes as Weight)))
 		// bitfields
-		.saturating_add((10_000_000 as Weight).saturating_mul(n_disputes as Weight))
+		.saturating_add((10_000_000 as Weight).saturating_mul(n_bitfields as Weight))
 		.saturating_add(weights.reads(10 as Weight))
 		.saturating_add(weights.reads((20 as Weight).saturating_mul(n_bitfields as Weight)))
 		.saturating_add(weights.writes(10 as Weight))
@@ -1450,8 +1450,7 @@ mod tests {
 						.collect::<Vec<_>>();
 
 					// the expected weight can always be computed by this formula
-					let expected_weight = MINIMAL_INCLUSION_INHERENT_WEIGHT +
-						(backed_candidates.len() as Weight * BACKED_CANDIDATE_WEIGHT);
+					let expected_weight = paras_inherent_total_weight::<Test>(backed_candidates.len() as u32, 0, 0);
 
 					// we've used half the block weight; there's plenty of margin
 					let max_block_weight =
@@ -1505,7 +1504,7 @@ mod tests {
 				let backed_candidates = vec![BackedCandidate::default(); 10];
 
 				// the expected weight with no blocks is just the minimum weight
-				let expected_weight = MINIMAL_INCLUSION_INHERENT_WEIGHT;
+				let expected_weight = paras_inherent_total_weight::<Test>(0, 0, 0);
 
 				// oops, looks like this mandatory call pushed the block weight over the limit
 				let max_block_weight =

From ca447a833b651762d4b22a6fb3093b27881a8ba2 Mon Sep 17 00:00:00 2001
From: Bernhard Schuster <bernhard@ahoi.io>
Date: Wed, 3 Nov 2021 14:21:38 +0100
Subject: [PATCH 061/107] address review comments and test failure

---
 runtime/parachains/src/paras_inherent.rs | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/runtime/parachains/src/paras_inherent.rs b/runtime/parachains/src/paras_inherent.rs
index 796ebecc2d29..cde303aecda4 100644
--- a/runtime/parachains/src/paras_inherent.rs
+++ b/runtime/parachains/src/paras_inherent.rs
@@ -595,6 +595,9 @@ fn apply_weight_limit<T: Config + inclusion::Config>(
 		weight_fn: F,
 		weight_limit: Weight,
 	) -> (Weight, Vec<usize>) {
+		if selectables.is_empty() {
+			return (0 as Weight, Vec::new())
+		}
 		let mut indices = (0..selectables.len()).into_iter().collect::<Vec<_>>();
 		let mut picked_indices = Vec::with_capacity(selectables.len().saturating_sub(1));
 
@@ -767,10 +770,14 @@ fn sanitize_backed_candidates<
 	// Assure the backed candidate's `ParaId`'s core is free.
 	// This holds under the assumption that `Scheduler::schedule` is called _before_.
 	// Also checks the candidate references the correct relay parent.
+
+	let scheduled_paras_set = scheduled
+		.into_iter()
+		.map(|core_assignment| core_assignment.para_id)
+		.collect::<BTreeSet<ParaId>>();
 	backed_candidates.retain(|backed_candidate| {
 		let desc = backed_candidate.descriptor();
-		desc.relay_parent == relay_parent &&
-			scheduled.iter().any(|core| core.para_id == desc.para_id)
+		desc.relay_parent == relay_parent && scheduled_paras_set.contains(&desc.para_id)
 	});
 	ensure2!(backed_candidates.len() == n, Error::<T>::CandidateConcludedInvalid, EARLY_RETURN);
 
@@ -1450,7 +1457,8 @@ mod tests {
 						.collect::<Vec<_>>();
 
 					// the expected weight can always be computed by this formula
-					let expected_weight = paras_inherent_total_weight::<Test>(backed_candidates.len() as u32, 0, 0);
+					let expected_weight =
+						paras_inherent_total_weight::<Test>(backed_candidates.len() as u32, 0, 0);
 
 					// we've used half the block weight; there's plenty of margin
 					let max_block_weight =

From 7b75f7ae3f8d5fbd5a850bf7637e7b039e304143 Mon Sep 17 00:00:00 2001
From: Bernhard Schuster <bernhard@ahoi.io>
Date: Wed, 3 Nov 2021 14:49:04 +0100
Subject: [PATCH 062/107] fix

---
 runtime/parachains/src/paras_inherent.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/parachains/src/paras_inherent.rs b/runtime/parachains/src/paras_inherent.rs
index cde303aecda4..493abae4f651 100644
--- a/runtime/parachains/src/paras_inherent.rs
+++ b/runtime/parachains/src/paras_inherent.rs
@@ -774,7 +774,7 @@ fn sanitize_backed_candidates<
 	let scheduled_paras_set = scheduled
 		.into_iter()
 		.map(|core_assignment| core_assignment.para_id)
-		.collect::<BTreeSet<ParaId>>();
+		.collect::<BTreeSet<_>>();
 	backed_candidates.retain(|backed_candidate| {
 		let desc = backed_candidate.descriptor();
 		desc.relay_parent == relay_parent && scheduled_paras_set.contains(&desc.para_id)

From d524b34c4756e3e001c3ef9710f2a1455bfdf3b7 Mon Sep 17 00:00:00 2001
From: Bernhard Schuster <bernhard@ahoi.io>
Date: Fri, 5 Nov 2021 09:16:17 +0100
Subject: [PATCH 063/107] fix: condition

---
 runtime/parachains/src/paras_inherent.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/parachains/src/paras_inherent.rs b/runtime/parachains/src/paras_inherent.rs
index 493abae4f651..1b5f2185d277 100644
--- a/runtime/parachains/src/paras_inherent.rs
+++ b/runtime/parachains/src/paras_inherent.rs
@@ -602,7 +602,7 @@ fn apply_weight_limit<T: Config + inclusion::Config>(
 		let mut picked_indices = Vec::with_capacity(selectables.len().saturating_sub(1));
 
 		let mut weight_acc = 0 as Weight;
-		while weight_acc < weight_limit || !selectables.is_empty() {
+		while weight_acc < weight_limit && !indices.is_empty() {
 			// randomly pick an index
 			let pick = rng.gen_range(0..indices.len());
 			// remove the index from the available set of indices

From 0e637ae81ae41057f8dcedd2835226382ec335d7 Mon Sep 17 00:00:00 2001
From: Lldenaurois <Ljdenaurois@gmail.com>
Date: Fri, 5 Nov 2021 11:25:47 +0100
Subject: [PATCH 064/107] Fix random_sel function

---
 runtime/parachains/src/paras_inherent.rs | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/runtime/parachains/src/paras_inherent.rs b/runtime/parachains/src/paras_inherent.rs
index 1b5f2185d277..3ea9c2ce240c 100644
--- a/runtime/parachains/src/paras_inherent.rs
+++ b/runtime/parachains/src/paras_inherent.rs
@@ -602,16 +602,20 @@ fn apply_weight_limit<T: Config + inclusion::Config>(
 		let mut picked_indices = Vec::with_capacity(selectables.len().saturating_sub(1));
 
 		let mut weight_acc = 0 as Weight;
-		while weight_acc < weight_limit && !indices.is_empty() {
+		while !indices.is_empty() {
 			// randomly pick an index
 			let pick = rng.gen_range(0..indices.len());
 			// remove the index from the available set of indices
 			let idx = indices.swap_remove(pick);
 
 			let item = &selectables[idx];
+			weight_acc += weight_fn(item);
+
+			if weight_acc > weight_limit {
+				break
+			}
 
 			picked_indices.push(idx);
-			weight_acc = weight_fn(item);
 		}
 		// sorting indices, so the ordering is retained
 		// unstable sorting is fine, since there are no duplicates

From 8113944e7ed1b84fa7f7369bc194ca6f1d1460d3 Mon Sep 17 00:00:00 2001
From: Lldenaurois <Ljdenaurois@gmail.com>
Date: Fri, 5 Nov 2021 11:27:48 +0100
Subject: [PATCH 065/107] Fix overlength block check

---
 runtime/parachains/src/paras_inherent.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/parachains/src/paras_inherent.rs b/runtime/parachains/src/paras_inherent.rs
index 3ea9c2ce240c..79a2605bd121 100644
--- a/runtime/parachains/src/paras_inherent.rs
+++ b/runtime/parachains/src/paras_inherent.rs
@@ -582,7 +582,7 @@ fn apply_weight_limit<T: Config + inclusion::Config>(
 	let total = total_bitfields_weight + total_candidates_weight;
 
 	// everything fits into the block
-	if max_weight < total {
+	if max_weight >= total {
 		return (total, candidates, bitfields)
 	}
 

From 9671728624b4821c8ca33153065a8e21fbe343ef Mon Sep 17 00:00:00 2001
From: Lldenaurois <Ljdenaurois@gmail.com>
Date: Sat, 6 Nov 2021 17:06:24 +0100
Subject: [PATCH 066/107] Zeke + Ladi commit for disputes filtering +
 integration test builder + runtime benchmarks + integration tests

---
 Cargo.lock                                    |    1 +
 primitives/Cargo.toml                         |    1 +
 primitives/src/v1/signed.rs                   |   21 +
 runtime/kusama/src/lib.rs                     |    6 +-
 runtime/kusama/src/weights/mod.rs             |    1 +
 .../runtime_parachains_paras_inherent.rs      |  148 ++
 runtime/parachains/Cargo.toml                 |    7 +-
 runtime/parachains/src/builder.rs             |  576 ++++++
 runtime/parachains/src/disputes.rs            |   25 +
 runtime/parachains/src/inclusion.rs           |   92 +-
 runtime/parachains/src/initializer.rs         |    2 +-
 runtime/parachains/src/lib.rs                 |    2 +
 runtime/parachains/src/mock.rs                |   12 +-
 runtime/parachains/src/paras.rs               |    7 +-
 runtime/parachains/src/paras_inherent.rs      | 1587 +++++++++++------
 .../src/paras_inherent/benchmarking.rs        |  147 ++
 runtime/polkadot/src/lib.rs                   |    7 +-
 runtime/polkadot/src/weights/mod.rs           |    1 +
 .../runtime_parachains_paras_inherent.rs      |   19 +
 runtime/rococo/Cargo.toml                     |    3 +-
 runtime/rococo/src/lib.rs                     |    6 +-
 runtime/rococo/src/weights/mod.rs             |    1 +
 .../runtime_parachains_paras_inherent.rs      |   19 +
 runtime/test-runtime/src/lib.rs               |    4 +-
 runtime/westend/src/lib.rs                    |    6 +-
 runtime/westend/src/weights/mod.rs            |    1 +
 .../runtime_parachains_paras_inherent.rs      |   19 +
 27 files changed, 2170 insertions(+), 551 deletions(-)
 create mode 100644 runtime/kusama/src/weights/runtime_parachains_paras_inherent.rs
 create mode 100644 runtime/parachains/src/builder.rs
 create mode 100644 runtime/parachains/src/paras_inherent/benchmarking.rs
 create mode 100644 runtime/polkadot/src/weights/runtime_parachains_paras_inherent.rs
 create mode 100644 runtime/rococo/src/weights/runtime_parachains_paras_inherent.rs
 create mode 100644 runtime/westend/src/weights/runtime_parachains_paras_inherent.rs

diff --git a/Cargo.lock b/Cargo.lock
index 0ccb75ed1755..2112bab3c9a5 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -6870,6 +6870,7 @@ dependencies = [
  "scale-info",
  "serde",
  "sp-api",
+ "sp-application-crypto",
  "sp-core",
  "sp-inherents",
  "sp-io",
diff --git a/primitives/Cargo.toml b/primitives/Cargo.toml
index c84c0f8f77a2..f58044bf67e5 100644
--- a/primitives/Cargo.toml
+++ b/primitives/Cargo.toml
@@ -56,3 +56,4 @@ std = [
 	"bitvec/std",
 	"frame-system/std",
 ]
+runtime-benchmarks = []
diff --git a/primitives/src/v1/signed.rs b/primitives/src/v1/signed.rs
index 7edce76cbe72..81ca09ce3891 100644
--- a/primitives/src/v1/signed.rs
+++ b/primitives/src/v1/signed.rs
@@ -260,6 +260,27 @@ impl<Payload: EncodeAs<RealPayload>, RealPayload: Encode> UncheckedSigned<Payloa
 			Err(())
 		}
 	}
+
+	/// Sign this payload with the given context and pair.
+	#[cfg(any(feature = "runtime-benchmarks", feature = "std"))]
+	pub fn benchmark_sign<H: Encode>(
+		public: &crate::v0::ValidatorId,
+		payload: Payload,
+		context: &SigningContext<H>,
+		validator_index: ValidatorIndex,
+	) -> Self {
+		use application_crypto::RuntimeAppPublic;
+		let data = Self::payload_data(&payload, context);
+		let signature = public.sign(&data).unwrap();
+
+		Self { payload, validator_index, signature, real_payload: sp_std::marker::PhantomData }
+	}
+
+	/// Immutably access the signature.
+	#[cfg(any(feature = "runtime-benchmarks", feature = "std"))]
+	pub fn benchmark_signature(&self) -> ValidatorSignature {
+		self.signature.clone()
+	}
 }
 
 impl<Payload, RealPayload> From<Signed<Payload, RealPayload>>
diff --git a/runtime/kusama/src/lib.rs b/runtime/kusama/src/lib.rs
index d924bd603064..29ab69fa185d 100644
--- a/runtime/kusama/src/lib.rs
+++ b/runtime/kusama/src/lib.rs
@@ -1201,7 +1201,9 @@ impl parachains_hrmp::Config for Runtime {
 	type Currency = Balances;
 }
 
-impl parachains_paras_inherent::Config for Runtime {}
+impl parachains_paras_inherent::Config for Runtime {
+	type WeightInfo = weights::runtime_parachains_paras_inherent::WeightInfo<Runtime>;
+}
 
 impl parachains_scheduler::Config for Runtime {}
 
@@ -1933,6 +1935,7 @@ sp_api::impl_runtime_apis! {
 			list_benchmark!(list, extra, runtime_common::paras_registrar, Registrar);
 			list_benchmark!(list, extra, runtime_parachains::configuration, Configuration);
 			list_benchmark!(list, extra, runtime_parachains::initializer, Initializer);
+			list_benchmark!(list, extra, runtime_parachains::paras_inherent, ParaInherent);
 			list_benchmark!(list, extra, runtime_parachains::paras, Paras);
 			// Substrate
 			list_benchmark!(list, extra, pallet_bags_list, BagsList);
@@ -2010,6 +2013,7 @@ sp_api::impl_runtime_apis! {
 			add_benchmark!(params, batches, runtime_common::paras_registrar, Registrar);
 			add_benchmark!(params, batches, runtime_parachains::configuration, Configuration);
 			add_benchmark!(params, batches, runtime_parachains::initializer, Initializer);
+			add_benchmark!(params, batches, runtime_parachains::paras_inherent, ParaInherent);
 			add_benchmark!(params, batches, runtime_parachains::paras, Paras);
 			// Substrate
 			add_benchmark!(params, batches, pallet_balances, Balances);
diff --git a/runtime/kusama/src/weights/mod.rs b/runtime/kusama/src/weights/mod.rs
index ea8c2fc7f58d..dcfd8ac5031b 100644
--- a/runtime/kusama/src/weights/mod.rs
+++ b/runtime/kusama/src/weights/mod.rs
@@ -47,3 +47,4 @@ pub mod runtime_common_slots;
 pub mod runtime_parachains_configuration;
 pub mod runtime_parachains_initializer;
 pub mod runtime_parachains_paras;
+pub mod runtime_parachains_paras_inherent;
diff --git a/runtime/kusama/src/weights/runtime_parachains_paras_inherent.rs b/runtime/kusama/src/weights/runtime_parachains_paras_inherent.rs
new file mode 100644
index 000000000000..8a628abfe132
--- /dev/null
+++ b/runtime/kusama/src/weights/runtime_parachains_paras_inherent.rs
@@ -0,0 +1,148 @@
+// Copyright 2017-2021 Parity Technologies (UK) Ltd.
+// This file is part of Polkadot.
+
+// Polkadot is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Polkadot is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
+//! Autogenerated weights for `runtime_parachains::paras_inherent`
+//!
+//! THIS FILE WAS AUTO-GENERATED USING THE SUBSTRATE BENCHMARK CLI VERSION 4.0.0-dev
+//! DATE: 2021-11-04, STEPS: `50`, REPEAT: 20, LOW RANGE: `[]`, HIGH RANGE: `[]`
+//! EXECUTION: Some(Wasm), WASM-EXECUTION: Compiled, CHAIN: Some("kusama-dev"), DB CACHE: 128
+
+// Executed Command:
+// target/release/polkadot
+// benchmark
+// --chain=kusama-dev
+// --steps=50
+// --repeat=20
+// --pallet=runtime_parachains::paras_inherent
+// --extrinsic=*
+// --execution=wasm
+// --wasm-execution=compiled
+// --heap-pages=4096
+// --header=./file_header.txt
+// --output=./runtime/kusama/src/weights/runtime_parachains_paras_inherent.rs
+
+
+#![cfg_attr(rustfmt, rustfmt_skip)]
+#![allow(unused_parens)]
+#![allow(unused_imports)]
+
+use frame_support::{traits::Get, weights::Weight};
+use sp_std::marker::PhantomData;
+
+/// Weight functions for `runtime_parachains::paras_inherent`.
+pub struct WeightInfo<T>(PhantomData<T>);
+impl<T: frame_system::Config> runtime_parachains::paras_inherent::WeightInfo for WeightInfo<T> {
+	// Storage: ParaInherent Included (r:1 w:1)
+	// Storage: System ParentHash (r:1 w:0)
+	// Storage: ParaScheduler AvailabilityCores (r:1 w:1)
+	// Storage: ParasShared CurrentSessionIndex (r:1 w:0)
+	// Storage: ParaInclusion PendingAvailability (r:2 w:1)
+	// Storage: ParasShared ActiveValidatorKeys (r:1 w:0)
+	// Storage: Paras Parachains (r:1 w:0)
+	// Storage: ParaInclusion PendingAvailabilityCommitments (r:1 w:1)
+	// Storage: Configuration ActiveConfig (r:1 w:0)
+	// Storage: Session Validators (r:1 w:0)
+	// Storage: ParasShared ActiveValidatorIndices (r:1 w:0)
+	// Storage: Staking ActiveEra (r:1 w:0)
+	// Storage: Staking ErasRewardPoints (r:1 w:1)
+	// Storage: Dmp DownwardMessageQueues (r:1 w:1)
+	// Storage: Hrmp HrmpChannelDigests (r:1 w:1)
+	// Storage: Paras FutureCodeUpgrades (r:1 w:0)
+	// Storage: ParaScheduler SessionStartBlock (r:1 w:0)
+	// Storage: ParaScheduler ParathreadQueue (r:1 w:1)
+	// Storage: ParaScheduler Scheduled (r:1 w:1)
+	// Storage: ParaScheduler ValidatorGroups (r:1 w:0)
+	// Storage: System BlockWeight (r:1 w:0)
+	// Storage: Ump NeedsDispatch (r:1 w:1)
+	// Storage: Ump NextDispatchRoundStartWith (r:1 w:1)
+	// Storage: ParaInherent OnChainVotes (r:0 w:1)
+	// Storage: Hrmp HrmpWatermarks (r:0 w:1)
+	// Storage: Paras Heads (r:0 w:1)
+	fn enter_variable_disputes(v: u32, ) -> Weight {
+		(224_853_000 as Weight)
+			// Standard Error: 2_000
+			.saturating_add((229_000 as Weight).saturating_mul(v as Weight))
+			.saturating_add(T::DbWeight::get().reads(24 as Weight))
+			.saturating_add(T::DbWeight::get().writes(14 as Weight))
+	}
+	// Storage: ParaInherent Included (r:1 w:1)
+	// Storage: System ParentHash (r:1 w:0)
+	// Storage: ParaScheduler AvailabilityCores (r:1 w:1)
+	// Storage: ParasShared CurrentSessionIndex (r:1 w:0)
+	// Storage: ParasShared ActiveValidatorKeys (r:1 w:0)
+	// Storage: Paras Parachains (r:1 w:0)
+	// Storage: ParaInclusion PendingAvailability (r:2 w:1)
+	// Storage: ParaInclusion PendingAvailabilityCommitments (r:1 w:1)
+	// Storage: Configuration ActiveConfig (r:1 w:0)
+	// Storage: Session Validators (r:1 w:0)
+	// Storage: ParasShared ActiveValidatorIndices (r:1 w:0)
+	// Storage: Staking ActiveEra (r:1 w:0)
+	// Storage: Staking ErasRewardPoints (r:1 w:1)
+	// Storage: Dmp DownwardMessageQueues (r:1 w:1)
+	// Storage: Hrmp HrmpChannelDigests (r:1 w:1)
+	// Storage: Paras FutureCodeUpgrades (r:1 w:0)
+	// Storage: ParaScheduler SessionStartBlock (r:1 w:0)
+	// Storage: ParaScheduler ParathreadQueue (r:1 w:1)
+	// Storage: ParaScheduler Scheduled (r:1 w:1)
+	// Storage: ParaScheduler ValidatorGroups (r:1 w:0)
+	// Storage: System BlockWeight (r:1 w:0)
+	// Storage: Ump NeedsDispatch (r:1 w:1)
+	// Storage: Ump NextDispatchRoundStartWith (r:1 w:1)
+	// Storage: ParaInclusion AvailabilityBitfields (r:0 w:1)
+	// Storage: ParaInherent OnChainVotes (r:0 w:1)
+	// Storage: Hrmp HrmpWatermarks (r:0 w:1)
+	// Storage: Paras Heads (r:0 w:1)
+	fn enter_bitfields() -> Weight {
+		(265_872_000 as Weight)
+			.saturating_add(T::DbWeight::get().reads(24 as Weight))
+			.saturating_add(T::DbWeight::get().writes(15 as Weight))
+	}
+	// Storage: ParaInherent Included (r:1 w:1)
+	// Storage: System ParentHash (r:1 w:0)
+	// Storage: ParaScheduler AvailabilityCores (r:1 w:1)
+	// Storage: ParasShared CurrentSessionIndex (r:1 w:0)
+	// Storage: ParasShared ActiveValidatorKeys (r:1 w:0)
+	// Storage: Paras Parachains (r:1 w:0)
+	// Storage: ParaInclusion PendingAvailability (r:2 w:1)
+	// Storage: ParaInclusion PendingAvailabilityCommitments (r:1 w:1)
+	// Storage: Configuration ActiveConfig (r:1 w:0)
+	// Storage: Session Validators (r:1 w:0)
+	// Storage: ParasShared ActiveValidatorIndices (r:1 w:0)
+	// Storage: Staking ActiveEra (r:1 w:0)
+	// Storage: Staking ErasRewardPoints (r:1 w:1)
+	// Storage: Dmp DownwardMessageQueues (r:1 w:1)
+	// Storage: Hrmp HrmpChannelDigests (r:1 w:1)
+	// Storage: Paras FutureCodeUpgrades (r:1 w:0)
+	// Storage: ParaScheduler SessionStartBlock (r:1 w:0)
+	// Storage: ParaScheduler ParathreadQueue (r:1 w:1)
+	// Storage: ParaScheduler Scheduled (r:1 w:1)
+	// Storage: ParaScheduler ValidatorGroups (r:1 w:0)
+	// Storage: System BlockWeight (r:1 w:0)
+	// Storage: Paras PastCodeMeta (r:1 w:0)
+	// Storage: Paras CurrentCodeHash (r:1 w:0)
+	// Storage: Ump RelayDispatchQueueSize (r:1 w:0)
+	// Storage: Ump NeedsDispatch (r:1 w:1)
+	// Storage: Ump NextDispatchRoundStartWith (r:1 w:1)
+	// Storage: ParaInherent OnChainVotes (r:0 w:1)
+	// Storage: Hrmp HrmpWatermarks (r:0 w:1)
+	// Storage: Paras Heads (r:0 w:1)
+	fn enter_backed_candidates_variable(v: u32, ) -> Weight {
+		(345_183_000 as Weight)
+			// Standard Error: 23_000
+			.saturating_add((49_234_000 as Weight).saturating_mul(v as Weight))
+			.saturating_add(T::DbWeight::get().reads(27 as Weight))
+			.saturating_add(T::DbWeight::get().writes(14 as Weight))
+	}
+}
diff --git a/runtime/parachains/Cargo.toml b/runtime/parachains/Cargo.toml
index 91801e5fd75e..4c955d8a9ee4 100644
--- a/runtime/parachains/Cargo.toml
+++ b/runtime/parachains/Cargo.toml
@@ -32,7 +32,7 @@ pallet-session = { git = "https://github.com/paritytech/substrate", branch = "ma
 pallet-staking = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false }
 pallet-timestamp = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false }
 pallet-vesting = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false }
-frame-benchmarking = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false, optional = true }
+frame-benchmarking = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false }
 frame-support = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false }
 frame-system = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false }
 
@@ -43,6 +43,8 @@ primitives = { package = "polkadot-primitives", path = "../../primitives", defau
 rand = { version = "0.8.3", default-features = false }
 rand_chacha = { version = "0.3.1", default-features = false }
 
+application-crypto = { package = "sp-application-crypto", git = "https://github.com/paritytech/substrate", branch = "master", default-features = false, optional = true }
+
 [dev-dependencies]
 futures = "0.3.17"
 hex-literal = "0.3.3"
@@ -83,9 +85,10 @@ std = [
 	"log/std",
 ]
 runtime-benchmarks = [
-	"frame-benchmarking",
 	"frame-support/runtime-benchmarks",
 	"frame-system/runtime-benchmarks",
+	"primitives/runtime-benchmarks",
+	"application-crypto/full_crypto"
 ]
 try-runtime = [
 	"frame-support/try-runtime",
diff --git a/runtime/parachains/src/builder.rs b/runtime/parachains/src/builder.rs
new file mode 100644
index 000000000000..dc837ea411cd
--- /dev/null
+++ b/runtime/parachains/src/builder.rs
@@ -0,0 +1,576 @@
+use crate::{
+	configuration, inclusion, initializer, paras,
+	paras_inherent::{self},
+	scheduler, session_info, shared,
+};
+use bitvec::{order::Lsb0 as BitOrderLsb0, vec::BitVec};
+use frame_benchmarking::{account, Vec};
+use frame_support::pallet_prelude::*;
+use primitives::v1::{
+	collator_signature_payload, AvailabilityBitfield, BackedCandidate, CandidateCommitments,
+	CandidateDescriptor, CandidateHash, CollatorId, CommittedCandidateReceipt, CompactStatement,
+	CoreIndex, CoreOccupied, DisputeStatement, DisputeStatementSet, GroupIndex, HeadData,
+	Id as ParaId, InherentData as ParachainsInherentData, InvalidDisputeStatementKind,
+	PersistedValidationData, SessionIndex, SigningContext, UncheckedSigned,
+	ValidDisputeStatementKind, ValidationCode, ValidatorId, ValidatorIndex, ValidityAttestation,
+};
+use sp_core::H256;
+use sp_runtime::{
+	generic::Digest,
+	traits::{Header as HeaderT, One, Zero},
+	RuntimeAppPublic,
+};
+use sp_std::{collections::btree_map::BTreeMap, convert::TryInto};
+
+fn byte32_slice_from(n: u32) -> [u8; 32] {
+	let mut slice = [0u8; 32];
+	slice[31] = (n % (1 << 8)) as u8;
+	slice[30] = ((n >> 8) % (1 << 8)) as u8;
+	slice[29] = ((n >> 16) % (1 << 8)) as u8;
+	slice[28] = ((n >> 24) % (1 << 8)) as u8;
+
+	slice
+}
+
+/// Paras inherent `enter` benchmark scenario builder.
+pub(crate) struct BenchBuilder<T: paras_inherent::Config> {
+	validators: Option<Vec<ValidatorId>>,
+	block_number: T::BlockNumber,
+	session: SessionIndex,
+	target_session: u32,
+	max_validators_per_core: Option<u32>,
+	max_validators: Option<u32>,
+	dispute_statements: BTreeMap<u32, u32>,
+	_phantom: sp_std::marker::PhantomData<T>,
+}
+
+/// Paras inherent `enter` benchmark scenario.
+#[cfg(any(feature = "runtime-benchmarks", test))]
+pub(crate) struct Bench<T: paras_inherent::Config> {
+	pub(crate) data: ParachainsInherentData<T::Header>,
+	pub(crate) session: u32,
+	pub(crate) block_number: T::BlockNumber,
+}
+
+impl<T: paras_inherent::Config> BenchBuilder<T> {
+	/// Create a new `BenchBuilder` with some opinionated values that should work with the rest
+	/// of the functions in this impl.å
+	pub(crate) fn new() -> Self {
+		BenchBuilder {
+			// Validators should be declared prior to all other setup.
+			validators: None,
+			// Starting block number; we expect it to get incremented on session setup.
+			block_number: Zero::zero(),
+			// Starting session; we expect it to get incremented on session setup.
+			session: SessionIndex::from(0u32),
+			// Session we want the scenario to take place in. We roll to to this session.
+			target_session: 2u32,
+			// Optionally set the max validators per core; otherwise uses the configuration value.
+			max_validators_per_core: None,
+			// Optionally set the max validators; otherwise uses the configuration value.
+			max_validators: None,
+			// Optionally set the number of dispute statements for each candidate,
+			dispute_statements: BTreeMap::new(),
+			_phantom: sp_std::marker::PhantomData::<T>,
+		}
+	}
+
+	/// Mock header.
+	pub(crate) fn header(block_number: T::BlockNumber) -> T::Header {
+		T::Header::new(
+			block_number,       // block_number,
+			Default::default(), // extrinsics_root,
+			Default::default(), // storage_root,
+			Default::default(), // parent_hash,
+			Default::default(), // digest,
+		)
+	}
+
+	/// Number of the relay parent block.
+	fn relay_parent_number(&self) -> u32 {
+		(self.block_number - One::one())
+			.try_into()
+			.map_err(|_| ())
+			.expect("self.block_number is u32")
+	}
+
+	/// Maximium number of validators that may be part of a validator group.
+	pub(crate) fn fallback_max_validators() -> u32 {
+		configuration::Pallet::<T>::config().max_validators.unwrap_or(200)
+	}
+
+	/// Maximum number of validators participating in parachains consensus (a.k.a. active validators).
+	fn max_validators(&self) -> u32 {
+		self.max_validators.unwrap_or(Self::fallback_max_validators())
+	}
+
+	#[cfg(test)]
+	pub(crate) fn set_max_validators(mut self, n: u32) -> Self {
+		self.max_validators = Some(n);
+		self
+	}
+
+	pub(crate) fn fallback_max_validators_per_core() -> u32 {
+		configuration::Pallet::<T>::config().max_validators_per_core.unwrap_or(5)
+	}
+
+	pub(crate) fn set_dispute_statements(mut self, m: BTreeMap<u32, u32>) -> Self {
+		self.dispute_statements = m;
+		self
+	}
+
+	fn max_validators_per_core(&self) -> u32 {
+		self.max_validators_per_core.unwrap_or(Self::fallback_max_validators_per_core())
+	}
+
+	/// Set maximum number of validators per core.
+	#[cfg(test)]
+	pub(crate) fn set_max_validators_per_core(mut self, n: u32) -> Self {
+		self.max_validators_per_core = Some(n);
+		self
+	}
+
+	/// Maximum number of cores we expect from this configuration.
+	pub(crate) fn max_cores(&self) -> u32 {
+		self.max_validators() / self.max_validators_per_core()
+	}
+
+	/// Minimum number of validity votes in order for a backed candidate to be included.
+	#[cfg(feature = "runtime-benchmarks")]
+	pub(crate) fn fallback_min_validity_votes() -> u32 {
+		(Self::fallback_max_validators() / 2) + 1
+	}
+
+	fn create_indexes(&self, seed: u32) -> (ParaId, CoreIndex, GroupIndex) {
+		let para_id = ParaId::from(seed);
+		let core_idx = CoreIndex(seed);
+		let group_idx =
+			scheduler::Pallet::<T>::group_assigned_to_core(core_idx, self.block_number).unwrap();
+
+		(para_id, core_idx, group_idx)
+	}
+
+	fn candidate_availability_mock(
+		group_idx: GroupIndex,
+		core_idx: CoreIndex,
+		candidate_hash: CandidateHash,
+		availability_votes: BitVec<BitOrderLsb0, u8>,
+	) -> inclusion::CandidatePendingAvailability<T::Hash, T::BlockNumber> {
+		inclusion::CandidatePendingAvailability::<T::Hash, T::BlockNumber>::new(
+			core_idx,           // core
+			candidate_hash,     // hash
+			Default::default(), // candidate descriptor
+			availability_votes, // availability votes
+			Default::default(), // backers
+			Zero::zero(),       // relay parent
+			One::one(),         // relay chain block this was backed in
+			group_idx,          // backing group
+		)
+	}
+
+	fn add_availability(
+		para_id: ParaId,
+		core_idx: CoreIndex,
+		group_idx: GroupIndex,
+		availability_votes: BitVec<BitOrderLsb0, u8>,
+		candidate_hash: CandidateHash,
+	) {
+		let candidate_availability = Self::candidate_availability_mock(
+			group_idx,
+			core_idx,
+			candidate_hash,
+			availability_votes,
+		);
+		// NOTE: commitments does not include any data that would lead to heavy code
+		// paths in `enact_candidate`. But enact_candidates does return a weight which will get
+		// taken into account.
+		let commitments = CandidateCommitments::<u32>::default();
+		inclusion::PendingAvailability::<T>::insert(para_id, candidate_availability);
+		inclusion::PendingAvailabilityCommitments::<T>::insert(&para_id, commitments);
+	}
+
+	fn availability_bitvec(concluding: &BTreeMap<u32, u32>, cores: u32) -> AvailabilityBitfield {
+		let mut bitfields = bitvec::bitvec![bitvec::order::Lsb0, u8; 0; 0];
+		for i in 0..cores {
+			if concluding.get(&(i as u32)).is_some() {
+				bitfields.push(true);
+			} else {
+				bitfields.push(false)
+			}
+		}
+
+		bitfields.into()
+	}
+
+	fn run_to_block(to: u32) {
+		let to = to.into();
+		while frame_system::Pallet::<T>::block_number() < to {
+			let b = frame_system::Pallet::<T>::block_number();
+			initializer::Pallet::<T>::on_finalize(b);
+
+			let b = b + One::one();
+			frame_system::Pallet::<T>::set_block_number(b);
+			initializer::Pallet::<T>::on_initialize(b);
+		}
+	}
+
+	/// Setup para ids.
+	/// * setup para_ids traverses each core,
+	/// * creates a ParaId for that CoreIndex,
+	/// * inserts ParaLifeCycle::Onboarding for that ParaId,
+	/// * inserts the upcoming paras genesis,
+	/// * inserts the ParaId into the ActionsQueue
+	fn setup_para_ids(cores: u32) {
+		// make sure parachains exist prior to session change.
+		for i in 0..cores {
+			let para_id = ParaId::from(i as u32);
+
+			paras::Pallet::<T>::schedule_para_initialize(
+				para_id,
+				paras::ParaGenesisArgs {
+					genesis_head: Default::default(),
+					validation_code: Default::default(),
+					parachain: true,
+				},
+			)
+			.unwrap();
+		}
+	}
+
+	/// Generate validator key pairs and account ids.
+	fn generate_validator_pairs(validator_count: u32) -> Vec<(T::AccountId, ValidatorId)> {
+		(0..validator_count)
+			.map(|i| {
+				let public = ValidatorId::generate_pair(None);
+
+				// The account Id is not actually used anywhere, just necessary to fulfill the
+				// expected type of the `validators` param of `test_trigger_on_new_session`.
+				let account: T::AccountId = account("validator", i, i);
+				(account, public)
+			})
+			.collect()
+	}
+
+	fn signing_context(&self) -> SigningContext<T::Hash> {
+		SigningContext {
+			parent_hash: Self::header(self.block_number.clone()).hash(),
+			session_index: self.session.clone(),
+		}
+	}
+
+	fn validator_availability_votes_yes(validators: usize) -> BitVec<bitvec::order::Lsb0, u8> {
+		// every validator confirms availability.
+		bitvec::bitvec![bitvec::order::Lsb0, u8; 1; validators as usize]
+	}
+
+	/// Setup session 1 and create `self.validators_map` and `self.validators`.
+	fn setup_session(
+		mut self,
+		target_session: SessionIndex,
+		validators: Vec<(T::AccountId, ValidatorId)>,
+		total_cores: u32,
+	) -> Self {
+		let mut block = 1;
+		for session in 0..=target_session {
+			initializer::Pallet::<T>::test_trigger_on_new_session(
+				false,
+				session,
+				validators.iter().map(|(a, v)| (a, v.clone())),
+				None,
+			);
+			block += 1;
+			Self::run_to_block(block);
+		}
+
+		let block_number = <T as frame_system::Config>::BlockNumber::from(block);
+		let header = Self::header(block_number.clone());
+
+		frame_system::Pallet::<T>::initialize(
+			&header.number(),
+			&header.hash(),
+			&Digest::<T::Hash> { logs: Vec::new() },
+			Default::default(),
+		);
+
+		assert_eq!(scheduler::ValidatorGroups::<T>::get().len(), total_cores as usize);
+		assert_eq!(<shared::Pallet<T>>::session_index(), target_session);
+
+		// We need to refetch validators since they have been shuffled.
+		let validators_shuffled: Vec<_> = session_info::Pallet::<T>::session_info(target_session)
+			.unwrap()
+			.validators
+			.clone();
+
+		self.validators = Some(validators_shuffled);
+		self.block_number = block_number;
+		self.session = target_session;
+		assert_eq!(paras::Pallet::<T>::parachains().len(), total_cores as usize);
+
+		self
+	}
+
+	fn create_availability_bitfields(
+		&self,
+		concluding_cores: &BTreeMap<u32, u32>,
+		total_cores: u32,
+	) -> Vec<UncheckedSigned<AvailabilityBitfield>> {
+		let validators =
+			self.validators.as_ref().expect("must have some validators prior to calling");
+
+		let availability_bitvec = Self::availability_bitvec(concluding_cores, total_cores);
+
+		let bitfields: Vec<UncheckedSigned<AvailabilityBitfield>> = validators
+			.iter()
+			.enumerate()
+			.map(|(i, public)| {
+				let unchecked_signed = UncheckedSigned::<AvailabilityBitfield>::benchmark_sign(
+					public,
+					availability_bitvec.clone(),
+					&self.signing_context(),
+					ValidatorIndex(i as u32),
+				);
+
+				unchecked_signed
+			})
+			.collect();
+
+		for (seed, _) in concluding_cores.iter() {
+			// make sure the candidates that are concluding by becoming available are marked as
+			// pending availability.
+			let (para_id, core_idx, group_idx) = self.create_indexes(seed.clone());
+			Self::add_availability(
+				para_id,
+				core_idx,
+				group_idx,
+				Self::validator_availability_votes_yes(validators.len()),
+				CandidateHash(H256::from(byte32_slice_from(seed.clone()))),
+			);
+		}
+
+		bitfields
+	}
+
+	/// Create backed candidates for `cores_with_backed_candidates`. You need these cores to be
+	/// scheduled _within_ paras inherent, which requires marking the available bitfields as fully
+	/// available.
+	/// - `cores_with_backed_candidates` Mapping of para_id/core_idx/group_idx seed to number of
+	/// validity votes.
+	fn create_backed_candidates(
+		&self,
+		cores_with_backed_candidates: &BTreeMap<u32, u32>,
+	) -> Vec<BackedCandidate<T::Hash>> {
+		let validators =
+			self.validators.as_ref().expect("must have some validators prior to calling");
+		let config = configuration::Pallet::<T>::config();
+
+		cores_with_backed_candidates
+			.iter()
+			.map(|(seed, num_votes)| {
+				assert!(*num_votes <= validators.len() as u32);
+				let (para_id, _core_idx, group_idx) = self.create_indexes(seed.clone());
+
+				// This generates a pair and adds it to the keystore, returning just the public.
+				let collator_public = CollatorId::generate_pair(None);
+				let header = Self::header(self.block_number.clone());
+				let relay_parent = header.hash();
+				let head_data: HeadData = Default::default();
+				let persisted_validation_data_hash = PersistedValidationData::<H256> {
+					parent_head: head_data.clone(),
+					relay_parent_number: self.relay_parent_number(),
+					relay_parent_storage_root: Default::default(),
+					max_pov_size: config.max_pov_size,
+				}
+				.hash();
+
+				let pov_hash = Default::default();
+				// NOTE: we use the default `ValidationCode` when setting it in `setup_para_ids`,
+				// so using the default again here makes sure things line up.
+				let validation_code_hash = ValidationCode::default().hash();
+				let payload = collator_signature_payload(
+					&relay_parent,
+					&para_id,
+					&persisted_validation_data_hash,
+					&pov_hash,
+					&validation_code_hash,
+				);
+				let signature = collator_public.sign(&payload).unwrap();
+
+				// Set the head data so it can be used while validating the signatures on the
+				// candidate receipt.
+				paras::Pallet::<T>::heads_insert(&para_id, head_data.clone());
+
+				let mut past_code_meta = paras::ParaPastCodeMeta::<T::BlockNumber>::default();
+				past_code_meta.note_replacement(0u32.into(), 0u32.into());
+
+				let group_validators = scheduler::Pallet::<T>::group_validators(group_idx).unwrap();
+
+				let candidate = CommittedCandidateReceipt::<T::Hash> {
+					descriptor: CandidateDescriptor::<T::Hash> {
+						para_id,
+						relay_parent,
+						collator: collator_public,
+						persisted_validation_data_hash,
+						pov_hash,
+						erasure_root: Default::default(),
+						signature,
+						para_head: head_data.hash(),
+						validation_code_hash,
+					},
+					commitments: CandidateCommitments::<u32> {
+						upward_messages: Vec::new(),
+						horizontal_messages: Vec::new(),
+						new_validation_code: None,
+						head_data,
+						processed_downward_messages: 0,
+						hrmp_watermark: self.relay_parent_number(),
+					},
+				};
+
+				let candidate_hash = candidate.hash();
+
+				let validity_votes: Vec<_> = group_validators
+					.iter()
+					.take(*num_votes as usize)
+					.map(|val_idx| {
+						let public = validators.get(val_idx.0 as usize).unwrap();
+						let sig = UncheckedSigned::<CompactStatement>::benchmark_sign(
+							public,
+							CompactStatement::Valid(candidate_hash.clone()),
+							&self.signing_context(),
+							*val_idx,
+						)
+						.benchmark_signature();
+
+						ValidityAttestation::Explicit(sig.clone())
+					})
+					.collect();
+
+				BackedCandidate::<T::Hash> {
+					candidate,
+					validity_votes,
+					validator_indices: bitvec::bitvec![bitvec::order::Lsb0, u8; 1; group_validators.len()],
+				}
+			})
+			.collect()
+	}
+
+	fn create_disputes_with_no_spam(
+		&self,
+		start: u32,
+		last: u32,
+		dispute_sessions: &[u32],
+	) -> Vec<DisputeStatementSet> {
+		let validators =
+			self.validators.as_ref().expect("must have some validators prior to calling");
+
+		(start..last)
+			.map(|seed| {
+				let session =
+					dispute_sessions.get(seed as usize).cloned().unwrap_or(self.target_session);
+
+				let (para_id, core_idx, group_idx) = self.create_indexes(seed);
+				let candidate_hash = CandidateHash(H256::from(byte32_slice_from(seed)));
+
+				Self::add_availability(
+					para_id,
+					core_idx,
+					group_idx,
+					Self::validator_availability_votes_yes(validators.len()),
+					candidate_hash,
+				);
+
+				let statements_len =
+					self.dispute_statements.get(&seed).cloned().unwrap_or(validators.len() as u32);
+				let statements = (0..statements_len)
+					.map(|validator_index| {
+						let validator_public = &validators.get(validator_index as usize).unwrap();
+
+						// We need dispute statements on each side. And we don't want a revert log
+						// so we make sure that we have a super majority with valid statements.
+						let dispute_statement = if validator_index % 4 == 0 {
+							DisputeStatement::Invalid(InvalidDisputeStatementKind::Explicit)
+						} else {
+							// Note that in the future we could use some availability votes as an
+							// implicit valid kind.
+							DisputeStatement::Valid(ValidDisputeStatementKind::Explicit)
+						};
+						let data = dispute_statement.payload_data(candidate_hash.clone(), session);
+						let statement_sig = validator_public.sign(&data).unwrap();
+
+						(dispute_statement, ValidatorIndex(validator_index), statement_sig)
+					})
+					.collect();
+
+				DisputeStatementSet { candidate_hash: candidate_hash.clone(), session, statements }
+			})
+			.collect()
+	}
+
+	/// Build a scenario for testing or benchmarks.
+	///
+	/// - `backed_and_concluding_cores`: Map from core/para id/group index seed to number of
+	/// validity votes.
+	/// - `dispute_sessions`: Session index of for each dispute. Index of slice corresponds to core.
+	/// The length of this must equal total cores used. Seed index for disputes starts at
+	/// `backed_and_concluding_cores.len()`, so `dispute_sessions` needs to be left padded by
+	/// `backed_and_concluding_cores.len()` values which effectively get ignored.
+	/// TODO we should fix this.
+	pub(crate) fn build(
+		self,
+		backed_and_concluding_cores: BTreeMap<u32, u32>,
+		dispute_sessions: &[u32],
+	) -> Bench<T> {
+		// Make sure relevant storage is cleared. This is just to get the asserts to work when
+		// running tests because it seems the storage is not cleared in between.
+		inclusion::PendingAvailabilityCommitments::<T>::remove_all(None);
+		inclusion::PendingAvailability::<T>::remove_all(None);
+
+		// We don't allow a core to have both disputes and be marked fully available at this block.
+		let cores = self.max_cores();
+		let used_cores = dispute_sessions.len() as u32;
+		assert!(used_cores <= cores);
+
+		// NOTE: there is an n+2 session delay for these actions to take effect
+		// We are currently in Session 0, so these changes will take effect in Session 2
+		Self::setup_para_ids(used_cores);
+
+		let validator_ids = Self::generate_validator_pairs(self.max_validators());
+		let target_session = SessionIndex::from(self.target_session);
+		let builder = self.setup_session(target_session, validator_ids, used_cores);
+
+		let bitfields =
+			builder.create_availability_bitfields(&backed_and_concluding_cores, used_cores);
+		let backed_candidates = builder.create_backed_candidates(&backed_and_concluding_cores);
+
+		let disputes = builder.create_disputes_with_no_spam(
+			backed_and_concluding_cores.len() as u32,
+			used_cores,
+			dispute_sessions,
+		);
+
+		assert_eq!(
+			inclusion::PendingAvailabilityCommitments::<T>::iter().count(),
+			used_cores as usize,
+		);
+		assert_eq!(inclusion::PendingAvailability::<T>::iter().count(), used_cores as usize,);
+
+		// Mark all the use cores as occupied. We expect that their are `backed_and_concluding_cores`
+		// that are pending availability and that there are `non_spam_dispute_cores` which are about
+		// to be disputed.
+		scheduler::AvailabilityCores::<T>::set(frame_benchmarking::vec![
+			Some(CoreOccupied::Parachain);
+			used_cores as usize
+		]);
+
+		Bench::<T> {
+			data: ParachainsInherentData {
+				bitfields,
+				backed_candidates,
+				disputes,
+				parent_header: Self::header(builder.block_number.clone()),
+			},
+			session: builder.target_session,
+			block_number: builder.block_number,
+		}
+	}
+}
diff --git a/runtime/parachains/src/disputes.rs b/runtime/parachains/src/disputes.rs
index 29a68451d01c..a487eaf98ee8 100644
--- a/runtime/parachains/src/disputes.rs
+++ b/runtime/parachains/src/disputes.rs
@@ -129,6 +129,9 @@ pub trait DisputesHandler<BlockNumber> {
 		included_in: BlockNumber,
 	);
 
+	/// Retrieve the included state of a given candidate in a particular session.
+	fn included_state(session: SessionIndex, candidate_hash: CandidateHash) -> Option<BlockNumber>;
+
 	/// Whether the given candidate concluded invalid in a dispute with supermajority.
 	fn concluded_invalid(session: SessionIndex, candidate_hash: CandidateHash) -> bool;
 
@@ -164,6 +167,13 @@ impl<BlockNumber> DisputesHandler<BlockNumber> for () {
 	) {
 	}
 
+	fn included_state(
+		_session: SessionIndex,
+		_candidate_hash: CandidateHash,
+	) -> Option<BlockNumber> {
+		None
+	}
+
 	fn concluded_invalid(_session: SessionIndex, _candidate_hash: CandidateHash) -> bool {
 		false
 	}
@@ -200,6 +210,13 @@ impl<T: Config> DisputesHandler<T::BlockNumber> for pallet::Pallet<T> {
 		pallet::Pallet::<T>::note_included(session, candidate_hash, included_in)
 	}
 
+	fn included_state(
+		session: SessionIndex,
+		candidate_hash: CandidateHash,
+	) -> Option<T::BlockNumber> {
+		pallet::Pallet::<T>::included_state(session, candidate_hash)
+	}
+
 	fn concluded_invalid(session: SessionIndex, candidate_hash: CandidateHash) -> bool {
 		pallet::Pallet::<T>::concluded_invalid(session, candidate_hash)
 	}
@@ -281,6 +298,7 @@ pub mod pallet {
 	///
 	/// The i'th entry of the vector corresponds to the i'th validator in the session.
 	#[pallet::storage]
+	#[pallet::getter(fn spam_slots)]
 	pub(super) type SpamSlots<T> = StorageMap<_, Twox64Concat, SessionIndex, Vec<u32>>;
 
 	/// Whether the chain is frozen. Starts as `None`. When this is `Some`,
@@ -1117,6 +1135,13 @@ impl<T: Config> Pallet<T> {
 		}
 	}
 
+	pub(crate) fn included_state(
+		session: SessionIndex,
+		candidate_hash: CandidateHash,
+	) -> Option<T::BlockNumber> {
+		<Included<T>>::get(session, candidate_hash)
+	}
+
 	pub(crate) fn concluded_invalid(session: SessionIndex, candidate_hash: CandidateHash) -> bool {
 		<Disputes<T>>::get(&session, &candidate_hash).map_or(false, |dispute| {
 			// A dispute that has concluded with supermajority-against.
diff --git a/runtime/parachains/src/inclusion.rs b/runtime/parachains/src/inclusion.rs
index 2ca0c62af075..e986c27d1ce5 100644
--- a/runtime/parachains/src/inclusion.rs
+++ b/runtime/parachains/src/inclusion.rs
@@ -58,7 +58,7 @@ pub struct AvailabilityBitfieldRecord<N> {
 
 /// A backed candidate pending availability.
 #[derive(Encode, Decode, PartialEq, TypeInfo)]
-#[cfg_attr(test, derive(Debug))]
+#[cfg_attr(test, derive(Debug, Default))]
 pub struct CandidatePendingAvailability<H, N> {
 	/// The availability core this is assigned to.
 	core: CoreIndex,
@@ -103,6 +103,29 @@ impl<H, N> CandidatePendingAvailability<H, N> {
 	pub(crate) fn candidate_descriptor(&self) -> &CandidateDescriptor<H> {
 		&self.descriptor
 	}
+
+	#[cfg(any(feature = "runtime-benchmarks", feature = "std"))]
+	pub(crate) fn new(
+		core: CoreIndex,
+		hash: CandidateHash,
+		descriptor: CandidateDescriptor<H>,
+		availability_votes: BitVec<BitOrderLsb0, u8>,
+		backers: BitVec<BitOrderLsb0, u8>,
+		relay_parent_number: N,
+		backed_in_number: N,
+		backing_group: GroupIndex,
+	) -> Self {
+		Self {
+			core,
+			hash,
+			descriptor,
+			availability_votes,
+			backers,
+			relay_parent_number,
+			backed_in_number,
+			backing_group,
+		}
+	}
 }
 
 /// A hook for applying validator rewards
@@ -272,19 +295,34 @@ impl<T: Config> Pallet<T> {
 		signed_bitfields: UncheckedSignedAvailabilityBitfields,
 		disputed_bitfield: DisputedBitfield,
 		core_lookup: impl Fn(CoreIndex) -> Option<ParaId>,
+		is_create_inherent: bool,
 	) -> Result<Vec<(CoreIndex, CandidateHash)>, DispatchError> {
 		let validators = shared::Pallet::<T>::active_validator_keys();
 		let session_index = shared::Pallet::<T>::session_index();
 		let parent_hash = frame_system::Pallet::<T>::parent_hash();
 
-		let checked_bitfields = sanitize_bitfields::<T, true>(
-			signed_bitfields,
-			disputed_bitfield,
-			expected_bits,
-			parent_hash,
-			session_index,
-			&validators[..],
-		)?;
+		let checked_bitfields = if is_create_inherent {
+			sanitize_bitfields::<T, false>(
+				signed_bitfields,
+				disputed_bitfield,
+				expected_bits,
+				parent_hash,
+				session_index,
+				&validators[..],
+			)
+			.expect(
+				"by convention, when called with `EARLY_RETURN=false`, will always return `Ok()`",
+			)
+		} else {
+			sanitize_bitfields::<T, true>(
+				signed_bitfields,
+				disputed_bitfield,
+				expected_bits,
+				parent_hash,
+				session_index,
+				&validators[..],
+			)?
+		};
 
 		let mut assigned_paras_record = (0..expected_bits)
 			.map(|bit_index| core_lookup(CoreIndex::from(bit_index as u32)))
@@ -353,18 +391,20 @@ impl<T: Config> Pallet<T> {
 					},
 				};
 
-				let receipt = CommittedCandidateReceipt {
-					descriptor: pending_availability.descriptor,
-					commitments,
-				};
-				Self::enact_candidate(
-					pending_availability.relay_parent_number,
-					receipt,
-					pending_availability.backers,
-					pending_availability.availability_votes,
-					pending_availability.core,
-					pending_availability.backing_group,
-				);
+				if !is_create_inherent {
+					let receipt = CommittedCandidateReceipt {
+						descriptor: pending_availability.descriptor,
+						commitments,
+					};
+					let _weight = Self::enact_candidate(
+						pending_availability.relay_parent_number,
+						receipt,
+						pending_availability.backers,
+						pending_availability.availability_votes,
+						pending_availability.core,
+						pending_availability.backing_group,
+					);
+				}
 
 				freed_cores.push((pending_availability.core, pending_availability.hash));
 			} else {
@@ -720,6 +760,7 @@ impl<T: Config> Pallet<T> {
 		}
 
 		// enact the messaging facet of the candidate.
+		// TODO check how to account for these
 		weight += <dmp::Pallet<T>>::prune_dmq(
 			receipt.descriptor.para_id,
 			commitments.processed_downward_messages,
@@ -1335,6 +1376,7 @@ pub(crate) mod tests {
 						vec![signed.into()],
 						DisputedBitfield::zeros(expected_bits()),
 						&core_lookup,
+						false
 					),
 					Error::<Test>::WrongBitfieldSize
 				);
@@ -1357,6 +1399,7 @@ pub(crate) mod tests {
 						vec![signed.into()],
 						DisputedBitfield::zeros(expected_bits()),
 						&core_lookup,
+						false
 					),
 					Error::<Test>::WrongBitfieldSize
 				);
@@ -1380,6 +1423,7 @@ pub(crate) mod tests {
 						vec![signed.clone(), signed],
 						DisputedBitfield::zeros(expected_bits()),
 						&core_lookup,
+						false
 					),
 					Error::<Test>::BitfieldDuplicateOrUnordered
 				);
@@ -1412,6 +1456,7 @@ pub(crate) mod tests {
 						vec![signed_1, signed_0],
 						DisputedBitfield::zeros(expected_bits()),
 						&core_lookup,
+						false
 					),
 					Error::<Test>::BitfieldDuplicateOrUnordered
 				);
@@ -1435,6 +1480,7 @@ pub(crate) mod tests {
 						vec![signed.into()],
 						DisputedBitfield::zeros(expected_bits()),
 						&core_lookup,
+						false
 					),
 					Ok(_)
 				);
@@ -1457,6 +1503,7 @@ pub(crate) mod tests {
 						vec![signed.into()],
 						DisputedBitfield::zeros(expected_bits()),
 						&core_lookup,
+						false
 					),
 					Ok(_)
 				);
@@ -1502,6 +1549,7 @@ pub(crate) mod tests {
 						vec![signed.into()],
 						DisputedBitfield::zeros(expected_bits()),
 						&core_lookup,
+						false
 					),
 					Ok(_)
 				);
@@ -1547,6 +1595,7 @@ pub(crate) mod tests {
 						vec![signed.into()],
 						DisputedBitfield::zeros(expected_bits()),
 						&core_lookup,
+						false
 					),
 					Ok(vec![])
 				);
@@ -1691,6 +1740,7 @@ pub(crate) mod tests {
 					signed_bitfields,
 					DisputedBitfield::zeros(expected_bits()),
 					&core_lookup,
+					false
 				),
 				Ok(_)
 			);
diff --git a/runtime/parachains/src/initializer.rs b/runtime/parachains/src/initializer.rs
index 95e3310e37fe..0467e10f1a4b 100644
--- a/runtime/parachains/src/initializer.rs
+++ b/runtime/parachains/src/initializer.rs
@@ -294,7 +294,7 @@ impl<T: Config> Pallet<T> {
 
 	// Allow to trigger on_new_session in tests, this is needed as long as pallet_session is not
 	// implemented in mock.
-	#[cfg(test)]
+	#[cfg(any(test, feature = "runtime-benchmarks"))]
 	pub(crate) fn test_trigger_on_new_session<'a, I: 'a>(
 		changed: bool,
 		session_index: SessionIndex,
diff --git a/runtime/parachains/src/lib.rs b/runtime/parachains/src/lib.rs
index ab48d693d601..f1d8473f8894 100644
--- a/runtime/parachains/src/lib.rs
+++ b/runtime/parachains/src/lib.rs
@@ -42,6 +42,8 @@ pub mod runtime_api_impl;
 
 mod util;
 
+#[cfg(any(feature = "runtime-benchmarks", test))]
+mod builder;
 #[cfg(test)]
 mod mock;
 
diff --git a/runtime/parachains/src/mock.rs b/runtime/parachains/src/mock.rs
index df5f4a8aa285..4d046e6eab0b 100644
--- a/runtime/parachains/src/mock.rs
+++ b/runtime/parachains/src/mock.rs
@@ -65,6 +65,7 @@ frame_support::construct_runtime!(
 		Hrmp: hrmp::{Pallet, Call, Storage, Event<T>},
 		SessionInfo: session_info::{Pallet, Storage},
 		Disputes: disputes::{Pallet, Storage, Event<T>},
+		Babe: pallet_babe::{Pallet, Call, Storage, Config, ValidateUnsigned},
 	}
 );
 
@@ -258,7 +259,9 @@ impl crate::inclusion::Config for Test {
 	type RewardValidators = TestRewardValidators;
 }
 
-impl crate::paras_inherent::Config for Test {}
+impl crate::paras_inherent::Config for Test {
+	type WeightInfo = crate::paras_inherent::TestWeightInfo;
+}
 
 impl crate::session_info::Config for Test {}
 
@@ -354,6 +357,8 @@ impl inclusion::RewardValidators for TestRewardValidators {
 
 /// Create a new set of test externalities.
 pub fn new_test_ext(state: MockGenesisConfig) -> TestExternalities {
+	use sp_keystore::{testing::KeyStore, KeystoreExt, SyncCryptoStorePtr};
+	use sp_std::sync::Arc;
 	BACKING_REWARDS.with(|r| r.borrow_mut().clear());
 	AVAILABILITY_REWARDS.with(|r| r.borrow_mut().clear());
 
@@ -361,7 +366,10 @@ pub fn new_test_ext(state: MockGenesisConfig) -> TestExternalities {
 	state.configuration.assimilate_storage(&mut t).unwrap();
 	GenesisBuild::<Test>::assimilate_storage(&state.paras, &mut t).unwrap();
 
-	t.into()
+	let mut ext: TestExternalities = t.into();
+	ext.register_extension(KeystoreExt(Arc::new(KeyStore::new()) as SyncCryptoStorePtr));
+
+	ext
 }
 
 #[derive(Default)]
diff --git a/runtime/parachains/src/paras.rs b/runtime/parachains/src/paras.rs
index 3c741b358641..df101fe0dbca 100644
--- a/runtime/parachains/src/paras.rs
+++ b/runtime/parachains/src/paras.rs
@@ -160,7 +160,7 @@ impl ParaLifecycle {
 
 impl<N: Ord + Copy + PartialEq> ParaPastCodeMeta<N> {
 	// note a replacement has occurred at a given block number.
-	fn note_replacement(&mut self, expected_at: N, activated_at: N) {
+	pub(crate) fn note_replacement(&mut self, expected_at: N, activated_at: N) {
 		self.upgrade_times.push(ReplacementTimes { expected_at, activated_at })
 	}
 
@@ -1168,6 +1168,11 @@ impl<T: Config> Pallet<T> {
 			..Default::default()
 		});
 	}
+
+	#[cfg(any(feature = "runtime-benchmarks", test))]
+	pub fn heads_insert(para_id: &ParaId, head_data: HeadData) {
+		Heads::<T>::insert(para_id, head_data);
+	}
 }
 
 #[cfg(test)]
diff --git a/runtime/parachains/src/paras_inherent.rs b/runtime/parachains/src/paras_inherent.rs
index 79a2605bd121..2db313099313 100644
--- a/runtime/parachains/src/paras_inherent.rs
+++ b/runtime/parachains/src/paras_inherent.rs
@@ -23,7 +23,7 @@
 
 use crate::{
 	disputes::DisputesHandler,
-	inclusion,
+	inclusion, initializer,
 	scheduler::{self, CoreAssignment, FreedReason},
 	shared, ump,
 };
@@ -35,74 +35,97 @@ use frame_support::{
 	traits::Randomness,
 };
 use frame_system::pallet_prelude::*;
-use pallet_babe::CurrentBlockRandomness;
+use pallet_babe::{self, CurrentBlockRandomness};
 use primitives::v1::{
-	BackedCandidate, CandidateHash, CoreIndex, InherentData as ParachainsInherentData,
-	ScrapedOnChainVotes, SessionIndex, SigningContext, UncheckedSignedAvailabilityBitfield,
+	BackedCandidate, CandidateHash, CoreIndex, DisputeStatementSet,
+	InherentData as ParachainsInherentData, MultiDisputeStatementSet, ScrapedOnChainVotes,
+	SessionIndex, SigningContext, UncheckedSignedAvailabilityBitfield,
 	UncheckedSignedAvailabilityBitfields, ValidatorId, PARACHAINS_INHERENT_IDENTIFIER,
 };
 use rand::Rng;
 use scale_info::TypeInfo;
 use sp_runtime::traits::Header as HeaderT;
 use sp_std::{
+	cmp::Ordering,
 	collections::{btree_map::BTreeMap, btree_set::BTreeSet},
 	prelude::*,
+	vec::Vec,
 };
-
-pub use pallet::*;
+#[cfg(feature = "runtime-benchmarks")]
+mod benchmarking;
 
 const LOG_TARGET: &str = "runtime::inclusion-inherent";
-// In the future, we should benchmark these consts; these are all untested assumptions for now.
-const INCLUSION_INHERENT_CLAIMED_WEIGHT: Weight = 1_000_000_000;
-// we assume that 75% of an paras inherent's weight is used processing backed candidates
-const MINIMAL_INCLUSION_INHERENT_WEIGHT: Weight = INCLUSION_INHERENT_CLAIMED_WEIGHT / 4;
 
-/// Weights
-///
-/// TODO replaced this with the true weights
-/// TODO as part of <https://github.com/paritytech/polkadot/pull/4044>
-/// TODO The current ones are _preliminary_ copies of those.
+pub trait WeightInfo {
+	/// Variant over `v`, the count of dispute statements in a dispute statement set. This gives the
+	/// weight of a single dispute statement set.
+	fn enter_variable_disputes(v: u32) -> Weight;
+	/// The weight of one bitfield.
+	fn enter_bitfields() -> Weight;
+	/// Variant over `v`, the count of validity votes for a backed candidate. This gives the weight
+	/// of a single backed candidate.
+	fn enter_backed_candidates_variable(v: u32) -> Weight;
+}
+
+pub struct TestWeightInfo;
+impl WeightInfo for TestWeightInfo {
+	fn enter_variable_disputes(v: u32) -> Weight {
+		// MAX Block Weight should fit 4 disputes
+		80_000 * v as Weight + 80_000
+	}
+	fn enter_bitfields() -> Weight {
+		// MAX Block Weight should fit 4 backed candidates
+		40_000 as Weight
+	}
+	fn enter_backed_candidates_variable(v: u32) -> Weight {
+		// MAX Block Weight should fit 4 backed candidates
+		40_000 * v as Weight + 40_000
+	}
+}
+
 fn paras_inherent_total_weight<T: Config>(
-	n_backed_candidates: u32,
-	n_bitfields: u32,
-	n_disputes: u32,
+	backed_candidates: &[BackedCandidate<<T as frame_system::Config>::Hash>],
+	bitfields: &[UncheckedSignedAvailabilityBitfield],
+	disputes: &[DisputeStatementSet],
 ) -> Weight {
-	let weights = T::DbWeight::get();
-	// static
-	(10_901_789_000 as Weight)
-		// backed candidates
-		.saturating_add((424_633_000 as Weight).saturating_mul(n_backed_candidates as Weight))
-		.saturating_add(weights.reads(58 as Weight))
-		.saturating_add(weights.reads((8 as Weight).saturating_mul(n_backed_candidates as Weight)))
-		.saturating_add(weights.writes(212 as Weight))
-		.saturating_add(weights.writes((2 as Weight).saturating_mul(n_backed_candidates as Weight)))
-		// disputes
-		.saturating_add((103_899_000 as Weight).saturating_mul(n_disputes as Weight))
-		.saturating_add(weights.reads(61 as Weight))
-		.saturating_add(weights.reads((1 as Weight).saturating_mul(n_disputes as Weight)))
-		.saturating_add(weights.writes(212 as Weight))
-		.saturating_add(weights.writes((2 as Weight).saturating_mul(n_disputes as Weight)))
-		// bitfields
-		.saturating_add((10_000_000 as Weight).saturating_mul(n_bitfields as Weight))
-		.saturating_add(weights.reads(10 as Weight))
-		.saturating_add(weights.reads((20 as Weight).saturating_mul(n_bitfields as Weight)))
-		.saturating_add(weights.writes(10 as Weight))
-		.saturating_add(weights.writes((20 as Weight).saturating_mul(n_bitfields as Weight)))
+	backed_candidates_weight::<T>(backed_candidates)
+		.saturating_add(signed_bitfields_weight::<T>(bitfields.len()))
+		.saturating_add(dispute_statements_weight::<T>(disputes))
 }
 
-/// Extract the static weight.
-fn static_weight<T: Config>() -> Weight {
-	paras_inherent_total_weight::<T>(0, 0, 0)
+fn minimal_inherent_weight<T: Config>() -> Weight {
+	// We just take the min of all our options. This can be changed in the future.
+	<T as Config>::WeightInfo::enter_bitfields()
+		.min(<T as Config>::WeightInfo::enter_variable_disputes(0))
+		.min(<T as Config>::WeightInfo::enter_backed_candidates_variable(0))
 }
 
-/// Extract the weight that is added _per_ bitfield.
-fn bitfield_weight<T: Config>() -> Weight {
-	paras_inherent_total_weight::<T>(0, 1, 0) - static_weight::<T>()
+fn dispute_statements_weight<T: Config>(disputes: &[DisputeStatementSet]) -> Weight {
+	disputes
+		.iter()
+		.map(|d| {
+			<<T as Config>::WeightInfo as WeightInfo>::enter_variable_disputes(
+				d.statements.len() as u32
+			)
+		})
+		.fold(0, |acc, x| acc.saturating_add(x))
 }
 
-/// Extract the weight that is adder _per_ backed candidate.
-fn backed_candidate_weight<T: Config>() -> Weight {
-	paras_inherent_total_weight::<T>(1, 0, 0) - static_weight::<T>()
+fn signed_bitfields_weight<T: Config>(bitfields_len: usize) -> Weight {
+	<<T as Config>::WeightInfo as WeightInfo>::enter_bitfields() * bitfields_len as Weight
+}
+
+fn backed_candidates_weight<T: frame_system::Config + Config>(
+	candidate: &[BackedCandidate<T::Hash>],
+) -> Weight {
+	candidate
+		.iter()
+		.map(|v| {
+			<<T as Config>::WeightInfo as WeightInfo>::enter_backed_candidates_variable(
+				v.validity_votes.len() as u32,
+			)
+		})
+		.sum()
 }
 
 /// A bitfield concerning concluded disputes for candidates
@@ -124,6 +147,8 @@ impl DisputedBitfield {
 	}
 }
 
+pub use pallet::*;
+
 #[frame_support::pallet]
 pub mod pallet {
 	use super::*;
@@ -134,7 +159,12 @@ pub mod pallet {
 
 	#[pallet::config]
 	#[pallet::disable_frame_system_supertrait_check]
-	pub trait Config: inclusion::Config + scheduler::Config + pallet_babe::Config {}
+	pub trait Config:
+		inclusion::Config + scheduler::Config + initializer::Config + pallet_babe::Config
+	{
+		/// Weight information for extrinsics in this pallet.
+		type WeightInfo: WeightInfo;
+	}
 
 	#[pallet::error]
 	pub enum Error<T> {
@@ -145,6 +175,8 @@ pub mod pallet {
 		InvalidParentHeader,
 		/// Disputed candidate that was concluded invalid.
 		CandidateConcludedInvalid,
+		/// The data given to the inherent will result in an overweight block.
+		InherentOverweight,
 	}
 
 	/// Whether the paras inherent was included within this block.
@@ -181,137 +213,7 @@ pub mod pallet {
 		const INHERENT_IDENTIFIER: InherentIdentifier = PARACHAINS_INHERENT_IDENTIFIER;
 
 		fn create_inherent(data: &InherentData) -> Option<Self::Call> {
-			let parent_hash = <frame_system::Pallet<T>>::parent_hash();
-
-			let ParachainsInherentData::<T::Header> {
-				bitfields,
-				backed_candidates,
-				mut disputes,
-				parent_header,
-			} = match data.get_data(&Self::INHERENT_IDENTIFIER) {
-				Ok(Some(d)) => d,
-				Ok(None) => return None,
-				Err(_) => {
-					log::warn!(target: LOG_TARGET, "ParachainsInherentData failed to decode");
-					return None
-				},
-			};
-
-			if parent_hash != parent_header.hash() {
-				log::warn!(
-					target: LOG_TARGET,
-					"ParachainsInherentData references a different parent header hash than frame"
-				);
-				return None
-			}
-
-			let current_session = <shared::Pallet<T>>::session_index();
-			let expected_bits = <scheduler::Pallet<T>>::availability_cores().len();
-			let validator_public = shared::Pallet::<T>::active_validator_keys();
-
-			T::DisputesHandler::filter_multi_dispute_data(&mut disputes);
-
-			let scheduled: Vec<CoreAssignment> = <scheduler::Pallet<T>>::scheduled();
-
-			let (freed_cores, concluded_invalid_disputes) =
-				frame_support::storage::with_transaction(|| {
-					// we don't care about fresh or not disputes
-					// this writes them to storage, so let's query it via those means
-					// if this fails for whatever reason, that's ok
-					let _ = T::DisputesHandler::provide_multi_dispute_data(disputes.clone())
-						.map_err(|e| {
-							log::warn!(
-								target: LOG_TARGET,
-								"MultiDisputesData failed to update: {:?}",
-								e
-							);
-							e
-						});
-
-					// current concluded invalid disputes, only including the current block's votes
-					let current_concluded_invalid_disputes = disputes
-						.iter()
-						.filter(|dss| dss.session == current_session)
-						.map(|dss| (dss.session, dss.candidate_hash))
-						.filter(|(session, candidate)| {
-							<T>::DisputesHandler::concluded_invalid(*session, *candidate)
-						})
-						.map(|(_session, candidate)| candidate)
-						.collect::<BTreeSet<CandidateHash>>();
-
-					let freed_cores = <inclusion::Pallet<T>>::collect_disputed(
-						&current_concluded_invalid_disputes,
-					);
-
-					// all concluded invalid disputes, that are relevant for the set of candidates
-					// the inherent provided
-					let concluded_invalid_disputes = backed_candidates
-						.iter()
-						.map(|backed_candidate| backed_candidate.hash())
-						.filter(|candidate| {
-							<T>::DisputesHandler::concluded_invalid(current_session, *candidate)
-						})
-						.collect::<BTreeSet<CandidateHash>>();
-
-					frame_support::storage::TransactionOutcome::Rollback((
-						freed_cores,
-						concluded_invalid_disputes,
-					))
-				});
-
-			let disputed_bitfield = create_disputed_bitfield(expected_bits, freed_cores.iter());
-
-			let bitfields = sanitize_bitfields::<T, false>(
-				bitfields,
-				disputed_bitfield,
-				expected_bits,
-				parent_hash,
-				current_session,
-				&validator_public[..],
-			)
-			.ok()?; // by convention, when called with `EARLY_RETURN=false`, will always return `Ok()`
-
-			let backed_candidates = sanitize_backed_candidates::<T, _, false>(
-				parent_hash,
-				backed_candidates,
-				move |candidate_hash: CandidateHash| -> bool {
-					concluded_invalid_disputes.contains(&candidate_hash)
-				},
-				&scheduled[..],
-			)
-			.ok()?; // by convention, when called with `EARLY_RETURN=false`, will always return `Ok()`
-
-			let entropy = {
-				const CANDIDATE_SEED_SUBJECT: [u8; 32] = *b"candidate-seed-selection-subject";
-				let vrf_random = CurrentBlockRandomness::<T>::random(&CANDIDATE_SEED_SUBJECT[..]).0;
-				let mut entropy: [u8; 32] = CANDIDATE_SEED_SUBJECT.clone();
-				if let Some(vrf_random) = vrf_random {
-					entropy.as_mut().copy_from_slice(vrf_random.as_ref());
-				} else {
-					// in case there is no vrf randomness present, we utilize the relay parent
-					// as seed, it's better than a static value.
-					log::warn!(
-						target: LOG_TARGET,
-						"CurrentBlockRandomness did not provide entropy"
-					);
-					entropy.as_mut().copy_from_slice(parent_hash.as_ref());
-				}
-				entropy
-			};
-
-			let remaining_weight = <T as frame_system::Config>::BlockWeights::get()
-				.max_block
-				.saturating_sub(static_weight::<T>());
-			let (_backed_candidates_weight, backed_candidates, bitfields) =
-				apply_weight_limit::<T>(backed_candidates, bitfields, entropy, remaining_weight);
-
-			let inherent_data = ParachainsInherentData::<T::Header> {
-				bitfields,
-				backed_candidates,
-				disputes,
-				parent_header,
-			};
-
+			let inherent_data = Self::create_inherent_inner(data)?;
 			// Sanity check: session changes can invalidate an inherent,
 			// and we _really_ don't want that to happen.
 			// See <https://github.com/paritytech/polkadot/issues/1327>
@@ -352,9 +254,9 @@ pub mod pallet {
 		/// Enter the paras inherent. This will process bitfields and backed candidates.
 		#[pallet::weight((
 			paras_inherent_total_weight::<T>(
-				data.backed_candidates.len() as u32,
-				data.bitfields.len() as u32,
-				data.disputes.len() as u32,
+				data.backed_candidates.as_slice(),
+				data.bitfields.as_slice(),
+				data.disputes.as_slice(),
 			),
 			DispatchClass::Mandatory,
 		))]
@@ -368,12 +270,18 @@ pub mod pallet {
 				parent_header,
 				disputes,
 			} = data;
-			let disputes_len = disputes.len() as u32;
-			let bitfields_len = signed_bitfields.len() as u32;
+			let total_weight =
+				paras_inherent_total_weight::<T>(&backed_candidates, &signed_bitfields, &disputes);
+
+			// Abort if the total weight of the block exceeds the max block weight
+			#[cfg(not(feature = "runtime-benchmarks"))]
+			ensure!(
+				total_weight <= <T as frame_system::Config>::BlockWeights::get().max_block,
+				Error::<T>::InherentOverweight
+			);
 
 			ensure_none(origin)?;
 			ensure!(!Included::<T>::exists(), Error::<T>::TooManyInclusionInherents);
-
 			// Check that the submitted parent header indeed corresponds to the previous block hash.
 			let parent_hash = <frame_system::Pallet<T>>::parent_hash();
 			ensure!(
@@ -396,7 +304,7 @@ pub mod pallet {
 				if T::DisputesHandler::is_frozen() {
 					// The relay chain we are currently on is invalid. Proceed no further on parachains.
 					Included::<T>::set(Some(()));
-					return Ok(Some(MINIMAL_INCLUSION_INHERENT_WEIGHT).into())
+					return Ok(Some(minimal_inherent_weight::<T>()).into())
 				}
 
 				let mut freed_disputed = if !new_current_dispute_sets.is_empty() {
@@ -443,6 +351,7 @@ pub mod pallet {
 				signed_bitfields,
 				disputed_bitfield,
 				<scheduler::Pallet<T>>::core_para,
+				false,
 			)?;
 
 			// Inform the disputes module of all included candidates.
@@ -467,7 +376,7 @@ pub mod pallet {
 				.collect::<BTreeMap<CoreIndex, FreedReason>>();
 
 			<scheduler::Pallet<T>>::clear();
-			<scheduler::Pallet<T>>::schedule(freed, <frame_system::Pallet<T>>::block_number());
+			<scheduler::Pallet<T>>::schedule(freed, now);
 
 			let scheduled = <scheduler::Pallet<T>>::scheduled();
 			let backed_candidates = sanitize_backed_candidates::<T, _, true>(
@@ -487,9 +396,6 @@ pub mod pallet {
 				Vec::new()
 			});
 
-			let backed_candidates = limit_backed_candidates::<T>(backed_candidates);
-			let backed_candidates_len = backed_candidates.len() as u32;
-
 			// Process backed candidates according to scheduled cores.
 			let parent_storage_root = parent_header.state_root().clone();
 			let inclusion::ProcessedCandidates::<<T::Header as HeaderT>::Hash> {
@@ -514,21 +420,217 @@ pub mod pallet {
 			<scheduler::Pallet<T>>::occupied(&occupied);
 
 			// Give some time slice to dispatch pending upward messages.
-			<ump::Pallet<T>>::process_pending_upward_messages();
+			// this is max config.ump_service_total_weight
+			let _ump_weight = <ump::Pallet<T>>::process_pending_upward_messages();
 
 			// And track that we've finished processing the inherent for this block.
 			Included::<T>::set(Some(()));
 
-			Ok(Some(paras_inherent_total_weight::<T>(
-				backed_candidates_len,
-				bitfields_len,
-				disputes_len,
-			))
-			.into())
+			Ok(Some(total_weight).into())
 		}
 	}
 }
 
+impl<T: Config> Pallet<T> {
+	/// Create the `ParachainsInherentData` that gets passed to `[`Self::enter`] in [`Self::create_inherent`].
+	/// This code is pulled out of [`Self::create_inherent`] so it can be unit tested.
+	fn create_inherent_inner(data: &InherentData) -> Option<ParachainsInherentData<T::Header>> {
+		let ParachainsInherentData::<T::Header> {
+			bitfields,
+			backed_candidates,
+			mut disputes,
+			parent_header,
+		} = match data.get_data(&Self::INHERENT_IDENTIFIER) {
+			Ok(Some(d)) => d,
+			Ok(None) => return None,
+			Err(_) => {
+				log::warn!(target: LOG_TARGET, "ParachainsInherentData failed to decode");
+				return None
+			},
+		};
+
+		let parent_hash = <frame_system::Pallet<T>>::parent_hash();
+		if parent_hash != parent_header.hash() {
+			log::warn!(
+				target: LOG_TARGET,
+				"ParachainsInherentData references a different parent header hash than frame"
+			);
+			return None
+		}
+
+		let current_session = <shared::Pallet<T>>::session_index();
+		let expected_bits = <scheduler::Pallet<T>>::availability_cores().len();
+		let validator_public = shared::Pallet::<T>::active_validator_keys();
+
+		T::DisputesHandler::filter_multi_dispute_data(&mut disputes);
+
+		let (concluded_invalid_disputes, disputed_bitfield, scheduled) =
+			frame_support::storage::with_transaction(|| {
+				// we don't care about fresh or not disputes
+				// this writes them to storage, so let's query it via those means
+				// if this fails for whatever reason, that's ok
+				let _ =
+					T::DisputesHandler::provide_multi_dispute_data(disputes.clone()).map_err(|e| {
+						log::warn!(
+							target: LOG_TARGET,
+							"MultiDisputesData failed to update: {:?}",
+							e
+						);
+						e
+					});
+
+				// concluded invalid disputes, only including the _current block's_ votes
+				let current_concluded_invalid_disputes = disputes
+					.iter()
+					.filter(|dss| dss.session == current_session)
+					.map(|dss| (dss.session, dss.candidate_hash))
+					.filter(|(session, candidate)| {
+						<T>::DisputesHandler::concluded_invalid(*session, *candidate)
+					})
+					.map(|(_session, candidate)| candidate)
+					.collect::<BTreeSet<CandidateHash>>();
+
+				// all concluded invalid disputes, that are relevant for the set of candidates
+				// the inherent provided
+				let concluded_invalid_disputes = backed_candidates
+					.iter()
+					.map(|backed_candidate| backed_candidate.hash())
+					.filter(|candidate| {
+						<T>::DisputesHandler::concluded_invalid(current_session, *candidate)
+					})
+					.collect::<BTreeSet<CandidateHash>>();
+
+				let dispute_freed_cores =
+					<inclusion::Pallet<T>>::collect_disputed(&current_concluded_invalid_disputes);
+				let disputed_bitfield =
+					create_disputed_bitfield(expected_bits, dispute_freed_cores.iter());
+
+				// Below Operations:
+				// * free disputed cores if any exist
+				// * get cores that have become free from processing fully bitfields
+				// * get cores that have become free from timing out
+				// * create  one collection of all the freed cores so we can schedule them
+				// * schedule freed cores based on collection of freed cores
+
+				{
+					let mut freed_disputed: Vec<_> = <inclusion::Pallet<T>>::collect_disputed(
+						&current_concluded_invalid_disputes,
+					)
+					.into_iter()
+					.map(|core| (core, FreedReason::Concluded))
+					.collect();
+
+					if !freed_disputed.is_empty() {
+						// There are freed disputed cores, so sort them and free them against the scheduler.
+
+						// unstable sort is fine, because core indices are unique
+						// i.e. the same candidate can't occupy 2 cores at once.
+						freed_disputed.sort_unstable_by_key(|pair| pair.0); // sort by core index
+
+						<scheduler::Pallet<T>>::free_cores(freed_disputed);
+					}
+				}
+
+				// Get cores that have become free from processing fully bitfields
+				let freed_concluded = <inclusion::Pallet<T>>::process_bitfields(
+					expected_bits,
+					bitfields.clone(),
+					disputed_bitfield.clone(),
+					<scheduler::Pallet<T>>::core_para,
+					true,
+				)
+				.unwrap_or_else(|err| {
+					log::error!(
+						target: LOG_TARGET,
+						"bitfields could not be processed while creating inherent: {:?}",
+						err,
+					);
+					Vec::new()
+				});
+
+				// Get cores that have become free from timing out
+				let availability_pred = <scheduler::Pallet<T>>::availability_timeout_predicate();
+				let freed_timeout = if let Some(pred) = availability_pred {
+					<inclusion::Pallet<T>>::collect_pending(pred)
+				} else {
+					Vec::new()
+				};
+
+				// Create one collection of cores freed from timeouts and availability conclusion ..
+				let timeout_and_concluded_freed_cores = freed_concluded
+					.into_iter()
+					.map(|(c, _hash)| (c, FreedReason::Concluded))
+					.chain(freed_timeout.into_iter().map(|c| (c, FreedReason::TimedOut)))
+					.collect::<BTreeMap<CoreIndex, FreedReason>>();
+
+				<scheduler::Pallet<T>>::clear();
+				// .. so we can schedule them.
+				<scheduler::Pallet<T>>::schedule(
+					timeout_and_concluded_freed_cores,
+					<frame_system::Pallet<T>>::block_number(),
+				);
+
+				// Return
+				frame_support::storage::TransactionOutcome::Rollback((
+					// * concluded disputes for backed candidates in this block,
+					concluded_invalid_disputes,
+					// * bitfield marking disputed cores,
+					disputed_bitfield,
+					// * and the newly created core schedule.
+					<scheduler::Pallet<T>>::scheduled(),
+				))
+			});
+
+		// TODO this is wasteful since its also called in `process_bitfield`. We should probably
+		// bubble up the bitfields from `process_bitfields` so this does not need to be called here.
+		let bitfields = sanitize_bitfields::<T, false>(
+			bitfields,
+			disputed_bitfield,
+			expected_bits,
+			parent_hash,
+			current_session,
+			&validator_public[..],
+		)
+		.ok()?; // by convention, when called with `EARLY_RETURN=false`, will always return `Ok()`
+
+		let backed_candidates = sanitize_backed_candidates::<T, _, false>(
+			parent_hash,
+			backed_candidates,
+			move |candidate_hash: CandidateHash| -> bool {
+				concluded_invalid_disputes.contains(&candidate_hash)
+			},
+			&scheduled[..],
+		)
+		.ok()?; // by convention, when called with `EARLY_RETURN=false`, will always return `Ok()`
+
+		let entropy = {
+			const CANDIDATE_SEED_SUBJECT: [u8; 32] = *b"candidate-seed-selection-subject";
+			let vrf_random = CurrentBlockRandomness::<T>::random(&CANDIDATE_SEED_SUBJECT[..]).0;
+			let mut entropy: [u8; 32] = CANDIDATE_SEED_SUBJECT.clone();
+			if let Some(vrf_random) = vrf_random {
+				entropy.as_mut().copy_from_slice(vrf_random.as_ref());
+			} else {
+				// in case there is no vrf randomness present, we utilize the relay parent
+				// as seed, it's better than a static value.
+				log::warn!(target: LOG_TARGET, "CurrentBlockRandomness did not provide entropy");
+				entropy.as_mut().copy_from_slice(parent_hash.as_ref());
+			}
+			entropy
+		};
+
+		let remaining_weight = limit_disputes::<T>(&mut disputes, entropy.clone());
+		let (_backed_candidates_weight, backed_candidates, bitfields) =
+			apply_weight_limit::<T>(backed_candidates, bitfields, entropy, remaining_weight);
+
+		Some(ParachainsInherentData::<T::Header> {
+			bitfields,
+			backed_candidates,
+			disputes,
+			parent_header,
+		})
+	}
+}
+
 /// Assures the `$condition` is `true`, and raises
 /// an error if `$action` is `true`.
 /// If `$action` is `false`, executes `$alt` if present.
@@ -560,6 +662,41 @@ pub(super) fn create_disputed_bitfield<'a, I: 'a + IntoIterator<Item = &'a CoreI
 	DisputedBitfield::from(bitvec)
 }
 
+fn random_sel<X, F: Fn(&X) -> Weight>(
+	rng: &mut rand_chacha::ChaChaRng,
+	selectables: Vec<X>,
+	weight_fn: F,
+	weight_limit: Weight,
+) -> (Weight, Vec<usize>) {
+	if selectables.is_empty() {
+		return (0 as Weight, Vec::new())
+	}
+	let mut indices = (0..selectables.len()).into_iter().collect::<Vec<_>>();
+	let mut picked_indices = Vec::with_capacity(selectables.len().saturating_sub(1));
+
+	let mut weight_acc = 0 as Weight;
+	while !indices.is_empty() {
+		// randomly pick an index
+		let pick = rng.gen_range(0..indices.len());
+		// remove the index from the available set of indices
+		let idx = indices.swap_remove(pick);
+
+		let item = &selectables[idx];
+		weight_acc += weight_fn(item);
+
+		if weight_acc > weight_limit {
+			break
+		}
+
+		picked_indices.push(idx);
+	}
+
+	// sorting indices, so the ordering is retained
+	// unstable sorting is fine, since there are no duplicates
+	picked_indices.sort_unstable();
+	(weight_acc, picked_indices)
+}
+
 /// Considers an upper threshold that the candidates must not exceed.
 ///
 /// If there is sufficient space, all bitfields and candidates will be included.
@@ -574,12 +711,10 @@ fn apply_weight_limit<T: Config + inclusion::Config>(
 	entropy: [u8; 32],
 	max_weight: Weight,
 ) -> (Weight, Vec<BackedCandidate<<T>::Hash>>, UncheckedSignedAvailabilityBitfields) {
-	let total_bitfields_weight = (bitfields.len() as Weight).saturating_mul(bitfield_weight::<T>());
-
-	let total_candidates_weight =
-		(candidates.len() as Weight).saturating_mul(backed_candidate_weight::<T>());
+	let total_candidates_weight = backed_candidates_weight::<T>(candidates.as_slice());
+	let total_bitfields_weight = signed_bitfields_weight::<T>(bitfields.len());
 
-	let total = total_bitfields_weight + total_candidates_weight;
+	let total = total_bitfields_weight.saturating_add(total_candidates_weight);
 
 	// everything fits into the block
 	if max_weight >= total {
@@ -589,54 +724,22 @@ fn apply_weight_limit<T: Config + inclusion::Config>(
 	use rand_chacha::rand_core::SeedableRng;
 	let mut rng = rand_chacha::ChaChaRng::from_seed(entropy.into());
 
-	fn random_sel<X, F: Fn(&X) -> Weight>(
-		rng: &mut rand_chacha::ChaChaRng,
-		selectables: &[X],
-		weight_fn: F,
-		weight_limit: Weight,
-	) -> (Weight, Vec<usize>) {
-		if selectables.is_empty() {
-			return (0 as Weight, Vec::new())
-		}
-		let mut indices = (0..selectables.len()).into_iter().collect::<Vec<_>>();
-		let mut picked_indices = Vec::with_capacity(selectables.len().saturating_sub(1));
-
-		let mut weight_acc = 0 as Weight;
-		while !indices.is_empty() {
-			// randomly pick an index
-			let pick = rng.gen_range(0..indices.len());
-			// remove the index from the available set of indices
-			let idx = indices.swap_remove(pick);
-
-			let item = &selectables[idx];
-			weight_acc += weight_fn(item);
-
-			if weight_acc > weight_limit {
-				break
-			}
-
-			picked_indices.push(idx);
-		}
-		// sorting indices, so the ordering is retained
-		// unstable sorting is fine, since there are no duplicates
-		picked_indices.sort_unstable();
-		(weight_acc, picked_indices)
-	}
-
 	// There is weight remaining to be consumed by a subset of candidates
 	// which are going to be picked now.
 	if let Some(remaining_weight) = max_weight.checked_sub(total_bitfields_weight) {
-		let (acc_candidate_weight, indices) = random_sel::<BackedCandidate<<T>::Hash>, _>(
+		let c = candidates.iter().map(|c| c.validity_votes.len() as u32).collect::<Vec<u32>>();
+
+		let (acc_candidate_weight, indices) = random_sel::<u32, _>(
 			&mut rng,
-			&candidates[..],
-			|_| backed_candidate_weight::<T>(),
+			c,
+			|v| <<T as Config>::WeightInfo as WeightInfo>::enter_backed_candidates_variable(*v),
 			remaining_weight,
 		);
 		let candidates =
 			indices.into_iter().map(move |idx| candidates[idx].clone()).collect::<Vec<_>>();
 		// pick all bitfields, and
 		// fill the remaining space with candidates
-		let total = acc_candidate_weight + total_bitfields_weight;
+		let total = acc_candidate_weight.saturating_add(total_bitfields_weight);
 		return (total, candidates, bitfields)
 	}
 
@@ -644,10 +747,11 @@ fn apply_weight_limit<T: Config + inclusion::Config>(
 	// into the block and skip the candidates entirely
 	let (total, indices) = random_sel::<UncheckedSignedAvailabilityBitfield, _>(
 		&mut rng,
-		&bitfields[..],
-		|_| bitfield_weight::<T>(),
+		bitfields.clone(),
+		|_| <<T as Config>::WeightInfo as WeightInfo>::enter_bitfields(),
 		max_weight,
 	);
+
 	let bitfields = indices.into_iter().map(move |idx| bitfields[idx].clone()).collect::<Vec<_>>();
 	(total, vec![], bitfields)
 }
@@ -745,7 +849,7 @@ pub(crate) fn sanitize_bitfields<T: crate::inclusion::Config, const EARLY_RETURN
 	Ok(bitfields)
 }
 
-/// Filter out any candidates, that have a concluded invalid dispute.
+/// Filter out any candidates that have a concluded invalid dispute.
 ///
 /// `scheduled` follows the same naming scheme as provided in the
 /// guide: Currently `free` but might become `occupied`.
@@ -789,111 +893,766 @@ fn sanitize_backed_candidates<
 	Ok(backed_candidates)
 }
 
-/// Limit the number of backed candidates processed in order to stay within block weight limits.
-///
-/// Use a configured assumption about the weight required to process a backed candidate and the
-/// current block weight as of the execution of this function to ensure that we don't overload
-/// the block with candidate processing.
-///
-/// If the backed candidates exceed the available block weight remaining, then skips all of them.
-/// This is somewhat less desirable than attempting to fit some of them, but is more fair in the
-/// even that we can't trust the provisioner to provide a fair / random ordering of candidates.
-fn limit_backed_candidates<T: Config>(
-	mut backed_candidates: Vec<BackedCandidate<T::Hash>>,
-) -> Vec<BackedCandidate<T::Hash>> {
-	const MAX_CODE_UPGRADES: usize = 1;
+fn limit_disputes<T: Config>(disputes: &mut MultiDisputeStatementSet, entropy: [u8; 32]) -> Weight {
+	let mut remaining_weight = <T as frame_system::Config>::BlockWeights::get().max_block;
+	let disputes_weight = dispute_statements_weight::<T>(&disputes);
+	if disputes_weight > remaining_weight {
+		// Sort the dispute statements according to the following prioritization:
+		//  1. Prioritize local disputes over remote disputes.
+		//  2. Prioritize older disputes over newer disputes.
+		disputes.sort_by(|a, b| {
+			let a_local_block = T::DisputesHandler::included_state(a.session, a.candidate_hash);
+			let b_local_block = T::DisputesHandler::included_state(b.session, b.candidate_hash);
+			match (a_local_block, b_local_block) {
+				// Prioritize local disputes over remote disputes.
+				(None, Some(_)) => Ordering::Greater,
+				(Some(_), None) => Ordering::Less,
+				// For local disputes, prioritize those that occur at an earlier height.
+				(Some(a_height), Some(b_height)) => a_height.cmp(&b_height),
+				// Prioritize earlier remote disputes using session as rough proxy.
+				(None, None) => a.session.cmp(&b.session),
+			}
+		});
 
-	// Ignore any candidates beyond one that contain code upgrades.
-	//
-	// This is an artificial limitation that does not appear in the guide as it is a practical
-	// concern around execution.
-	{
-		let mut code_upgrades = 0;
-		backed_candidates.retain(|c| {
-			if c.candidate.commitments.new_validation_code.is_some() {
-				if code_upgrades >= MAX_CODE_UPGRADES {
-					return false
+		// Since the disputes array is sorted, we may use binary search to find the beginning of
+		// remote disputes
+		let idx = disputes
+			.binary_search_by(|probe| {
+				if T::DisputesHandler::included_state(probe.session, probe.candidate_hash).is_some()
+				{
+					Ordering::Greater
+				} else {
+					Ordering::Less
 				}
-
-				code_upgrades += 1;
+			})
+			// The above predicate will never find an item and therefore we are guaranteed to obtain
+			// an error, which we can safely unwrap. QED.
+			.unwrap_err();
+
+		// Due to the binary search predicate above, the index computed will constitute the beginning
+		// of the remote disputes sub-array
+		let remote_disputes = disputes.split_off(idx);
+
+		// Select disputes in-order until the remaining weight is attained
+		disputes.retain(|d| {
+			let dispute_weight = <<T as Config>::WeightInfo as WeightInfo>::enter_variable_disputes(
+				d.statements.len() as u32,
+			);
+			if remaining_weight > dispute_weight {
+				remaining_weight -= dispute_weight;
+				true
+			} else {
+				false
 			}
-
-			true
 		});
-	}
 
-	// the weight of the paras inherent is already included in the current block weight,
-	// so our operation is simple: if the block is currently overloaded, make this intrinsic smaller
-	if frame_system::Pallet::<T>::block_weight().total() >
-		<T as frame_system::Config>::BlockWeights::get().max_block
-	{
-		Vec::new()
-	} else {
-		backed_candidates
+		// Compute the statements length of all remote disputes
+		let d = remote_disputes.iter().map(|d| d.statements.len() as u32).collect::<Vec<u32>>();
+
+		use rand_chacha::rand_core::SeedableRng;
+		let mut rng = rand_chacha::ChaChaRng::from_seed(entropy.into());
+
+		// Select remote disputes at random until the block is full
+		let (acc_remote_disputes_weight, indices) = random_sel::<u32, _>(
+			&mut rng,
+			d,
+			|v| <<T as Config>::WeightInfo as WeightInfo>::enter_variable_disputes(*v),
+			remaining_weight,
+		);
+
+		// Collect all remote disputes
+		let mut remote_disputes =
+			indices.into_iter().map(|idx| disputes[idx].clone()).collect::<Vec<_>>();
+
+		// Construct the full list of selected disputes
+		disputes.append(&mut remote_disputes);
+
+		// Update the remaining weight
+		remaining_weight = remaining_weight.saturating_sub(acc_remote_disputes_weight);
 	}
+
+	remaining_weight
 }
 
 #[cfg(test)]
 mod tests {
 	use super::*;
 
-	use crate::mock::{new_test_ext, MockGenesisConfig, System, Test};
+	use crate::{
+		builder::{Bench, BenchBuilder},
+		mock::{new_test_ext, MockGenesisConfig, Test},
+	};
+	use frame_support::{assert_err, assert_ok};
+	use sp_std::collections::btree_map::BTreeMap;
 
 	mod limit_backed_candidates {
 		use super::*;
 
+		struct TestConfig {
+			dispute_statements: BTreeMap<u32, u32>,
+			dispute_sessions: Vec<u32>,
+			backed_and_concluding: BTreeMap<u32, u32>,
+			num_validators_per_core: u32,
+			includes_code_upgrade: bool,
+		}
+
+		fn make_inherent_data(
+			TestConfig {
+				dispute_statements,
+				dispute_sessions,
+				backed_and_concluding,
+				num_validators_per_core,
+				includes_code_upgrade,
+			}: TestConfig,
+		) -> Bench<Test> {
+			BenchBuilder::<Test>::new()
+				.set_max_validators((dispute_sessions.len() as u32) * num_validators_per_core)
+				.set_max_validators_per_core(num_validators_per_core)
+				.set_dispute_statements(dispute_statements)
+				.build(backed_and_concluding, dispute_sessions.as_slice())
+		}
+
 		#[test]
-		fn does_not_truncate_on_empty_block() {
+        // Validate that if we create 2 backed candidates which are assigned to 2 cores that will be freed via
+        // becoming fully available, the backed candidates will not be filtered out in `create_inherent` and
+        // will not cause `enter` to exit early.
+		fn include_backed_candidates() {
 			new_test_ext(MockGenesisConfig::default()).execute_with(|| {
-				let backed_candidates = vec![BackedCandidate::default()];
-				System::set_block_consumed_resources(0, 0);
-				assert_eq!(limit_backed_candidates::<Test>(backed_candidates).len(), 1);
+				let dispute_statements = BTreeMap::new();
+
+				let mut backed_and_concluding = BTreeMap::new();
+				backed_and_concluding.insert(0, 1);
+				backed_and_concluding.insert(1, 1);
+
+				let scenario = make_inherent_data(TestConfig {
+					dispute_statements,
+					dispute_sessions: vec![0, 0],
+					backed_and_concluding,
+					num_validators_per_core: 1,
+					includes_code_upgrade: false,
+				});
+
+				// We expect the scenario to have cores 0 & 1 with pending availability. The backed
+				// candidates are also created for cores 0 & 1, so once the pending available
+				// become fully available those cores are marked as free and scheduled for the backed
+				// candidates.
+				let expected_para_inherent_data = scenario.data.clone();
+
+				// Check the para inherent data is as expected:
+				// * 1 bitfield per validator (2 validators)
+				assert_eq!(expected_para_inherent_data.bitfields.len(), 2);
+				// * 1 backed candidate per core (2 cores)
+				assert_eq!(expected_para_inherent_data.backed_candidates.len(), 2);
+				// * 0 disputes.
+				assert_eq!(expected_para_inherent_data.disputes.len(), 0);
+				let mut inherent_data = InherentData::new();
+				inherent_data
+					.put_data(PARACHAINS_INHERENT_IDENTIFIER, &expected_para_inherent_data)
+					.unwrap();
+
+				// The current schedule is empty prior to calling `create_inherent_enter`.
+				assert_eq!(<scheduler::Pallet<Test>>::scheduled(), vec![]);
+
+				// Nothing is filtered out (including the backed candidates.)
+				assert_eq!(
+					Pallet::<Test>::create_inherent_inner(&inherent_data.clone()).unwrap(),
+					expected_para_inherent_data
+				);
+
+				// The schedule is still empty prior to calling `enter`. (`create_inherent_inner` should not
+				// alter storage, but just double checking for sanity).
+				assert_eq!(<scheduler::Pallet<Test>>::scheduled(), vec![]);
+
+				assert_eq!(Pallet::<Test>::on_chain_votes(), None);
+				// Call enter with our 2 backed candidates
+				assert_ok!(Pallet::<Test>::enter(
+					frame_system::RawOrigin::None.into(),
+					expected_para_inherent_data
+				));
+				assert_eq!(
+					// The length of this vec is equal to the number of candidates, so we know our 2
+					// backed candidates did not get filtered out
+					Pallet::<Test>::on_chain_votes()
+						.unwrap()
+						.backing_validators_per_candidate
+						.len(),
+					2
+				);
 			});
 		}
 
 		#[test]
-		fn does_not_truncate_on_exactly_full_block() {
+		// Ensure that disputes are filtered out if the session is in the future.
+		fn filter_multi_dispute_data() {
 			new_test_ext(MockGenesisConfig::default()).execute_with(|| {
-				let backed_candidates = vec![BackedCandidate::default()];
-				let max_block_weight =
-					<Test as frame_system::Config>::BlockWeights::get().max_block;
-				// if the consumed resources are precisely equal to the max block weight, we do not truncate.
-				System::set_block_consumed_resources(max_block_weight, 0);
-				assert_eq!(limit_backed_candidates::<Test>(backed_candidates).len(), 1);
+				// Create the inherent data for this block
+				let dispute_statements = BTreeMap::new();
+
+				let backed_and_concluding = BTreeMap::new();
+
+				let scenario = make_inherent_data(TestConfig {
+					dispute_statements,
+					dispute_sessions: vec![
+						1, 2, 3, /* Session 3 too new, will get filtered out */
+					],
+					backed_and_concluding,
+					num_validators_per_core: 5,
+					includes_code_upgrade: false,
+				});
+
+				let expected_para_inherent_data = scenario.data.clone();
+
+				// Check the para inherent data is as expected:
+				// * 1 bitfield per validator (5 validators per core, 3 disputes => 3 cores, 15 validators)
+				assert_eq!(expected_para_inherent_data.bitfields.len(), 15);
+				// * 0 backed candidate per core
+				assert_eq!(expected_para_inherent_data.backed_candidates.len(), 0);
+				// * 3 disputes.
+				assert_eq!(expected_para_inherent_data.disputes.len(), 3);
+				let mut inherent_data = InherentData::new();
+				inherent_data
+					.put_data(PARACHAINS_INHERENT_IDENTIFIER, &expected_para_inherent_data)
+					.unwrap();
+
+				// The current schedule is empty prior to calling `create_inherent_enter`.
+				assert_eq!(<scheduler::Pallet<Test>>::scheduled(), vec![]);
+
+				let multi_dispute_inherent_data =
+					Pallet::<Test>::create_inherent_inner(&inherent_data.clone()).unwrap();
+				// Dispute for session that lies too far in the future should be filtered out
+				assert!(multi_dispute_inherent_data != expected_para_inherent_data);
+
+				assert_eq!(multi_dispute_inherent_data.disputes.len(), 2);
+
+				// Assert that the first 2 disputes are included
+				assert_eq!(
+					&multi_dispute_inherent_data.disputes[..2],
+					&expected_para_inherent_data.disputes[..2],
+				);
+
+				// The schedule is still empty prior to calling `enter`. (`create_inherent_inner` should not
+				// alter storage, but just double checking for sanity).
+				assert_eq!(<scheduler::Pallet<Test>>::scheduled(), vec![]);
+
+				assert_eq!(Pallet::<Test>::on_chain_votes(), None);
+				// Call enter with our 2 disputes
+				assert_ok!(Pallet::<Test>::enter(
+					frame_system::RawOrigin::None.into(),
+					multi_dispute_inherent_data,
+				));
+
+				assert_eq!(
+					// The length of this vec is equal to the number of candidates, so we know there
+					// where no backed candidates included
+					Pallet::<Test>::on_chain_votes()
+						.unwrap()
+						.backing_validators_per_candidate
+						.len(),
+					0
+				);
 			});
 		}
 
 		#[test]
-		fn truncates_on_over_full_block() {
+		// Ensure that when dispute data establishes an over weight block that we adequately
+		// filter out disputes according to our prioritization rule
+		fn limit_dispute_data() {
 			new_test_ext(MockGenesisConfig::default()).execute_with(|| {
-				let backed_candidates = vec![BackedCandidate::default()];
-				let max_block_weight =
-					<Test as frame_system::Config>::BlockWeights::get().max_block;
-				// if the consumed resources are precisely equal to the max block weight, we do not truncate.
-				System::set_block_consumed_resources(max_block_weight + 1, 0);
-				assert_eq!(limit_backed_candidates::<Test>(backed_candidates).len(), 0);
+				// Create the inherent data for this block
+				let dispute_statements = BTreeMap::new();
+				// No backed and concluding cores, so all cores will be fileld with disputesw
+				let backed_and_concluding = BTreeMap::new();
+
+				let scenario = make_inherent_data(TestConfig {
+					dispute_statements,
+					dispute_sessions: vec![2, 2, 1], // 3 cores, all disputes
+					backed_and_concluding,
+					num_validators_per_core: 6,
+					includes_code_upgrade: false,
+				});
+
+				let expected_para_inherent_data = scenario.data.clone();
+
+				// Check the para inherent data is as expected:
+				// * 1 bitfield per validator (6 validators per core, 3 disputes => 18 validators)
+				assert_eq!(expected_para_inherent_data.bitfields.len(), 18);
+				// * 0 backed candidate per core
+				assert_eq!(expected_para_inherent_data.backed_candidates.len(), 0);
+				// * 3 disputes.
+				assert_eq!(expected_para_inherent_data.disputes.len(), 3);
+				let mut inherent_data = InherentData::new();
+				inherent_data
+					.put_data(PARACHAINS_INHERENT_IDENTIFIER, &expected_para_inherent_data)
+					.unwrap();
+
+				// The current schedule is empty prior to calling `create_inherent_enter`.
+				assert_eq!(<scheduler::Pallet<Test>>::scheduled(), vec![]);
+
+				let limit_inherent_data =
+					Pallet::<Test>::create_inherent_inner(&inherent_data.clone()).unwrap();
+				// Expect that inherent data is filtered to include only 2 disputes
+				assert!(limit_inherent_data != expected_para_inherent_data);
+
+				// Ensure that the included disputes are sorted by session
+				assert_eq!(limit_inherent_data.disputes.len(), 2);
+				assert_eq!(limit_inherent_data.disputes[0].session, 1);
+				assert_eq!(limit_inherent_data.disputes[1].session, 2);
+
+				// The schedule is still empty prior to calling `enter`. (`create_inherent_inner` should not
+				// alter storage, but just double checking for sanity).
+				assert_eq!(<scheduler::Pallet<Test>>::scheduled(), vec![]);
+
+				assert_eq!(Pallet::<Test>::on_chain_votes(), None);
+				// Call enter with our 2 disputes
+				assert_ok!(Pallet::<Test>::enter(
+					frame_system::RawOrigin::None.into(),
+					limit_inherent_data,
+				));
+
+				assert_eq!(
+					// Ensure that our inherent data did not included backed candidates as expected
+					Pallet::<Test>::on_chain_votes()
+						.unwrap()
+						.backing_validators_per_candidate
+						.len(),
+					0
+				);
 			});
 		}
 
 		#[test]
-		fn all_backed_candidates_get_truncated() {
+		// Ensure that when dispute data establishes an over weight block that we abort
+		// due to an over weight block
+		fn limit_dispute_data_failure() {
 			new_test_ext(MockGenesisConfig::default()).execute_with(|| {
-				let backed_candidates = vec![BackedCandidate::default(); 10];
-				let max_block_weight =
-					<Test as frame_system::Config>::BlockWeights::get().max_block;
-				// if the consumed resources are precisely equal to the max block weight, we do not truncate.
-				System::set_block_consumed_resources(max_block_weight + 1, 0);
-				assert_eq!(limit_backed_candidates::<Test>(backed_candidates).len(), 0);
+				// Create the inherent data for this block
+				let dispute_statements = BTreeMap::new();
+				// No backed and concluding cores, so all cores will be fileld with disputesw
+				let backed_and_concluding = BTreeMap::new();
+
+				let scenario = make_inherent_data(TestConfig {
+					dispute_statements,
+					dispute_sessions: vec![2, 2, 1], // 3 cores, all disputes
+					backed_and_concluding,
+					num_validators_per_core: 6,
+					includes_code_upgrade: false,
+				});
+
+				let expected_para_inherent_data = scenario.data.clone();
+
+				// Check the para inherent data is as expected:
+				// * 1 bitfield per validator (6 validators per core, 3 disputes => 18 validators)
+				assert_eq!(expected_para_inherent_data.bitfields.len(), 18);
+				// * 0 backed candidate per core
+				assert_eq!(expected_para_inherent_data.backed_candidates.len(), 0);
+				// * 3 disputes.
+				assert_eq!(expected_para_inherent_data.disputes.len(), 3);
+				let mut inherent_data = InherentData::new();
+				inherent_data
+					.put_data(PARACHAINS_INHERENT_IDENTIFIER, &expected_para_inherent_data)
+					.unwrap();
+
+				// The current schedule is empty prior to calling `create_inherent_enter`.
+				assert_eq!(<scheduler::Pallet<Test>>::scheduled(), vec![]);
+
+				// Ensure that calling enter with 3 disputes would cause an over weight block
+				assert_err!(
+					Pallet::<Test>::enter(
+						frame_system::RawOrigin::None.into(),
+						expected_para_inherent_data,
+					),
+					DispatchError::from(Error::<Test>::InherentOverweight),
+				);
 			});
 		}
 
 		#[test]
-		fn ignores_subsequent_code_upgrades() {
+		// Ensure that when a block is over weight due to disputes, but there is still sufficient
+		// block weight to include a number of signed bitfields, the inherent data is filtered
+		// as expected
+		fn limit_dispute_data_ignore_backed_candidates() {
 			new_test_ext(MockGenesisConfig::default()).execute_with(|| {
-				let mut backed = BackedCandidate::default();
-				backed.candidate.commitments.new_validation_code = Some(Vec::new().into());
-				let backed_candidates = (0..3).map(|_| backed.clone()).collect();
-				assert_eq!(limit_backed_candidates::<Test>(backed_candidates).len(), 1);
+				// Create the inherent data for this block
+				let dispute_statements = BTreeMap::new();
+
+				let mut backed_and_concluding = BTreeMap::new();
+				// 2 backed candidates shall be scheduled
+				backed_and_concluding.insert(0, 2);
+				backed_and_concluding.insert(1, 2);
+
+				let scenario = make_inherent_data(TestConfig {
+					dispute_statements,
+					// 2 backed candidates + 3 disputes (at sessions 2, 1 and 1)
+					dispute_sessions: vec![0, 0, 2, 2, 1],
+					backed_and_concluding,
+					num_validators_per_core: 4,
+					includes_code_upgrade: false,
+				});
+
+				let expected_para_inherent_data = scenario.data.clone();
+
+				// Check the para inherent data is as expected:
+				// * 1 bitfield per validator (4 validators per core, 2 backed candidates, 3 disputes => 4*5 = 20)
+				assert_eq!(expected_para_inherent_data.bitfields.len(), 20);
+				// * 2 backed candidates
+				assert_eq!(expected_para_inherent_data.backed_candidates.len(), 2);
+				// * 3 disputes.
+				assert_eq!(expected_para_inherent_data.disputes.len(), 3);
+				let mut inherent_data = InherentData::new();
+				inherent_data
+					.put_data(PARACHAINS_INHERENT_IDENTIFIER, &expected_para_inherent_data)
+					.unwrap();
+
+				// The current schedule is empty prior to calling `create_inherent_enter`.
+				assert_eq!(<scheduler::Pallet<Test>>::scheduled(), vec![]);
+
+				// Nothing is filtered out (including the backed candidates.)
+				let limit_inherent_data =
+					Pallet::<Test>::create_inherent_inner(&inherent_data.clone()).unwrap();
+				assert!(limit_inherent_data != expected_para_inherent_data);
+
+				// Three disputes is over weight (see previous test), so we expect to only see 2 disputes
+				assert_eq!(limit_inherent_data.disputes.len(), 2);
+				// Ensure disputes are filtered as expected
+				assert_eq!(limit_inherent_data.disputes[0].session, 1);
+				assert_eq!(limit_inherent_data.disputes[1].session, 2);
+				// Ensure all bitfields are included as these are still not over weight
+				assert_eq!(
+					limit_inherent_data.bitfields.len(),
+					expected_para_inherent_data.bitfields.len()
+				);
+				// Ensure that all backed candidates are filtered out as either would make the block over weight
+				assert_eq!(limit_inherent_data.backed_candidates.len(), 0);
+
+				// The schedule is still empty prior to calling `enter`. (`create_inherent_inner` should not
+				// alter storage, but just double checking for sanity).
+				assert_eq!(<scheduler::Pallet<Test>>::scheduled(), vec![]);
+
+				assert_eq!(Pallet::<Test>::on_chain_votes(), None);
+				// Call enter with our 2 disputes
+				assert_ok!(Pallet::<Test>::enter(
+					frame_system::RawOrigin::None.into(),
+					limit_inherent_data,
+				));
+
+				assert_eq!(
+					// The length of this vec is equal to the number of candidates, so we know
+					// all of our candidates got filtered out
+					Pallet::<Test>::on_chain_votes()
+						.unwrap()
+						.backing_validators_per_candidate
+						.len(),
+					0,
+				);
+			});
+		}
+
+		#[test]
+		// Ensure that we abort if we encounter an over weight block for disputes + bitfields
+		fn limit_dispute_data_ignore_backed_candidates_failure() {
+			new_test_ext(MockGenesisConfig::default()).execute_with(|| {
+				// Create the inherent data for this block
+				let dispute_statements = BTreeMap::new();
+
+				let mut backed_and_concluding = BTreeMap::new();
+				// 2 backed candidates shall be scheduled
+				backed_and_concluding.insert(0, 2);
+				backed_and_concluding.insert(1, 2);
+
+				let scenario = make_inherent_data(TestConfig {
+					dispute_statements,
+					// 2 backed candidates + 3 disputes (at sessions 2, 1 and 1)
+					dispute_sessions: vec![0, 0, 2, 2, 1],
+					backed_and_concluding,
+					num_validators_per_core: 4,
+					includes_code_upgrade: false,
+				});
+
+				let expected_para_inherent_data = scenario.data.clone();
+
+				// Check the para inherent data is as expected:
+				// * 1 bitfield per validator (4 validators per core, 2 backed candidates, 3 disputes => 4*5 = 20)
+				assert_eq!(expected_para_inherent_data.bitfields.len(), 20);
+				// * 2 backed candidates
+				assert_eq!(expected_para_inherent_data.backed_candidates.len(), 2);
+				// * 3 disputes.
+				assert_eq!(expected_para_inherent_data.disputes.len(), 3);
+				let mut inherent_data = InherentData::new();
+				inherent_data
+					.put_data(PARACHAINS_INHERENT_IDENTIFIER, &expected_para_inherent_data)
+					.unwrap();
+
+				// The current schedule is empty prior to calling `create_inherent_enter`.
+				assert_eq!(<scheduler::Pallet<Test>>::scheduled(), vec![]);
+
+				// Ensure that calling enter with 3 disputes and 2 candidates is over weight
+				assert_err!(
+					Pallet::<Test>::enter(
+						frame_system::RawOrigin::None.into(),
+						expected_para_inherent_data,
+					),
+					DispatchError::from(Error::<Test>::InherentOverweight),
+				);
+
+				// No on chain votes are recorded because we bailed early
+				assert!(
+					Pallet::<Test>::on_chain_votes().is_none()
+				);
+			});
+		}
+
+		#[test]
+		// Ensure that when a block is over weight due to disputes and bitfields, the bitfields are
+		// filtered to accommodate the block size and no backed candidates are included.
+		fn limit_bitfields() {
+			new_test_ext(MockGenesisConfig::default()).execute_with(|| {
+				// Create the inherent data for this block
+				let mut dispute_statements = BTreeMap::new();
+				// Cap the number of statements per dispute to 20 in order to ensure we have enough
+				// space in the block for some (but not all) bitfields
+				dispute_statements.insert(2, 20);
+				dispute_statements.insert(3, 20);
+				dispute_statements.insert(4, 20);
+
+				let mut backed_and_concluding = BTreeMap::new();
+				// Schedule 2 backed candidates
+				backed_and_concluding.insert(0, 2);
+				backed_and_concluding.insert(1, 2);
+
+				let scenario = make_inherent_data(TestConfig {
+					dispute_statements,
+					// 2 backed candidates + 3 disputes (at sessions 2, 1 and 1)
+					dispute_sessions: vec![0, 0, 2, 2, 1],
+					backed_and_concluding,
+					num_validators_per_core: 5,
+					includes_code_upgrade: false,
+				});
+
+				let expected_para_inherent_data = scenario.data.clone();
+
+				// Check the para inherent data is as expected:
+				// * 1 bitfield per validator (5 validators per core, 2 backed candidates, 3 disputes => 4*5 = 20),
+				assert_eq!(expected_para_inherent_data.bitfields.len(), 25);
+				// * 2 backed candidates,
+				assert_eq!(expected_para_inherent_data.backed_candidates.len(), 2);
+				// * 3 disputes.
+				assert_eq!(expected_para_inherent_data.disputes.len(), 3);
+				let mut inherent_data = InherentData::new();
+				inherent_data
+					.put_data(PARACHAINS_INHERENT_IDENTIFIER, &expected_para_inherent_data)
+					.unwrap();
+
+				// The current schedule is empty prior to calling `create_inherent_enter`.
+				assert_eq!(<scheduler::Pallet<Test>>::scheduled(), vec![]);
+
+				// Nothing is filtered out (including the backed candidates.)
+				let limit_inherent_data =
+					Pallet::<Test>::create_inherent_inner(&inherent_data.clone()).unwrap();
+				assert!(limit_inherent_data != expected_para_inherent_data);
+
+				// Three disputes is over weight (see previous test), so we expect to only see 2 disputes
+				assert_eq!(limit_inherent_data.disputes.len(), 2);
+				// Ensure disputes are filtered as expected
+				assert_eq!(limit_inherent_data.disputes[0].session, 1);
+				assert_eq!(limit_inherent_data.disputes[1].session, 2);
+				// Ensure all bitfields are included as these are still not over weight
+				assert_eq!(limit_inherent_data.bitfields.len(), 20,);
+				// Ensure that all backed candidates are filtered out as either would make the block over weight
+				assert_eq!(limit_inherent_data.backed_candidates.len(), 0);
+
+				// The schedule is still empty prior to calling `enter`. (`create_inherent_inner` should not
+				// alter storage, but just double checking for sanity).
+				assert_eq!(<scheduler::Pallet<Test>>::scheduled(), vec![]);
+
+				assert_eq!(Pallet::<Test>::on_chain_votes(), None);
+				// Call enter with our 2 disputes
+				assert_ok!(Pallet::<Test>::enter(
+					frame_system::RawOrigin::None.into(),
+					limit_inherent_data,
+				));
+
+				assert_eq!(
+					// The length of this vec is equal to the number of candidates, so we know
+					// all of our candidates got filtered out
+					Pallet::<Test>::on_chain_votes()
+						.unwrap()
+						.backing_validators_per_candidate
+						.len(),
+					0,
+				);
+			});
+		}
+
+		#[test]
+		// Ensure that when a block is over weight due to disputes and bitfields, we abort
+		fn limit_bitfields_failure() {
+			new_test_ext(MockGenesisConfig::default()).execute_with(|| {
+				// Create the inherent data for this block
+				let mut dispute_statements = BTreeMap::new();
+				// Control the number of statements per dispute to ensure we have enough space
+				// in the block for some (but not all) bitfields
+				dispute_statements.insert(2, 20);
+				dispute_statements.insert(3, 20);
+				dispute_statements.insert(4, 20);
+
+				let mut backed_and_concluding = BTreeMap::new();
+				// 2 backed candidates shall be scheduled
+				backed_and_concluding.insert(0, 2);
+				backed_and_concluding.insert(1, 2);
+
+				let scenario = make_inherent_data(TestConfig {
+					dispute_statements,
+					// 2 backed candidates + 3 disputes (at sessions 2, 1 and 1)
+					dispute_sessions: vec![0, 0, 2, 2, 1],
+					backed_and_concluding,
+					num_validators_per_core: 5,
+					includes_code_upgrade: false,
+				});
+
+				let expected_para_inherent_data = scenario.data.clone();
+
+				// Check the para inherent data is as expected:
+				// * 1 bitfield per validator (5 validators per core, 2 backed candidates, 3 disputes => 5*5 = 25)
+				assert_eq!(expected_para_inherent_data.bitfields.len(), 25);
+				// * 2 backed candidates
+				assert_eq!(expected_para_inherent_data.backed_candidates.len(), 2);
+				// * 3 disputes.
+				assert_eq!(expected_para_inherent_data.disputes.len(), 3);
+				let mut inherent_data = InherentData::new();
+				inherent_data
+					.put_data(PARACHAINS_INHERENT_IDENTIFIER, &expected_para_inherent_data)
+					.unwrap();
+
+				// The current schedule is empty prior to calling `create_inherent_enter`.
+				assert_eq!(<scheduler::Pallet<Test>>::scheduled(), vec![]);
+
+				assert_err!(
+					Pallet::<Test>::enter(
+						frame_system::RawOrigin::None.into(),
+						expected_para_inherent_data,
+					),
+					DispatchError::from(Error::<Test>::InherentOverweight),
+				);
+
+				assert!(Pallet::<Test>::on_chain_votes().is_none());
+			});
+		}
+
+		#[test]
+		// Ensure that when a block is over weight due to disputes and bitfields, we abort
+		fn limit_candidates_over_weight() {
+			new_test_ext(MockGenesisConfig::default()).execute_with(|| {
+				// Create the inherent data for this block
+				let mut dispute_statements = BTreeMap::new();
+				// Control the number of statements per dispute to ensure we have enough space
+				// in the block for some (but not all) bitfields
+				dispute_statements.insert(2, 17);
+				dispute_statements.insert(3, 17);
+				dispute_statements.insert(4, 17);
+
+				let mut backed_and_concluding = BTreeMap::new();
+				// 2 backed candidates shall be scheduled
+				backed_and_concluding.insert(0, 16);
+				backed_and_concluding.insert(1, 25);
+
+				let scenario = make_inherent_data(TestConfig {
+					dispute_statements,
+					dispute_sessions: vec![0, 0, 2, 2, 1], // 2 backed candidates, 3 disputes at sessions 2, 1 and 1 respectively
+					backed_and_concluding,
+					num_validators_per_core: 5,
+					includes_code_upgrade: false,
+				});
+
+				let expected_para_inherent_data = scenario.data.clone();
+
+				// Check the para inherent data is as expected:
+				// * 1 bitfield per validator (5 validators per core, 2 backed candidates, 3 disputes => 5*5 = 25)
+				assert_eq!(expected_para_inherent_data.bitfields.len(), 25);
+				// * 2 backed candidates
+				assert_eq!(expected_para_inherent_data.backed_candidates.len(), 2);
+				// * 3 disputes.
+				assert_eq!(expected_para_inherent_data.disputes.len(), 3);
+				let mut inherent_data = InherentData::new();
+				inherent_data
+					.put_data(PARACHAINS_INHERENT_IDENTIFIER, &expected_para_inherent_data)
+					.unwrap();
+
+				let limit_inherent_data =
+					Pallet::<Test>::create_inherent_inner(&inherent_data.clone()).unwrap();
+				// Expect that inherent data is filtered to include only 1 backed candidate and 2 disputes
+				assert!(limit_inherent_data != expected_para_inherent_data);
+
+				// * 1 bitfields
+				assert_eq!(limit_inherent_data.bitfields.len(), 25);
+				// * 2 backed candidates
+				assert_eq!(limit_inherent_data.backed_candidates.len(), 1);
+				// * 3 disputes.
+				assert_eq!(limit_inherent_data.disputes.len(), 2);
+
+				// The current schedule is empty prior to calling `create_inherent_enter`.
+				assert_eq!(<scheduler::Pallet<Test>>::scheduled(), vec![]);
+
+				assert_ok!(Pallet::<Test>::enter(
+					frame_system::RawOrigin::None.into(),
+					limit_inherent_data,
+				),);
+
+				assert_eq!(
+					// The length of this vec is equal to the number of candidates, so we know our 2
+					// backed candidates did not get filtered out
+					Pallet::<Test>::on_chain_votes()
+						.unwrap()
+						.backing_validators_per_candidate
+						.len(),
+					1
+				);
+			});
+		}
+
+		#[test]
+		// Ensure that when a block is over weight due to disputes and bitfields, we abort
+		fn limit_candidates_over_weight_failure() {
+			new_test_ext(MockGenesisConfig::default()).execute_with(|| {
+				// Create the inherent data for this block
+				let mut dispute_statements = BTreeMap::new();
+				// Control the number of statements per dispute to ensure we have enough space
+				// in the block for some (but not all) bitfields
+				dispute_statements.insert(2, 17);
+				dispute_statements.insert(3, 17);
+				dispute_statements.insert(4, 17);
+
+				let mut backed_and_concluding = BTreeMap::new();
+				// 2 backed candidates shall be scheduled
+				backed_and_concluding.insert(0, 16);
+				backed_and_concluding.insert(1, 25);
+
+				let scenario = make_inherent_data(TestConfig {
+					dispute_statements,
+					dispute_sessions: vec![0, 0, 2, 2, 1], // 2 backed candidates, 3 disputes at sessions 2, 1 and 1 respectively
+					backed_and_concluding,
+					num_validators_per_core: 5,
+					includes_code_upgrade: false,
+				});
+
+				let expected_para_inherent_data = scenario.data.clone();
+
+				// Check the para inherent data is as expected:
+				// * 1 bitfield per validator (5 validators per core, 2 backed candidates, 3 disputes => 5*5 = 25)
+				assert_eq!(expected_para_inherent_data.bitfields.len(), 25);
+				// * 2 backed candidates
+				assert_eq!(expected_para_inherent_data.backed_candidates.len(), 2);
+				// * 3 disputes.
+				assert_eq!(expected_para_inherent_data.disputes.len(), 3);
+
+				assert_err!(
+					Pallet::<Test>::enter(
+						frame_system::RawOrigin::None.into(),
+						expected_para_inherent_data,
+					),
+					DispatchError::from(Error::<Test>::InherentOverweight),
+				);
+
+				assert!(Pallet::<Test>::on_chain_votes().is_none());
 			});
 		}
 	}
@@ -1337,210 +2096,4 @@ mod tests {
 			}
 		}
 	}
-
-	mod paras_inherent_weight {
-		use super::*;
-		use crate::inclusion::tests::*;
-
-		use primitives::v1::{GroupIndex, Hash, Id as ParaId, ValidatorIndex};
-
-		use crate::mock::{new_test_ext, MockGenesisConfig, System, Test};
-		use frame_support::traits::UnfilteredDispatchable;
-		use futures::executor::block_on;
-		use primitives::v0::PARACHAIN_KEY_TYPE_ID;
-		use sc_keystore::LocalKeystore;
-		use sp_keystore::{SyncCryptoStore, SyncCryptoStorePtr};
-		use std::sync::Arc;
-
-		/// We expect the weight of the paras inherent not to change when no truncation occurs:
-		/// its weight is dynamically computed from the size of the backed candidates list, and is
-		/// already incorporated into the current block weight when it is selected by the provisioner.
-		#[test]
-		fn weight_does_not_change_on_happy_path() {
-			let chain_a = ParaId::from(1);
-			let chain_b = ParaId::from(2);
-			let chains = vec![chain_a, chain_b];
-
-			new_test_ext(genesis_config(vec![(chain_a, true), (chain_b, true)])).execute_with(
-				|| {
-					let header = default_header();
-					System::set_block_number(1);
-					System::set_parent_hash(header.hash());
-					let session_index = SessionIndex::from(0_u32);
-					crate::shared::CurrentSessionIndex::<Test>::set(session_index);
-					let keystore: SyncCryptoStorePtr = Arc::new(LocalKeystore::in_memory());
-					let validators = vec![
-						keyring::Sr25519Keyring::Alice,
-						keyring::Sr25519Keyring::Bob,
-						keyring::Sr25519Keyring::Charlie,
-						keyring::Sr25519Keyring::Dave,
-					];
-					for validator in validators.iter() {
-						SyncCryptoStore::sr25519_generate_new(
-							&*keystore,
-							PARACHAIN_KEY_TYPE_ID,
-							Some(&validator.to_seed()),
-						)
-						.unwrap();
-					}
-					let validator_public = validator_pubkeys(&validators);
-
-					shared::Pallet::<Test>::set_active_validators_ascending(
-						validator_public.clone(),
-					);
-
-					let group_validators = |group_index: GroupIndex| {
-						match group_index {
-							group_index if group_index == GroupIndex::from(0) => vec![0, 1],
-							group_index if group_index == GroupIndex::from(1) => vec![2, 3],
-							x => panic!("Group index out of bounds for 2 parachains and 1 parathread core {}", x.0),
-						}
-						.into_iter().map(ValidatorIndex).collect::<Vec<_>>()
-					};
-
-					let signing_context =
-						SigningContext { parent_hash: System::parent_hash(), session_index };
-
-					// number of bitfields doesn't affect the paras inherent weight, so we can mock it with an empty one
-					let signed_bitfields = Vec::new();
-					// backed candidates must not be empty, so we can demonstrate that the weight has not changed
-					crate::paras::Parachains::<Test>::set(chains.clone());
-
-					crate::scheduler::ValidatorGroups::<Test>::set(vec![
-						group_validators(GroupIndex::from(0)),
-						group_validators(GroupIndex::from(1)),
-					]);
-
-					crate::scheduler::AvailabilityCores::<Test>::set(vec![None, None]);
-
-					let core_a = CoreIndex::from(0);
-					let grp_idx_a = crate::scheduler::Pallet::<Test>::group_assigned_to_core(
-						core_a,
-						System::block_number(),
-					)
-					.unwrap();
-
-					let core_b = CoreIndex::from(1);
-					let grp_idx_b = crate::scheduler::Pallet::<Test>::group_assigned_to_core(
-						core_b,
-						System::block_number(),
-					)
-					.unwrap();
-
-					let grp_indices = vec![grp_idx_a.clone(), grp_idx_b.clone()];
-
-					let backed_candidates = chains
-						.iter()
-						.cloned()
-						.enumerate()
-						.map(|(idx, para_id)| {
-							let mut candidate = TestCandidateBuilder {
-								para_id,
-								relay_parent: header.hash(),
-								pov_hash: Hash::repeat_byte(1_u8 + idx as u8),
-								persisted_validation_data_hash: make_vdata_hash(para_id).unwrap(),
-								hrmp_watermark: 0,
-								..Default::default()
-							}
-							.build();
-
-							collator_sign_candidate(keyring::Sr25519Keyring::One, &mut candidate);
-
-							let group_to_use_for_signing = grp_indices[idx].clone();
-							let backing_validators = group_validators(group_to_use_for_signing);
-							let backed_candidate = block_on(back_candidate(
-								candidate,
-								&validators,
-								backing_validators.as_slice(),
-								&keystore,
-								&signing_context,
-								BackingKind::Threshold,
-							));
-							backed_candidate
-						})
-						.collect::<Vec<_>>();
-
-					// the expected weight can always be computed by this formula
-					let expected_weight =
-						paras_inherent_total_weight::<Test>(backed_candidates.len() as u32, 0, 0);
-
-					// we've used half the block weight; there's plenty of margin
-					let max_block_weight =
-						<Test as frame_system::Config>::BlockWeights::get().max_block;
-
-					let used_block_weight = max_block_weight / 2;
-					System::set_block_consumed_resources(used_block_weight, 0);
-
-					// no need to schedule anything, `schedule(..)` is called
-					// form the `fn` under test.
-
-					// execute the paras inherent
-					let post_info = Call::<Test>::enter {
-						data: ParachainsInherentData {
-							bitfields: signed_bitfields,
-							backed_candidates,
-							disputes: Vec::new(),
-							parent_header: header,
-						},
-					}
-					.dispatch_bypass_filter(None.into())
-					.unwrap();
-
-					// we don't directly check the block's weight post-call. Instead, we check that the
-					// call has returned the appropriate post-dispatch weight for refund, and trust
-					// Substrate to do the right thing with that information.
-					//
-					// In this case, the weight system can update the actual weight with the same amount,
-					// or return `None` to indicate that the pre-computed weight should not change.
-					// Either option is acceptable for our purposes.
-					if let Some(actual_weight) = post_info.actual_weight {
-						assert_eq!(actual_weight, expected_weight);
-					}
-				},
-			);
-		}
-
-		/// We expect the weight of the paras inherent to change when truncation occurs: its
-		/// weight was initially dynamically computed from the size of the backed candidates list,
-		/// but was reduced by truncation.
-		#[test]
-		fn weight_changes_when_backed_candidates_are_truncated() {
-			new_test_ext(MockGenesisConfig::default()).execute_with(|| {
-				let header = default_header();
-				System::set_block_number(1);
-				System::set_parent_hash(header.hash());
-
-				// number of bitfields doesn't affect the paras inherent weight, so we can mock it with an empty one
-				let signed_bitfields = Vec::new();
-				// backed candidates must not be empty, so we can demonstrate that the weight has not changed
-				let backed_candidates = vec![BackedCandidate::default(); 10];
-
-				// the expected weight with no blocks is just the minimum weight
-				let expected_weight = paras_inherent_total_weight::<Test>(0, 0, 0);
-
-				// oops, looks like this mandatory call pushed the block weight over the limit
-				let max_block_weight =
-					<Test as frame_system::Config>::BlockWeights::get().max_block;
-				let used_block_weight = max_block_weight + 1;
-				System::set_block_consumed_resources(used_block_weight, 0);
-
-				// execute the paras inherent
-				let post_info = Call::<Test>::enter {
-					data: ParachainsInherentData {
-						bitfields: signed_bitfields,
-						backed_candidates,
-						disputes: Vec::new(),
-						parent_header: header,
-					},
-				}
-				.dispatch_bypass_filter(None.into())
-				.unwrap();
-
-				// we don't directly check the block's weight post-call. Instead, we check that the
-				// call has returned the appropriate post-dispatch weight for refund, and trust
-				// Substrate to do the right thing with that information.
-				assert_eq!(post_info.actual_weight, Some(expected_weight));
-			});
-		}
-	}
 }
diff --git a/runtime/parachains/src/paras_inherent/benchmarking.rs b/runtime/parachains/src/paras_inherent/benchmarking.rs
new file mode 100644
index 000000000000..1795af2f7453
--- /dev/null
+++ b/runtime/parachains/src/paras_inherent/benchmarking.rs
@@ -0,0 +1,147 @@
+// Copyright 2021 Parity Technologies (UK) Ltd.
+// This file is part of Polkadot.
+
+// Polkadot is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Polkadot is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
+
+use super::*;
+use crate::{inclusion, ParaId};
+use frame_benchmarking::{benchmarks, impl_benchmark_test_suite};
+use frame_system::RawOrigin;
+use sp_std::collections::btree_map::BTreeMap;
+
+use crate::builder::BenchBuilder;
+
+benchmarks! {
+	// Variant over `v`, the number of dispute statements in a dispute statement set. This gives the
+	// weight of a single dispute statement set.
+	enter_variable_disputes {
+		let v in 10..BenchBuilder::<T>::fallback_max_validators();
+
+		let scenario = BenchBuilder::<T>::new()
+			.build(Default::default(), &[2]);
+
+		let mut benchmark = scenario.data.clone();
+		let dispute = benchmark.disputes.pop().unwrap();
+
+		benchmark.bitfields.clear();
+		benchmark.backed_candidates.clear();
+		benchmark.disputes.clear();
+
+		benchmark.disputes.push(dispute);
+		benchmark.disputes.get_mut(0).unwrap().statements.drain(v as usize..);
+	}: enter(RawOrigin::None, benchmark)
+	verify {
+		// Assert that the block was not discarded
+		assert!(Included::<T>::get().is_some());
+		// Assert that there are on-chain votes that got scraped
+		let onchain_votes = OnChainVotes::<T>::get();
+		assert!(onchain_votes.is_some());
+		let vote = onchain_votes.unwrap();
+		// Ensure that the votes are for the correct session
+		assert_eq!(vote.session, scenario.session);
+	}
+
+	// The weight of one bitfield.
+	enter_bitfields {
+		let cores_with_backed: BTreeMap<_, _>
+			= vec![(0, BenchBuilder::<T>::fallback_max_validators())]
+				.into_iter()
+				.collect();
+
+		let scenario = BenchBuilder::<T>::new()
+			.build(cores_with_backed, &[1]);
+
+		let mut benchmark = scenario.data.clone();
+		let bitfield = benchmark.bitfields.pop().unwrap();
+
+		benchmark.bitfields.clear();
+		benchmark.backed_candidates.clear();
+		benchmark.disputes.clear();
+
+		benchmark.bitfields.push(bitfield);
+	}: enter(RawOrigin::None, benchmark)
+	verify {
+		// Assert that the block was not discarded
+		assert!(Included::<T>::get().is_some());
+		// Assert that there are on-chain votes that got scraped
+		let onchain_votes = OnChainVotes::<T>::get();
+		assert!(onchain_votes.is_some());
+		let vote = onchain_votes.unwrap();
+		// Ensure that the votes are for the correct session
+		assert_eq!(vote.session, scenario.session);
+	}
+
+	// Variant over `v`, the amount of validity votes for a backed candidate. This gives the weight
+	// of a single backed candidate.
+	enter_backed_candidates_variable {
+		// NOTE: the starting value must be over half of `max_validators` so the backed candidate is
+		// not rejected.
+		let v
+			in (BenchBuilder::<T>::fallback_min_validity_votes())
+				..BenchBuilder::<T>::fallback_max_validators();
+
+		let cores_with_backed: BTreeMap<_, _>
+			= vec![(0, v)] // The backed candidate will have `v` validity votes.
+				.into_iter()
+				.collect();
+
+		let scenario = BenchBuilder::<T>::new()
+			.build(cores_with_backed.clone(), &[1]);
+
+		let mut benchmark = scenario.data.clone();
+
+		// There is 1 backed,
+		assert_eq!(benchmark.backed_candidates.len(), 1);
+		// with `v` validity votes.
+		assert_eq!(benchmark.backed_candidates.get(0).unwrap().validity_votes.len(), v as usize);
+
+		benchmark.bitfields.clear();
+		benchmark.disputes.clear();
+	}: enter(RawOrigin::None, benchmark)
+	verify {
+		let max_validators_per_core = BenchBuilder::<T>::fallback_max_validators_per_core();
+		// Assert that the block was not discarded
+		assert!(Included::<T>::get().is_some());
+		// Assert that there are on-chain votes that got scraped
+		let onchain_votes = OnChainVotes::<T>::get();
+		assert!(onchain_votes.is_some());
+		let vote = onchain_votes.unwrap();
+		// Ensure that the votes are for the correct session
+		assert_eq!(vote.session, scenario.session);
+		// Ensure that there are an expected number of candidates
+		let header = BenchBuilder::<T>::header(scenario.block_number.clone());
+		// Traverse candidates and assert descriptors are as expected
+		for (para_id, backing_validators) in vote.backing_validators_per_candidate.iter().enumerate() {
+			let descriptor = backing_validators.0.descriptor();
+			assert_eq!(ParaId::from(para_id), descriptor.para_id);
+			assert_eq!(header.hash(), descriptor.relay_parent);
+			assert_eq!(backing_validators.1.len(), v as usize);
+		}
+
+		assert_eq!(
+			inclusion::PendingAvailabilityCommitments::<T>::iter().count(),
+			cores_with_backed.len()
+		);
+		assert_eq!(
+			inclusion::PendingAvailability::<T>::iter().count(),
+			cores_with_backed.len()
+		);
+	}
+}
+
+impl_benchmark_test_suite!(
+	Pallet,
+	crate::mock::new_test_ext(Default::default()),
+	crate::mock::Test
+);
diff --git a/runtime/polkadot/src/lib.rs b/runtime/polkadot/src/lib.rs
index 2b1c53c9a771..b235c3463633 100644
--- a/runtime/polkadot/src/lib.rs
+++ b/runtime/polkadot/src/lib.rs
@@ -1182,7 +1182,9 @@ impl parachains_hrmp::Config for Runtime {
 	type Currency = Balances;
 }
 
-impl parachains_paras_inherent::Config for Runtime {}
+impl parachains_paras_inherent::Config for Runtime {
+	type WeightInfo = weights::runtime_parachains_paras_inherent::WeightInfo<Runtime>;
+}
 
 impl parachains_scheduler::Config for Runtime {}
 
@@ -1755,6 +1757,7 @@ sp_api::impl_runtime_apis! {
 			list_benchmark!(list, extra, runtime_parachains::configuration, Configuration);
 			list_benchmark!(list, extra, runtime_parachains::initializer, Initializer);
 			list_benchmark!(list, extra, runtime_parachains::paras, Paras);
+			list_benchmark!(list, extra, runtime_parachains::paras_inherent, ParaInherent);
 			// Substrate
 			list_benchmark!(list, extra, pallet_bags_list, BagsList);
 			list_benchmark!(list, extra, pallet_balances, Balances);
@@ -1831,6 +1834,7 @@ sp_api::impl_runtime_apis! {
 			add_benchmark!(params, batches, runtime_parachains::configuration, Configuration);
 			add_benchmark!(params, batches, runtime_parachains::initializer, Initializer);
 			add_benchmark!(params, batches, runtime_parachains::paras, Paras);
+			add_benchmark!(params, batches, runtime_parachains::paras_inherent, ParaInherent);
 			// Substrate
 			add_benchmark!(params, batches, pallet_bags_list, BagsList);
 			add_benchmark!(params, batches, pallet_balances, Balances);
@@ -1880,7 +1884,6 @@ mod test_fees {
 				MaxNominatorRewardedPerValidator::get(),
 			) as f64;
 		let block_weight = BlockWeights::get().max_block as f64;
-
 		println!(
 			"a full payout takes {:.2} of the block weight [{} / {}]",
 			payout_weight / block_weight,
diff --git a/runtime/polkadot/src/weights/mod.rs b/runtime/polkadot/src/weights/mod.rs
index c913094df553..bf4d83844fd2 100644
--- a/runtime/polkadot/src/weights/mod.rs
+++ b/runtime/polkadot/src/weights/mod.rs
@@ -46,3 +46,4 @@ pub mod runtime_common_slots;
 pub mod runtime_parachains_configuration;
 pub mod runtime_parachains_initializer;
 pub mod runtime_parachains_paras;
+pub mod runtime_parachains_paras_inherent;
diff --git a/runtime/polkadot/src/weights/runtime_parachains_paras_inherent.rs b/runtime/polkadot/src/weights/runtime_parachains_paras_inherent.rs
new file mode 100644
index 000000000000..cce2e26d50c1
--- /dev/null
+++ b/runtime/polkadot/src/weights/runtime_parachains_paras_inherent.rs
@@ -0,0 +1,19 @@
+#![cfg_attr(rustfmt, rustfmt_skip)]
+#![allow(unused_parens)]
+#![allow(unused_imports)]
+
+use frame_support::{traits::Get, weights::Weight};
+use sp_std::marker::PhantomData;
+
+pub struct WeightInfo<T>(PhantomData<T>);
+impl<T: frame_system::Config> runtime_parachains::paras_inherent::WeightInfo for WeightInfo<T> {
+	fn enter_variable_disputes(_v: u32) -> Weight {
+		Weight::MAX
+	}
+	fn enter_bitfields() -> Weight {
+		Weight::MAX
+	}
+	fn enter_backed_candidates_variable(_v: u32) -> Weight {
+		Weight::MAX
+	}
+}
diff --git a/runtime/rococo/Cargo.toml b/runtime/rococo/Cargo.toml
index 388d49101c16..251bc210965d 100644
--- a/runtime/rococo/Cargo.toml
+++ b/runtime/rococo/Cargo.toml
@@ -177,7 +177,8 @@ runtime-benchmarks = [
 	"xcm-builder/runtime-benchmarks",
 	"pallet-multisig/runtime-benchmarks",
 	"frame-benchmarking",
-	"hex-literal"
+	"hex-literal",
+	"runtime-parachains/runtime-benchmarks"
 ]
 try-runtime = [
 	"frame-executive/try-runtime",
diff --git a/runtime/rococo/src/lib.rs b/runtime/rococo/src/lib.rs
index 750850ba5f3f..5d88333cc464 100644
--- a/runtime/rococo/src/lib.rs
+++ b/runtime/rococo/src/lib.rs
@@ -750,7 +750,9 @@ impl parachains_hrmp::Config for Runtime {
 	type Currency = Balances;
 }
 
-impl parachains_paras_inherent::Config for Runtime {}
+impl parachains_paras_inherent::Config for Runtime {
+	type WeightInfo = weights::runtime_parachains_paras_inherent::WeightInfo<Runtime>;
+}
 
 impl parachains_scheduler::Config for Runtime {}
 
@@ -1616,6 +1618,7 @@ sp_api::impl_runtime_apis! {
 
 			list_benchmark!(list, extra, runtime_parachains::configuration, Configuration);
 			list_benchmark!(list, extra, runtime_parachains::disputes, ParasDisputes);
+			list_benchmark!(list, extra, runtime_parachains::paras_inherent, ParaInherent);
 			list_benchmark!(list, extra, runtime_parachains::paras, Paras);
 
 			let storage_info = AllPalletsWithSystem::storage_info();
@@ -1648,6 +1651,7 @@ sp_api::impl_runtime_apis! {
 
 			add_benchmark!(params, batches, runtime_parachains::configuration, Configuration);
 			add_benchmark!(params, batches, runtime_parachains::disputes, ParasDisputes);
+			add_benchmark!(params, batches, runtime_parachains::paras_inherent, ParaInherent);
 			add_benchmark!(params, batches, runtime_parachains::paras, Paras);
 
 			if batches.is_empty() { return Err("Benchmark not found for this pallet.".into()) }
diff --git a/runtime/rococo/src/weights/mod.rs b/runtime/rococo/src/weights/mod.rs
index 0c18d1ff9201..ccd1e3b0cda4 100644
--- a/runtime/rococo/src/weights/mod.rs
+++ b/runtime/rococo/src/weights/mod.rs
@@ -18,3 +18,4 @@
 pub mod runtime_parachains_configuration;
 pub mod runtime_parachains_disputes;
 pub mod runtime_parachains_paras;
+pub mod runtime_parachains_paras_inherent;
diff --git a/runtime/rococo/src/weights/runtime_parachains_paras_inherent.rs b/runtime/rococo/src/weights/runtime_parachains_paras_inherent.rs
new file mode 100644
index 000000000000..cce2e26d50c1
--- /dev/null
+++ b/runtime/rococo/src/weights/runtime_parachains_paras_inherent.rs
@@ -0,0 +1,19 @@
+#![cfg_attr(rustfmt, rustfmt_skip)]
+#![allow(unused_parens)]
+#![allow(unused_imports)]
+
+use frame_support::{traits::Get, weights::Weight};
+use sp_std::marker::PhantomData;
+
+pub struct WeightInfo<T>(PhantomData<T>);
+impl<T: frame_system::Config> runtime_parachains::paras_inherent::WeightInfo for WeightInfo<T> {
+	fn enter_variable_disputes(_v: u32) -> Weight {
+		Weight::MAX
+	}
+	fn enter_bitfields() -> Weight {
+		Weight::MAX
+	}
+	fn enter_backed_candidates_variable(_v: u32) -> Weight {
+		Weight::MAX
+	}
+}
diff --git a/runtime/test-runtime/src/lib.rs b/runtime/test-runtime/src/lib.rs
index 773c50920c52..f92f2d476e33 100644
--- a/runtime/test-runtime/src/lib.rs
+++ b/runtime/test-runtime/src/lib.rs
@@ -476,7 +476,9 @@ impl parachains_disputes::Config for Runtime {
 	type WeightInfo = parachains_disputes::TestWeightInfo;
 }
 
-impl parachains_paras_inherent::Config for Runtime {}
+impl parachains_paras_inherent::Config for Runtime {
+	type WeightInfo = parachains_paras_inherent::TestWeightInfo;
+}
 
 impl parachains_initializer::Config for Runtime {
 	type Randomness = pallet_babe::RandomnessFromOneEpochAgo<Runtime>;
diff --git a/runtime/westend/src/lib.rs b/runtime/westend/src/lib.rs
index 8ff636abf51e..40ac1e8ad1db 100644
--- a/runtime/westend/src/lib.rs
+++ b/runtime/westend/src/lib.rs
@@ -846,7 +846,9 @@ impl parachains_hrmp::Config for Runtime {
 	type Currency = Balances;
 }
 
-impl parachains_paras_inherent::Config for Runtime {}
+impl parachains_paras_inherent::Config for Runtime {
+	type WeightInfo = weights::runtime_parachains_paras_inherent::WeightInfo<Runtime>;
+}
 
 impl parachains_scheduler::Config for Runtime {}
 
@@ -1482,6 +1484,7 @@ sp_api::impl_runtime_apis! {
 			list_benchmark!(list, extra, runtime_common::slots, Slots);
 			list_benchmark!(list, extra, runtime_parachains::configuration, Configuration);
 			list_benchmark!(list, extra, runtime_parachains::initializer, Initializer);
+			list_benchmark!(list, extra, runtime_parachains::paras_inherent, ParaInherent);
 			list_benchmark!(list, extra, runtime_parachains::paras, Paras);
 
 			// Substrate
@@ -1595,6 +1598,7 @@ sp_api::impl_runtime_apis! {
 			add_benchmark!(params, batches, runtime_parachains::configuration, Configuration);
 			add_benchmark!(params, batches, runtime_parachains::initializer, Initializer);
 			add_benchmark!(params, batches, runtime_parachains::paras, Paras);
+			add_benchmark!(params, batches, runtime_parachains::paras_inherent, ParaInherent);
 
 			// Substrate
 			add_benchmark!(params, batches, pallet_bags_list, BagsList);
diff --git a/runtime/westend/src/weights/mod.rs b/runtime/westend/src/weights/mod.rs
index 923245b2fc35..8e7c4c4e0d55 100644
--- a/runtime/westend/src/weights/mod.rs
+++ b/runtime/westend/src/weights/mod.rs
@@ -37,4 +37,5 @@ pub mod runtime_common_slots;
 pub mod runtime_parachains_configuration;
 pub mod runtime_parachains_initializer;
 pub mod runtime_parachains_paras;
+pub mod runtime_parachains_paras_inherent;
 pub mod xcm;
diff --git a/runtime/westend/src/weights/runtime_parachains_paras_inherent.rs b/runtime/westend/src/weights/runtime_parachains_paras_inherent.rs
new file mode 100644
index 000000000000..cce2e26d50c1
--- /dev/null
+++ b/runtime/westend/src/weights/runtime_parachains_paras_inherent.rs
@@ -0,0 +1,19 @@
+#![cfg_attr(rustfmt, rustfmt_skip)]
+#![allow(unused_parens)]
+#![allow(unused_imports)]
+
+use frame_support::{traits::Get, weights::Weight};
+use sp_std::marker::PhantomData;
+
+pub struct WeightInfo<T>(PhantomData<T>);
+impl<T: frame_system::Config> runtime_parachains::paras_inherent::WeightInfo for WeightInfo<T> {
+	fn enter_variable_disputes(_v: u32) -> Weight {
+		Weight::MAX
+	}
+	fn enter_bitfields() -> Weight {
+		Weight::MAX
+	}
+	fn enter_backed_candidates_variable(_v: u32) -> Weight {
+		Weight::MAX
+	}
+}

From a8b4771ba9e40a71234e6da09aa45620201ea15f Mon Sep 17 00:00:00 2001
From: Lldenaurois <Ljdenaurois@gmail.com>
Date: Sat, 6 Nov 2021 18:14:53 +0100
Subject: [PATCH 067/107] Add benchmarks for code upgrades

---
 parachain/src/primitives.rs                   | 14 ++++-
 primitives/src/v1/mod.rs                      | 12 ++--
 runtime/parachains/src/builder.rs             | 14 ++++-
 runtime/parachains/src/paras_inherent.rs      | 34 +++++------
 .../src/paras_inherent/benchmarking.rs        | 61 ++++++++++++++++++-
 5 files changed, 103 insertions(+), 32 deletions(-)

diff --git a/parachain/src/primitives.rs b/parachain/src/primitives.rs
index bda56bf59e8c..df5b6d299bc8 100644
--- a/parachain/src/primitives.rs
+++ b/parachain/src/primitives.rs
@@ -41,9 +41,19 @@ pub use polkadot_core_primitives::BlockNumber as RelayChainBlockNumber;
 
 /// Parachain head data included in the chain.
 #[derive(
-	PartialEq, Eq, Clone, PartialOrd, Ord, Encode, Decode, RuntimeDebug, derive_more::From, TypeInfo,
+	Default,
+	PartialEq,
+	Eq,
+	Clone,
+	PartialOrd,
+	Ord,
+	Encode,
+	Decode,
+	RuntimeDebug,
+	derive_more::From,
+	TypeInfo,
 )]
-#[cfg_attr(feature = "std", derive(Serialize, Deserialize, Default, Hash, MallocSizeOf))]
+#[cfg_attr(feature = "std", derive(Serialize, Deserialize, Hash, MallocSizeOf))]
 pub struct HeadData(#[cfg_attr(feature = "std", serde(with = "bytes"))] pub Vec<u8>);
 
 impl HeadData {
diff --git a/primitives/src/v1/mod.rs b/primitives/src/v1/mod.rs
index 84b6af38e2d2..f3e77d101653 100644
--- a/primitives/src/v1/mod.rs
+++ b/primitives/src/v1/mod.rs
@@ -327,8 +327,8 @@ fn check_collator_signature<H: AsRef<[u8]>>(
 }
 
 /// A unique descriptor of the candidate receipt.
-#[derive(PartialEq, Eq, Clone, Encode, Decode, TypeInfo)]
-#[cfg_attr(feature = "std", derive(Debug, Default, Hash, MallocSizeOf))]
+#[derive(Default, PartialEq, Eq, Clone, Encode, Decode, TypeInfo)]
+#[cfg_attr(feature = "std", derive(Debug, Hash, MallocSizeOf))]
 pub struct CandidateDescriptor<H = Hash> {
 	/// The ID of the para this is a candidate for.
 	pub para_id: Id,
@@ -407,8 +407,8 @@ pub struct FullCandidateReceipt<H = Hash, N = BlockNumber> {
 }
 
 /// A candidate-receipt with commitments directly included.
-#[derive(PartialEq, Eq, Clone, Encode, Decode, TypeInfo)]
-#[cfg_attr(feature = "std", derive(Debug, Default, Hash, MallocSizeOf))]
+#[derive(Default, PartialEq, Eq, Clone, Encode, Decode, TypeInfo)]
+#[cfg_attr(feature = "std", derive(Debug, Hash, MallocSizeOf))]
 pub struct CommittedCandidateReceipt<H = Hash> {
 	/// The descriptor of the candidate.
 	pub descriptor: CandidateDescriptor<H>,
@@ -509,8 +509,8 @@ impl<H: Encode, N: Encode> PersistedValidationData<H, N> {
 }
 
 /// Commitments made in a `CandidateReceipt`. Many of these are outputs of validation.
-#[derive(PartialEq, Eq, Clone, Encode, Decode, TypeInfo)]
-#[cfg_attr(feature = "std", derive(Debug, Default, Hash, MallocSizeOf))]
+#[derive(Default, PartialEq, Eq, Clone, Encode, Decode, TypeInfo)]
+#[cfg_attr(feature = "std", derive(Debug, Hash, MallocSizeOf))]
 pub struct CandidateCommitments<N = BlockNumber> {
 	/// Messages destined to be interpreted by the Relay chain itself.
 	pub upward_messages: Vec<UpwardMessage>,
diff --git a/runtime/parachains/src/builder.rs b/runtime/parachains/src/builder.rs
index dc837ea411cd..2792f3eba8a5 100644
--- a/runtime/parachains/src/builder.rs
+++ b/runtime/parachains/src/builder.rs
@@ -20,7 +20,7 @@ use sp_runtime::{
 	traits::{Header as HeaderT, One, Zero},
 	RuntimeAppPublic,
 };
-use sp_std::{collections::btree_map::BTreeMap, convert::TryInto};
+use sp_std::{collections::btree_map::BTreeMap, convert::TryInto, iter};
 
 fn byte32_slice_from(n: u32) -> [u8; 32] {
 	let mut slice = [0u8; 32];
@@ -114,6 +114,7 @@ impl<T: paras_inherent::Config> BenchBuilder<T> {
 		configuration::Pallet::<T>::config().max_validators_per_core.unwrap_or(5)
 	}
 
+	#[cfg(test)]
 	pub(crate) fn set_dispute_statements(mut self, m: BTreeMap<u32, u32>) -> Self {
 		self.dispute_statements = m;
 		self
@@ -358,6 +359,7 @@ impl<T: paras_inherent::Config> BenchBuilder<T> {
 	fn create_backed_candidates(
 		&self,
 		cores_with_backed_candidates: &BTreeMap<u32, u32>,
+		includes_code_upgrade: Option<u32>,
 	) -> Vec<BackedCandidate<T::Hash>> {
 		let validators =
 			self.validators.as_ref().expect("must have some validators prior to calling");
@@ -419,7 +421,11 @@ impl<T: paras_inherent::Config> BenchBuilder<T> {
 					commitments: CandidateCommitments::<u32> {
 						upward_messages: Vec::new(),
 						horizontal_messages: Vec::new(),
-						new_validation_code: None,
+						new_validation_code: includes_code_upgrade.map(|v| {
+							ValidationCode(
+								iter::repeat(0u8).take(2usize.pow(v)).collect::<Vec<u8>>(),
+							)
+						}),
 						head_data,
 						processed_downward_messages: 0,
 						hrmp_watermark: self.relay_parent_number(),
@@ -519,6 +525,7 @@ impl<T: paras_inherent::Config> BenchBuilder<T> {
 		self,
 		backed_and_concluding_cores: BTreeMap<u32, u32>,
 		dispute_sessions: &[u32],
+		includes_code_upgrade: Option<u32>,
 	) -> Bench<T> {
 		// Make sure relevant storage is cleared. This is just to get the asserts to work when
 		// running tests because it seems the storage is not cleared in between.
@@ -540,7 +547,8 @@ impl<T: paras_inherent::Config> BenchBuilder<T> {
 
 		let bitfields =
 			builder.create_availability_bitfields(&backed_and_concluding_cores, used_cores);
-		let backed_candidates = builder.create_backed_candidates(&backed_and_concluding_cores);
+		let backed_candidates =
+			builder.create_backed_candidates(&backed_and_concluding_cores, includes_code_upgrade);
 
 		let disputes = builder.create_disputes_with_no_spam(
 			backed_and_concluding_cores.len() as u32,
diff --git a/runtime/parachains/src/paras_inherent.rs b/runtime/parachains/src/paras_inherent.rs
index 2db313099313..74562d8ea2fc 100644
--- a/runtime/parachains/src/paras_inherent.rs
+++ b/runtime/parachains/src/paras_inherent.rs
@@ -993,7 +993,7 @@ mod tests {
 			dispute_sessions: Vec<u32>,
 			backed_and_concluding: BTreeMap<u32, u32>,
 			num_validators_per_core: u32,
-			includes_code_upgrade: bool,
+			includes_code_upgrade: Option<u32>,
 		}
 
 		fn make_inherent_data(
@@ -1009,13 +1009,13 @@ mod tests {
 				.set_max_validators((dispute_sessions.len() as u32) * num_validators_per_core)
 				.set_max_validators_per_core(num_validators_per_core)
 				.set_dispute_statements(dispute_statements)
-				.build(backed_and_concluding, dispute_sessions.as_slice())
+				.build(backed_and_concluding, dispute_sessions.as_slice(), includes_code_upgrade)
 		}
 
 		#[test]
-        // Validate that if we create 2 backed candidates which are assigned to 2 cores that will be freed via
-        // becoming fully available, the backed candidates will not be filtered out in `create_inherent` and
-        // will not cause `enter` to exit early.
+		// Validate that if we create 2 backed candidates which are assigned to 2 cores that will be freed via
+		// becoming fully available, the backed candidates will not be filtered out in `create_inherent` and
+		// will not cause `enter` to exit early.
 		fn include_backed_candidates() {
 			new_test_ext(MockGenesisConfig::default()).execute_with(|| {
 				let dispute_statements = BTreeMap::new();
@@ -1029,7 +1029,7 @@ mod tests {
 					dispute_sessions: vec![0, 0],
 					backed_and_concluding,
 					num_validators_per_core: 1,
-					includes_code_upgrade: false,
+					includes_code_upgrade: None,
 				});
 
 				// We expect the scenario to have cores 0 & 1 with pending availability. The backed
@@ -1097,7 +1097,7 @@ mod tests {
 					],
 					backed_and_concluding,
 					num_validators_per_core: 5,
-					includes_code_upgrade: false,
+					includes_code_upgrade: None,
 				});
 
 				let expected_para_inherent_data = scenario.data.clone();
@@ -1168,7 +1168,7 @@ mod tests {
 					dispute_sessions: vec![2, 2, 1], // 3 cores, all disputes
 					backed_and_concluding,
 					num_validators_per_core: 6,
-					includes_code_upgrade: false,
+					includes_code_upgrade: None,
 				});
 
 				let expected_para_inherent_data = scenario.data.clone();
@@ -1235,7 +1235,7 @@ mod tests {
 					dispute_sessions: vec![2, 2, 1], // 3 cores, all disputes
 					backed_and_concluding,
 					num_validators_per_core: 6,
-					includes_code_upgrade: false,
+					includes_code_upgrade: None,
 				});
 
 				let expected_para_inherent_data = scenario.data.clone();
@@ -1286,7 +1286,7 @@ mod tests {
 					dispute_sessions: vec![0, 0, 2, 2, 1],
 					backed_and_concluding,
 					num_validators_per_core: 4,
-					includes_code_upgrade: false,
+					includes_code_upgrade: None,
 				});
 
 				let expected_para_inherent_data = scenario.data.clone();
@@ -1365,7 +1365,7 @@ mod tests {
 					dispute_sessions: vec![0, 0, 2, 2, 1],
 					backed_and_concluding,
 					num_validators_per_core: 4,
-					includes_code_upgrade: false,
+					includes_code_upgrade: None,
 				});
 
 				let expected_para_inherent_data = scenario.data.clone();
@@ -1395,9 +1395,7 @@ mod tests {
 				);
 
 				// No on chain votes are recorded because we bailed early
-				assert!(
-					Pallet::<Test>::on_chain_votes().is_none()
-				);
+				assert!(Pallet::<Test>::on_chain_votes().is_none());
 			});
 		}
 
@@ -1425,7 +1423,7 @@ mod tests {
 					dispute_sessions: vec![0, 0, 2, 2, 1],
 					backed_and_concluding,
 					num_validators_per_core: 5,
-					includes_code_upgrade: false,
+					includes_code_upgrade: None,
 				});
 
 				let expected_para_inherent_data = scenario.data.clone();
@@ -1506,7 +1504,7 @@ mod tests {
 					dispute_sessions: vec![0, 0, 2, 2, 1],
 					backed_and_concluding,
 					num_validators_per_core: 5,
-					includes_code_upgrade: false,
+					includes_code_upgrade: None,
 				});
 
 				let expected_para_inherent_data = scenario.data.clone();
@@ -1560,7 +1558,7 @@ mod tests {
 					dispute_sessions: vec![0, 0, 2, 2, 1], // 2 backed candidates, 3 disputes at sessions 2, 1 and 1 respectively
 					backed_and_concluding,
 					num_validators_per_core: 5,
-					includes_code_upgrade: false,
+					includes_code_upgrade: None,
 				});
 
 				let expected_para_inherent_data = scenario.data.clone();
@@ -1631,7 +1629,7 @@ mod tests {
 					dispute_sessions: vec![0, 0, 2, 2, 1], // 2 backed candidates, 3 disputes at sessions 2, 1 and 1 respectively
 					backed_and_concluding,
 					num_validators_per_core: 5,
-					includes_code_upgrade: false,
+					includes_code_upgrade: None,
 				});
 
 				let expected_para_inherent_data = scenario.data.clone();
diff --git a/runtime/parachains/src/paras_inherent/benchmarking.rs b/runtime/parachains/src/paras_inherent/benchmarking.rs
index 1795af2f7453..49d2b00d98d3 100644
--- a/runtime/parachains/src/paras_inherent/benchmarking.rs
+++ b/runtime/parachains/src/paras_inherent/benchmarking.rs
@@ -29,7 +29,7 @@ benchmarks! {
 		let v in 10..BenchBuilder::<T>::fallback_max_validators();
 
 		let scenario = BenchBuilder::<T>::new()
-			.build(Default::default(), &[2]);
+			.build(Default::default(), &[2], None);
 
 		let mut benchmark = scenario.data.clone();
 		let dispute = benchmark.disputes.pop().unwrap();
@@ -60,7 +60,7 @@ benchmarks! {
 				.collect();
 
 		let scenario = BenchBuilder::<T>::new()
-			.build(cores_with_backed, &[1]);
+			.build(cores_with_backed, &[1], None);
 
 		let mut benchmark = scenario.data.clone();
 		let bitfield = benchmark.bitfields.pop().unwrap();
@@ -97,7 +97,62 @@ benchmarks! {
 				.collect();
 
 		let scenario = BenchBuilder::<T>::new()
-			.build(cores_with_backed.clone(), &[1]);
+			.build(cores_with_backed.clone(), &[1], None);
+
+		let mut benchmark = scenario.data.clone();
+
+		// There is 1 backed,
+		assert_eq!(benchmark.backed_candidates.len(), 1);
+		// with `v` validity votes.
+		assert_eq!(benchmark.backed_candidates.get(0).unwrap().validity_votes.len(), v as usize);
+
+		benchmark.bitfields.clear();
+		benchmark.disputes.clear();
+	}: enter(RawOrigin::None, benchmark)
+	verify {
+		let max_validators_per_core = BenchBuilder::<T>::fallback_max_validators_per_core();
+		// Assert that the block was not discarded
+		assert!(Included::<T>::get().is_some());
+		// Assert that there are on-chain votes that got scraped
+		let onchain_votes = OnChainVotes::<T>::get();
+		assert!(onchain_votes.is_some());
+		let vote = onchain_votes.unwrap();
+		// Ensure that the votes are for the correct session
+		assert_eq!(vote.session, scenario.session);
+		// Ensure that there are an expected number of candidates
+		let header = BenchBuilder::<T>::header(scenario.block_number.clone());
+		// Traverse candidates and assert descriptors are as expected
+		for (para_id, backing_validators) in vote.backing_validators_per_candidate.iter().enumerate() {
+			let descriptor = backing_validators.0.descriptor();
+			assert_eq!(ParaId::from(para_id), descriptor.para_id);
+			assert_eq!(header.hash(), descriptor.relay_parent);
+			assert_eq!(backing_validators.1.len(), v as usize);
+		}
+
+		assert_eq!(
+			inclusion::PendingAvailabilityCommitments::<T>::iter().count(),
+			cores_with_backed.len()
+		);
+		assert_eq!(
+			inclusion::PendingAvailability::<T>::iter().count(),
+			cores_with_backed.len()
+		);
+	}
+
+	// Variant over `v`, the amount of validity votes for a backed candidate. This gives the weight
+	// of a single backed candidate.
+	enter_backed_candidates_code_upgrades_variable {
+		// NOTE: the starting value must be over half of `max_validators` so the backed candidate is
+		// not rejected.
+		let v in 0..24;
+
+		let cores_with_backed: BTreeMap<_, _>
+			= vec![(0, BenchBuilder::<T>::fallback_min_validity_votes())] // The backed candidate will have `v` validity votes.
+				.into_iter()
+				.collect();
+
+		let scenario = BenchBuilder::<T>::new()
+			.build(cores_with_backed.clone(), &[1], Some(v));
 
 		let mut benchmark = scenario.data.clone();
 

From 70feb6e2f2f9d18079615e9f3e33f5769f43fd45 Mon Sep 17 00:00:00 2001
From: emostov <32168567+emostov@users.noreply.github.com>
Date: Sat, 6 Nov 2021 20:10:46 +0100
Subject: [PATCH 068/107] Code upgrade bench; Feature gate TestWeightInfo

---
 .../runtime_parachains_paras_inherent.rs      |   4 +
 runtime/parachains/src/builder.rs             |   9 +-
 runtime/parachains/src/paras.rs               |   2 +-
 runtime/parachains/src/paras/benchmarking.rs  |   2 +-
 runtime/parachains/src/paras_inherent.rs      |  49 ++++--
 .../src/paras_inherent/benchmarking.rs        |  34 +++--
 .../runtime_parachains_paras_inherent.rs      | 143 +++++++++++++++++-
 .../runtime_parachains_paras_inherent.rs      | 143 +++++++++++++++++-
 .../runtime_parachains_paras_inherent.rs      | 143 +++++++++++++++++-
 9 files changed, 479 insertions(+), 50 deletions(-)

diff --git a/runtime/kusama/src/weights/runtime_parachains_paras_inherent.rs b/runtime/kusama/src/weights/runtime_parachains_paras_inherent.rs
index 8a628abfe132..033af3be5067 100644
--- a/runtime/kusama/src/weights/runtime_parachains_paras_inherent.rs
+++ b/runtime/kusama/src/weights/runtime_parachains_paras_inherent.rs
@@ -145,4 +145,8 @@ impl<T: frame_system::Config> runtime_parachains::paras_inherent::WeightInfo for
 			.saturating_add(T::DbWeight::get().reads(27 as Weight))
 			.saturating_add(T::DbWeight::get().writes(14 as Weight))
 	}
+	fn enter_backed_candidate_code_upgrade() -> Weight {
+		0
+	}
+
 }
diff --git a/runtime/parachains/src/builder.rs b/runtime/parachains/src/builder.rs
index 2792f3eba8a5..3a0c13b22261 100644
--- a/runtime/parachains/src/builder.rs
+++ b/runtime/parachains/src/builder.rs
@@ -20,7 +20,7 @@ use sp_runtime::{
 	traits::{Header as HeaderT, One, Zero},
 	RuntimeAppPublic,
 };
-use sp_std::{collections::btree_map::BTreeMap, convert::TryInto, iter};
+use sp_std::{collections::btree_map::BTreeMap, convert::TryInto};
 
 fn byte32_slice_from(n: u32) -> [u8; 32] {
 	let mut slice = [0u8; 32];
@@ -421,11 +421,8 @@ impl<T: paras_inherent::Config> BenchBuilder<T> {
 					commitments: CandidateCommitments::<u32> {
 						upward_messages: Vec::new(),
 						horizontal_messages: Vec::new(),
-						new_validation_code: includes_code_upgrade.map(|v| {
-							ValidationCode(
-								iter::repeat(0u8).take(2usize.pow(v)).collect::<Vec<u8>>(),
-							)
-						}),
+						new_validation_code: includes_code_upgrade
+							.map(|v| ValidationCode(vec![0u8; v as usize])),
 						head_data,
 						processed_downward_messages: 0,
 						hrmp_watermark: self.relay_parent_number(),
diff --git a/runtime/parachains/src/paras.rs b/runtime/parachains/src/paras.rs
index df101fe0dbca..7c786360e57c 100644
--- a/runtime/parachains/src/paras.rs
+++ b/runtime/parachains/src/paras.rs
@@ -42,7 +42,7 @@ use serde::{Deserialize, Serialize};
 pub use crate::Origin as ParachainOrigin;
 
 #[cfg(feature = "runtime-benchmarks")]
-mod benchmarking;
+pub(crate) mod benchmarking;
 
 pub use pallet::*;
 
diff --git a/runtime/parachains/src/paras/benchmarking.rs b/runtime/parachains/src/paras/benchmarking.rs
index b37ee5b83618..9c3de5cf9b2c 100644
--- a/runtime/parachains/src/paras/benchmarking.rs
+++ b/runtime/parachains/src/paras/benchmarking.rs
@@ -47,7 +47,7 @@ fn generate_disordered_pruning<T: Config>() {
 	<Pallet<T> as Store>::PastCodePruning::put(needs_pruning);
 }
 
-fn generate_disordered_upgrades<T: Config>() {
+pub(crate) fn generate_disordered_upgrades<T: Config>() {
 	let mut upgrades = Vec::new();
 	let mut cooldowns = Vec::new();
 
diff --git a/runtime/parachains/src/paras_inherent.rs b/runtime/parachains/src/paras_inherent.rs
index 74562d8ea2fc..d43e3de544fc 100644
--- a/runtime/parachains/src/paras_inherent.rs
+++ b/runtime/parachains/src/paras_inherent.rs
@@ -65,9 +65,14 @@ pub trait WeightInfo {
 	/// Variant over `v`, the count of validity votes for a backed candidate. This gives the weight
 	/// of a single backed candidate.
 	fn enter_backed_candidates_variable(v: u32) -> Weight;
+	/// The weight of a single backed candidate with a code upgrade.
+	fn enter_backed_candidate_code_upgrade() -> Weight;
 }
 
 pub struct TestWeightInfo;
+// `WeightInfo` impl for unit and integration tests. Based off of the `max_block` weight for the
+//  mock.
+#[cfg(not(feature = "runtime-benchmarks"))]
 impl WeightInfo for TestWeightInfo {
 	fn enter_variable_disputes(v: u32) -> Weight {
 		// MAX Block Weight should fit 4 disputes
@@ -81,6 +86,27 @@ impl WeightInfo for TestWeightInfo {
 		// MAX Block Weight should fit 4 backed candidates
 		40_000 * v as Weight + 40_000
 	}
+	fn enter_backed_candidate_code_upgrade() -> Weight {
+		0
+	}
+}
+// To simplify benchmarks running as tests, we set all the weights to 0. `enter` will exit early
+// when if the data causes it to be over weight, but we don't want that to block a benchmark from
+// running as a test.
+#[cfg(feature = "runtime-benchmarks")]
+impl WeightInfo for TestWeightInfo {
+	fn enter_variable_disputes(_v: u32) -> Weight {
+		0
+	}
+	fn enter_bitfields() -> Weight {
+		0
+	}
+	fn enter_backed_candidates_variable(_v: u32) -> Weight {
+		0
+	}
+	fn enter_backed_candidate_code_upgrade() -> Weight {
+		0
+	}
 }
 
 fn paras_inherent_total_weight<T: Config>(
@@ -238,7 +264,7 @@ pub mod pallet {
 							disputes: Vec::new(),
 							parent_header: inherent_data.parent_header,
 						}
-					},
+					}
 				};
 
 			Some(Call::enter { data: inherent_data })
@@ -274,7 +300,6 @@ pub mod pallet {
 				paras_inherent_total_weight::<T>(&backed_candidates, &signed_bitfields, &disputes);
 
 			// Abort if the total weight of the block exceeds the max block weight
-			#[cfg(not(feature = "runtime-benchmarks"))]
 			ensure!(
 				total_weight <= <T as frame_system::Config>::BlockWeights::get().max_block,
 				Error::<T>::InherentOverweight
@@ -304,7 +329,7 @@ pub mod pallet {
 				if T::DisputesHandler::is_frozen() {
 					// The relay chain we are currently on is invalid. Proceed no further on parachains.
 					Included::<T>::set(Some(()));
-					return Ok(Some(minimal_inherent_weight::<T>()).into())
+					return Ok(Some(minimal_inherent_weight::<T>()).into());
 				}
 
 				let mut freed_disputed = if !new_current_dispute_sets.is_empty() {
@@ -445,8 +470,8 @@ impl<T: Config> Pallet<T> {
 			Ok(None) => return None,
 			Err(_) => {
 				log::warn!(target: LOG_TARGET, "ParachainsInherentData failed to decode");
-				return None
-			},
+				return None;
+			}
 		};
 
 		let parent_hash = <frame_system::Pallet<T>>::parent_hash();
@@ -455,7 +480,7 @@ impl<T: Config> Pallet<T> {
 				target: LOG_TARGET,
 				"ParachainsInherentData references a different parent header hash than frame"
 			);
-			return None
+			return None;
 		}
 
 		let current_session = <shared::Pallet<T>>::session_index();
@@ -669,7 +694,7 @@ fn random_sel<X, F: Fn(&X) -> Weight>(
 	weight_limit: Weight,
 ) -> (Weight, Vec<usize>) {
 	if selectables.is_empty() {
-		return (0 as Weight, Vec::new())
+		return (0 as Weight, Vec::new());
 	}
 	let mut indices = (0..selectables.len()).into_iter().collect::<Vec<_>>();
 	let mut picked_indices = Vec::with_capacity(selectables.len().saturating_sub(1));
@@ -685,7 +710,7 @@ fn random_sel<X, F: Fn(&X) -> Weight>(
 		weight_acc += weight_fn(item);
 
 		if weight_acc > weight_limit {
-			break
+			break;
 		}
 
 		picked_indices.push(idx);
@@ -718,7 +743,7 @@ fn apply_weight_limit<T: Config + inclusion::Config>(
 
 	// everything fits into the block
 	if max_weight >= total {
-		return (total, candidates, bitfields)
+		return (total, candidates, bitfields);
 	}
 
 	use rand_chacha::rand_core::SeedableRng;
@@ -740,7 +765,7 @@ fn apply_weight_limit<T: Config + inclusion::Config>(
 		// pick all bitfields, and
 		// fill the remaining space with candidates
 		let total = acc_candidate_weight.saturating_add(total_bitfields_weight);
-		return (total, candidates, bitfields)
+		return (total, candidates, bitfields);
 	}
 
 	// insufficient space for even the bitfields alone, so only try to fit as many of those
@@ -805,8 +830,8 @@ pub(crate) fn sanitize_bitfields<T: crate::inclusion::Config, const EARLY_RETURN
 		);
 
 		ensure2!(
-			unchecked_bitfield.unchecked_payload().0.clone() & disputed_bitfield.0.clone() ==
-				all_zeros,
+			unchecked_bitfield.unchecked_payload().0.clone() & disputed_bitfield.0.clone()
+				== all_zeros,
 			crate::inclusion::Error::<T>::BitfieldReferencesFreedCore,
 			EARLY_RETURN,
 			continue
diff --git a/runtime/parachains/src/paras_inherent/benchmarking.rs b/runtime/parachains/src/paras_inherent/benchmarking.rs
index 49d2b00d98d3..0229ebf5e2aa 100644
--- a/runtime/parachains/src/paras_inherent/benchmarking.rs
+++ b/runtime/parachains/src/paras_inherent/benchmarking.rs
@@ -139,15 +139,12 @@ benchmarks! {
 		);
 	}
 
-	// Variant over `v`, the amount of validity votes for a backed candidate. This gives the weight
-	// of a single backed candidate.
-	enter_backed_candidates_code_upgrades_variable {
-		// NOTE: the starting value must be over half of `max_validators` so the backed candidate is
-		// not rejected.
-		let v in 0..24;
+	enter_backed_candidate_code_upgrade {
+		// For now we always assume worst case code size. In the future we could vary over this.
+		let v = crate::configuration::Pallet::<T>::config().max_code_size;
 
 		let cores_with_backed: BTreeMap<_, _>
-			= vec![(0, BenchBuilder::<T>::fallback_min_validity_votes())] // The backed candidate will have `v` validity votes.
+			= vec![(0, BenchBuilder::<T>::fallback_min_validity_votes())]
 				.into_iter()
 				.collect();
 
@@ -158,11 +155,14 @@ benchmarks! {
 
 		// There is 1 backed,
 		assert_eq!(benchmark.backed_candidates.len(), 1);
-		// with `v` validity votes.
-		assert_eq!(benchmark.backed_candidates.get(0).unwrap().validity_votes.len(), v as usize);
+		assert_eq!(
+			benchmark.backed_candidates.get(0).unwrap().validity_votes.len() as u32,
+			BenchBuilder::<T>::fallback_min_validity_votes()
+		);
 
 		benchmark.bitfields.clear();
 		benchmark.disputes.clear();
+		crate::paras::benchmarking::generate_disordered_upgrades::<T>();
 	}: enter(RawOrigin::None, benchmark)
 	verify {
 		let max_validators_per_core = BenchBuilder::<T>::fallback_max_validators_per_core();
@@ -177,12 +177,16 @@ benchmarks! {
 		// Ensure that there are an expected number of candidates
 		let header = BenchBuilder::<T>::header(scenario.block_number.clone());
 		// Traverse candidates and assert descriptors are as expected
-		for (para_id, backing_validators) in vote.backing_validators_per_candidate.iter().enumerate() {
-			let descriptor = backing_validators.0.descriptor();
-			assert_eq!(ParaId::from(para_id), descriptor.para_id);
-			assert_eq!(header.hash(), descriptor.relay_parent);
-			assert_eq!(backing_validators.1.len(), v as usize);
-		}
+		for (para_id, backing_validators)
+			in vote.backing_validators_per_candidate.iter().enumerate() {
+				let descriptor = backing_validators.0.descriptor();
+				assert_eq!(ParaId::from(para_id), descriptor.para_id);
+				assert_eq!(header.hash(), descriptor.relay_parent);
+				assert_eq!(
+					backing_validators.1.len() as u32,
+					BenchBuilder::<T>::fallback_min_validity_votes()
+				);
+			}
 
 		assert_eq!(
 			inclusion::PendingAvailabilityCommitments::<T>::iter().count(),
diff --git a/runtime/polkadot/src/weights/runtime_parachains_paras_inherent.rs b/runtime/polkadot/src/weights/runtime_parachains_paras_inherent.rs
index cce2e26d50c1..033af3be5067 100644
--- a/runtime/polkadot/src/weights/runtime_parachains_paras_inherent.rs
+++ b/runtime/polkadot/src/weights/runtime_parachains_paras_inherent.rs
@@ -1,3 +1,39 @@
+// Copyright 2017-2021 Parity Technologies (UK) Ltd.
+// This file is part of Polkadot.
+
+// Polkadot is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Polkadot is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
+//! Autogenerated weights for `runtime_parachains::paras_inherent`
+//!
+//! THIS FILE WAS AUTO-GENERATED USING THE SUBSTRATE BENCHMARK CLI VERSION 4.0.0-dev
+//! DATE: 2021-11-04, STEPS: `50`, REPEAT: 20, LOW RANGE: `[]`, HIGH RANGE: `[]`
+//! EXECUTION: Some(Wasm), WASM-EXECUTION: Compiled, CHAIN: Some("kusama-dev"), DB CACHE: 128
+
+// Executed Command:
+// target/release/polkadot
+// benchmark
+// --chain=kusama-dev
+// --steps=50
+// --repeat=20
+// --pallet=runtime_parachains::paras_inherent
+// --extrinsic=*
+// --execution=wasm
+// --wasm-execution=compiled
+// --heap-pages=4096
+// --header=./file_header.txt
+// --output=./runtime/kusama/src/weights/runtime_parachains_paras_inherent.rs
+
+
 #![cfg_attr(rustfmt, rustfmt_skip)]
 #![allow(unused_parens)]
 #![allow(unused_imports)]
@@ -5,15 +41,112 @@
 use frame_support::{traits::Get, weights::Weight};
 use sp_std::marker::PhantomData;
 
+/// Weight functions for `runtime_parachains::paras_inherent`.
 pub struct WeightInfo<T>(PhantomData<T>);
 impl<T: frame_system::Config> runtime_parachains::paras_inherent::WeightInfo for WeightInfo<T> {
-	fn enter_variable_disputes(_v: u32) -> Weight {
-		Weight::MAX
+	// Storage: ParaInherent Included (r:1 w:1)
+	// Storage: System ParentHash (r:1 w:0)
+	// Storage: ParaScheduler AvailabilityCores (r:1 w:1)
+	// Storage: ParasShared CurrentSessionIndex (r:1 w:0)
+	// Storage: ParaInclusion PendingAvailability (r:2 w:1)
+	// Storage: ParasShared ActiveValidatorKeys (r:1 w:0)
+	// Storage: Paras Parachains (r:1 w:0)
+	// Storage: ParaInclusion PendingAvailabilityCommitments (r:1 w:1)
+	// Storage: Configuration ActiveConfig (r:1 w:0)
+	// Storage: Session Validators (r:1 w:0)
+	// Storage: ParasShared ActiveValidatorIndices (r:1 w:0)
+	// Storage: Staking ActiveEra (r:1 w:0)
+	// Storage: Staking ErasRewardPoints (r:1 w:1)
+	// Storage: Dmp DownwardMessageQueues (r:1 w:1)
+	// Storage: Hrmp HrmpChannelDigests (r:1 w:1)
+	// Storage: Paras FutureCodeUpgrades (r:1 w:0)
+	// Storage: ParaScheduler SessionStartBlock (r:1 w:0)
+	// Storage: ParaScheduler ParathreadQueue (r:1 w:1)
+	// Storage: ParaScheduler Scheduled (r:1 w:1)
+	// Storage: ParaScheduler ValidatorGroups (r:1 w:0)
+	// Storage: System BlockWeight (r:1 w:0)
+	// Storage: Ump NeedsDispatch (r:1 w:1)
+	// Storage: Ump NextDispatchRoundStartWith (r:1 w:1)
+	// Storage: ParaInherent OnChainVotes (r:0 w:1)
+	// Storage: Hrmp HrmpWatermarks (r:0 w:1)
+	// Storage: Paras Heads (r:0 w:1)
+	fn enter_variable_disputes(v: u32, ) -> Weight {
+		(224_853_000 as Weight)
+			// Standard Error: 2_000
+			.saturating_add((229_000 as Weight).saturating_mul(v as Weight))
+			.saturating_add(T::DbWeight::get().reads(24 as Weight))
+			.saturating_add(T::DbWeight::get().writes(14 as Weight))
 	}
+	// Storage: ParaInherent Included (r:1 w:1)
+	// Storage: System ParentHash (r:1 w:0)
+	// Storage: ParaScheduler AvailabilityCores (r:1 w:1)
+	// Storage: ParasShared CurrentSessionIndex (r:1 w:0)
+	// Storage: ParasShared ActiveValidatorKeys (r:1 w:0)
+	// Storage: Paras Parachains (r:1 w:0)
+	// Storage: ParaInclusion PendingAvailability (r:2 w:1)
+	// Storage: ParaInclusion PendingAvailabilityCommitments (r:1 w:1)
+	// Storage: Configuration ActiveConfig (r:1 w:0)
+	// Storage: Session Validators (r:1 w:0)
+	// Storage: ParasShared ActiveValidatorIndices (r:1 w:0)
+	// Storage: Staking ActiveEra (r:1 w:0)
+	// Storage: Staking ErasRewardPoints (r:1 w:1)
+	// Storage: Dmp DownwardMessageQueues (r:1 w:1)
+	// Storage: Hrmp HrmpChannelDigests (r:1 w:1)
+	// Storage: Paras FutureCodeUpgrades (r:1 w:0)
+	// Storage: ParaScheduler SessionStartBlock (r:1 w:0)
+	// Storage: ParaScheduler ParathreadQueue (r:1 w:1)
+	// Storage: ParaScheduler Scheduled (r:1 w:1)
+	// Storage: ParaScheduler ValidatorGroups (r:1 w:0)
+	// Storage: System BlockWeight (r:1 w:0)
+	// Storage: Ump NeedsDispatch (r:1 w:1)
+	// Storage: Ump NextDispatchRoundStartWith (r:1 w:1)
+	// Storage: ParaInclusion AvailabilityBitfields (r:0 w:1)
+	// Storage: ParaInherent OnChainVotes (r:0 w:1)
+	// Storage: Hrmp HrmpWatermarks (r:0 w:1)
+	// Storage: Paras Heads (r:0 w:1)
 	fn enter_bitfields() -> Weight {
-		Weight::MAX
+		(265_872_000 as Weight)
+			.saturating_add(T::DbWeight::get().reads(24 as Weight))
+			.saturating_add(T::DbWeight::get().writes(15 as Weight))
 	}
-	fn enter_backed_candidates_variable(_v: u32) -> Weight {
-		Weight::MAX
+	// Storage: ParaInherent Included (r:1 w:1)
+	// Storage: System ParentHash (r:1 w:0)
+	// Storage: ParaScheduler AvailabilityCores (r:1 w:1)
+	// Storage: ParasShared CurrentSessionIndex (r:1 w:0)
+	// Storage: ParasShared ActiveValidatorKeys (r:1 w:0)
+	// Storage: Paras Parachains (r:1 w:0)
+	// Storage: ParaInclusion PendingAvailability (r:2 w:1)
+	// Storage: ParaInclusion PendingAvailabilityCommitments (r:1 w:1)
+	// Storage: Configuration ActiveConfig (r:1 w:0)
+	// Storage: Session Validators (r:1 w:0)
+	// Storage: ParasShared ActiveValidatorIndices (r:1 w:0)
+	// Storage: Staking ActiveEra (r:1 w:0)
+	// Storage: Staking ErasRewardPoints (r:1 w:1)
+	// Storage: Dmp DownwardMessageQueues (r:1 w:1)
+	// Storage: Hrmp HrmpChannelDigests (r:1 w:1)
+	// Storage: Paras FutureCodeUpgrades (r:1 w:0)
+	// Storage: ParaScheduler SessionStartBlock (r:1 w:0)
+	// Storage: ParaScheduler ParathreadQueue (r:1 w:1)
+	// Storage: ParaScheduler Scheduled (r:1 w:1)
+	// Storage: ParaScheduler ValidatorGroups (r:1 w:0)
+	// Storage: System BlockWeight (r:1 w:0)
+	// Storage: Paras PastCodeMeta (r:1 w:0)
+	// Storage: Paras CurrentCodeHash (r:1 w:0)
+	// Storage: Ump RelayDispatchQueueSize (r:1 w:0)
+	// Storage: Ump NeedsDispatch (r:1 w:1)
+	// Storage: Ump NextDispatchRoundStartWith (r:1 w:1)
+	// Storage: ParaInherent OnChainVotes (r:0 w:1)
+	// Storage: Hrmp HrmpWatermarks (r:0 w:1)
+	// Storage: Paras Heads (r:0 w:1)
+	fn enter_backed_candidates_variable(v: u32, ) -> Weight {
+		(345_183_000 as Weight)
+			// Standard Error: 23_000
+			.saturating_add((49_234_000 as Weight).saturating_mul(v as Weight))
+			.saturating_add(T::DbWeight::get().reads(27 as Weight))
+			.saturating_add(T::DbWeight::get().writes(14 as Weight))
 	}
+	fn enter_backed_candidate_code_upgrade() -> Weight {
+		0
+	}
+
 }
diff --git a/runtime/rococo/src/weights/runtime_parachains_paras_inherent.rs b/runtime/rococo/src/weights/runtime_parachains_paras_inherent.rs
index cce2e26d50c1..033af3be5067 100644
--- a/runtime/rococo/src/weights/runtime_parachains_paras_inherent.rs
+++ b/runtime/rococo/src/weights/runtime_parachains_paras_inherent.rs
@@ -1,3 +1,39 @@
+// Copyright 2017-2021 Parity Technologies (UK) Ltd.
+// This file is part of Polkadot.
+
+// Polkadot is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Polkadot is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
+//! Autogenerated weights for `runtime_parachains::paras_inherent`
+//!
+//! THIS FILE WAS AUTO-GENERATED USING THE SUBSTRATE BENCHMARK CLI VERSION 4.0.0-dev
+//! DATE: 2021-11-04, STEPS: `50`, REPEAT: 20, LOW RANGE: `[]`, HIGH RANGE: `[]`
+//! EXECUTION: Some(Wasm), WASM-EXECUTION: Compiled, CHAIN: Some("kusama-dev"), DB CACHE: 128
+
+// Executed Command:
+// target/release/polkadot
+// benchmark
+// --chain=kusama-dev
+// --steps=50
+// --repeat=20
+// --pallet=runtime_parachains::paras_inherent
+// --extrinsic=*
+// --execution=wasm
+// --wasm-execution=compiled
+// --heap-pages=4096
+// --header=./file_header.txt
+// --output=./runtime/kusama/src/weights/runtime_parachains_paras_inherent.rs
+
+
 #![cfg_attr(rustfmt, rustfmt_skip)]
 #![allow(unused_parens)]
 #![allow(unused_imports)]
@@ -5,15 +41,112 @@
 use frame_support::{traits::Get, weights::Weight};
 use sp_std::marker::PhantomData;
 
+/// Weight functions for `runtime_parachains::paras_inherent`.
 pub struct WeightInfo<T>(PhantomData<T>);
 impl<T: frame_system::Config> runtime_parachains::paras_inherent::WeightInfo for WeightInfo<T> {
-	fn enter_variable_disputes(_v: u32) -> Weight {
-		Weight::MAX
+	// Storage: ParaInherent Included (r:1 w:1)
+	// Storage: System ParentHash (r:1 w:0)
+	// Storage: ParaScheduler AvailabilityCores (r:1 w:1)
+	// Storage: ParasShared CurrentSessionIndex (r:1 w:0)
+	// Storage: ParaInclusion PendingAvailability (r:2 w:1)
+	// Storage: ParasShared ActiveValidatorKeys (r:1 w:0)
+	// Storage: Paras Parachains (r:1 w:0)
+	// Storage: ParaInclusion PendingAvailabilityCommitments (r:1 w:1)
+	// Storage: Configuration ActiveConfig (r:1 w:0)
+	// Storage: Session Validators (r:1 w:0)
+	// Storage: ParasShared ActiveValidatorIndices (r:1 w:0)
+	// Storage: Staking ActiveEra (r:1 w:0)
+	// Storage: Staking ErasRewardPoints (r:1 w:1)
+	// Storage: Dmp DownwardMessageQueues (r:1 w:1)
+	// Storage: Hrmp HrmpChannelDigests (r:1 w:1)
+	// Storage: Paras FutureCodeUpgrades (r:1 w:0)
+	// Storage: ParaScheduler SessionStartBlock (r:1 w:0)
+	// Storage: ParaScheduler ParathreadQueue (r:1 w:1)
+	// Storage: ParaScheduler Scheduled (r:1 w:1)
+	// Storage: ParaScheduler ValidatorGroups (r:1 w:0)
+	// Storage: System BlockWeight (r:1 w:0)
+	// Storage: Ump NeedsDispatch (r:1 w:1)
+	// Storage: Ump NextDispatchRoundStartWith (r:1 w:1)
+	// Storage: ParaInherent OnChainVotes (r:0 w:1)
+	// Storage: Hrmp HrmpWatermarks (r:0 w:1)
+	// Storage: Paras Heads (r:0 w:1)
+	fn enter_variable_disputes(v: u32, ) -> Weight {
+		(224_853_000 as Weight)
+			// Standard Error: 2_000
+			.saturating_add((229_000 as Weight).saturating_mul(v as Weight))
+			.saturating_add(T::DbWeight::get().reads(24 as Weight))
+			.saturating_add(T::DbWeight::get().writes(14 as Weight))
 	}
+	// Storage: ParaInherent Included (r:1 w:1)
+	// Storage: System ParentHash (r:1 w:0)
+	// Storage: ParaScheduler AvailabilityCores (r:1 w:1)
+	// Storage: ParasShared CurrentSessionIndex (r:1 w:0)
+	// Storage: ParasShared ActiveValidatorKeys (r:1 w:0)
+	// Storage: Paras Parachains (r:1 w:0)
+	// Storage: ParaInclusion PendingAvailability (r:2 w:1)
+	// Storage: ParaInclusion PendingAvailabilityCommitments (r:1 w:1)
+	// Storage: Configuration ActiveConfig (r:1 w:0)
+	// Storage: Session Validators (r:1 w:0)
+	// Storage: ParasShared ActiveValidatorIndices (r:1 w:0)
+	// Storage: Staking ActiveEra (r:1 w:0)
+	// Storage: Staking ErasRewardPoints (r:1 w:1)
+	// Storage: Dmp DownwardMessageQueues (r:1 w:1)
+	// Storage: Hrmp HrmpChannelDigests (r:1 w:1)
+	// Storage: Paras FutureCodeUpgrades (r:1 w:0)
+	// Storage: ParaScheduler SessionStartBlock (r:1 w:0)
+	// Storage: ParaScheduler ParathreadQueue (r:1 w:1)
+	// Storage: ParaScheduler Scheduled (r:1 w:1)
+	// Storage: ParaScheduler ValidatorGroups (r:1 w:0)
+	// Storage: System BlockWeight (r:1 w:0)
+	// Storage: Ump NeedsDispatch (r:1 w:1)
+	// Storage: Ump NextDispatchRoundStartWith (r:1 w:1)
+	// Storage: ParaInclusion AvailabilityBitfields (r:0 w:1)
+	// Storage: ParaInherent OnChainVotes (r:0 w:1)
+	// Storage: Hrmp HrmpWatermarks (r:0 w:1)
+	// Storage: Paras Heads (r:0 w:1)
 	fn enter_bitfields() -> Weight {
-		Weight::MAX
+		(265_872_000 as Weight)
+			.saturating_add(T::DbWeight::get().reads(24 as Weight))
+			.saturating_add(T::DbWeight::get().writes(15 as Weight))
 	}
-	fn enter_backed_candidates_variable(_v: u32) -> Weight {
-		Weight::MAX
+	// Storage: ParaInherent Included (r:1 w:1)
+	// Storage: System ParentHash (r:1 w:0)
+	// Storage: ParaScheduler AvailabilityCores (r:1 w:1)
+	// Storage: ParasShared CurrentSessionIndex (r:1 w:0)
+	// Storage: ParasShared ActiveValidatorKeys (r:1 w:0)
+	// Storage: Paras Parachains (r:1 w:0)
+	// Storage: ParaInclusion PendingAvailability (r:2 w:1)
+	// Storage: ParaInclusion PendingAvailabilityCommitments (r:1 w:1)
+	// Storage: Configuration ActiveConfig (r:1 w:0)
+	// Storage: Session Validators (r:1 w:0)
+	// Storage: ParasShared ActiveValidatorIndices (r:1 w:0)
+	// Storage: Staking ActiveEra (r:1 w:0)
+	// Storage: Staking ErasRewardPoints (r:1 w:1)
+	// Storage: Dmp DownwardMessageQueues (r:1 w:1)
+	// Storage: Hrmp HrmpChannelDigests (r:1 w:1)
+	// Storage: Paras FutureCodeUpgrades (r:1 w:0)
+	// Storage: ParaScheduler SessionStartBlock (r:1 w:0)
+	// Storage: ParaScheduler ParathreadQueue (r:1 w:1)
+	// Storage: ParaScheduler Scheduled (r:1 w:1)
+	// Storage: ParaScheduler ValidatorGroups (r:1 w:0)
+	// Storage: System BlockWeight (r:1 w:0)
+	// Storage: Paras PastCodeMeta (r:1 w:0)
+	// Storage: Paras CurrentCodeHash (r:1 w:0)
+	// Storage: Ump RelayDispatchQueueSize (r:1 w:0)
+	// Storage: Ump NeedsDispatch (r:1 w:1)
+	// Storage: Ump NextDispatchRoundStartWith (r:1 w:1)
+	// Storage: ParaInherent OnChainVotes (r:0 w:1)
+	// Storage: Hrmp HrmpWatermarks (r:0 w:1)
+	// Storage: Paras Heads (r:0 w:1)
+	fn enter_backed_candidates_variable(v: u32, ) -> Weight {
+		(345_183_000 as Weight)
+			// Standard Error: 23_000
+			.saturating_add((49_234_000 as Weight).saturating_mul(v as Weight))
+			.saturating_add(T::DbWeight::get().reads(27 as Weight))
+			.saturating_add(T::DbWeight::get().writes(14 as Weight))
 	}
+	fn enter_backed_candidate_code_upgrade() -> Weight {
+		0
+	}
+
 }
diff --git a/runtime/westend/src/weights/runtime_parachains_paras_inherent.rs b/runtime/westend/src/weights/runtime_parachains_paras_inherent.rs
index cce2e26d50c1..033af3be5067 100644
--- a/runtime/westend/src/weights/runtime_parachains_paras_inherent.rs
+++ b/runtime/westend/src/weights/runtime_parachains_paras_inherent.rs
@@ -1,3 +1,39 @@
+// Copyright 2017-2021 Parity Technologies (UK) Ltd.
+// This file is part of Polkadot.
+
+// Polkadot is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Polkadot is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
+//! Autogenerated weights for `runtime_parachains::paras_inherent`
+//!
+//! THIS FILE WAS AUTO-GENERATED USING THE SUBSTRATE BENCHMARK CLI VERSION 4.0.0-dev
+//! DATE: 2021-11-04, STEPS: `50`, REPEAT: 20, LOW RANGE: `[]`, HIGH RANGE: `[]`
+//! EXECUTION: Some(Wasm), WASM-EXECUTION: Compiled, CHAIN: Some("kusama-dev"), DB CACHE: 128
+
+// Executed Command:
+// target/release/polkadot
+// benchmark
+// --chain=kusama-dev
+// --steps=50
+// --repeat=20
+// --pallet=runtime_parachains::paras_inherent
+// --extrinsic=*
+// --execution=wasm
+// --wasm-execution=compiled
+// --heap-pages=4096
+// --header=./file_header.txt
+// --output=./runtime/kusama/src/weights/runtime_parachains_paras_inherent.rs
+
+
 #![cfg_attr(rustfmt, rustfmt_skip)]
 #![allow(unused_parens)]
 #![allow(unused_imports)]
@@ -5,15 +41,112 @@
 use frame_support::{traits::Get, weights::Weight};
 use sp_std::marker::PhantomData;
 
+/// Weight functions for `runtime_parachains::paras_inherent`.
 pub struct WeightInfo<T>(PhantomData<T>);
 impl<T: frame_system::Config> runtime_parachains::paras_inherent::WeightInfo for WeightInfo<T> {
-	fn enter_variable_disputes(_v: u32) -> Weight {
-		Weight::MAX
+	// Storage: ParaInherent Included (r:1 w:1)
+	// Storage: System ParentHash (r:1 w:0)
+	// Storage: ParaScheduler AvailabilityCores (r:1 w:1)
+	// Storage: ParasShared CurrentSessionIndex (r:1 w:0)
+	// Storage: ParaInclusion PendingAvailability (r:2 w:1)
+	// Storage: ParasShared ActiveValidatorKeys (r:1 w:0)
+	// Storage: Paras Parachains (r:1 w:0)
+	// Storage: ParaInclusion PendingAvailabilityCommitments (r:1 w:1)
+	// Storage: Configuration ActiveConfig (r:1 w:0)
+	// Storage: Session Validators (r:1 w:0)
+	// Storage: ParasShared ActiveValidatorIndices (r:1 w:0)
+	// Storage: Staking ActiveEra (r:1 w:0)
+	// Storage: Staking ErasRewardPoints (r:1 w:1)
+	// Storage: Dmp DownwardMessageQueues (r:1 w:1)
+	// Storage: Hrmp HrmpChannelDigests (r:1 w:1)
+	// Storage: Paras FutureCodeUpgrades (r:1 w:0)
+	// Storage: ParaScheduler SessionStartBlock (r:1 w:0)
+	// Storage: ParaScheduler ParathreadQueue (r:1 w:1)
+	// Storage: ParaScheduler Scheduled (r:1 w:1)
+	// Storage: ParaScheduler ValidatorGroups (r:1 w:0)
+	// Storage: System BlockWeight (r:1 w:0)
+	// Storage: Ump NeedsDispatch (r:1 w:1)
+	// Storage: Ump NextDispatchRoundStartWith (r:1 w:1)
+	// Storage: ParaInherent OnChainVotes (r:0 w:1)
+	// Storage: Hrmp HrmpWatermarks (r:0 w:1)
+	// Storage: Paras Heads (r:0 w:1)
+	fn enter_variable_disputes(v: u32, ) -> Weight {
+		(224_853_000 as Weight)
+			// Standard Error: 2_000
+			.saturating_add((229_000 as Weight).saturating_mul(v as Weight))
+			.saturating_add(T::DbWeight::get().reads(24 as Weight))
+			.saturating_add(T::DbWeight::get().writes(14 as Weight))
 	}
+	// Storage: ParaInherent Included (r:1 w:1)
+	// Storage: System ParentHash (r:1 w:0)
+	// Storage: ParaScheduler AvailabilityCores (r:1 w:1)
+	// Storage: ParasShared CurrentSessionIndex (r:1 w:0)
+	// Storage: ParasShared ActiveValidatorKeys (r:1 w:0)
+	// Storage: Paras Parachains (r:1 w:0)
+	// Storage: ParaInclusion PendingAvailability (r:2 w:1)
+	// Storage: ParaInclusion PendingAvailabilityCommitments (r:1 w:1)
+	// Storage: Configuration ActiveConfig (r:1 w:0)
+	// Storage: Session Validators (r:1 w:0)
+	// Storage: ParasShared ActiveValidatorIndices (r:1 w:0)
+	// Storage: Staking ActiveEra (r:1 w:0)
+	// Storage: Staking ErasRewardPoints (r:1 w:1)
+	// Storage: Dmp DownwardMessageQueues (r:1 w:1)
+	// Storage: Hrmp HrmpChannelDigests (r:1 w:1)
+	// Storage: Paras FutureCodeUpgrades (r:1 w:0)
+	// Storage: ParaScheduler SessionStartBlock (r:1 w:0)
+	// Storage: ParaScheduler ParathreadQueue (r:1 w:1)
+	// Storage: ParaScheduler Scheduled (r:1 w:1)
+	// Storage: ParaScheduler ValidatorGroups (r:1 w:0)
+	// Storage: System BlockWeight (r:1 w:0)
+	// Storage: Ump NeedsDispatch (r:1 w:1)
+	// Storage: Ump NextDispatchRoundStartWith (r:1 w:1)
+	// Storage: ParaInclusion AvailabilityBitfields (r:0 w:1)
+	// Storage: ParaInherent OnChainVotes (r:0 w:1)
+	// Storage: Hrmp HrmpWatermarks (r:0 w:1)
+	// Storage: Paras Heads (r:0 w:1)
 	fn enter_bitfields() -> Weight {
-		Weight::MAX
+		(265_872_000 as Weight)
+			.saturating_add(T::DbWeight::get().reads(24 as Weight))
+			.saturating_add(T::DbWeight::get().writes(15 as Weight))
 	}
-	fn enter_backed_candidates_variable(_v: u32) -> Weight {
-		Weight::MAX
+	// Storage: ParaInherent Included (r:1 w:1)
+	// Storage: System ParentHash (r:1 w:0)
+	// Storage: ParaScheduler AvailabilityCores (r:1 w:1)
+	// Storage: ParasShared CurrentSessionIndex (r:1 w:0)
+	// Storage: ParasShared ActiveValidatorKeys (r:1 w:0)
+	// Storage: Paras Parachains (r:1 w:0)
+	// Storage: ParaInclusion PendingAvailability (r:2 w:1)
+	// Storage: ParaInclusion PendingAvailabilityCommitments (r:1 w:1)
+	// Storage: Configuration ActiveConfig (r:1 w:0)
+	// Storage: Session Validators (r:1 w:0)
+	// Storage: ParasShared ActiveValidatorIndices (r:1 w:0)
+	// Storage: Staking ActiveEra (r:1 w:0)
+	// Storage: Staking ErasRewardPoints (r:1 w:1)
+	// Storage: Dmp DownwardMessageQueues (r:1 w:1)
+	// Storage: Hrmp HrmpChannelDigests (r:1 w:1)
+	// Storage: Paras FutureCodeUpgrades (r:1 w:0)
+	// Storage: ParaScheduler SessionStartBlock (r:1 w:0)
+	// Storage: ParaScheduler ParathreadQueue (r:1 w:1)
+	// Storage: ParaScheduler Scheduled (r:1 w:1)
+	// Storage: ParaScheduler ValidatorGroups (r:1 w:0)
+	// Storage: System BlockWeight (r:1 w:0)
+	// Storage: Paras PastCodeMeta (r:1 w:0)
+	// Storage: Paras CurrentCodeHash (r:1 w:0)
+	// Storage: Ump RelayDispatchQueueSize (r:1 w:0)
+	// Storage: Ump NeedsDispatch (r:1 w:1)
+	// Storage: Ump NextDispatchRoundStartWith (r:1 w:1)
+	// Storage: ParaInherent OnChainVotes (r:0 w:1)
+	// Storage: Hrmp HrmpWatermarks (r:0 w:1)
+	// Storage: Paras Heads (r:0 w:1)
+	fn enter_backed_candidates_variable(v: u32, ) -> Weight {
+		(345_183_000 as Weight)
+			// Standard Error: 23_000
+			.saturating_add((49_234_000 as Weight).saturating_mul(v as Weight))
+			.saturating_add(T::DbWeight::get().reads(27 as Weight))
+			.saturating_add(T::DbWeight::get().writes(14 as Weight))
 	}
+	fn enter_backed_candidate_code_upgrade() -> Weight {
+		0
+	}
+
 }

From f861c2cc09fa60b0f98da316f65cce7e2c0eafb0 Mon Sep 17 00:00:00 2001
From: emostov <32168567+emostov@users.noreply.github.com>
Date: Sat, 6 Nov 2021 20:26:09 +0100
Subject: [PATCH 069/107] Try and make CI happier

---
 runtime/parachains/src/builder.rs        |  2 +-
 runtime/parachains/src/paras_inherent.rs | 22 +++++++++++-----------
 2 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/runtime/parachains/src/builder.rs b/runtime/parachains/src/builder.rs
index 3a0c13b22261..f393e78b47fa 100644
--- a/runtime/parachains/src/builder.rs
+++ b/runtime/parachains/src/builder.rs
@@ -4,7 +4,7 @@ use crate::{
 	scheduler, session_info, shared,
 };
 use bitvec::{order::Lsb0 as BitOrderLsb0, vec::BitVec};
-use frame_benchmarking::{account, Vec};
+use frame_benchmarking::{account, vec, Vec};
 use frame_support::pallet_prelude::*;
 use primitives::v1::{
 	collator_signature_payload, AvailabilityBitfield, BackedCandidate, CandidateCommitments,
diff --git a/runtime/parachains/src/paras_inherent.rs b/runtime/parachains/src/paras_inherent.rs
index d43e3de544fc..1db0625537bc 100644
--- a/runtime/parachains/src/paras_inherent.rs
+++ b/runtime/parachains/src/paras_inherent.rs
@@ -264,7 +264,7 @@ pub mod pallet {
 							disputes: Vec::new(),
 							parent_header: inherent_data.parent_header,
 						}
-					}
+					},
 				};
 
 			Some(Call::enter { data: inherent_data })
@@ -329,7 +329,7 @@ pub mod pallet {
 				if T::DisputesHandler::is_frozen() {
 					// The relay chain we are currently on is invalid. Proceed no further on parachains.
 					Included::<T>::set(Some(()));
-					return Ok(Some(minimal_inherent_weight::<T>()).into());
+					return Ok(Some(minimal_inherent_weight::<T>()).into())
 				}
 
 				let mut freed_disputed = if !new_current_dispute_sets.is_empty() {
@@ -470,8 +470,8 @@ impl<T: Config> Pallet<T> {
 			Ok(None) => return None,
 			Err(_) => {
 				log::warn!(target: LOG_TARGET, "ParachainsInherentData failed to decode");
-				return None;
-			}
+				return None
+			},
 		};
 
 		let parent_hash = <frame_system::Pallet<T>>::parent_hash();
@@ -480,7 +480,7 @@ impl<T: Config> Pallet<T> {
 				target: LOG_TARGET,
 				"ParachainsInherentData references a different parent header hash than frame"
 			);
-			return None;
+			return None
 		}
 
 		let current_session = <shared::Pallet<T>>::session_index();
@@ -694,7 +694,7 @@ fn random_sel<X, F: Fn(&X) -> Weight>(
 	weight_limit: Weight,
 ) -> (Weight, Vec<usize>) {
 	if selectables.is_empty() {
-		return (0 as Weight, Vec::new());
+		return (0 as Weight, Vec::new())
 	}
 	let mut indices = (0..selectables.len()).into_iter().collect::<Vec<_>>();
 	let mut picked_indices = Vec::with_capacity(selectables.len().saturating_sub(1));
@@ -710,7 +710,7 @@ fn random_sel<X, F: Fn(&X) -> Weight>(
 		weight_acc += weight_fn(item);
 
 		if weight_acc > weight_limit {
-			break;
+			break
 		}
 
 		picked_indices.push(idx);
@@ -743,7 +743,7 @@ fn apply_weight_limit<T: Config + inclusion::Config>(
 
 	// everything fits into the block
 	if max_weight >= total {
-		return (total, candidates, bitfields);
+		return (total, candidates, bitfields)
 	}
 
 	use rand_chacha::rand_core::SeedableRng;
@@ -765,7 +765,7 @@ fn apply_weight_limit<T: Config + inclusion::Config>(
 		// pick all bitfields, and
 		// fill the remaining space with candidates
 		let total = acc_candidate_weight.saturating_add(total_bitfields_weight);
-		return (total, candidates, bitfields);
+		return (total, candidates, bitfields)
 	}
 
 	// insufficient space for even the bitfields alone, so only try to fit as many of those
@@ -830,8 +830,8 @@ pub(crate) fn sanitize_bitfields<T: crate::inclusion::Config, const EARLY_RETURN
 		);
 
 		ensure2!(
-			unchecked_bitfield.unchecked_payload().0.clone() & disputed_bitfield.0.clone()
-				== all_zeros,
+			unchecked_bitfield.unchecked_payload().0.clone() & disputed_bitfield.0.clone() ==
+				all_zeros,
 			crate::inclusion::Error::<T>::BitfieldReferencesFreedCore,
 			EARLY_RETURN,
 			continue

From 35407fc7749e6d24b2b33f618b5f36cc319e4861 Mon Sep 17 00:00:00 2001
From: emostov <32168567+emostov@users.noreply.github.com>
Date: Mon, 8 Nov 2021 12:34:33 +0100
Subject: [PATCH 070/107] Feature gate enter test to not(benchmarks)

---
 runtime/parachains/Cargo.toml            |  2 +-
 runtime/parachains/src/paras_inherent.rs | 28 ++++++++++++++----------
 2 files changed, 17 insertions(+), 13 deletions(-)

diff --git a/runtime/parachains/Cargo.toml b/runtime/parachains/Cargo.toml
index 4c955d8a9ee4..f6b8e6455ac3 100644
--- a/runtime/parachains/Cargo.toml
+++ b/runtime/parachains/Cargo.toml
@@ -88,7 +88,7 @@ runtime-benchmarks = [
 	"frame-support/runtime-benchmarks",
 	"frame-system/runtime-benchmarks",
 	"primitives/runtime-benchmarks",
-	"application-crypto/full_crypto"
+	"application-crypto/full_crypto",
 ]
 try-runtime = [
 	"frame-support/try-runtime",
diff --git a/runtime/parachains/src/paras_inherent.rs b/runtime/parachains/src/paras_inherent.rs
index 1db0625537bc..aa224d36c7c5 100644
--- a/runtime/parachains/src/paras_inherent.rs
+++ b/runtime/parachains/src/paras_inherent.rs
@@ -264,7 +264,7 @@ pub mod pallet {
 							disputes: Vec::new(),
 							parent_header: inherent_data.parent_header,
 						}
-					},
+					}
 				};
 
 			Some(Call::enter { data: inherent_data })
@@ -329,7 +329,7 @@ pub mod pallet {
 				if T::DisputesHandler::is_frozen() {
 					// The relay chain we are currently on is invalid. Proceed no further on parachains.
 					Included::<T>::set(Some(()));
-					return Ok(Some(minimal_inherent_weight::<T>()).into())
+					return Ok(Some(minimal_inherent_weight::<T>()).into());
 				}
 
 				let mut freed_disputed = if !new_current_dispute_sets.is_empty() {
@@ -470,8 +470,8 @@ impl<T: Config> Pallet<T> {
 			Ok(None) => return None,
 			Err(_) => {
 				log::warn!(target: LOG_TARGET, "ParachainsInherentData failed to decode");
-				return None
-			},
+				return None;
+			}
 		};
 
 		let parent_hash = <frame_system::Pallet<T>>::parent_hash();
@@ -480,7 +480,7 @@ impl<T: Config> Pallet<T> {
 				target: LOG_TARGET,
 				"ParachainsInherentData references a different parent header hash than frame"
 			);
-			return None
+			return None;
 		}
 
 		let current_session = <shared::Pallet<T>>::session_index();
@@ -694,7 +694,7 @@ fn random_sel<X, F: Fn(&X) -> Weight>(
 	weight_limit: Weight,
 ) -> (Weight, Vec<usize>) {
 	if selectables.is_empty() {
-		return (0 as Weight, Vec::new())
+		return (0 as Weight, Vec::new());
 	}
 	let mut indices = (0..selectables.len()).into_iter().collect::<Vec<_>>();
 	let mut picked_indices = Vec::with_capacity(selectables.len().saturating_sub(1));
@@ -710,7 +710,7 @@ fn random_sel<X, F: Fn(&X) -> Weight>(
 		weight_acc += weight_fn(item);
 
 		if weight_acc > weight_limit {
-			break
+			break;
 		}
 
 		picked_indices.push(idx);
@@ -743,7 +743,7 @@ fn apply_weight_limit<T: Config + inclusion::Config>(
 
 	// everything fits into the block
 	if max_weight >= total {
-		return (total, candidates, bitfields)
+		return (total, candidates, bitfields);
 	}
 
 	use rand_chacha::rand_core::SeedableRng;
@@ -765,7 +765,7 @@ fn apply_weight_limit<T: Config + inclusion::Config>(
 		// pick all bitfields, and
 		// fill the remaining space with candidates
 		let total = acc_candidate_weight.saturating_add(total_bitfields_weight);
-		return (total, candidates, bitfields)
+		return (total, candidates, bitfields);
 	}
 
 	// insufficient space for even the bitfields alone, so only try to fit as many of those
@@ -830,8 +830,8 @@ pub(crate) fn sanitize_bitfields<T: crate::inclusion::Config, const EARLY_RETURN
 		);
 
 		ensure2!(
-			unchecked_bitfield.unchecked_payload().0.clone() & disputed_bitfield.0.clone() ==
-				all_zeros,
+			unchecked_bitfield.unchecked_payload().0.clone() & disputed_bitfield.0.clone()
+				== all_zeros,
 			crate::inclusion::Error::<T>::BitfieldReferencesFreedCore,
 			EARLY_RETURN,
 			continue
@@ -1010,7 +1010,11 @@ mod tests {
 	use frame_support::{assert_err, assert_ok};
 	use sp_std::collections::btree_map::BTreeMap;
 
-	mod limit_backed_candidates {
+	// In order to facilitate benchmarks as tests we have a benchmark feature gated `WeightInfo` impl
+	// that uses 0 for all the weights. Because all the weights are 0, the tests that rely on
+	// weights for limiting data will fail, so we don't run them when using the benchmark feature.
+	#[cfg(not(feature = "runtime-benchmarks"))]
+	mod enter {
 		use super::*;
 
 		struct TestConfig {

From 6c0c078b6d1f8512b0b2ea103a71b1da3d1052ab Mon Sep 17 00:00:00 2001
From: emostov <32168567+emostov@users.noreply.github.com>
Date: Mon, 8 Nov 2021 13:28:42 +0100
Subject: [PATCH 071/107] Make sure no unused imports/fn

---
 runtime/parachains/src/builder.rs        |  6 +++---
 runtime/parachains/src/paras_inherent.rs | 13 ++++++-------
 2 files changed, 9 insertions(+), 10 deletions(-)

diff --git a/runtime/parachains/src/builder.rs b/runtime/parachains/src/builder.rs
index f393e78b47fa..2ec0d4504d19 100644
--- a/runtime/parachains/src/builder.rs
+++ b/runtime/parachains/src/builder.rs
@@ -104,7 +104,7 @@ impl<T: paras_inherent::Config> BenchBuilder<T> {
 		self.max_validators.unwrap_or(Self::fallback_max_validators())
 	}
 
-	#[cfg(test)]
+	#[cfg(not(feature = "runtime-benchmarks"))]
 	pub(crate) fn set_max_validators(mut self, n: u32) -> Self {
 		self.max_validators = Some(n);
 		self
@@ -114,7 +114,7 @@ impl<T: paras_inherent::Config> BenchBuilder<T> {
 		configuration::Pallet::<T>::config().max_validators_per_core.unwrap_or(5)
 	}
 
-	#[cfg(test)]
+	#[cfg(not(feature = "runtime-benchmarks"))]
 	pub(crate) fn set_dispute_statements(mut self, m: BTreeMap<u32, u32>) -> Self {
 		self.dispute_statements = m;
 		self
@@ -125,7 +125,7 @@ impl<T: paras_inherent::Config> BenchBuilder<T> {
 	}
 
 	/// Set maximum number of validators per core.
-	#[cfg(test)]
+	#[cfg(not(feature = "runtime-benchmarks"))]
 	pub(crate) fn set_max_validators_per_core(mut self, n: u32) -> Self {
 		self.max_validators_per_core = Some(n);
 		self
diff --git a/runtime/parachains/src/paras_inherent.rs b/runtime/parachains/src/paras_inherent.rs
index aa224d36c7c5..e40443ea39cb 100644
--- a/runtime/parachains/src/paras_inherent.rs
+++ b/runtime/parachains/src/paras_inherent.rs
@@ -1003,19 +1003,18 @@ fn limit_disputes<T: Config>(disputes: &mut MultiDisputeStatementSet, entropy: [
 mod tests {
 	use super::*;
 
-	use crate::{
-		builder::{Bench, BenchBuilder},
-		mock::{new_test_ext, MockGenesisConfig, Test},
-	};
-	use frame_support::{assert_err, assert_ok};
-	use sp_std::collections::btree_map::BTreeMap;
-
 	// In order to facilitate benchmarks as tests we have a benchmark feature gated `WeightInfo` impl
 	// that uses 0 for all the weights. Because all the weights are 0, the tests that rely on
 	// weights for limiting data will fail, so we don't run them when using the benchmark feature.
 	#[cfg(not(feature = "runtime-benchmarks"))]
 	mod enter {
 		use super::*;
+		use crate::{
+			builder::{Bench, BenchBuilder},
+			mock::{new_test_ext, MockGenesisConfig, Test},
+		};
+		use frame_support::{assert_err, assert_ok};
+		use sp_std::collections::btree_map::BTreeMap;
 
 		struct TestConfig {
 			dispute_statements: BTreeMap<u32, u32>,

From 0e8894832aca4711476638d6a8443765dd788c17 Mon Sep 17 00:00:00 2001
From: Bernhard Schuster <bernhard@ahoi.io>
Date: Fri, 5 Nov 2021 11:47:33 +0100
Subject: [PATCH 072/107] refactor, re-use, the beginning

---
 runtime/parachains/src/inclusion.rs      | 102 +++++++++++++----------
 runtime/parachains/src/paras_inherent.rs |  38 +++++----
 2 files changed, 80 insertions(+), 60 deletions(-)

diff --git a/runtime/parachains/src/inclusion.rs b/runtime/parachains/src/inclusion.rs
index e986c27d1ce5..b9a5f1c59c7a 100644
--- a/runtime/parachains/src/inclusion.rs
+++ b/runtime/parachains/src/inclusion.rs
@@ -33,7 +33,7 @@ use primitives::v1::{
 	AvailabilityBitfield, BackedCandidate, CandidateCommitments, CandidateDescriptor,
 	CandidateHash, CandidateReceipt, CommittedCandidateReceipt, CoreIndex, GroupIndex, Hash,
 	HeadData, Id as ParaId, SigningContext, UncheckedSignedAvailabilityBitfields, ValidatorIndex,
-	ValidityAttestation,
+	ValidityAttestation, ValidatorId,
 };
 use scale_info::TypeInfo;
 use sp_runtime::{
@@ -286,43 +286,20 @@ impl<T: Config> Pallet<T> {
 		for _ in <AvailabilityBitfields<T>>::drain() {}
 	}
 
-	/// Process a set of incoming bitfields.
+	/// Extract the freed cores based on cores tht became available.
 	///
-	/// Returns a `Vec` of `CandidateHash`es and their respective `AvailabilityCore`s that became available,
-	/// and cores free.
-	pub(crate) fn process_bitfields(
+	/// Updates storage items `PendingAvailability` and `AvailabilityBitfields`.
+	pub(crate) fn update_pending_availability_and_get_freed_cores<IS_CREATE_INHERENT: bool>(
 		expected_bits: usize,
+		validators: &[ValidatorId],
 		signed_bitfields: UncheckedSignedAvailabilityBitfields,
 		disputed_bitfield: DisputedBitfield,
 		core_lookup: impl Fn(CoreIndex) -> Option<ParaId>,
-		is_create_inherent: bool,
-	) -> Result<Vec<(CoreIndex, CandidateHash)>, DispatchError> {
+	) -> Vec<(CoreIndex, CandidateHash)> {
 		let validators = shared::Pallet::<T>::active_validator_keys();
 		let session_index = shared::Pallet::<T>::session_index();
 		let parent_hash = frame_system::Pallet::<T>::parent_hash();
 
-		let checked_bitfields = if is_create_inherent {
-			sanitize_bitfields::<T, false>(
-				signed_bitfields,
-				disputed_bitfield,
-				expected_bits,
-				parent_hash,
-				session_index,
-				&validators[..],
-			)
-			.expect(
-				"by convention, when called with `EARLY_RETURN=false`, will always return `Ok()`",
-			)
-		} else {
-			sanitize_bitfields::<T, true>(
-				signed_bitfields,
-				disputed_bitfield,
-				expected_bits,
-				parent_hash,
-				session_index,
-				&validators[..],
-			)?
-		};
 
 		let mut assigned_paras_record = (0..expected_bits)
 			.map(|bit_index| core_lookup(CoreIndex::from(bit_index as u32)))
@@ -333,7 +310,7 @@ impl<T: Config> Pallet<T> {
 
 		let now = <frame_system::Pallet<T>>::block_number();
 		for (checked_bitfield, validator_index) in
-			checked_bitfields.into_iter().map(|signed_bitfield| {
+			signed_bitfields.into_iter().map(|signed_bitfield| {
 				// extracting unchecked data, since it's checked in `fn sanitize_bitfields` already.
 				let validator_idx = signed_bitfield.unchecked_validator_index();
 				let checked_bitfield = signed_bitfield.unchecked_into_payload();
@@ -390,28 +367,61 @@ impl<T: Config> Pallet<T> {
 						continue
 					},
 				};
-
-				if !is_create_inherent {
-					let receipt = CommittedCandidateReceipt {
-						descriptor: pending_availability.descriptor,
-						commitments,
-					};
-					let _weight = Self::enact_candidate(
-						pending_availability.relay_parent_number,
-						receipt,
-						pending_availability.backers,
-						pending_availability.availability_votes,
-						pending_availability.core,
-						pending_availability.backing_group,
-					);
-				}
-
 				freed_cores.push((pending_availability.core, pending_availability.hash));
 			} else {
 				<PendingAvailability<T>>::insert(&para_id, &pending_availability);
 			}
+
+			if !IS_CREATE_INHERENT {
+				let receipt = CommittedCandidateReceipt {
+					descriptor: pending_availability.descriptor,
+					commitments,
+				};
+				let _weight = Self::enact_candidate(
+					pending_availability.relay_parent_number,
+					receipt,
+					pending_availability.backers,
+					pending_availability.availability_votes,
+					pending_availability.core,
+					pending_availability.backing_group,
+				);
+			}
 		}
 
+		freed_cores
+	}
+
+	/// Process a set of incoming bitfields.
+	///
+	/// Returns a `Vec` of `CandidateHash`es and their respective `AvailabilityCore`s that became available,
+	/// and cores free.
+	pub(crate) fn process_bitfields<IS_CREATE_INHERENT: bool>(
+		expected_bits: usize,
+		signed_bitfields: UncheckedSignedAvailabilityBitfields,
+		disputed_bitfield: DisputedBitfield,
+		core_lookup: impl Fn(CoreIndex) -> Option<ParaId>,
+	) -> Result<Vec<(CoreIndex, ParaId)>, DispatchError> {
+		let validators = shared::Pallet::<T>::active_validator_keys();
+		let session_index = shared::Pallet::<T>::session_index();
+		let parent_hash = frame_system::Pallet::<T>::parent_hash();
+
+		let checked_bitfields = sanitize_bitfields::<T, IS_CREATE_INHERENT>(
+			signed_bitfields,
+			disputed_bitfield,
+			expected_bits,
+			parent_hash,
+			session_index,
+			&validators[..],
+		)?;
+
+		let freed_cores = Self::update_pending_availability_and_get_freed_cores::<false>(
+			expected_bits,
+			&validators[..],
+			disputed_bitfield,
+			core_lookup,
+		);
+
+
 		Ok(freed_cores)
 	}
 
diff --git a/runtime/parachains/src/paras_inherent.rs b/runtime/parachains/src/paras_inherent.rs
index e40443ea39cb..728abdb31a9a 100644
--- a/runtime/parachains/src/paras_inherent.rs
+++ b/runtime/parachains/src/paras_inherent.rs
@@ -275,6 +275,29 @@ pub mod pallet {
 		}
 	}
 
+	/// Collect all freed cores based on storage data.
+	///
+	/// The paramter `freed_concluded` contains all core indicies that became
+	/// free due to candidate that became available.
+	pub(crate) fn collect_all_freed_cores<T>(freed_concluded: Vec<CoreIndex>) -> BTreeMap<CoreIndex, FreedReason> {
+
+		// Handle timeouts for any availability core work.
+		let availability_pred = <scheduler::Pallet<T>>::availability_timeout_predicate();
+		let freed_timeout = if let Some(pred) = availability_pred {
+			<inclusion::Pallet<T>>::collect_pending(pred)
+		} else {
+			Vec::new()
+		};
+
+		// Schedule paras again, given freed cores, and reasons for freeing.
+		let freed = freed_concluded
+			.into_iter()
+			.map(|(c, _hash)| (c, FreedReason::Concluded))
+			.chain(freed_timeout.into_iter().map(|c| (c, FreedReason::TimedOut)))
+			.collect::<BTreeMap<CoreIndex, FreedReason>>();
+		freed
+	}
+
 	#[pallet::call]
 	impl<T: Config> Pallet<T> {
 		/// Enter the paras inherent. This will process bitfields and backed candidates.
@@ -385,20 +408,7 @@ pub mod pallet {
 				T::DisputesHandler::note_included(current_session, *candidate_hash, now);
 			}
 
-			// Handle timeouts for any availability core work.
-			let availability_pred = <scheduler::Pallet<T>>::availability_timeout_predicate();
-			let freed_timeout = if let Some(pred) = availability_pred {
-				<inclusion::Pallet<T>>::collect_pending(pred)
-			} else {
-				Vec::new()
-			};
-
-			// Schedule paras again, given freed cores, and reasons for freeing.
-			let freed = freed_concluded
-				.into_iter()
-				.map(|(c, _hash)| (c, FreedReason::Concluded))
-				.chain(freed_timeout.into_iter().map(|c| (c, FreedReason::TimedOut)))
-				.collect::<BTreeMap<CoreIndex, FreedReason>>();
+			let freed = collect_all_freed_cores::<T>(freed_concluded);
 
 			<scheduler::Pallet<T>>::clear();
 			<scheduler::Pallet<T>>::schedule(freed, now);

From aecdd96a5645ed6fa00e785ec97e25e7fc2d4b64 Mon Sep 17 00:00:00 2001
From: emostov <32168567+emostov@users.noreply.github.com>
Date: Mon, 8 Nov 2021 17:39:59 +0100
Subject: [PATCH 073/107] Fix issue with frame benchmarking dep compilation

---
 runtime/kusama/Cargo.toml         |  1 +
 runtime/parachains/Cargo.toml     |  3 ++-
 runtime/parachains/src/builder.rs | 18 ++++++++++++++++--
 runtime/parachains/src/shared.rs  |  2 +-
 runtime/polkadot/Cargo.toml       |  1 +
 runtime/rococo/Cargo.toml         |  2 +-
 runtime/westend/Cargo.toml        |  1 +
 7 files changed, 23 insertions(+), 5 deletions(-)

diff --git a/runtime/kusama/Cargo.toml b/runtime/kusama/Cargo.toml
index 57657b817d0e..416e38cd55fc 100644
--- a/runtime/kusama/Cargo.toml
+++ b/runtime/kusama/Cargo.toml
@@ -214,6 +214,7 @@ runtime-benchmarks = [
 	"xcm-builder/runtime-benchmarks",
 	"frame-election-provider-support/runtime-benchmarks",
 	"pallet-bags-list/runtime-benchmarks",
+	"runtime-parachains/runtime-benchmarks",
 ]
 try-runtime = [
 	"frame-executive/try-runtime",
diff --git a/runtime/parachains/Cargo.toml b/runtime/parachains/Cargo.toml
index f6b8e6455ac3..d708682e2e10 100644
--- a/runtime/parachains/Cargo.toml
+++ b/runtime/parachains/Cargo.toml
@@ -32,7 +32,6 @@ pallet-session = { git = "https://github.com/paritytech/substrate", branch = "ma
 pallet-staking = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false }
 pallet-timestamp = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false }
 pallet-vesting = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false }
-frame-benchmarking = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false }
 frame-support = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false }
 frame-system = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false }
 
@@ -44,6 +43,7 @@ rand = { version = "0.8.3", default-features = false }
 rand_chacha = { version = "0.3.1", default-features = false }
 
 application-crypto = { package = "sp-application-crypto", git = "https://github.com/paritytech/substrate", branch = "master", default-features = false, optional = true }
+frame-benchmarking = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false, optional = true }
 
 [dev-dependencies]
 futures = "0.3.17"
@@ -85,6 +85,7 @@ std = [
 	"log/std",
 ]
 runtime-benchmarks = [
+	"frame-benchmarking",
 	"frame-support/runtime-benchmarks",
 	"frame-system/runtime-benchmarks",
 	"primitives/runtime-benchmarks",
diff --git a/runtime/parachains/src/builder.rs b/runtime/parachains/src/builder.rs
index 2ec0d4504d19..78d601346f2c 100644
--- a/runtime/parachains/src/builder.rs
+++ b/runtime/parachains/src/builder.rs
@@ -4,7 +4,7 @@ use crate::{
 	scheduler, session_info, shared,
 };
 use bitvec::{order::Lsb0 as BitOrderLsb0, vec::BitVec};
-use frame_benchmarking::{account, vec, Vec};
+use sp_std::{vec, prelude::Vec};
 use frame_support::pallet_prelude::*;
 use primitives::v1::{
 	collator_signature_payload, AvailabilityBitfield, BackedCandidate, CandidateCommitments,
@@ -22,6 +22,20 @@ use sp_runtime::{
 };
 use sp_std::{collections::btree_map::BTreeMap, convert::TryInto};
 
+/// Grab an account, seeded by a name and index.
+///
+/// This is directly from frame-benchmarking. Copy/Pasted so we can use it when not compiling with
+/// "features = runtime-benchmarks"
+fn account<AccountId: Decode + Default>(
+	name: &'static str,
+	index: u32,
+	seed: u32,
+) -> AccountId {
+	let entropy = (name, index, seed).using_encoded(sp_core::blake2_256);
+	AccountId::decode(&mut &entropy[..]).unwrap_or_default()
+}
+
+/// Create a 32 byte slice based on the given number.
 fn byte32_slice_from(n: u32) -> [u8; 32] {
 	let mut slice = [0u8; 32];
 	slice[31] = (n % (1 << 8)) as u8;
@@ -562,7 +576,7 @@ impl<T: paras_inherent::Config> BenchBuilder<T> {
 		// Mark all the use cores as occupied. We expect that their are `backed_and_concluding_cores`
 		// that are pending availability and that there are `non_spam_dispute_cores` which are about
 		// to be disputed.
-		scheduler::AvailabilityCores::<T>::set(frame_benchmarking::vec![
+		scheduler::AvailabilityCores::<T>::set(vec![
 			Some(CoreOccupied::Parachain);
 			used_cores as usize
 		]);
diff --git a/runtime/parachains/src/shared.rs b/runtime/parachains/src/shared.rs
index 28242e1a7637..9b5aa5c59629 100644
--- a/runtime/parachains/src/shared.rs
+++ b/runtime/parachains/src/shared.rs
@@ -49,7 +49,7 @@ pub mod pallet {
 	/// The current session index.
 	#[pallet::storage]
 	#[pallet::getter(fn session_index)]
-	pub(crate) type CurrentSessionIndex<T: Config> = StorageValue<_, SessionIndex, ValueQuery>;
+	pub(super) type CurrentSessionIndex<T: Config> = StorageValue<_, SessionIndex, ValueQuery>;
 
 	/// All the validators actively participating in parachain consensus.
 	/// Indices are into the broader validator set.
diff --git a/runtime/polkadot/Cargo.toml b/runtime/polkadot/Cargo.toml
index 0217cc334428..f96203a51089 100644
--- a/runtime/polkadot/Cargo.toml
+++ b/runtime/polkadot/Cargo.toml
@@ -192,6 +192,7 @@ runtime-benchmarks = [
 	"frame-system-benchmarking",
 	"hex-literal",
 	"frame-election-provider-support/runtime-benchmarks",
+	"runtime-parachains/runtime-benchmarks",
 ]
 try-runtime = [
 	"frame-executive/try-runtime",
diff --git a/runtime/rococo/Cargo.toml b/runtime/rococo/Cargo.toml
index 251bc210965d..61878123125c 100644
--- a/runtime/rococo/Cargo.toml
+++ b/runtime/rococo/Cargo.toml
@@ -178,7 +178,7 @@ runtime-benchmarks = [
 	"pallet-multisig/runtime-benchmarks",
 	"frame-benchmarking",
 	"hex-literal",
-	"runtime-parachains/runtime-benchmarks"
+	"runtime-parachains/runtime-benchmarks",
 ]
 try-runtime = [
 	"frame-executive/try-runtime",
diff --git a/runtime/westend/Cargo.toml b/runtime/westend/Cargo.toml
index 65a43599440d..358ae192014c 100644
--- a/runtime/westend/Cargo.toml
+++ b/runtime/westend/Cargo.toml
@@ -206,6 +206,7 @@ runtime-benchmarks = [
 	"pallet-xcm-benchmarks",
 	"frame-election-provider-support/runtime-benchmarks",
 	"pallet-bags-list/runtime-benchmarks",
+	"runtime-parachains/runtime-benchmarks",
 ]
 try-runtime = [
 	"frame-executive/try-runtime",

From ef35d97e1ec9acd43404d7a14ea63e9d65659faf Mon Sep 17 00:00:00 2001
From: emostov <32168567+emostov@users.noreply.github.com>
Date: Mon, 8 Nov 2021 17:52:23 +0100
Subject: [PATCH 074/107] More precise feature gating for some derives

---
 parachain/src/primitives.rs              | 15 ++----
 primitives/src/v1/mod.rs                 | 58 ++++++++++++++----------
 runtime/parachains/src/builder.rs        |  9 +---
 runtime/parachains/src/paras_inherent.rs | 22 ++++-----
 4 files changed, 49 insertions(+), 55 deletions(-)

diff --git a/parachain/src/primitives.rs b/parachain/src/primitives.rs
index df5b6d299bc8..98135df0802b 100644
--- a/parachain/src/primitives.rs
+++ b/parachain/src/primitives.rs
@@ -41,19 +41,10 @@ pub use polkadot_core_primitives::BlockNumber as RelayChainBlockNumber;
 
 /// Parachain head data included in the chain.
 #[derive(
-	Default,
-	PartialEq,
-	Eq,
-	Clone,
-	PartialOrd,
-	Ord,
-	Encode,
-	Decode,
-	RuntimeDebug,
-	derive_more::From,
-	TypeInfo,
+	PartialEq, Eq, Clone, PartialOrd, Ord, Encode, Decode, RuntimeDebug, derive_more::From, TypeInfo,
 )]
 #[cfg_attr(feature = "std", derive(Serialize, Deserialize, Hash, MallocSizeOf))]
+#[cfg_attr(any(feature = "std", feature = "runtime-benchmarks"), derive(Default))]
 pub struct HeadData(#[cfg_attr(feature = "std", serde(with = "bytes"))] pub Vec<u8>);
 
 impl HeadData {
@@ -326,7 +317,7 @@ impl<T: Encode + Decode + Default> AccountIdConversion<T> for Id {
 	fn try_from_account(x: &T) -> Option<Self> {
 		x.using_encoded(|d| {
 			if &d[0..4] != b"para" {
-				return None
+				return None;
 			}
 			let mut cursor = &d[4..];
 			let result = Decode::decode(&mut cursor).ok()?;
diff --git a/primitives/src/v1/mod.rs b/primitives/src/v1/mod.rs
index f3e77d101653..293373821bdd 100644
--- a/primitives/src/v1/mod.rs
+++ b/primitives/src/v1/mod.rs
@@ -327,8 +327,9 @@ fn check_collator_signature<H: AsRef<[u8]>>(
 }
 
 /// A unique descriptor of the candidate receipt.
-#[derive(Default, PartialEq, Eq, Clone, Encode, Decode, TypeInfo)]
+#[derive(PartialEq, Eq, Clone, Encode, Decode, TypeInfo)]
 #[cfg_attr(feature = "std", derive(Debug, Hash, MallocSizeOf))]
+#[cfg_attr(any(feature = "std", feature = "runtime-benchmarks"), derive(Default))]
 pub struct CandidateDescriptor<H = Hash> {
 	/// The ID of the para this is a candidate for.
 	pub para_id: Id,
@@ -407,8 +408,9 @@ pub struct FullCandidateReceipt<H = Hash, N = BlockNumber> {
 }
 
 /// A candidate-receipt with commitments directly included.
-#[derive(Default, PartialEq, Eq, Clone, Encode, Decode, TypeInfo)]
+#[derive(PartialEq, Eq, Clone, Encode, Decode, TypeInfo)]
 #[cfg_attr(feature = "std", derive(Debug, Hash, MallocSizeOf))]
+#[cfg_attr(any(feature = "std", feature = "runtime-benchmarks"), derive(Default))]
 pub struct CommittedCandidateReceipt<H = Hash> {
 	/// The descriptor of the candidate.
 	pub descriptor: CandidateDescriptor<H>,
@@ -509,8 +511,9 @@ impl<H: Encode, N: Encode> PersistedValidationData<H, N> {
 }
 
 /// Commitments made in a `CandidateReceipt`. Many of these are outputs of validation.
-#[derive(Default, PartialEq, Eq, Clone, Encode, Decode, TypeInfo)]
+#[derive(PartialEq, Eq, Clone, Encode, Decode, TypeInfo)]
 #[cfg_attr(feature = "std", derive(Debug, Hash, MallocSizeOf))]
+#[cfg_attr(any(feature = "std", feature = "runtime-benchmarks"), derive(Default))]
 pub struct CandidateCommitments<N = BlockNumber> {
 	/// Messages destined to be interpreted by the Relay chain itself.
 	pub upward_messages: Vec<UpwardMessage>,
@@ -610,11 +613,11 @@ pub fn check_candidate_backing<H: AsRef<[u8]> + Clone + Encode>(
 	validator_lookup: impl Fn(usize) -> Option<ValidatorId>,
 ) -> Result<usize, ()> {
 	if backed.validator_indices.len() != group_len {
-		return Err(())
+		return Err(());
 	}
 
 	if backed.validity_votes.len() > group_len {
-		return Err(())
+		return Err(());
 	}
 
 	// this is known, even in runtime, to be blake2-256.
@@ -635,12 +638,12 @@ pub fn check_candidate_backing<H: AsRef<[u8]> + Clone + Encode>(
 		if sig.verify(&payload[..], &validator_id) {
 			signed += 1;
 		} else {
-			return Err(())
+			return Err(());
 		}
 	}
 
 	if signed != backed.validity_votes.len() {
-		return Err(())
+		return Err(());
 	}
 
 	Ok(signed)
@@ -712,10 +715,10 @@ impl GroupRotationInfo {
 	/// `core_index` should be less than `cores`, which is capped at `u32::max()`.
 	pub fn group_for_core(&self, core_index: CoreIndex, cores: usize) -> GroupIndex {
 		if self.group_rotation_frequency == 0 {
-			return GroupIndex(core_index.0)
+			return GroupIndex(core_index.0);
 		}
 		if cores == 0 {
-			return GroupIndex(0)
+			return GroupIndex(0);
 		}
 
 		let cores = sp_std::cmp::min(cores, u32::MAX as usize);
@@ -734,10 +737,10 @@ impl GroupRotationInfo {
 	/// `core_index` should be less than `cores`, which is capped at `u32::max()`.
 	pub fn core_for_group(&self, group_index: GroupIndex, cores: usize) -> CoreIndex {
 		if self.group_rotation_frequency == 0 {
-			return CoreIndex(group_index.0)
+			return CoreIndex(group_index.0);
 		}
 		if cores == 0 {
-			return CoreIndex(0)
+			return CoreIndex(0);
 		}
 
 		let cores = sp_std::cmp::min(cores, u32::MAX as usize);
@@ -769,15 +772,15 @@ impl<N: Saturating + BaseArithmetic + Copy> GroupRotationInfo<N> {
 	/// is 10 and the rotation frequency is 5, this should return 15.
 	pub fn next_rotation_at(&self) -> N {
 		let cycle_once = self.now + self.group_rotation_frequency;
-		cycle_once -
-			(cycle_once.saturating_sub(self.session_start_block) % self.group_rotation_frequency)
+		cycle_once
+			- (cycle_once.saturating_sub(self.session_start_block) % self.group_rotation_frequency)
 	}
 
 	/// Returns the block number of the last rotation before or including the current block. If the
 	/// current block is 10 and the rotation frequency is 5, this should return 10.
 	pub fn last_rotation_at(&self) -> N {
-		self.now -
-			(self.now.saturating_sub(self.session_start_block) % self.group_rotation_frequency)
+		self.now
+			- (self.now.saturating_sub(self.session_start_block) % self.group_rotation_frequency)
 	}
 }
 
@@ -1194,8 +1197,9 @@ impl ConsensusLog {
 		digest_item: &runtime_primitives::DigestItem<H>,
 	) -> Result<Option<Self>, parity_scale_codec::Error> {
 		match digest_item {
-			runtime_primitives::DigestItem::Consensus(id, encoded) if id == &POLKADOT_ENGINE_ID =>
-				Ok(Some(Self::decode(&mut &encoded[..])?)),
+			runtime_primitives::DigestItem::Consensus(id, encoded) if id == &POLKADOT_ENGINE_ID => {
+				Ok(Some(Self::decode(&mut &encoded[..])?))
+			}
 			_ => Ok(None),
 		}
 	}
@@ -1225,23 +1229,27 @@ impl DisputeStatement {
 	/// Get the payload data for this type of dispute statement.
 	pub fn payload_data(&self, candidate_hash: CandidateHash, session: SessionIndex) -> Vec<u8> {
 		match *self {
-			DisputeStatement::Valid(ValidDisputeStatementKind::Explicit) =>
-				ExplicitDisputeStatement { valid: true, candidate_hash, session }.signing_payload(),
+			DisputeStatement::Valid(ValidDisputeStatementKind::Explicit) => {
+				ExplicitDisputeStatement { valid: true, candidate_hash, session }.signing_payload()
+			}
 			DisputeStatement::Valid(ValidDisputeStatementKind::BackingSeconded(
 				inclusion_parent,
 			)) => CompactStatement::Seconded(candidate_hash).signing_payload(&SigningContext {
 				session_index: session,
 				parent_hash: inclusion_parent,
 			}),
-			DisputeStatement::Valid(ValidDisputeStatementKind::BackingValid(inclusion_parent)) =>
+			DisputeStatement::Valid(ValidDisputeStatementKind::BackingValid(inclusion_parent)) => {
 				CompactStatement::Valid(candidate_hash).signing_payload(&SigningContext {
 					session_index: session,
 					parent_hash: inclusion_parent,
-				}),
-			DisputeStatement::Valid(ValidDisputeStatementKind::ApprovalChecking) =>
-				ApprovalVote(candidate_hash).signing_payload(session),
-			DisputeStatement::Invalid(InvalidDisputeStatementKind::Explicit) =>
-				ExplicitDisputeStatement { valid: false, candidate_hash, session }.signing_payload(),
+				})
+			}
+			DisputeStatement::Valid(ValidDisputeStatementKind::ApprovalChecking) => {
+				ApprovalVote(candidate_hash).signing_payload(session)
+			}
+			DisputeStatement::Invalid(InvalidDisputeStatementKind::Explicit) => {
+				ExplicitDisputeStatement { valid: false, candidate_hash, session }.signing_payload()
+			}
 		}
 	}
 
diff --git a/runtime/parachains/src/builder.rs b/runtime/parachains/src/builder.rs
index 78d601346f2c..81153701b019 100644
--- a/runtime/parachains/src/builder.rs
+++ b/runtime/parachains/src/builder.rs
@@ -4,7 +4,6 @@ use crate::{
 	scheduler, session_info, shared,
 };
 use bitvec::{order::Lsb0 as BitOrderLsb0, vec::BitVec};
-use sp_std::{vec, prelude::Vec};
 use frame_support::pallet_prelude::*;
 use primitives::v1::{
 	collator_signature_payload, AvailabilityBitfield, BackedCandidate, CandidateCommitments,
@@ -20,17 +19,13 @@ use sp_runtime::{
 	traits::{Header as HeaderT, One, Zero},
 	RuntimeAppPublic,
 };
-use sp_std::{collections::btree_map::BTreeMap, convert::TryInto};
+use sp_std::{collections::btree_map::BTreeMap, convert::TryInto, prelude::Vec, vec};
 
 /// Grab an account, seeded by a name and index.
 ///
 /// This is directly from frame-benchmarking. Copy/Pasted so we can use it when not compiling with
 /// "features = runtime-benchmarks"
-fn account<AccountId: Decode + Default>(
-	name: &'static str,
-	index: u32,
-	seed: u32,
-) -> AccountId {
+fn account<AccountId: Decode + Default>(name: &'static str, index: u32, seed: u32) -> AccountId {
 	let entropy = (name, index, seed).using_encoded(sp_core::blake2_256);
 	AccountId::decode(&mut &entropy[..]).unwrap_or_default()
 }
diff --git a/runtime/parachains/src/paras_inherent.rs b/runtime/parachains/src/paras_inherent.rs
index e40443ea39cb..89d045f25a7c 100644
--- a/runtime/parachains/src/paras_inherent.rs
+++ b/runtime/parachains/src/paras_inherent.rs
@@ -264,7 +264,7 @@ pub mod pallet {
 							disputes: Vec::new(),
 							parent_header: inherent_data.parent_header,
 						}
-					}
+					},
 				};
 
 			Some(Call::enter { data: inherent_data })
@@ -329,7 +329,7 @@ pub mod pallet {
 				if T::DisputesHandler::is_frozen() {
 					// The relay chain we are currently on is invalid. Proceed no further on parachains.
 					Included::<T>::set(Some(()));
-					return Ok(Some(minimal_inherent_weight::<T>()).into());
+					return Ok(Some(minimal_inherent_weight::<T>()).into())
 				}
 
 				let mut freed_disputed = if !new_current_dispute_sets.is_empty() {
@@ -470,8 +470,8 @@ impl<T: Config> Pallet<T> {
 			Ok(None) => return None,
 			Err(_) => {
 				log::warn!(target: LOG_TARGET, "ParachainsInherentData failed to decode");
-				return None;
-			}
+				return None
+			},
 		};
 
 		let parent_hash = <frame_system::Pallet<T>>::parent_hash();
@@ -480,7 +480,7 @@ impl<T: Config> Pallet<T> {
 				target: LOG_TARGET,
 				"ParachainsInherentData references a different parent header hash than frame"
 			);
-			return None;
+			return None
 		}
 
 		let current_session = <shared::Pallet<T>>::session_index();
@@ -694,7 +694,7 @@ fn random_sel<X, F: Fn(&X) -> Weight>(
 	weight_limit: Weight,
 ) -> (Weight, Vec<usize>) {
 	if selectables.is_empty() {
-		return (0 as Weight, Vec::new());
+		return (0 as Weight, Vec::new())
 	}
 	let mut indices = (0..selectables.len()).into_iter().collect::<Vec<_>>();
 	let mut picked_indices = Vec::with_capacity(selectables.len().saturating_sub(1));
@@ -710,7 +710,7 @@ fn random_sel<X, F: Fn(&X) -> Weight>(
 		weight_acc += weight_fn(item);
 
 		if weight_acc > weight_limit {
-			break;
+			break
 		}
 
 		picked_indices.push(idx);
@@ -743,7 +743,7 @@ fn apply_weight_limit<T: Config + inclusion::Config>(
 
 	// everything fits into the block
 	if max_weight >= total {
-		return (total, candidates, bitfields);
+		return (total, candidates, bitfields)
 	}
 
 	use rand_chacha::rand_core::SeedableRng;
@@ -765,7 +765,7 @@ fn apply_weight_limit<T: Config + inclusion::Config>(
 		// pick all bitfields, and
 		// fill the remaining space with candidates
 		let total = acc_candidate_weight.saturating_add(total_bitfields_weight);
-		return (total, candidates, bitfields);
+		return (total, candidates, bitfields)
 	}
 
 	// insufficient space for even the bitfields alone, so only try to fit as many of those
@@ -830,8 +830,8 @@ pub(crate) fn sanitize_bitfields<T: crate::inclusion::Config, const EARLY_RETURN
 		);
 
 		ensure2!(
-			unchecked_bitfield.unchecked_payload().0.clone() & disputed_bitfield.0.clone()
-				== all_zeros,
+			unchecked_bitfield.unchecked_payload().0.clone() & disputed_bitfield.0.clone() ==
+				all_zeros,
 			crate::inclusion::Error::<T>::BitfieldReferencesFreedCore,
 			EARLY_RETURN,
 			continue

From 45367411cd217c6709a33c357bec5efd41237b7d Mon Sep 17 00:00:00 2001
From: Bernhard Schuster <bernhard@ahoi.io>
Date: Mon, 8 Nov 2021 17:46:48 +0100
Subject: [PATCH 075/107] integrate piece-wise

---
 .../src/requester/mod.rs                      |   2 +-
 runtime/parachains/src/inclusion.rs           |   6 +-
 runtime/parachains/src/paras_inherent.rs      | 132 ++++++++----------
 3 files changed, 61 insertions(+), 79 deletions(-)

diff --git a/node/network/availability-distribution/src/requester/mod.rs b/node/network/availability-distribution/src/requester/mod.rs
index f678a768d61f..53804aae8723 100644
--- a/node/network/availability-distribution/src/requester/mod.rs
+++ b/node/network/availability-distribution/src/requester/mod.rs
@@ -159,7 +159,7 @@ impl Requester {
 				// Just book keeping - we are already requesting that chunk:
 				{
 					e.get_mut().add_leaf(leaf);
-				}
+				},
 				Entry::Vacant(e) => {
 					let tx = self.tx.clone();
 					let metrics = self.metrics.clone();
diff --git a/runtime/parachains/src/inclusion.rs b/runtime/parachains/src/inclusion.rs
index b9a5f1c59c7a..ec04be087f81 100644
--- a/runtime/parachains/src/inclusion.rs
+++ b/runtime/parachains/src/inclusion.rs
@@ -32,8 +32,8 @@ use parity_scale_codec::{Decode, Encode};
 use primitives::v1::{
 	AvailabilityBitfield, BackedCandidate, CandidateCommitments, CandidateDescriptor,
 	CandidateHash, CandidateReceipt, CommittedCandidateReceipt, CoreIndex, GroupIndex, Hash,
-	HeadData, Id as ParaId, SigningContext, UncheckedSignedAvailabilityBitfields, ValidatorIndex,
-	ValidityAttestation, ValidatorId,
+	HeadData, Id as ParaId, SigningContext, UncheckedSignedAvailabilityBitfields, ValidatorId,
+	ValidatorIndex, ValidityAttestation,
 };
 use scale_info::TypeInfo;
 use sp_runtime::{
@@ -300,7 +300,6 @@ impl<T: Config> Pallet<T> {
 		let session_index = shared::Pallet::<T>::session_index();
 		let parent_hash = frame_system::Pallet::<T>::parent_hash();
 
-
 		let mut assigned_paras_record = (0..expected_bits)
 			.map(|bit_index| core_lookup(CoreIndex::from(bit_index as u32)))
 			.map(|opt_para_id| {
@@ -421,7 +420,6 @@ impl<T: Config> Pallet<T> {
 			core_lookup,
 		);
 
-
 		Ok(freed_cores)
 	}
 
diff --git a/runtime/parachains/src/paras_inherent.rs b/runtime/parachains/src/paras_inherent.rs
index 728abdb31a9a..89932707b18d 100644
--- a/runtime/parachains/src/paras_inherent.rs
+++ b/runtime/parachains/src/paras_inherent.rs
@@ -279,8 +279,9 @@ pub mod pallet {
 	///
 	/// The paramter `freed_concluded` contains all core indicies that became
 	/// free due to candidate that became available.
-	pub(crate) fn collect_all_freed_cores<T>(freed_concluded: Vec<CoreIndex>) -> BTreeMap<CoreIndex, FreedReason> {
-
+	pub(crate) fn collect_all_freed_cores<T>(
+		freed_concluded: Vec<CoreIndex>,
+	) -> BTreeMap<CoreIndex, FreedReason> {
 		// Handle timeouts for any availability core work.
 		let availability_pred = <scheduler::Pallet<T>>::availability_timeout_predicate();
 		let freed_timeout = if let Some(pred) = availability_pred {
@@ -352,7 +353,7 @@ pub mod pallet {
 				if T::DisputesHandler::is_frozen() {
 					// The relay chain we are currently on is invalid. Proceed no further on parachains.
 					Included::<T>::set(Some(()));
-					return Ok(Some(minimal_inherent_weight::<T>()).into());
+					return Ok(Some(minimal_inherent_weight::<T>()).into())
 				}
 
 				let mut freed_disputed = if !new_current_dispute_sets.is_empty() {
@@ -485,6 +486,21 @@ impl<T: Config> Pallet<T> {
 		};
 
 		let parent_hash = <frame_system::Pallet<T>>::parent_hash();
+
+		let ParachainsInherentData::<T::Header> {
+			bitfields,
+			backed_candidates,
+			mut disputes,
+			parent_header,
+		} = match data.get_data(&Self::INHERENT_IDENTIFIER) {
+			Ok(Some(d)) => d,
+			Ok(None) => return None,
+			Err(_) => {
+				log::warn!(target: LOG_TARGET, "ParachainsInherentData failed to decode");
+				return None
+			},
+		};
+
 		if parent_hash != parent_header.hash() {
 			log::warn!(
 				target: LOG_TARGET,
@@ -514,7 +530,7 @@ impl<T: Config> Pallet<T> {
 						e
 					});
 
-				// concluded invalid disputes, only including the _current block's_ votes
+				// current concluded invalid disputes, only including the current block's votes
 				let current_concluded_invalid_disputes = disputes
 					.iter()
 					.filter(|dss| dss.session == current_session)
@@ -535,75 +551,56 @@ impl<T: Config> Pallet<T> {
 					})
 					.collect::<BTreeSet<CandidateHash>>();
 
-				let dispute_freed_cores =
-					<inclusion::Pallet<T>>::collect_disputed(&current_concluded_invalid_disputes);
-				let disputed_bitfield =
-					create_disputed_bitfield(expected_bits, dispute_freed_cores.iter());
-
-				// Below Operations:
-				// * free disputed cores if any exist
-				// * get cores that have become free from processing fully bitfields
-				// * get cores that have become free from timing out
-				// * create  one collection of all the freed cores so we can schedule them
-				// * schedule freed cores based on collection of freed cores
-
-				{
-					let mut freed_disputed: Vec<_> = <inclusion::Pallet<T>>::collect_disputed(
-						&current_concluded_invalid_disputes,
+				let freed_disputed: Vec<_> = <inclusion::Pallet<T>>::collect_disputed(
+						&current_concluded_invalid_disputes
 					)
 					.into_iter()
 					.map(|core| (core, FreedReason::Concluded))
 					.collect();
 
-					if !freed_disputed.is_empty() {
-						// There are freed disputed cores, so sort them and free them against the scheduler.
-
-						// unstable sort is fine, because core indices are unique
-						// i.e. the same candidate can't occupy 2 cores at once.
-						freed_disputed.sort_unstable_by_key(|pair| pair.0); // sort by core index
+				let disputed_bitfield = create_disputed_bitfield(expected_bits, freed_disputed.iter());
 
-						<scheduler::Pallet<T>>::free_cores(freed_disputed);
-					}
+				if !freed_disputed.is_empty() {
+					// unstable sort is fine, because core indices are unique
+					// i.e. the same candidate can't occupy 2 cores at once.
+					freed_disputed.sort_unstable_by_key(|pair| pair.0); // sort by core index
+					<scheduler::Pallet<T>>::free_cores(freed_disputed.clone());
 				}
 
-				// Get cores that have become free from processing fully bitfields
-				let freed_concluded = <inclusion::Pallet<T>>::process_bitfields(
+				// The following 3 calls are equiv to a call to `process_bitfields`
+				// but we can retain access to `bitfields`.
+				let bitfields = match sanitize_bitfields::<T, false>(
+					bitfields,
+					disputed_bitfield,
 					expected_bits,
-					bitfields.clone(),
-					disputed_bitfield.clone(),
-					<scheduler::Pallet<T>>::core_para,
-					true,
-				)
-				.unwrap_or_else(|err| {
-					log::error!(
-						target: LOG_TARGET,
-						"bitfields could not be processed while creating inherent: {:?}",
-						err,
-					);
-					Vec::new()
-				});
-
-				// Get cores that have become free from timing out
-				let availability_pred = <scheduler::Pallet<T>>::availability_timeout_predicate();
-				let freed_timeout = if let Some(pred) = availability_pred {
-					<inclusion::Pallet<T>>::collect_pending(pred)
-				} else {
-					Vec::new()
+					parent_hash,
+					current_session,
+					&validator_public[..],
+				) {
+					Ok(bitfields) => bitfields,
+					Err(err) => {
+						// by convention, when called with `EARLY_RETURN=false`, will always return `Ok()`
+						log::error!(target: LOG_TARGET, ?err, "BUG: convention violation in create_inherent");
+						vec![]
+					},
 				};
 
-				// Create one collection of cores freed from timeouts and availability conclusion ..
-				let timeout_and_concluded_freed_cores = freed_concluded
-					.into_iter()
-					.map(|(c, _hash)| (c, FreedReason::Concluded))
-					.chain(freed_timeout.into_iter().map(|c| (c, FreedReason::TimedOut)))
-					.collect::<BTreeMap<CoreIndex, FreedReason>>();
+				let freed_concluded =
+					<inclusion::Pallet<T>>::update_pending_availability_and_get_freed_cores(
+						expected_bits,
+						&validator_public[..],
+						bitfields,
+						disputed_bitfield,
+						<scheduler::Pallet<T>>::core_para,
+					);
+				let freed = <crate::paras_inherent::Pallet<T>>::collect_all_freed_cores::<T>(
+					freed_concluded,
+				);
 
 				<scheduler::Pallet<T>>::clear();
-				// .. so we can schedule them.
-				<scheduler::Pallet<T>>::schedule(
-					timeout_and_concluded_freed_cores,
-					<frame_system::Pallet<T>>::block_number(),
-				);
+				<scheduler::Pallet<T>>::schedule(freed, <frame_system::Pallet<T>>::block_number());
+
+				let scheduled = <scheduler::Pallet<T>>::scheduled();
 
 				// Return
 				frame_support::storage::TransactionOutcome::Rollback((
@@ -611,23 +608,10 @@ impl<T: Config> Pallet<T> {
 					concluded_invalid_disputes,
 					// * bitfield marking disputed cores,
 					disputed_bitfield,
-					// * and the newly created core schedule.
-					<scheduler::Pallet<T>>::scheduled(),
+					scheduled,
 				))
 			});
 
-		// TODO this is wasteful since its also called in `process_bitfield`. We should probably
-		// bubble up the bitfields from `process_bitfields` so this does not need to be called here.
-		let bitfields = sanitize_bitfields::<T, false>(
-			bitfields,
-			disputed_bitfield,
-			expected_bits,
-			parent_hash,
-			current_session,
-			&validator_public[..],
-		)
-		.ok()?; // by convention, when called with `EARLY_RETURN=false`, will always return `Ok()`
-
 		let backed_candidates = sanitize_backed_candidates::<T, _, false>(
 			parent_hash,
 			backed_candidates,

From 5b1f1c0c0da1ed9736ba52c0394e9b332df6ff4e Mon Sep 17 00:00:00 2001
From: Bernhard Schuster <bernhard@ahoi.io>
Date: Mon, 8 Nov 2021 18:26:49 +0100
Subject: [PATCH 076/107] foo

---
 runtime/parachains/src/inclusion.rs      | 6 +++---
 runtime/parachains/src/paras_inherent.rs | 2 +-
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/runtime/parachains/src/inclusion.rs b/runtime/parachains/src/inclusion.rs
index ec04be087f81..dd492e069b1c 100644
--- a/runtime/parachains/src/inclusion.rs
+++ b/runtime/parachains/src/inclusion.rs
@@ -289,7 +289,7 @@ impl<T: Config> Pallet<T> {
 	/// Extract the freed cores based on cores tht became available.
 	///
 	/// Updates storage items `PendingAvailability` and `AvailabilityBitfields`.
-	pub(crate) fn update_pending_availability_and_get_freed_cores<IS_CREATE_INHERENT: bool>(
+	pub(crate) fn update_pending_availability_and_get_freed_cores<const IS_CREATE_INHERENT: bool>(
 		expected_bits: usize,
 		validators: &[ValidatorId],
 		signed_bitfields: UncheckedSignedAvailabilityBitfields,
@@ -394,7 +394,7 @@ impl<T: Config> Pallet<T> {
 	///
 	/// Returns a `Vec` of `CandidateHash`es and their respective `AvailabilityCore`s that became available,
 	/// and cores free.
-	pub(crate) fn process_bitfields<IS_CREATE_INHERENT: bool>(
+	pub(crate) fn process_bitfields(
 		expected_bits: usize,
 		signed_bitfields: UncheckedSignedAvailabilityBitfields,
 		disputed_bitfield: DisputedBitfield,
@@ -404,7 +404,7 @@ impl<T: Config> Pallet<T> {
 		let session_index = shared::Pallet::<T>::session_index();
 		let parent_hash = frame_system::Pallet::<T>::parent_hash();
 
-		let checked_bitfields = sanitize_bitfields::<T, IS_CREATE_INHERENT>(
+		let checked_bitfields = sanitize_bitfields::<T, false>(
 			signed_bitfields,
 			disputed_bitfield,
 			expected_bits,
diff --git a/runtime/parachains/src/paras_inherent.rs b/runtime/parachains/src/paras_inherent.rs
index 89932707b18d..8eee53b07c07 100644
--- a/runtime/parachains/src/paras_inherent.rs
+++ b/runtime/parachains/src/paras_inherent.rs
@@ -580,7 +580,7 @@ impl<T: Config> Pallet<T> {
 					Ok(bitfields) => bitfields,
 					Err(err) => {
 						// by convention, when called with `EARLY_RETURN=false`, will always return `Ok()`
-						log::error!(target: LOG_TARGET, ?err, "BUG: convention violation in create_inherent");
+						log::error!(target: LOG_TARGET, "BUG: convention violation in create_inherent: {:?}", err);
 						vec![]
 					},
 				};

From fa0de2e38b700fba1dd1f3c5ebda4c72230491f4 Mon Sep 17 00:00:00 2001
From: Bernhard Schuster <bernhard@ahoi.io>
Date: Mon, 8 Nov 2021 19:28:32 +0100
Subject: [PATCH 077/107] fixins

---
 runtime/parachains/src/inclusion.rs      | 60 ++++++++----------
 runtime/parachains/src/paras_inherent.rs | 78 +++++++++++-------------
 runtime/parachains/src/scheduler.rs      |  1 +
 3 files changed, 63 insertions(+), 76 deletions(-)

diff --git a/runtime/parachains/src/inclusion.rs b/runtime/parachains/src/inclusion.rs
index dd492e069b1c..cd2637dd245b 100644
--- a/runtime/parachains/src/inclusion.rs
+++ b/runtime/parachains/src/inclusion.rs
@@ -289,17 +289,15 @@ impl<T: Config> Pallet<T> {
 	/// Extract the freed cores based on cores tht became available.
 	///
 	/// Updates storage items `PendingAvailability` and `AvailabilityBitfields`.
-	pub(crate) fn update_pending_availability_and_get_freed_cores<const IS_CREATE_INHERENT: bool>(
+	pub(crate) fn update_pending_availability_and_get_freed_cores<F, const IS_CREATE_INHERENT: bool>(
 		expected_bits: usize,
 		validators: &[ValidatorId],
 		signed_bitfields: UncheckedSignedAvailabilityBitfields,
-		disputed_bitfield: DisputedBitfield,
-		core_lookup: impl Fn(CoreIndex) -> Option<ParaId>,
-	) -> Vec<(CoreIndex, CandidateHash)> {
-		let validators = shared::Pallet::<T>::active_validator_keys();
-		let session_index = shared::Pallet::<T>::session_index();
-		let parent_hash = frame_system::Pallet::<T>::parent_hash();
-
+		core_lookup: F,
+	) -> Vec<(CoreIndex, CandidateHash)>
+	where
+		F:  Fn(CoreIndex) -> Option<ParaId>,
+	{
 		let mut assigned_paras_record = (0..expected_bits)
 			.map(|bit_index| core_lookup(CoreIndex::from(bit_index as u32)))
 			.map(|opt_para_id| {
@@ -366,25 +364,26 @@ impl<T: Config> Pallet<T> {
 						continue
 					},
 				};
+
+				if !IS_CREATE_INHERENT {
+					let receipt = CommittedCandidateReceipt {
+						descriptor: pending_availability.descriptor,
+						commitments,
+					};
+					let _weight = Self::enact_candidate(
+						pending_availability.relay_parent_number,
+						receipt,
+						pending_availability.backers,
+						pending_availability.availability_votes,
+						pending_availability.core,
+						pending_availability.backing_group,
+					);
+				}
+
 				freed_cores.push((pending_availability.core, pending_availability.hash));
 			} else {
 				<PendingAvailability<T>>::insert(&para_id, &pending_availability);
 			}
-
-			if !IS_CREATE_INHERENT {
-				let receipt = CommittedCandidateReceipt {
-					descriptor: pending_availability.descriptor,
-					commitments,
-				};
-				let _weight = Self::enact_candidate(
-					pending_availability.relay_parent_number,
-					receipt,
-					pending_availability.backers,
-					pending_availability.availability_votes,
-					pending_availability.core,
-					pending_availability.backing_group,
-				);
-			}
 		}
 
 		freed_cores
@@ -399,7 +398,7 @@ impl<T: Config> Pallet<T> {
 		signed_bitfields: UncheckedSignedAvailabilityBitfields,
 		disputed_bitfield: DisputedBitfield,
 		core_lookup: impl Fn(CoreIndex) -> Option<ParaId>,
-	) -> Result<Vec<(CoreIndex, ParaId)>, DispatchError> {
+	) -> Result<Vec<(CoreIndex, CandidateHash)>, DispatchError> {
 		let validators = shared::Pallet::<T>::active_validator_keys();
 		let session_index = shared::Pallet::<T>::session_index();
 		let parent_hash = frame_system::Pallet::<T>::parent_hash();
@@ -413,10 +412,10 @@ impl<T: Config> Pallet<T> {
 			&validators[..],
 		)?;
 
-		let freed_cores = Self::update_pending_availability_and_get_freed_cores::<false>(
+		let freed_cores = Self::update_pending_availability_and_get_freed_cores::<_, false>(
 			expected_bits,
 			&validators[..],
-			disputed_bitfield,
+			checked_bitfields,
 			core_lookup,
 		);
 
@@ -1384,7 +1383,6 @@ pub(crate) mod tests {
 						vec![signed.into()],
 						DisputedBitfield::zeros(expected_bits()),
 						&core_lookup,
-						false
 					),
 					Error::<Test>::WrongBitfieldSize
 				);
@@ -1407,7 +1405,6 @@ pub(crate) mod tests {
 						vec![signed.into()],
 						DisputedBitfield::zeros(expected_bits()),
 						&core_lookup,
-						false
 					),
 					Error::<Test>::WrongBitfieldSize
 				);
@@ -1431,7 +1428,6 @@ pub(crate) mod tests {
 						vec![signed.clone(), signed],
 						DisputedBitfield::zeros(expected_bits()),
 						&core_lookup,
-						false
 					),
 					Error::<Test>::BitfieldDuplicateOrUnordered
 				);
@@ -1464,7 +1460,6 @@ pub(crate) mod tests {
 						vec![signed_1, signed_0],
 						DisputedBitfield::zeros(expected_bits()),
 						&core_lookup,
-						false
 					),
 					Error::<Test>::BitfieldDuplicateOrUnordered
 				);
@@ -1488,7 +1483,6 @@ pub(crate) mod tests {
 						vec![signed.into()],
 						DisputedBitfield::zeros(expected_bits()),
 						&core_lookup,
-						false
 					),
 					Ok(_)
 				);
@@ -1511,7 +1505,6 @@ pub(crate) mod tests {
 						vec![signed.into()],
 						DisputedBitfield::zeros(expected_bits()),
 						&core_lookup,
-						false
 					),
 					Ok(_)
 				);
@@ -1557,7 +1550,6 @@ pub(crate) mod tests {
 						vec![signed.into()],
 						DisputedBitfield::zeros(expected_bits()),
 						&core_lookup,
-						false
 					),
 					Ok(_)
 				);
@@ -1603,7 +1595,6 @@ pub(crate) mod tests {
 						vec![signed.into()],
 						DisputedBitfield::zeros(expected_bits()),
 						&core_lookup,
-						false
 					),
 					Ok(vec![])
 				);
@@ -1748,7 +1739,6 @@ pub(crate) mod tests {
 					signed_bitfields,
 					DisputedBitfield::zeros(expected_bits()),
 					&core_lookup,
-					false
 				),
 				Ok(_)
 			);
diff --git a/runtime/parachains/src/paras_inherent.rs b/runtime/parachains/src/paras_inherent.rs
index 8eee53b07c07..8fb3436edf4f 100644
--- a/runtime/parachains/src/paras_inherent.rs
+++ b/runtime/parachains/src/paras_inherent.rs
@@ -279,9 +279,13 @@ pub mod pallet {
 	///
 	/// The paramter `freed_concluded` contains all core indicies that became
 	/// free due to candidate that became available.
-	pub(crate) fn collect_all_freed_cores<T>(
-		freed_concluded: Vec<CoreIndex>,
-	) -> BTreeMap<CoreIndex, FreedReason> {
+	pub(crate) fn collect_all_freed_cores<T, I>(
+		freed_concluded: I,
+	) -> BTreeMap<CoreIndex, FreedReason>
+	where
+		I: core::iter::IntoIterator<Item=(CoreIndex, CandidateHash)>,
+		T: Config,
+	{
 		// Handle timeouts for any availability core work.
 		let availability_pred = <scheduler::Pallet<T>>::availability_timeout_predicate();
 		let freed_timeout = if let Some(pred) = availability_pred {
@@ -400,7 +404,6 @@ pub mod pallet {
 				signed_bitfields,
 				disputed_bitfield,
 				<scheduler::Pallet<T>>::core_para,
-				false,
 			)?;
 
 			// Inform the disputes module of all included candidates.
@@ -409,7 +412,7 @@ pub mod pallet {
 				T::DisputesHandler::note_included(current_session, *candidate_hash, now);
 			}
 
-			let freed = collect_all_freed_cores::<T>(freed_concluded);
+			let freed = collect_all_freed_cores::<T,_>(freed_concluded.iter().cloned());
 
 			<scheduler::Pallet<T>>::clear();
 			<scheduler::Pallet<T>>::schedule(freed, now);
@@ -471,22 +474,6 @@ impl<T: Config> Pallet<T> {
 	/// Create the `ParachainsInherentData` that gets passed to `[`Self::enter`] in [`Self::create_inherent`].
 	/// This code is pulled out of [`Self::create_inherent`] so it can be unit tested.
 	fn create_inherent_inner(data: &InherentData) -> Option<ParachainsInherentData<T::Header>> {
-		let ParachainsInherentData::<T::Header> {
-			bitfields,
-			backed_candidates,
-			mut disputes,
-			parent_header,
-		} = match data.get_data(&Self::INHERENT_IDENTIFIER) {
-			Ok(Some(d)) => d,
-			Ok(None) => return None,
-			Err(_) => {
-				log::warn!(target: LOG_TARGET, "ParachainsInherentData failed to decode");
-				return None;
-			}
-		};
-
-		let parent_hash = <frame_system::Pallet<T>>::parent_hash();
-
 		let ParachainsInherentData::<T::Header> {
 			bitfields,
 			backed_candidates,
@@ -501,6 +488,8 @@ impl<T: Config> Pallet<T> {
 			},
 		};
 
+		let parent_hash = <frame_system::Pallet<T>>::parent_hash();
+
 		if parent_hash != parent_header.hash() {
 			log::warn!(
 				target: LOG_TARGET,
@@ -515,7 +504,7 @@ impl<T: Config> Pallet<T> {
 
 		T::DisputesHandler::filter_multi_dispute_data(&mut disputes);
 
-		let (concluded_invalid_disputes, disputed_bitfield, scheduled) =
+		let (concluded_invalid_disputes, bitfields, scheduled) =
 			frame_support::storage::with_transaction(|| {
 				// we don't care about fresh or not disputes
 				// this writes them to storage, so let's query it via those means
@@ -551,14 +540,14 @@ impl<T: Config> Pallet<T> {
 					})
 					.collect::<BTreeSet<CandidateHash>>();
 
-				let freed_disputed: Vec<_> = <inclusion::Pallet<T>>::collect_disputed(
-						&current_concluded_invalid_disputes
-					)
-					.into_iter()
-					.map(|core| (core, FreedReason::Concluded))
-					.collect();
+				let mut freed_disputed: Vec<_> =
+					<inclusion::Pallet<T>>::collect_disputed(&current_concluded_invalid_disputes)
+						.into_iter()
+						.map(|core| (core, FreedReason::Concluded))
+						.collect();
 
-				let disputed_bitfield = create_disputed_bitfield(expected_bits, freed_disputed.iter());
+				let disputed_bitfield =
+					create_disputed_bitfield(expected_bits, freed_disputed.iter().map(|(x,_)| x));
 
 				if !freed_disputed.is_empty() {
 					// unstable sort is fine, because core indices are unique
@@ -580,21 +569,25 @@ impl<T: Config> Pallet<T> {
 					Ok(bitfields) => bitfields,
 					Err(err) => {
 						// by convention, when called with `EARLY_RETURN=false`, will always return `Ok()`
-						log::error!(target: LOG_TARGET, "BUG: convention violation in create_inherent: {:?}", err);
+						log::error!(
+							target: LOG_TARGET,
+							"BUG: convention violation in create_inherent: {:?}",
+							err
+						);
 						vec![]
 					},
 				};
 
 				let freed_concluded =
-					<inclusion::Pallet<T>>::update_pending_availability_and_get_freed_cores(
+					<inclusion::Pallet<T>>::update_pending_availability_and_get_freed_cores::<_, false>(
 						expected_bits,
 						&validator_public[..],
-						bitfields,
-						disputed_bitfield,
+						bitfields.clone(),
 						<scheduler::Pallet<T>>::core_para,
 					);
-				let freed = <crate::paras_inherent::Pallet<T>>::collect_all_freed_cores::<T>(
-					freed_concluded,
+
+				let freed = collect_all_freed_cores::<T,_>(
+					freed_concluded.iter().cloned(),
 				);
 
 				<scheduler::Pallet<T>>::clear();
@@ -602,12 +595,12 @@ impl<T: Config> Pallet<T> {
 
 				let scheduled = <scheduler::Pallet<T>>::scheduled();
 
-				// Return
 				frame_support::storage::TransactionOutcome::Rollback((
-					// * concluded disputes for backed candidates in this block,
+					// concluded disputes for backed candidates in this block
 					concluded_invalid_disputes,
-					// * bitfield marking disputed cores,
-					disputed_bitfield,
+					// filtered bitfields,
+					bitfields,
+					// updated schedule
 					scheduled,
 				))
 			});
@@ -667,10 +660,13 @@ macro_rules! ensure2 {
 }
 
 /// Derive a bitfield from dispute
-pub(super) fn create_disputed_bitfield<'a, I: 'a + IntoIterator<Item = &'a CoreIndex>>(
+pub(super) fn create_disputed_bitfield<'a, I>(
 	expected_bits: usize,
 	freed_cores: I,
-) -> DisputedBitfield {
+) -> DisputedBitfield
+where
+	I: 'a + IntoIterator<Item = &'a CoreIndex>
+{
 	let mut bitvec = BitVec::repeat(false, expected_bits);
 	for core_idx in freed_cores {
 		let core_idx = core_idx.0 as usize;
diff --git a/runtime/parachains/src/scheduler.rs b/runtime/parachains/src/scheduler.rs
index 8e948e3b5529..ca658be2cc28 100644
--- a/runtime/parachains/src/scheduler.rs
+++ b/runtime/parachains/src/scheduler.rs
@@ -90,6 +90,7 @@ impl ParathreadClaimQueue {
 }
 
 /// Reasons a core might be freed
+#[derive(Clone, Copy)]
 pub enum FreedReason {
 	/// The core's work concluded and the parablock assigned to it is considered available.
 	Concluded,

From 55a7f71598b8cee40b3e9543e5fd98cff15ca382 Mon Sep 17 00:00:00 2001
From: Bernhard Schuster <bernhard@ahoi.io>
Date: Mon, 8 Nov 2021 19:30:33 +0100
Subject: [PATCH 078/107] chore fmt

---
 runtime/parachains/src/inclusion.rs      |  7 +++++--
 runtime/parachains/src/paras_inherent.rs | 17 +++++++++--------
 2 files changed, 14 insertions(+), 10 deletions(-)

diff --git a/runtime/parachains/src/inclusion.rs b/runtime/parachains/src/inclusion.rs
index cd2637dd245b..6fa6d2931bc6 100644
--- a/runtime/parachains/src/inclusion.rs
+++ b/runtime/parachains/src/inclusion.rs
@@ -289,14 +289,17 @@ impl<T: Config> Pallet<T> {
 	/// Extract the freed cores based on cores tht became available.
 	///
 	/// Updates storage items `PendingAvailability` and `AvailabilityBitfields`.
-	pub(crate) fn update_pending_availability_and_get_freed_cores<F, const IS_CREATE_INHERENT: bool>(
+	pub(crate) fn update_pending_availability_and_get_freed_cores<
+		F,
+		const IS_CREATE_INHERENT: bool,
+	>(
 		expected_bits: usize,
 		validators: &[ValidatorId],
 		signed_bitfields: UncheckedSignedAvailabilityBitfields,
 		core_lookup: F,
 	) -> Vec<(CoreIndex, CandidateHash)>
 	where
-		F:  Fn(CoreIndex) -> Option<ParaId>,
+		F: Fn(CoreIndex) -> Option<ParaId>,
 	{
 		let mut assigned_paras_record = (0..expected_bits)
 			.map(|bit_index| core_lookup(CoreIndex::from(bit_index as u32)))
diff --git a/runtime/parachains/src/paras_inherent.rs b/runtime/parachains/src/paras_inherent.rs
index 8fb3436edf4f..b7a0a92257d2 100644
--- a/runtime/parachains/src/paras_inherent.rs
+++ b/runtime/parachains/src/paras_inherent.rs
@@ -283,7 +283,7 @@ pub mod pallet {
 		freed_concluded: I,
 	) -> BTreeMap<CoreIndex, FreedReason>
 	where
-		I: core::iter::IntoIterator<Item=(CoreIndex, CandidateHash)>,
+		I: core::iter::IntoIterator<Item = (CoreIndex, CandidateHash)>,
 		T: Config,
 	{
 		// Handle timeouts for any availability core work.
@@ -412,7 +412,7 @@ pub mod pallet {
 				T::DisputesHandler::note_included(current_session, *candidate_hash, now);
 			}
 
-			let freed = collect_all_freed_cores::<T,_>(freed_concluded.iter().cloned());
+			let freed = collect_all_freed_cores::<T, _>(freed_concluded.iter().cloned());
 
 			<scheduler::Pallet<T>>::clear();
 			<scheduler::Pallet<T>>::schedule(freed, now);
@@ -547,7 +547,7 @@ impl<T: Config> Pallet<T> {
 						.collect();
 
 				let disputed_bitfield =
-					create_disputed_bitfield(expected_bits, freed_disputed.iter().map(|(x,_)| x));
+					create_disputed_bitfield(expected_bits, freed_disputed.iter().map(|(x, _)| x));
 
 				if !freed_disputed.is_empty() {
 					// unstable sort is fine, because core indices are unique
@@ -579,16 +579,17 @@ impl<T: Config> Pallet<T> {
 				};
 
 				let freed_concluded =
-					<inclusion::Pallet<T>>::update_pending_availability_and_get_freed_cores::<_, false>(
+					<inclusion::Pallet<T>>::update_pending_availability_and_get_freed_cores::<
+						_,
+						false,
+					>(
 						expected_bits,
 						&validator_public[..],
 						bitfields.clone(),
 						<scheduler::Pallet<T>>::core_para,
 					);
 
-				let freed = collect_all_freed_cores::<T,_>(
-					freed_concluded.iter().cloned(),
-				);
+				let freed = collect_all_freed_cores::<T, _>(freed_concluded.iter().cloned());
 
 				<scheduler::Pallet<T>>::clear();
 				<scheduler::Pallet<T>>::schedule(freed, <frame_system::Pallet<T>>::block_number());
@@ -665,7 +666,7 @@ pub(super) fn create_disputed_bitfield<'a, I>(
 	freed_cores: I,
 ) -> DisputedBitfield
 where
-	I: 'a + IntoIterator<Item = &'a CoreIndex>
+	I: 'a + IntoIterator<Item = &'a CoreIndex>,
 {
 	let mut bitvec = BitVec::repeat(false, expected_bits);
 	for core_idx in freed_cores {

From dac3b8e0656f40b5636ecf1341711e831519d89f Mon Sep 17 00:00:00 2001
From: Bernhard Schuster <bernhard@ahoi.io>
Date: Mon, 8 Nov 2021 19:39:41 +0100
Subject: [PATCH 079/107] fixins

---
 runtime/parachains/src/inclusion.rs | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/runtime/parachains/src/inclusion.rs b/runtime/parachains/src/inclusion.rs
index 6fa6d2931bc6..0b7dbf7775a8 100644
--- a/runtime/parachains/src/inclusion.rs
+++ b/runtime/parachains/src/inclusion.rs
@@ -291,7 +291,7 @@ impl<T: Config> Pallet<T> {
 	/// Updates storage items `PendingAvailability` and `AvailabilityBitfields`.
 	pub(crate) fn update_pending_availability_and_get_freed_cores<
 		F,
-		const IS_CREATE_INHERENT: bool,
+		const EARLY_RETURN: bool,
 	>(
 		expected_bits: usize,
 		validators: &[ValidatorId],
@@ -368,7 +368,7 @@ impl<T: Config> Pallet<T> {
 					},
 				};
 
-				if !IS_CREATE_INHERENT {
+				if EARLY_RETURN {
 					let receipt = CommittedCandidateReceipt {
 						descriptor: pending_availability.descriptor,
 						commitments,
@@ -406,7 +406,7 @@ impl<T: Config> Pallet<T> {
 		let session_index = shared::Pallet::<T>::session_index();
 		let parent_hash = frame_system::Pallet::<T>::parent_hash();
 
-		let checked_bitfields = sanitize_bitfields::<T, false>(
+		let checked_bitfields = sanitize_bitfields::<T, true>(
 			signed_bitfields,
 			disputed_bitfield,
 			expected_bits,
@@ -415,7 +415,7 @@ impl<T: Config> Pallet<T> {
 			&validators[..],
 		)?;
 
-		let freed_cores = Self::update_pending_availability_and_get_freed_cores::<_, false>(
+		let freed_cores = Self::update_pending_availability_and_get_freed_cores::<_, true>(
 			expected_bits,
 			&validators[..],
 			checked_bitfields,

From 61db8a7ad169f0b7dc49cbc2dc9bb8ec4e2cb5b1 Mon Sep 17 00:00:00 2001
From: Bernhard Schuster <bernhard@ahoi.io>
Date: Mon, 8 Nov 2021 19:44:18 +0100
Subject: [PATCH 080/107] rename const generic

---
 runtime/parachains/src/inclusion.rs      | 4 ++--
 runtime/parachains/src/paras_inherent.rs | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/runtime/parachains/src/inclusion.rs b/runtime/parachains/src/inclusion.rs
index 0b7dbf7775a8..a59fb643aed4 100644
--- a/runtime/parachains/src/inclusion.rs
+++ b/runtime/parachains/src/inclusion.rs
@@ -291,7 +291,7 @@ impl<T: Config> Pallet<T> {
 	/// Updates storage items `PendingAvailability` and `AvailabilityBitfields`.
 	pub(crate) fn update_pending_availability_and_get_freed_cores<
 		F,
-		const EARLY_RETURN: bool,
+		const ON_CHAIN_USE: bool,
 	>(
 		expected_bits: usize,
 		validators: &[ValidatorId],
@@ -368,7 +368,7 @@ impl<T: Config> Pallet<T> {
 					},
 				};
 
-				if EARLY_RETURN {
+				if ON_CHAIN_USE {
 					let receipt = CommittedCandidateReceipt {
 						descriptor: pending_availability.descriptor,
 						commitments,
diff --git a/runtime/parachains/src/paras_inherent.rs b/runtime/parachains/src/paras_inherent.rs
index b7a0a92257d2..0831f8f4e7f1 100644
--- a/runtime/parachains/src/paras_inherent.rs
+++ b/runtime/parachains/src/paras_inherent.rs
@@ -568,7 +568,7 @@ impl<T: Config> Pallet<T> {
 				) {
 					Ok(bitfields) => bitfields,
 					Err(err) => {
-						// by convention, when called with `EARLY_RETURN=false`, will always return `Ok()`
+						// by convention, when called with `ON_CHAIN_USE=false`, will always return `Ok()`
 						log::error!(
 							target: LOG_TARGET,
 							"BUG: convention violation in create_inherent: {:?}",
@@ -614,7 +614,7 @@ impl<T: Config> Pallet<T> {
 			},
 			&scheduled[..],
 		)
-		.ok()?; // by convention, when called with `EARLY_RETURN=false`, will always return `Ok()`
+		.ok()?; // by convention, when called with `ON_CHAIN_USE=false`, will always return `Ok()`
 
 		let entropy = {
 			const CANDIDATE_SEED_SUBJECT: [u8; 32] = *b"candidate-seed-selection-subject";

From 7f1293b405455866e957690a23fb7d3fd2423c9e Mon Sep 17 00:00:00 2001
From: Bernhard Schuster <bernhard@ahoi.io>
Date: Wed, 10 Nov 2021 13:31:00 +0100
Subject: [PATCH 081/107] Update runtime/parachains/src/paras_inherent.rs

Co-authored-by: Zeke Mostov <z.mostov@gmail.com>
---
 runtime/parachains/src/paras_inherent.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/parachains/src/paras_inherent.rs b/runtime/parachains/src/paras_inherent.rs
index aa8b7a157cc3..f6c3bb4028ba 100644
--- a/runtime/parachains/src/paras_inherent.rs
+++ b/runtime/parachains/src/paras_inherent.rs
@@ -138,7 +138,7 @@ fn dispute_statements_weight<T: Config>(disputes: &[DisputeStatementSet]) -> Wei
 }
 
 fn signed_bitfields_weight<T: Config>(bitfields_len: usize) -> Weight {
-	<<T as Config>::WeightInfo as WeightInfo>::enter_bitfields() * bitfields_len as Weight
+	<<T as Config>::WeightInfo as WeightInfo>::enter_bitfields().saturating_mul(bitfields_len as Weight)
 }
 
 fn backed_candidates_weight<T: frame_system::Config + Config>(

From 60a9be5cb961b2f3c03449632c62f2554e5dd647 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bastian=20K=C3=B6cher?= <info@kchr.de>
Date: Thu, 11 Nov 2021 09:35:10 +0100
Subject: [PATCH 082/107] Fix compilation

---
 parachain/Cargo.toml  | 1 +
 primitives/Cargo.toml | 6 ++++--
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/parachain/Cargo.toml b/parachain/Cargo.toml
index 77dcd13b5ce5..196e20e529ab 100644
--- a/parachain/Cargo.toml
+++ b/parachain/Cargo.toml
@@ -36,3 +36,4 @@ std = [
 	"polkadot-core-primitives/std",
 	"frame-support/std",
 ]
+runtime-benchmarks = []
diff --git a/primitives/Cargo.toml b/primitives/Cargo.toml
index f58044bf67e5..e5242b25de23 100644
--- a/primitives/Cargo.toml
+++ b/primitives/Cargo.toml
@@ -29,7 +29,6 @@ frame-system = { git = "https://github.com/paritytech/substrate", branch = "mast
 hex-literal = "0.3.3"
 parity-util-mem = { version = "0.10.0", default-features = false, optional = true }
 
-
 [features]
 default = ["std"]
 std = [
@@ -56,4 +55,7 @@ std = [
 	"bitvec/std",
 	"frame-system/std",
 ]
-runtime-benchmarks = []
+runtime-benchmarks = [
+	"polkadot-parachain/runtime-benchmarks",
+	"frame-system/runtime-benchmarks",
+]

From e36e12a6b99d6130229df03696f616a5948a49d4 Mon Sep 17 00:00:00 2001
From: Bernhard Schuster <bernhard@ahoi.io>
Date: Thu, 11 Nov 2021 11:22:42 +0100
Subject: [PATCH 083/107] limit to test

---
 runtime/parachains/src/inclusion.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/parachains/src/inclusion.rs b/runtime/parachains/src/inclusion.rs
index a59fb643aed4..a38db577b087 100644
--- a/runtime/parachains/src/inclusion.rs
+++ b/runtime/parachains/src/inclusion.rs
@@ -104,7 +104,7 @@ impl<H, N> CandidatePendingAvailability<H, N> {
 		&self.descriptor
 	}
 
-	#[cfg(any(feature = "runtime-benchmarks", feature = "std"))]
+	#[cfg(any(feature = "runtime-benchmarks", test))]
 	pub(crate) fn new(
 		core: CoreIndex,
 		hash: CandidateHash,

From a2dcd206ceca2b65926387cc022217c6798287d9 Mon Sep 17 00:00:00 2001
From: Bernhard Schuster <bernhard@ahoi.io>
Date: Thu, 11 Nov 2021 11:30:51 +0100
Subject: [PATCH 084/107] remove unused spam slots

---
 runtime/parachains/src/disputes.rs | 1 -
 1 file changed, 1 deletion(-)

diff --git a/runtime/parachains/src/disputes.rs b/runtime/parachains/src/disputes.rs
index a487eaf98ee8..da8d2cab23dc 100644
--- a/runtime/parachains/src/disputes.rs
+++ b/runtime/parachains/src/disputes.rs
@@ -298,7 +298,6 @@ pub mod pallet {
 	///
 	/// The i'th entry of the vector corresponds to the i'th validator in the session.
 	#[pallet::storage]
-	#[pallet::getter(fn spam_slots)]
 	pub(super) type SpamSlots<T> = StorageMap<_, Twox64Concat, SessionIndex, Vec<u32>>;
 
 	/// Whether the chain is frozen. Starts as `None`. When this is `Some`,

From 9a9eb2200888fdb6ec438b89e2d45d6bda567b3f Mon Sep 17 00:00:00 2001
From: Bernhard Schuster <bernhard@ahoi.io>
Date: Thu, 11 Nov 2021 12:27:47 +0100
Subject: [PATCH 085/107] spellcheck

---
 runtime/parachains/src/builder.rs | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/runtime/parachains/src/builder.rs b/runtime/parachains/src/builder.rs
index 81153701b019..ffd56bda1069 100644
--- a/runtime/parachains/src/builder.rs
+++ b/runtime/parachains/src/builder.rs
@@ -63,7 +63,7 @@ pub(crate) struct Bench<T: paras_inherent::Config> {
 
 impl<T: paras_inherent::Config> BenchBuilder<T> {
 	/// Create a new `BenchBuilder` with some opinionated values that should work with the rest
-	/// of the functions in this impl.å
+	/// of the functions in this implementation.
 	pub(crate) fn new() -> Self {
 		BenchBuilder {
 			// Validators should be declared prior to all other setup.
@@ -103,7 +103,7 @@ impl<T: paras_inherent::Config> BenchBuilder<T> {
 			.expect("self.block_number is u32")
 	}
 
-	/// Maximium number of validators that may be part of a validator group.
+	/// Maximum number of validators that may be part of a validator group.
 	pub(crate) fn fallback_max_validators() -> u32 {
 		configuration::Pallet::<T>::config().max_validators.unwrap_or(200)
 	}
@@ -225,11 +225,11 @@ impl<T: paras_inherent::Config> BenchBuilder<T> {
 	}
 
 	/// Setup para ids.
-	/// * setup para_ids traverses each core,
+	/// * setup `para_ids` traverses each core,
 	/// * creates a ParaId for that CoreIndex,
 	/// * inserts ParaLifeCycle::Onboarding for that ParaId,
 	/// * inserts the upcoming paras genesis,
-	/// * inserts the ParaId into the ActionsQueue
+	/// * inserts the ParaId into the `ActionsQueue`
 	fn setup_para_ids(cores: u32) {
 		// make sure parachains exist prior to session change.
 		for i in 0..cores {
@@ -363,7 +363,7 @@ impl<T: paras_inherent::Config> BenchBuilder<T> {
 	/// Create backed candidates for `cores_with_backed_candidates`. You need these cores to be
 	/// scheduled _within_ paras inherent, which requires marking the available bitfields as fully
 	/// available.
-	/// - `cores_with_backed_candidates` Mapping of para_id/core_idx/group_idx seed to number of
+	/// - `cores_with_backed_candidates` Mapping of `para_id`/`core_idx`/`group_idx` seed to number of
 	/// validity votes.
 	fn create_backed_candidates(
 		&self,

From f1ef3e194523a99cf1d12e46b04ea131f2c6af53 Mon Sep 17 00:00:00 2001
From: Bernhard Schuster <bernhard@ahoi.io>
Date: Thu, 11 Nov 2021 17:12:52 +0100
Subject: [PATCH 086/107] remove a tick, fix a typo

---
 runtime/parachains/src/paras_inherent.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/parachains/src/paras_inherent.rs b/runtime/parachains/src/paras_inherent.rs
index f6c3bb4028ba..d010bdece60b 100644
--- a/runtime/parachains/src/paras_inherent.rs
+++ b/runtime/parachains/src/paras_inherent.rs
@@ -277,7 +277,7 @@ pub mod pallet {
 
 	/// Collect all freed cores based on storage data.
 	///
-	/// The paramter `freed_concluded` contains all core indicies that became
+	/// The parameter `freed_concluded` contains all core indicies that became
 	/// free due to candidate that became available.
 	pub(crate) fn collect_all_freed_cores<T, I>(
 		freed_concluded: I,

From c9e5872cabe8588b12e7b54aab47f80c7fe42272 Mon Sep 17 00:00:00 2001
From: Lldenaurois <Ljdenaurois@gmail.com>
Date: Thu, 11 Nov 2021 17:57:24 +0100
Subject: [PATCH 087/107] Add Code upgrade weights

---
 runtime/parachains/src/builder.rs             |  8 +++---
 runtime/parachains/src/inclusion.rs           |  5 +---
 runtime/parachains/src/paras_inherent.rs      | 26 ++++++++++++-------
 .../src/paras_inherent/benchmarking.rs        | 12 ++++-----
 4 files changed, 28 insertions(+), 23 deletions(-)

diff --git a/runtime/parachains/src/builder.rs b/runtime/parachains/src/builder.rs
index ffd56bda1069..76a40544a5a2 100644
--- a/runtime/parachains/src/builder.rs
+++ b/runtime/parachains/src/builder.rs
@@ -57,8 +57,8 @@ pub(crate) struct BenchBuilder<T: paras_inherent::Config> {
 #[cfg(any(feature = "runtime-benchmarks", test))]
 pub(crate) struct Bench<T: paras_inherent::Config> {
 	pub(crate) data: ParachainsInherentData<T::Header>,
-	pub(crate) session: u32,
-	pub(crate) block_number: T::BlockNumber,
+	pub(crate) _session: u32,
+	pub(crate) _block_number: T::BlockNumber,
 }
 
 impl<T: paras_inherent::Config> BenchBuilder<T> {
@@ -583,8 +583,8 @@ impl<T: paras_inherent::Config> BenchBuilder<T> {
 				disputes,
 				parent_header: Self::header(builder.block_number.clone()),
 			},
-			session: builder.target_session,
-			block_number: builder.block_number,
+			_session: target_session,
+			_block_number: builder.block_number,
 		}
 	}
 }
diff --git a/runtime/parachains/src/inclusion.rs b/runtime/parachains/src/inclusion.rs
index a38db577b087..03d88535e5e5 100644
--- a/runtime/parachains/src/inclusion.rs
+++ b/runtime/parachains/src/inclusion.rs
@@ -289,10 +289,7 @@ impl<T: Config> Pallet<T> {
 	/// Extract the freed cores based on cores tht became available.
 	///
 	/// Updates storage items `PendingAvailability` and `AvailabilityBitfields`.
-	pub(crate) fn update_pending_availability_and_get_freed_cores<
-		F,
-		const ON_CHAIN_USE: bool,
-	>(
+	pub(crate) fn update_pending_availability_and_get_freed_cores<F, const ON_CHAIN_USE: bool>(
 		expected_bits: usize,
 		validators: &[ValidatorId],
 		signed_bitfields: UncheckedSignedAvailabilityBitfields,
diff --git a/runtime/parachains/src/paras_inherent.rs b/runtime/parachains/src/paras_inherent.rs
index d010bdece60b..6e23bfd87356 100644
--- a/runtime/parachains/src/paras_inherent.rs
+++ b/runtime/parachains/src/paras_inherent.rs
@@ -138,7 +138,8 @@ fn dispute_statements_weight<T: Config>(disputes: &[DisputeStatementSet]) -> Wei
 }
 
 fn signed_bitfields_weight<T: Config>(bitfields_len: usize) -> Weight {
-	<<T as Config>::WeightInfo as WeightInfo>::enter_bitfields().saturating_mul(bitfields_len as Weight)
+	<<T as Config>::WeightInfo as WeightInfo>::enter_bitfields()
+		.saturating_mul(bitfields_len as Weight)
 }
 
 fn backed_candidates_weight<T: frame_system::Config + Config>(
@@ -743,14 +744,21 @@ fn apply_weight_limit<T: Config + inclusion::Config>(
 	// There is weight remaining to be consumed by a subset of candidates
 	// which are going to be picked now.
 	if let Some(remaining_weight) = max_weight.checked_sub(total_bitfields_weight) {
-		let c = candidates.iter().map(|c| c.validity_votes.len() as u32).collect::<Vec<u32>>();
-
-		let (acc_candidate_weight, indices) = random_sel::<u32, _>(
-			&mut rng,
-			c,
-			|v| <<T as Config>::WeightInfo as WeightInfo>::enter_backed_candidates_variable(*v),
-			remaining_weight,
-		);
+		let (acc_candidate_weight, indices) =
+			random_sel::<BackedCandidate<<T as frame_system::Config>::Hash>, _>(
+				&mut rng,
+				candidates.clone(),
+				|c| {
+					<<T as Config>::WeightInfo as WeightInfo>::enter_backed_candidates_variable(
+						c.validity_votes.len() as u32,
+					) + if c.candidate.commitments.new_validation_code.is_some() {
+						<<T as Config>::WeightInfo as WeightInfo>::enter_backed_candidate_code_upgrade()
+					} else {
+						0
+					}
+				},
+				remaining_weight,
+			);
 		let candidates =
 			indices.into_iter().map(move |idx| candidates[idx].clone()).collect::<Vec<_>>();
 		// pick all bitfields, and
diff --git a/runtime/parachains/src/paras_inherent/benchmarking.rs b/runtime/parachains/src/paras_inherent/benchmarking.rs
index 0229ebf5e2aa..34ea81633199 100644
--- a/runtime/parachains/src/paras_inherent/benchmarking.rs
+++ b/runtime/parachains/src/paras_inherent/benchmarking.rs
@@ -49,7 +49,7 @@ benchmarks! {
 		assert!(onchain_votes.is_some());
 		let vote = onchain_votes.unwrap();
 		// Ensure that the votes are for the correct session
-		assert_eq!(vote.session, scenario.session);
+		assert_eq!(vote.session, scenario._session);
 	}
 
 	// The weight of one bitfield.
@@ -79,7 +79,7 @@ benchmarks! {
 		assert!(onchain_votes.is_some());
 		let vote = onchain_votes.unwrap();
 		// Ensure that the votes are for the correct session
-		assert_eq!(vote.session, scenario.session);
+		assert_eq!(vote.session, scenario._session);
 	}
 
 	// Variant over `v`, the amount of validity votes for a backed candidate. This gives the weight
@@ -118,9 +118,9 @@ benchmarks! {
 		assert!(onchain_votes.is_some());
 		let vote = onchain_votes.unwrap();
 		// Ensure that the votes are for the correct session
-		assert_eq!(vote.session, scenario.session);
+		assert_eq!(vote.session, scenario._session);
 		// Ensure that there are an expected number of candidates
-		let header = BenchBuilder::<T>::header(scenario.block_number.clone());
+		let header = BenchBuilder::<T>::header(scenario._block_number.clone());
 		// Traverse candidates and assert descriptors are as expected
 		for (para_id, backing_validators) in vote.backing_validators_per_candidate.iter().enumerate() {
 			let descriptor = backing_validators.0.descriptor();
@@ -173,9 +173,9 @@ benchmarks! {
 		assert!(onchain_votes.is_some());
 		let vote = onchain_votes.unwrap();
 		// Ensure that the votes are for the correct session
-		assert_eq!(vote.session, scenario.session);
+		assert_eq!(vote.session, scenario._session);
 		// Ensure that there are an expected number of candidates
-		let header = BenchBuilder::<T>::header(scenario.block_number.clone());
+		let header = BenchBuilder::<T>::header(scenario._block_number.clone());
 		// Traverse candidates and assert descriptors are as expected
 		for (para_id, backing_validators)
 			in vote.backing_validators_per_candidate.iter().enumerate() {

From 8f1ad09609d6eb049e176d07a958c45faf35d305 Mon Sep 17 00:00:00 2001
From: Bernhard Schuster <bernhard@ahoi.io>
Date: Fri, 12 Nov 2021 17:35:27 +0100
Subject: [PATCH 088/107] comment improvements + >=

Co-authored-by: Zeke Mostov <z.mostov@gmail.com>
---
 runtime/parachains/src/builder.rs        | 8 ++++++--
 runtime/parachains/src/disputes.rs       | 3 ++-
 runtime/parachains/src/inclusion.rs      | 3 +--
 runtime/parachains/src/paras_inherent.rs | 9 +++++----
 4 files changed, 14 insertions(+), 9 deletions(-)

diff --git a/runtime/parachains/src/builder.rs b/runtime/parachains/src/builder.rs
index 76a40544a5a2..d408723d6622 100644
--- a/runtime/parachains/src/builder.rs
+++ b/runtime/parachains/src/builder.rs
@@ -23,8 +23,8 @@ use sp_std::{collections::btree_map::BTreeMap, convert::TryInto, prelude::Vec, v
 
 /// Grab an account, seeded by a name and index.
 ///
-/// This is directly from frame-benchmarking. Copy/Pasted so we can use it when not compiling with
-/// "features = runtime-benchmarks"
+/// This is directly from frame-benchmarking. Copy/pasted so we can use it when not compiling with
+/// "features = runtime-benchmarks".
 fn account<AccountId: Decode + Default>(name: &'static str, index: u32, seed: u32) -> AccountId {
 	let entropy = (name, index, seed).using_encoded(sp_core::blake2_256);
 	AccountId::decode(&mut &entropy[..]).unwrap_or_default()
@@ -123,6 +123,10 @@ impl<T: paras_inherent::Config> BenchBuilder<T> {
 		configuration::Pallet::<T>::config().max_validators_per_core.unwrap_or(5)
 	}
 
+        /// Specify a mapping of core_idx/para_id/group_idx seed to the number of dispute statements for the
+        /// corresponding dispute statement set. Note that if the number of disputes is not specified it fallbacks
+        /// to having a dispute per every validator. Additionally, an entry is not guaranteed to have a dispute - it 
+        /// must line up with the cores marked as disputed as defined in `Self::Build`.
 	#[cfg(not(feature = "runtime-benchmarks"))]
 	pub(crate) fn set_dispute_statements(mut self, m: BTreeMap<u32, u32>) -> Self {
 		self.dispute_statements = m;
diff --git a/runtime/parachains/src/disputes.rs b/runtime/parachains/src/disputes.rs
index da8d2cab23dc..21990f8b93f1 100644
--- a/runtime/parachains/src/disputes.rs
+++ b/runtime/parachains/src/disputes.rs
@@ -129,7 +129,8 @@ pub trait DisputesHandler<BlockNumber> {
 		included_in: BlockNumber,
 	);
 
-	/// Retrieve the included state of a given candidate in a particular session.
+/// Retrieve the included state of a given candidate in a particular session. If it
+/// returns `Some`, then we have a local dispute for the given `candidate_hash`.
 	fn included_state(session: SessionIndex, candidate_hash: CandidateHash) -> Option<BlockNumber>;
 
 	/// Whether the given candidate concluded invalid in a dispute with supermajority.
diff --git a/runtime/parachains/src/inclusion.rs b/runtime/parachains/src/inclusion.rs
index 03d88535e5e5..a52c6d7509c4 100644
--- a/runtime/parachains/src/inclusion.rs
+++ b/runtime/parachains/src/inclusion.rs
@@ -286,7 +286,7 @@ impl<T: Config> Pallet<T> {
 		for _ in <AvailabilityBitfields<T>>::drain() {}
 	}
 
-	/// Extract the freed cores based on cores tht became available.
+	/// Extract the freed cores based on cores that became available.
 	///
 	/// Updates storage items `PendingAvailability` and `AvailabilityBitfields`.
 	pub(crate) fn update_pending_availability_and_get_freed_cores<F, const ON_CHAIN_USE: bool>(
@@ -767,7 +767,6 @@ impl<T: Config> Pallet<T> {
 		}
 
 		// enact the messaging facet of the candidate.
-		// TODO check how to account for these
 		weight += <dmp::Pallet<T>>::prune_dmq(
 			receipt.descriptor.para_id,
 			commitments.processed_downward_messages,
diff --git a/runtime/parachains/src/paras_inherent.rs b/runtime/parachains/src/paras_inherent.rs
index 6e23bfd87356..43fe4f287fd3 100644
--- a/runtime/parachains/src/paras_inherent.rs
+++ b/runtime/parachains/src/paras_inherent.rs
@@ -245,9 +245,10 @@ pub mod pallet {
 			// and we _really_ don't want that to happen.
 			// See <https://github.com/paritytech/polkadot/issues/1327>
 
-			// Calling `Self::enter` here is a safe-guard, to avoid any discrepancy
-			// between on-chain logic and the logic that is executed
-			// at the block producer off-chain (this code path).
+			// Calling `Self::enter` here is a safe-guard, to avoid any discrepancy between on-chain checks
+			// (`enter`) and the off-chain checks by the block author (this function). Once we are confident
+			// in all the logic in this module this check should be removed to optimize performance.
+
 			let inherent_data =
 				match Self::enter(frame_system::RawOrigin::None.into(), inherent_data.clone()) {
 					Ok(_) => inherent_data,
@@ -962,7 +963,7 @@ fn limit_disputes<T: Config>(disputes: &mut MultiDisputeStatementSet, entropy: [
 			let dispute_weight = <<T as Config>::WeightInfo as WeightInfo>::enter_variable_disputes(
 				d.statements.len() as u32,
 			);
-			if remaining_weight > dispute_weight {
+			if remaining_weight >= dispute_weight {
 				remaining_weight -= dispute_weight;
 				true
 			} else {

From 8ac5e56c6132ba197123f644dedd10212fa34bd2 Mon Sep 17 00:00:00 2001
From: Bernhard Schuster <bernhard@ahoi.io>
Date: Fri, 12 Nov 2021 17:07:44 +0100
Subject: [PATCH 089/107] remove another tick

---
 runtime/parachains/src/paras_inherent.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/parachains/src/paras_inherent.rs b/runtime/parachains/src/paras_inherent.rs
index 43fe4f287fd3..641e00a05b78 100644
--- a/runtime/parachains/src/paras_inherent.rs
+++ b/runtime/parachains/src/paras_inherent.rs
@@ -473,7 +473,7 @@ pub mod pallet {
 }
 
 impl<T: Config> Pallet<T> {
-	/// Create the `ParachainsInherentData` that gets passed to `[`Self::enter`] in [`Self::create_inherent`].
+	/// Create the `ParachainsInherentData` that gets passed to [`Self::enter`] in [`Self::create_inherent`].
 	/// This code is pulled out of [`Self::create_inherent`] so it can be unit tested.
 	fn create_inherent_inner(data: &InherentData) -> Option<ParachainsInherentData<T::Header>> {
 		let ParachainsInherentData::<T::Header> {

From ee89736c88f635dcc3f33dd7ccde42b45d309568 Mon Sep 17 00:00:00 2001
From: Bernhard Schuster <bernhard@ahoi.io>
Date: Fri, 12 Nov 2021 17:44:03 +0100
Subject: [PATCH 090/107] Update
 runtime/parachains/src/paras_inherent/benchmarking.rs

Co-authored-by: Zeke Mostov <z.mostov@gmail.com>
---
 runtime/parachains/src/paras_inherent/benchmarking.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/parachains/src/paras_inherent/benchmarking.rs b/runtime/parachains/src/paras_inherent/benchmarking.rs
index 34ea81633199..d5aea6dde331 100644
--- a/runtime/parachains/src/paras_inherent/benchmarking.rs
+++ b/runtime/parachains/src/paras_inherent/benchmarking.rs
@@ -153,7 +153,7 @@ benchmarks! {
 
 		let mut benchmark = scenario.data.clone();
 
-		// There is 1 backed,
+		// There is 1 backed
 		assert_eq!(benchmark.backed_candidates.len(), 1);
 		assert_eq!(
 			benchmark.backed_candidates.get(0).unwrap().validity_votes.len() as u32,

From ca5cc54db0a842c6a7ce3be7da0f2f4c6d0d42a0 Mon Sep 17 00:00:00 2001
From: Bernhard Schuster <bernhard@ahoi.io>
Date: Fri, 12 Nov 2021 17:57:19 +0100
Subject: [PATCH 091/107] saturating fixins + some spaces

---
 runtime/parachains/src/builder.rs        |  8 ++++----
 runtime/parachains/src/disputes.rs       |  4 ++--
 runtime/parachains/src/paras_inherent.rs | 21 +++++++++++++--------
 3 files changed, 19 insertions(+), 14 deletions(-)

diff --git a/runtime/parachains/src/builder.rs b/runtime/parachains/src/builder.rs
index d408723d6622..67d025a0bc62 100644
--- a/runtime/parachains/src/builder.rs
+++ b/runtime/parachains/src/builder.rs
@@ -123,10 +123,10 @@ impl<T: paras_inherent::Config> BenchBuilder<T> {
 		configuration::Pallet::<T>::config().max_validators_per_core.unwrap_or(5)
 	}
 
-        /// Specify a mapping of core_idx/para_id/group_idx seed to the number of dispute statements for the
-        /// corresponding dispute statement set. Note that if the number of disputes is not specified it fallbacks
-        /// to having a dispute per every validator. Additionally, an entry is not guaranteed to have a dispute - it 
-        /// must line up with the cores marked as disputed as defined in `Self::Build`.
+	/// Specify a mapping of core_idx/para_id/group_idx seed to the number of dispute statements for the
+	/// corresponding dispute statement set. Note that if the number of disputes is not specified it fallbacks
+	/// to having a dispute per every validator. Additionally, an entry is not guaranteed to have a dispute - it
+	/// must line up with the cores marked as disputed as defined in `Self::Build`.
 	#[cfg(not(feature = "runtime-benchmarks"))]
 	pub(crate) fn set_dispute_statements(mut self, m: BTreeMap<u32, u32>) -> Self {
 		self.dispute_statements = m;
diff --git a/runtime/parachains/src/disputes.rs b/runtime/parachains/src/disputes.rs
index 21990f8b93f1..1f49f0f4d25d 100644
--- a/runtime/parachains/src/disputes.rs
+++ b/runtime/parachains/src/disputes.rs
@@ -129,8 +129,8 @@ pub trait DisputesHandler<BlockNumber> {
 		included_in: BlockNumber,
 	);
 
-/// Retrieve the included state of a given candidate in a particular session. If it
-/// returns `Some`, then we have a local dispute for the given `candidate_hash`.
+	/// Retrieve the included state of a given candidate in a particular session. If it
+	/// returns `Some`, then we have a local dispute for the given `candidate_hash`.
 	fn included_state(session: SessionIndex, candidate_hash: CandidateHash) -> Option<BlockNumber>;
 
 	/// Whether the given candidate concluded invalid in a dispute with supermajority.
diff --git a/runtime/parachains/src/paras_inherent.rs b/runtime/parachains/src/paras_inherent.rs
index 641e00a05b78..fa25895a7742 100644
--- a/runtime/parachains/src/paras_inherent.rs
+++ b/runtime/parachains/src/paras_inherent.rs
@@ -152,7 +152,7 @@ fn backed_candidates_weight<T: frame_system::Config + Config>(
 				v.validity_votes.len() as u32,
 			)
 		})
-		.sum()
+		.fold(0, |acc, x| acc.saturating_add(x))
 }
 
 /// A bitfield concerning concluded disputes for candidates
@@ -749,14 +749,19 @@ fn apply_weight_limit<T: Config + inclusion::Config>(
 			random_sel::<BackedCandidate<<T as frame_system::Config>::Hash>, _>(
 				&mut rng,
 				candidates.clone(),
-				|c| {
+				|backed_candidate| {
+					let code_weight = backed_candidate
+						.candidate
+						.commitments
+						.new_validation_code
+						.map(|_code| {
+							<<T as Config>::WeightInfo as WeightInfo>::enter_backed_candidate_code_upgrade()
+						})
+						.unwrap_or(0);
 					<<T as Config>::WeightInfo as WeightInfo>::enter_backed_candidates_variable(
-						c.validity_votes.len() as u32,
-					) + if c.candidate.commitments.new_validation_code.is_some() {
-						<<T as Config>::WeightInfo as WeightInfo>::enter_backed_candidate_code_upgrade()
-					} else {
-						0
-					}
+						backed_candidate.validity_votes.len() as u32,
+					)
+					.saturating_add(code_weight)
 				},
 				remaining_weight,
 			);

From c18e141b601a68f7fd2ab82b245d4601146efdf4 Mon Sep 17 00:00:00 2001
From: Bernhard Schuster <bernhard@ahoi.io>
Date: Sat, 13 Nov 2021 19:50:49 +0100
Subject: [PATCH 092/107] fix

---
 runtime/parachains/src/paras_inherent.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/runtime/parachains/src/paras_inherent.rs b/runtime/parachains/src/paras_inherent.rs
index fa25895a7742..2da4dd7ec839 100644
--- a/runtime/parachains/src/paras_inherent.rs
+++ b/runtime/parachains/src/paras_inherent.rs
@@ -754,6 +754,7 @@ fn apply_weight_limit<T: Config + inclusion::Config>(
 						.candidate
 						.commitments
 						.new_validation_code
+						.as_ref()
 						.map(|_code| {
 							<<T as Config>::WeightInfo as WeightInfo>::enter_backed_candidate_code_upgrade()
 						})

From ba34d063e9045ca7339a46daf9440f5d3957f691 Mon Sep 17 00:00:00 2001
From: Bernhard Schuster <bernhard@ahoi.io>
Date: Sun, 14 Nov 2021 14:04:37 +0100
Subject: [PATCH 093/107] benchmarking - preliminary results

---
 .../runtime_parachains_paras_inherent.rs      | 60 +++++++++++++------
 1 file changed, 43 insertions(+), 17 deletions(-)

diff --git a/runtime/kusama/src/weights/runtime_parachains_paras_inherent.rs b/runtime/kusama/src/weights/runtime_parachains_paras_inherent.rs
index 033af3be5067..e43221b98995 100644
--- a/runtime/kusama/src/weights/runtime_parachains_paras_inherent.rs
+++ b/runtime/kusama/src/weights/runtime_parachains_paras_inherent.rs
@@ -16,7 +16,7 @@
 //! Autogenerated weights for `runtime_parachains::paras_inherent`
 //!
 //! THIS FILE WAS AUTO-GENERATED USING THE SUBSTRATE BENCHMARK CLI VERSION 4.0.0-dev
-//! DATE: 2021-11-04, STEPS: `50`, REPEAT: 20, LOW RANGE: `[]`, HIGH RANGE: `[]`
+//! DATE: 2021-11-14, STEPS: `50`, REPEAT: 3, LOW RANGE: `[]`, HIGH RANGE: `[]`
 //! EXECUTION: Some(Wasm), WASM-EXECUTION: Compiled, CHAIN: Some("kusama-dev"), DB CACHE: 128
 
 // Executed Command:
@@ -24,7 +24,7 @@
 // benchmark
 // --chain=kusama-dev
 // --steps=50
-// --repeat=20
+// --repeat=3
 // --pallet=runtime_parachains::paras_inherent
 // --extrinsic=*
 // --execution=wasm
@@ -64,17 +64,16 @@ impl<T: frame_system::Config> runtime_parachains::paras_inherent::WeightInfo for
 	// Storage: ParaScheduler ParathreadQueue (r:1 w:1)
 	// Storage: ParaScheduler Scheduled (r:1 w:1)
 	// Storage: ParaScheduler ValidatorGroups (r:1 w:0)
-	// Storage: System BlockWeight (r:1 w:0)
 	// Storage: Ump NeedsDispatch (r:1 w:1)
 	// Storage: Ump NextDispatchRoundStartWith (r:1 w:1)
 	// Storage: ParaInherent OnChainVotes (r:0 w:1)
 	// Storage: Hrmp HrmpWatermarks (r:0 w:1)
 	// Storage: Paras Heads (r:0 w:1)
 	fn enter_variable_disputes(v: u32, ) -> Weight {
-		(224_853_000 as Weight)
-			// Standard Error: 2_000
-			.saturating_add((229_000 as Weight).saturating_mul(v as Weight))
-			.saturating_add(T::DbWeight::get().reads(24 as Weight))
+		(316_331_000 as Weight)
+			// Standard Error: 112_000
+			.saturating_add((325_000 as Weight).saturating_mul(v as Weight))
+			.saturating_add(T::DbWeight::get().reads(23 as Weight))
 			.saturating_add(T::DbWeight::get().writes(14 as Weight))
 	}
 	// Storage: ParaInherent Included (r:1 w:1)
@@ -97,7 +96,6 @@ impl<T: frame_system::Config> runtime_parachains::paras_inherent::WeightInfo for
 	// Storage: ParaScheduler ParathreadQueue (r:1 w:1)
 	// Storage: ParaScheduler Scheduled (r:1 w:1)
 	// Storage: ParaScheduler ValidatorGroups (r:1 w:0)
-	// Storage: System BlockWeight (r:1 w:0)
 	// Storage: Ump NeedsDispatch (r:1 w:1)
 	// Storage: Ump NextDispatchRoundStartWith (r:1 w:1)
 	// Storage: ParaInclusion AvailabilityBitfields (r:0 w:1)
@@ -105,8 +103,8 @@ impl<T: frame_system::Config> runtime_parachains::paras_inherent::WeightInfo for
 	// Storage: Hrmp HrmpWatermarks (r:0 w:1)
 	// Storage: Paras Heads (r:0 w:1)
 	fn enter_bitfields() -> Weight {
-		(265_872_000 as Weight)
-			.saturating_add(T::DbWeight::get().reads(24 as Weight))
+		(352_749_000 as Weight)
+			.saturating_add(T::DbWeight::get().reads(23 as Weight))
 			.saturating_add(T::DbWeight::get().writes(15 as Weight))
 	}
 	// Storage: ParaInherent Included (r:1 w:1)
@@ -129,7 +127,6 @@ impl<T: frame_system::Config> runtime_parachains::paras_inherent::WeightInfo for
 	// Storage: ParaScheduler ParathreadQueue (r:1 w:1)
 	// Storage: ParaScheduler Scheduled (r:1 w:1)
 	// Storage: ParaScheduler ValidatorGroups (r:1 w:0)
-	// Storage: System BlockWeight (r:1 w:0)
 	// Storage: Paras PastCodeMeta (r:1 w:0)
 	// Storage: Paras CurrentCodeHash (r:1 w:0)
 	// Storage: Ump RelayDispatchQueueSize (r:1 w:0)
@@ -139,14 +136,43 @@ impl<T: frame_system::Config> runtime_parachains::paras_inherent::WeightInfo for
 	// Storage: Hrmp HrmpWatermarks (r:0 w:1)
 	// Storage: Paras Heads (r:0 w:1)
 	fn enter_backed_candidates_variable(v: u32, ) -> Weight {
-		(345_183_000 as Weight)
-			// Standard Error: 23_000
-			.saturating_add((49_234_000 as Weight).saturating_mul(v as Weight))
-			.saturating_add(T::DbWeight::get().reads(27 as Weight))
+		(88_047_000 as Weight)
+			// Standard Error: 3_275_000
+			.saturating_add((68_499_000 as Weight).saturating_mul(v as Weight))
+			.saturating_add(T::DbWeight::get().reads(26 as Weight))
 			.saturating_add(T::DbWeight::get().writes(14 as Weight))
 	}
+	// Storage: ParaInherent Included (r:1 w:1)
+	// Storage: System ParentHash (r:1 w:0)
+	// Storage: ParaScheduler AvailabilityCores (r:1 w:1)
+	// Storage: ParasShared CurrentSessionIndex (r:1 w:0)
+	// Storage: ParasShared ActiveValidatorKeys (r:1 w:0)
+	// Storage: Paras Parachains (r:1 w:0)
+	// Storage: ParaInclusion PendingAvailability (r:2 w:1)
+	// Storage: ParaInclusion PendingAvailabilityCommitments (r:1 w:1)
+	// Storage: Configuration ActiveConfig (r:1 w:0)
+	// Storage: Session Validators (r:1 w:0)
+	// Storage: ParasShared ActiveValidatorIndices (r:1 w:0)
+	// Storage: Staking ActiveEra (r:1 w:0)
+	// Storage: Staking ErasRewardPoints (r:1 w:1)
+	// Storage: Dmp DownwardMessageQueues (r:1 w:1)
+	// Storage: Hrmp HrmpChannelDigests (r:1 w:1)
+	// Storage: Paras FutureCodeUpgrades (r:1 w:0)
+	// Storage: ParaScheduler SessionStartBlock (r:1 w:0)
+	// Storage: ParaScheduler ParathreadQueue (r:1 w:1)
+	// Storage: ParaScheduler Scheduled (r:1 w:1)
+	// Storage: ParaScheduler ValidatorGroups (r:1 w:0)
+	// Storage: Paras PastCodeMeta (r:1 w:0)
+	// Storage: Paras CurrentCodeHash (r:1 w:0)
+	// Storage: Ump RelayDispatchQueueSize (r:1 w:0)
+	// Storage: Ump NeedsDispatch (r:1 w:1)
+	// Storage: Ump NextDispatchRoundStartWith (r:1 w:1)
+	// Storage: ParaInherent OnChainVotes (r:0 w:1)
+	// Storage: Hrmp HrmpWatermarks (r:0 w:1)
+	// Storage: Paras Heads (r:0 w:1)
 	fn enter_backed_candidate_code_upgrade() -> Weight {
-		0
+		(53_728_168_000 as Weight)
+			.saturating_add(T::DbWeight::get().reads(26 as Weight))
+			.saturating_add(T::DbWeight::get().writes(14 as Weight))
 	}
-
 }

From f71e233678729a0c01c6c1df1ce331bc1a433ae8 Mon Sep 17 00:00:00 2001
From: Lldenaurois <Ljdenaurois@gmail.com>
Date: Fri, 12 Nov 2021 16:31:22 +0100
Subject: [PATCH 094/107] Add training wheels

---
 parachain/src/primitives.rs              |   2 +-
 primitives/Cargo.toml                    |   1 -
 primitives/src/v1/mod.rs                 |  49 ++++---
 runtime/parachains/src/paras_inherent.rs | 155 ++++++++++++++---------
 4 files changed, 116 insertions(+), 91 deletions(-)

diff --git a/parachain/src/primitives.rs b/parachain/src/primitives.rs
index 98135df0802b..c766ca357cb6 100644
--- a/parachain/src/primitives.rs
+++ b/parachain/src/primitives.rs
@@ -317,7 +317,7 @@ impl<T: Encode + Decode + Default> AccountIdConversion<T> for Id {
 	fn try_from_account(x: &T) -> Option<Self> {
 		x.using_encoded(|d| {
 			if &d[0..4] != b"para" {
-				return None;
+				return None
 			}
 			let mut cursor = &d[4..];
 			let result = Decode::decode(&mut cursor).ok()?;
diff --git a/primitives/Cargo.toml b/primitives/Cargo.toml
index e5242b25de23..6614f6d17e65 100644
--- a/primitives/Cargo.toml
+++ b/primitives/Cargo.toml
@@ -57,5 +57,4 @@ std = [
 ]
 runtime-benchmarks = [
 	"polkadot-parachain/runtime-benchmarks",
-	"frame-system/runtime-benchmarks",
 ]
diff --git a/primitives/src/v1/mod.rs b/primitives/src/v1/mod.rs
index 293373821bdd..5de50aa2d4d3 100644
--- a/primitives/src/v1/mod.rs
+++ b/primitives/src/v1/mod.rs
@@ -613,11 +613,11 @@ pub fn check_candidate_backing<H: AsRef<[u8]> + Clone + Encode>(
 	validator_lookup: impl Fn(usize) -> Option<ValidatorId>,
 ) -> Result<usize, ()> {
 	if backed.validator_indices.len() != group_len {
-		return Err(());
+		return Err(())
 	}
 
 	if backed.validity_votes.len() > group_len {
-		return Err(());
+		return Err(())
 	}
 
 	// this is known, even in runtime, to be blake2-256.
@@ -638,12 +638,12 @@ pub fn check_candidate_backing<H: AsRef<[u8]> + Clone + Encode>(
 		if sig.verify(&payload[..], &validator_id) {
 			signed += 1;
 		} else {
-			return Err(());
+			return Err(())
 		}
 	}
 
 	if signed != backed.validity_votes.len() {
-		return Err(());
+		return Err(())
 	}
 
 	Ok(signed)
@@ -715,10 +715,10 @@ impl GroupRotationInfo {
 	/// `core_index` should be less than `cores`, which is capped at `u32::max()`.
 	pub fn group_for_core(&self, core_index: CoreIndex, cores: usize) -> GroupIndex {
 		if self.group_rotation_frequency == 0 {
-			return GroupIndex(core_index.0);
+			return GroupIndex(core_index.0)
 		}
 		if cores == 0 {
-			return GroupIndex(0);
+			return GroupIndex(0)
 		}
 
 		let cores = sp_std::cmp::min(cores, u32::MAX as usize);
@@ -737,10 +737,10 @@ impl GroupRotationInfo {
 	/// `core_index` should be less than `cores`, which is capped at `u32::max()`.
 	pub fn core_for_group(&self, group_index: GroupIndex, cores: usize) -> CoreIndex {
 		if self.group_rotation_frequency == 0 {
-			return CoreIndex(group_index.0);
+			return CoreIndex(group_index.0)
 		}
 		if cores == 0 {
-			return CoreIndex(0);
+			return CoreIndex(0)
 		}
 
 		let cores = sp_std::cmp::min(cores, u32::MAX as usize);
@@ -772,15 +772,15 @@ impl<N: Saturating + BaseArithmetic + Copy> GroupRotationInfo<N> {
 	/// is 10 and the rotation frequency is 5, this should return 15.
 	pub fn next_rotation_at(&self) -> N {
 		let cycle_once = self.now + self.group_rotation_frequency;
-		cycle_once
-			- (cycle_once.saturating_sub(self.session_start_block) % self.group_rotation_frequency)
+		cycle_once -
+			(cycle_once.saturating_sub(self.session_start_block) % self.group_rotation_frequency)
 	}
 
 	/// Returns the block number of the last rotation before or including the current block. If the
 	/// current block is 10 and the rotation frequency is 5, this should return 10.
 	pub fn last_rotation_at(&self) -> N {
-		self.now
-			- (self.now.saturating_sub(self.session_start_block) % self.group_rotation_frequency)
+		self.now -
+			(self.now.saturating_sub(self.session_start_block) % self.group_rotation_frequency)
 	}
 }
 
@@ -1197,9 +1197,8 @@ impl ConsensusLog {
 		digest_item: &runtime_primitives::DigestItem<H>,
 	) -> Result<Option<Self>, parity_scale_codec::Error> {
 		match digest_item {
-			runtime_primitives::DigestItem::Consensus(id, encoded) if id == &POLKADOT_ENGINE_ID => {
-				Ok(Some(Self::decode(&mut &encoded[..])?))
-			}
+			runtime_primitives::DigestItem::Consensus(id, encoded) if id == &POLKADOT_ENGINE_ID =>
+				Ok(Some(Self::decode(&mut &encoded[..])?)),
 			_ => Ok(None),
 		}
 	}
@@ -1229,27 +1228,23 @@ impl DisputeStatement {
 	/// Get the payload data for this type of dispute statement.
 	pub fn payload_data(&self, candidate_hash: CandidateHash, session: SessionIndex) -> Vec<u8> {
 		match *self {
-			DisputeStatement::Valid(ValidDisputeStatementKind::Explicit) => {
-				ExplicitDisputeStatement { valid: true, candidate_hash, session }.signing_payload()
-			}
+			DisputeStatement::Valid(ValidDisputeStatementKind::Explicit) =>
+				ExplicitDisputeStatement { valid: true, candidate_hash, session }.signing_payload(),
 			DisputeStatement::Valid(ValidDisputeStatementKind::BackingSeconded(
 				inclusion_parent,
 			)) => CompactStatement::Seconded(candidate_hash).signing_payload(&SigningContext {
 				session_index: session,
 				parent_hash: inclusion_parent,
 			}),
-			DisputeStatement::Valid(ValidDisputeStatementKind::BackingValid(inclusion_parent)) => {
+			DisputeStatement::Valid(ValidDisputeStatementKind::BackingValid(inclusion_parent)) =>
 				CompactStatement::Valid(candidate_hash).signing_payload(&SigningContext {
 					session_index: session,
 					parent_hash: inclusion_parent,
-				})
-			}
-			DisputeStatement::Valid(ValidDisputeStatementKind::ApprovalChecking) => {
-				ApprovalVote(candidate_hash).signing_payload(session)
-			}
-			DisputeStatement::Invalid(InvalidDisputeStatementKind::Explicit) => {
-				ExplicitDisputeStatement { valid: false, candidate_hash, session }.signing_payload()
-			}
+				}),
+			DisputeStatement::Valid(ValidDisputeStatementKind::ApprovalChecking) =>
+				ApprovalVote(candidate_hash).signing_payload(session),
+			DisputeStatement::Invalid(InvalidDisputeStatementKind::Explicit) =>
+				ExplicitDisputeStatement { valid: false, candidate_hash, session }.signing_payload(),
 		}
 	}
 
diff --git a/runtime/parachains/src/paras_inherent.rs b/runtime/parachains/src/paras_inherent.rs
index 2da4dd7ec839..c266a298c5c2 100644
--- a/runtime/parachains/src/paras_inherent.rs
+++ b/runtime/parachains/src/paras_inherent.rs
@@ -142,15 +142,26 @@ fn signed_bitfields_weight<T: Config>(bitfields_len: usize) -> Weight {
 		.saturating_mul(bitfields_len as Weight)
 }
 
+fn backed_candidate_weight<T: frame_system::Config + Config>(
+	candidate: &BackedCandidate<T::Hash>,
+) -> Weight {
+	<<T as Config>::WeightInfo as WeightInfo>::enter_backed_candidates_variable(
+		candidate.validity_votes.len() as u32,
+	)
+	.saturating_add(if candidate.candidate.commitments.new_validation_code.is_some() {
+		<<T as Config>::WeightInfo as WeightInfo>::enter_backed_candidate_code_upgrade()
+	} else {
+		0
+	})
+}
+
 fn backed_candidates_weight<T: frame_system::Config + Config>(
-	candidate: &[BackedCandidate<T::Hash>],
+	candidates: &[BackedCandidate<T::Hash>],
 ) -> Weight {
-	candidate
+	candidates
 		.iter()
-		.map(|v| {
-			<<T as Config>::WeightInfo as WeightInfo>::enter_backed_candidates_variable(
-				v.validity_votes.len() as u32,
-			)
+		.map(|c| {
+			backed_candidate_weight::<T>(c)
 		})
 		.fold(0, |acc, x| acc.saturating_add(x))
 }
@@ -321,19 +332,33 @@ pub mod pallet {
 			data: ParachainsInherentData<T::Header>,
 		) -> DispatchResultWithPostInfo {
 			let ParachainsInherentData {
-				bitfields: signed_bitfields,
-				backed_candidates,
+				bitfields: mut signed_bitfields,
+				mut backed_candidates,
 				parent_header,
-				disputes,
+				mut disputes,
 			} = data;
-			let total_weight =
-				paras_inherent_total_weight::<T>(&backed_candidates, &signed_bitfields, &disputes);
+
+			let mut candidate_weight = backed_candidates_weight::<T>(&backed_candidates);
+			let mut bitfields_weight = signed_bitfields_weight::<T>(signed_bitfields.len());
+			let disputes_weight = dispute_statements_weight::<T>(&disputes);
+
+			let max_block_weight = <T as frame_system::Config>::BlockWeights::get().max_block;
 
 			// Abort if the total weight of the block exceeds the max block weight
-			ensure!(
-				total_weight <= <T as frame_system::Config>::BlockWeights::get().max_block,
-				Error::<T>::InherentOverweight
-			);
+			if candidate_weight.saturating_add(bitfields_weight).saturating_add(disputes_weight) > max_block_weight {
+				backed_candidates.clear();
+				candidate_weight = 0;
+				signed_bitfields.clear();
+				bitfields_weight = 0;
+			}
+
+			let total_weight = if disputes_weight > max_block_weight {
+				let parent_hash = <frame_system::Pallet<T>>::parent_hash();
+				let entropy = compute_entropy::<T>(parent_hash);
+				max_block_weight.saturating_sub(limit_disputes::<T>(&mut disputes, entropy))
+			} else {
+				candidate_weight.saturating_add(bitfields_weight).saturating_add(disputes_weight)
+			};
 
 			ensure_none(origin)?;
 			ensure!(!Included::<T>::exists(), Error::<T>::TooManyInclusionInherents);
@@ -618,20 +643,7 @@ impl<T: Config> Pallet<T> {
 		)
 		.ok()?; // by convention, when called with `ON_CHAIN_USE=false`, will always return `Ok()`
 
-		let entropy = {
-			const CANDIDATE_SEED_SUBJECT: [u8; 32] = *b"candidate-seed-selection-subject";
-			let vrf_random = CurrentBlockRandomness::<T>::random(&CANDIDATE_SEED_SUBJECT[..]).0;
-			let mut entropy: [u8; 32] = CANDIDATE_SEED_SUBJECT.clone();
-			if let Some(vrf_random) = vrf_random {
-				entropy.as_mut().copy_from_slice(vrf_random.as_ref());
-			} else {
-				// in case there is no vrf randomness present, we utilize the relay parent
-				// as seed, it's better than a static value.
-				log::warn!(target: LOG_TARGET, "CurrentBlockRandomness did not provide entropy");
-				entropy.as_mut().copy_from_slice(parent_hash.as_ref());
-			}
-			entropy
-		};
+		let entropy = compute_entropy::<T>(parent_hash);
 
 		let remaining_weight = limit_disputes::<T>(&mut disputes, entropy.clone());
 		let (_backed_candidates_weight, backed_candidates, bitfields) =
@@ -749,21 +761,7 @@ fn apply_weight_limit<T: Config + inclusion::Config>(
 			random_sel::<BackedCandidate<<T as frame_system::Config>::Hash>, _>(
 				&mut rng,
 				candidates.clone(),
-				|backed_candidate| {
-					let code_weight = backed_candidate
-						.candidate
-						.commitments
-						.new_validation_code
-						.as_ref()
-						.map(|_code| {
-							<<T as Config>::WeightInfo as WeightInfo>::enter_backed_candidate_code_upgrade()
-						})
-						.unwrap_or(0);
-					<<T as Config>::WeightInfo as WeightInfo>::enter_backed_candidates_variable(
-						backed_candidate.validity_votes.len() as u32,
-					)
-					.saturating_add(code_weight)
-				},
+				|c| backed_candidate_weight::<T>(c),
 				remaining_weight,
 			);
 		let candidates =
@@ -924,6 +922,21 @@ fn sanitize_backed_candidates<
 	Ok(backed_candidates)
 }
 
+fn compute_entropy<T: Config>(parent_hash: T::Hash) -> [u8; 32] {
+	const CANDIDATE_SEED_SUBJECT: [u8; 32] = *b"candidate-seed-selection-subject";
+	let vrf_random = CurrentBlockRandomness::<T>::random(&CANDIDATE_SEED_SUBJECT[..]).0;
+	let mut entropy: [u8; 32] = CANDIDATE_SEED_SUBJECT.clone();
+	if let Some(vrf_random) = vrf_random {
+		entropy.as_mut().copy_from_slice(vrf_random.as_ref());
+	} else {
+		// in case there is no vrf randomness present, we utilize the relay parent
+		// as seed, it's better than a static value.
+		log::warn!(target: LOG_TARGET, "CurrentBlockRandomness did not provide entropy");
+		entropy.as_mut().copy_from_slice(parent_hash.as_ref());
+	}
+	entropy
+}
+
 fn limit_disputes<T: Config>(disputes: &mut MultiDisputeStatementSet, entropy: [u8; 32]) -> Weight {
 	let mut remaining_weight = <T as frame_system::Config>::BlockWeights::get().max_block;
 	let disputes_weight = dispute_statements_weight::<T>(&disputes);
@@ -1019,7 +1032,7 @@ mod tests {
 			builder::{Bench, BenchBuilder},
 			mock::{new_test_ext, MockGenesisConfig, Test},
 		};
-		use frame_support::{assert_err, assert_ok};
+		use frame_support::assert_ok;
 		use sp_std::collections::btree_map::BTreeMap;
 
 		struct TestConfig {
@@ -1257,7 +1270,7 @@ mod tests {
 		#[test]
 		// Ensure that when dispute data establishes an over weight block that we abort
 		// due to an over weight block
-		fn limit_dispute_data_failure() {
+		fn limit_dispute_data_overweight() {
 			new_test_ext(MockGenesisConfig::default()).execute_with(|| {
 				// Create the inherent data for this block
 				let dispute_statements = BTreeMap::new();
@@ -1289,13 +1302,11 @@ mod tests {
 				// The current schedule is empty prior to calling `create_inherent_enter`.
 				assert_eq!(<scheduler::Pallet<Test>>::scheduled(), vec![]);
 
-				// Ensure that calling enter with 3 disputes would cause an over weight block
-				assert_err!(
+				assert_ok!(
 					Pallet::<Test>::enter(
 						frame_system::RawOrigin::None.into(),
 						expected_para_inherent_data,
 					),
-					DispatchError::from(Error::<Test>::InherentOverweight),
 				);
 			});
 		}
@@ -1383,7 +1394,7 @@ mod tests {
 
 		#[test]
 		// Ensure that we abort if we encounter an over weight block for disputes + bitfields
-		fn limit_dispute_data_ignore_backed_candidates_failure() {
+		fn limit_dispute_data_ignore_backed_candidates_overweight() {
 			new_test_ext(MockGenesisConfig::default()).execute_with(|| {
 				// Create the inherent data for this block
 				let dispute_statements = BTreeMap::new();
@@ -1420,16 +1431,22 @@ mod tests {
 				assert_eq!(<scheduler::Pallet<Test>>::scheduled(), vec![]);
 
 				// Ensure that calling enter with 3 disputes and 2 candidates is over weight
-				assert_err!(
+				assert_ok!(
 					Pallet::<Test>::enter(
 						frame_system::RawOrigin::None.into(),
 						expected_para_inherent_data,
 					),
-					DispatchError::from(Error::<Test>::InherentOverweight),
 				);
 
-				// No on chain votes are recorded because we bailed early
-				assert!(Pallet::<Test>::on_chain_votes().is_none());
+				assert_eq!(
+					// The length of this vec is equal to the number of candidates, so we know
+					// all of our candidates got filtered out
+					Pallet::<Test>::on_chain_votes()
+						.unwrap()
+						.backing_validators_per_candidate
+						.len(),
+					0,
+				);
 			});
 		}
 
@@ -1517,7 +1534,7 @@ mod tests {
 
 		#[test]
 		// Ensure that when a block is over weight due to disputes and bitfields, we abort
-		fn limit_bitfields_failure() {
+		fn limit_bitfields_overweight() {
 			new_test_ext(MockGenesisConfig::default()).execute_with(|| {
 				// Create the inherent data for this block
 				let mut dispute_statements = BTreeMap::new();
@@ -1558,15 +1575,22 @@ mod tests {
 				// The current schedule is empty prior to calling `create_inherent_enter`.
 				assert_eq!(<scheduler::Pallet<Test>>::scheduled(), vec![]);
 
-				assert_err!(
+				assert_ok!(
 					Pallet::<Test>::enter(
 						frame_system::RawOrigin::None.into(),
 						expected_para_inherent_data,
 					),
-					DispatchError::from(Error::<Test>::InherentOverweight),
 				);
 
-				assert!(Pallet::<Test>::on_chain_votes().is_none());
+				assert_eq!(
+					// The length of this vec is equal to the number of candidates, so we know
+					// all of our candidates got filtered out
+					Pallet::<Test>::on_chain_votes()
+						.unwrap()
+						.backing_validators_per_candidate
+						.len(),
+					0,
+				);
 			});
 		}
 
@@ -1643,7 +1667,7 @@ mod tests {
 
 		#[test]
 		// Ensure that when a block is over weight due to disputes and bitfields, we abort
-		fn limit_candidates_over_weight_failure() {
+		fn limit_candidates_over_weight_overweight() {
 			new_test_ext(MockGenesisConfig::default()).execute_with(|| {
 				// Create the inherent data for this block
 				let mut dispute_statements = BTreeMap::new();
@@ -1676,15 +1700,22 @@ mod tests {
 				// * 3 disputes.
 				assert_eq!(expected_para_inherent_data.disputes.len(), 3);
 
-				assert_err!(
+				assert_ok!(
 					Pallet::<Test>::enter(
 						frame_system::RawOrigin::None.into(),
 						expected_para_inherent_data,
 					),
-					DispatchError::from(Error::<Test>::InherentOverweight),
 				);
 
-				assert!(Pallet::<Test>::on_chain_votes().is_none());
+				assert_eq!(
+					// The length of this vec is equal to the number of candidates, so we know our 2
+					// backed candidates did not get filtered out
+					Pallet::<Test>::on_chain_votes()
+						.unwrap()
+						.backing_validators_per_candidate
+						.len(),
+					0
+				);
 			});
 		}
 	}

From 8a421713097bb7d940b62ef9c64572bf0251d0ea Mon Sep 17 00:00:00 2001
From: emostov <32168567+emostov@users.noreply.github.com>
Date: Mon, 15 Nov 2021 12:54:38 +0100
Subject: [PATCH 095/107] Refactor some early exit logic for enter

---
 runtime/parachains/src/paras_inherent.rs | 37 +++++++++---------------
 1 file changed, 14 insertions(+), 23 deletions(-)

diff --git a/runtime/parachains/src/paras_inherent.rs b/runtime/parachains/src/paras_inherent.rs
index c266a298c5c2..74a86f7c9db5 100644
--- a/runtime/parachains/src/paras_inherent.rs
+++ b/runtime/parachains/src/paras_inherent.rs
@@ -119,13 +119,6 @@ fn paras_inherent_total_weight<T: Config>(
 		.saturating_add(dispute_statements_weight::<T>(disputes))
 }
 
-fn minimal_inherent_weight<T: Config>() -> Weight {
-	// We just take the min of all our options. This can be changed in the future.
-	<T as Config>::WeightInfo::enter_bitfields()
-		.min(<T as Config>::WeightInfo::enter_variable_disputes(0))
-		.min(<T as Config>::WeightInfo::enter_backed_candidates_variable(0))
-}
-
 fn dispute_statements_weight<T: Config>(disputes: &[DisputeStatementSet]) -> Weight {
 	disputes
 		.iter()
@@ -331,6 +324,11 @@ pub mod pallet {
 			origin: OriginFor<T>,
 			data: ParachainsInherentData<T::Header>,
 		) -> DispatchResultWithPostInfo {
+			ensure_none(origin)?;
+			ensure!(!Included::<T>::exists(), Error::<T>::TooManyInclusionInherents);
+			// Once we are sure we can proceed, mark as included
+			Included::<T>::set(Some(()));
+
 			let ParachainsInherentData {
 				bitfields: mut signed_bitfields,
 				mut backed_candidates,
@@ -338,6 +336,13 @@ pub mod pallet {
 				mut disputes,
 			} = data;
 
+			let parent_hash = <frame_system::Pallet<T>>::parent_hash();
+			// Check that the submitted parent header indeed corresponds to the previous block hash.
+			ensure!(
+				parent_header.hash().as_ref() == parent_hash.as_ref(),
+				Error::<T>::InvalidParentHeader,
+			);
+
 			let mut candidate_weight = backed_candidates_weight::<T>(&backed_candidates);
 			let mut bitfields_weight = signed_bitfields_weight::<T>(signed_bitfields.len());
 			let disputes_weight = dispute_statements_weight::<T>(&disputes);
@@ -352,23 +357,14 @@ pub mod pallet {
 				bitfields_weight = 0;
 			}
 
+
 			let total_weight = if disputes_weight > max_block_weight {
-				let parent_hash = <frame_system::Pallet<T>>::parent_hash();
 				let entropy = compute_entropy::<T>(parent_hash);
 				max_block_weight.saturating_sub(limit_disputes::<T>(&mut disputes, entropy))
 			} else {
 				candidate_weight.saturating_add(bitfields_weight).saturating_add(disputes_weight)
 			};
 
-			ensure_none(origin)?;
-			ensure!(!Included::<T>::exists(), Error::<T>::TooManyInclusionInherents);
-			// Check that the submitted parent header indeed corresponds to the previous block hash.
-			let parent_hash = <frame_system::Pallet<T>>::parent_hash();
-			ensure!(
-				parent_header.hash().as_ref() == parent_hash.as_ref(),
-				Error::<T>::InvalidParentHeader,
-			);
-
 			let expected_bits = <scheduler::Pallet<T>>::availability_cores().len();
 
 			// Handle disputes logic.
@@ -383,8 +379,7 @@ pub mod pallet {
 				let _ = T::DisputesHandler::provide_multi_dispute_data(disputes.clone())?;
 				if T::DisputesHandler::is_frozen() {
 					// The relay chain we are currently on is invalid. Proceed no further on parachains.
-					Included::<T>::set(Some(()));
-					return Ok(Some(minimal_inherent_weight::<T>()).into())
+					return Ok(Some(dispute_statements_weight::<T>(&disputes)).into())
 				}
 
 				let mut freed_disputed = if !new_current_dispute_sets.is_empty() {
@@ -489,9 +484,6 @@ pub mod pallet {
 			// this is max config.ump_service_total_weight
 			let _ump_weight = <ump::Pallet<T>>::process_pending_upward_messages();
 
-			// And track that we've finished processing the inherent for this block.
-			Included::<T>::set(Some(()));
-
 			Ok(Some(total_weight).into())
 		}
 	}
@@ -644,7 +636,6 @@ impl<T: Config> Pallet<T> {
 		.ok()?; // by convention, when called with `ON_CHAIN_USE=false`, will always return `Ok()`
 
 		let entropy = compute_entropy::<T>(parent_hash);
-
 		let remaining_weight = limit_disputes::<T>(&mut disputes, entropy.clone());
 		let (_backed_candidates_weight, backed_candidates, bitfields) =
 			apply_weight_limit::<T>(backed_candidates, bitfields, entropy, remaining_weight);

From ac0c325ffda9d4b1ea1e64ba943b297eba186833 Mon Sep 17 00:00:00 2001
From: Zeke Mostov <z.mostov@gmail.com>
Date: Mon, 15 Nov 2021 18:21:04 +0100
Subject: [PATCH 096/107] Gracefully handle filtering bitfields & candidates
 (#4280)

This updates the logic for sanitize_bitfields and sanitize_backed_candidates to never error when there is an issue, but instead to simply skip the problematic items.
---
 Cargo.lock                               |   1 -
 primitives/src/v1/signed.rs              |   6 +
 runtime/parachains/Cargo.toml            |   1 -
 runtime/parachains/src/inclusion.rs      | 201 +++++++----
 runtime/parachains/src/paras_inherent.rs | 406 ++++++++++-------------
 5 files changed, 305 insertions(+), 310 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 54e8dc602c03..1c2211ca5ff7 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -6877,7 +6877,6 @@ dependencies = [
 name = "polkadot-runtime-parachains"
 version = "0.9.12"
 dependencies = [
- "assert_matches",
  "bitflags",
  "bitvec 0.20.1",
  "derive_more",
diff --git a/primitives/src/v1/signed.rs b/primitives/src/v1/signed.rs
index 81ca09ce3891..cc84448f88f8 100644
--- a/primitives/src/v1/signed.rs
+++ b/primitives/src/v1/signed.rs
@@ -281,6 +281,12 @@ impl<Payload: EncodeAs<RealPayload>, RealPayload: Encode> UncheckedSigned<Payloa
 	pub fn benchmark_signature(&self) -> ValidatorSignature {
 		self.signature.clone()
 	}
+
+	/// Set the signature. Only should be used for creating testing mocks.
+	#[cfg(feature = "std")]
+	pub fn set_signature(&mut self, signature: ValidatorSignature) {
+		self.signature = signature
+	}
 }
 
 impl<Payload, RealPayload> From<Signed<Payload, RealPayload>>
diff --git a/runtime/parachains/Cargo.toml b/runtime/parachains/Cargo.toml
index d708682e2e10..8c89508ff8d7 100644
--- a/runtime/parachains/Cargo.toml
+++ b/runtime/parachains/Cargo.toml
@@ -51,7 +51,6 @@ hex-literal = "0.3.3"
 keyring = { package = "sp-keyring", git = "https://github.com/paritytech/substrate", branch = "master" }
 frame-support-test = { git = "https://github.com/paritytech/substrate", branch = "master" }
 sc-keystore = { git = "https://github.com/paritytech/substrate", branch = "master" }
-assert_matches = "1"
 
 [features]
 default = ["std"]
diff --git a/runtime/parachains/src/inclusion.rs b/runtime/parachains/src/inclusion.rs
index a52c6d7509c4..e1a5e0efc258 100644
--- a/runtime/parachains/src/inclusion.rs
+++ b/runtime/parachains/src/inclusion.rs
@@ -22,7 +22,7 @@
 
 use crate::{
 	configuration, disputes, dmp, hrmp, paras,
-	paras_inherent::{sanitize_bitfields, DisputedBitfield},
+	paras_inherent::{sanitize_bitfields, DisputedBitfield, VERIFY_SIGS},
 	scheduler::CoreAssignment,
 	shared, ump,
 };
@@ -398,19 +398,19 @@ impl<T: Config> Pallet<T> {
 		signed_bitfields: UncheckedSignedAvailabilityBitfields,
 		disputed_bitfield: DisputedBitfield,
 		core_lookup: impl Fn(CoreIndex) -> Option<ParaId>,
-	) -> Result<Vec<(CoreIndex, CandidateHash)>, DispatchError> {
+	) -> Vec<(CoreIndex, CandidateHash)> {
 		let validators = shared::Pallet::<T>::active_validator_keys();
 		let session_index = shared::Pallet::<T>::session_index();
 		let parent_hash = frame_system::Pallet::<T>::parent_hash();
 
-		let checked_bitfields = sanitize_bitfields::<T, true>(
+		let checked_bitfields = sanitize_bitfields::<T, VERIFY_SIGS>(
 			signed_bitfields,
 			disputed_bitfield,
 			expected_bits,
 			parent_hash,
 			session_index,
 			&validators[..],
-		)?;
+		);
 
 		let freed_cores = Self::update_pending_availability_and_get_freed_cores::<_, true>(
 			expected_bits,
@@ -419,7 +419,7 @@ impl<T: Config> Pallet<T> {
 			core_lookup,
 		);
 
-		Ok(freed_cores)
+		freed_cores
 	}
 
 	/// Process candidates that have been backed. Provide the relay storage root, a set of candidates
@@ -1008,7 +1008,6 @@ pub(crate) mod tests {
 		paras_inherent::DisputedBitfield,
 		scheduler::AssignmentKind,
 	};
-	use assert_matches::assert_matches;
 	use frame_support::assert_noop;
 	use futures::executor::block_on;
 	use keyring::Sr25519Keyring;
@@ -1349,7 +1348,7 @@ pub(crate) mod tests {
 		}
 		let validator_public = validator_pubkeys(&validators);
 
-		new_test_ext(genesis_config(paras)).execute_with(|| {
+		new_test_ext(genesis_config(paras.clone())).execute_with(|| {
 			shared::Pallet::<Test>::set_active_validators_ascending(validator_public.clone());
 			shared::Pallet::<Test>::set_session_index(5);
 
@@ -1364,6 +1363,19 @@ pub(crate) mod tests {
 				_ => panic!("out of bounds for testing"),
 			};
 
+			// mark all candidates as pending availability
+			let set_pending_av = || {
+				for (p_id, _) in paras {
+					PendingAvailability::<Test>::insert(
+						p_id,
+						CandidatePendingAvailability {
+							availability_votes: default_availability_votes(),
+							..Default::default()
+						},
+					)
+				}
+			};
+
 			// too many bits in bitfield
 			{
 				let mut bare_bitfield = default_bitfield();
@@ -1376,14 +1388,14 @@ pub(crate) mod tests {
 					&signing_context,
 				));
 
-				assert_noop!(
+				assert_eq!(
 					ParaInclusion::process_bitfields(
 						expected_bits(),
 						vec![signed.into()],
 						DisputedBitfield::zeros(expected_bits()),
 						&core_lookup,
 					),
-					Error::<Test>::WrongBitfieldSize
+					vec![]
 				);
 			}
 
@@ -1398,48 +1410,77 @@ pub(crate) mod tests {
 					&signing_context,
 				));
 
-				assert_noop!(
+				assert_eq!(
 					ParaInclusion::process_bitfields(
 						expected_bits() + 1,
 						vec![signed.into()],
 						DisputedBitfield::zeros(expected_bits()),
 						&core_lookup,
 					),
-					Error::<Test>::WrongBitfieldSize
+					vec![]
 				);
 			}
 
 			// duplicate.
 			{
-				let bare_bitfield = default_bitfield();
+				set_pending_av.clone()();
+				let back_core_0_bitfield = {
+					let mut b = default_bitfield();
+					b.0.set(0, true);
+					b
+				};
 				let signed: UncheckedSignedAvailabilityBitfield = block_on(sign_bitfield(
 					&keystore,
 					&validators[0],
 					ValidatorIndex(0),
-					bare_bitfield,
+					back_core_0_bitfield,
 					&signing_context,
 				))
 				.into();
 
-				assert_noop!(
-					ParaInclusion::process_bitfields(
-						expected_bits(),
-						vec![signed.clone(), signed],
-						DisputedBitfield::zeros(expected_bits()),
-						&core_lookup,
-					),
-					Error::<Test>::BitfieldDuplicateOrUnordered
+				assert_eq!(
+					<PendingAvailability<Test>>::get(chain_a)
+						.unwrap()
+						.availability_votes
+						.count_ones(),
+					0
+				);
+
+				// the threshold to free a core is 4 availability votes, but we only expect 1 valid
+				// valid bitfield.
+				assert!(ParaInclusion::process_bitfields(
+					expected_bits(),
+					vec![signed.clone(), signed],
+					DisputedBitfield::zeros(expected_bits()),
+					&core_lookup,
+				)
+				.is_empty());
+
+				assert_eq!(
+					<PendingAvailability<Test>>::get(chain_a)
+						.unwrap()
+						.availability_votes
+						.count_ones(),
+					1
 				);
+
+				// clean up
+				PendingAvailability::<Test>::remove_all(None);
 			}
 
 			// out of order.
 			{
-				let bare_bitfield = default_bitfield();
+				set_pending_av.clone()();
+				let back_core_0_bitfield = {
+					let mut b = default_bitfield();
+					b.0.set(0, true);
+					b
+				};
 				let signed_0 = block_on(sign_bitfield(
 					&keystore,
 					&validators[0],
 					ValidatorIndex(0),
-					bare_bitfield.clone(),
+					back_core_0_bitfield.clone(),
 					&signing_context,
 				))
 				.into();
@@ -1448,20 +1489,38 @@ pub(crate) mod tests {
 					&keystore,
 					&validators[1],
 					ValidatorIndex(1),
-					bare_bitfield,
+					back_core_0_bitfield,
 					&signing_context,
 				))
 				.into();
 
-				assert_noop!(
-					ParaInclusion::process_bitfields(
-						expected_bits(),
-						vec![signed_1, signed_0],
-						DisputedBitfield::zeros(expected_bits()),
-						&core_lookup,
-					),
-					Error::<Test>::BitfieldDuplicateOrUnordered
+				assert_eq!(
+					<PendingAvailability<Test>>::get(chain_a)
+						.unwrap()
+						.availability_votes
+						.count_ones(),
+					0
+				);
+
+				// the threshold to free a core is 4 availability votes, but we only expect 1 valid
+				// valid bitfield because `signed_0` will get skipped for being out of order.
+				assert!(ParaInclusion::process_bitfields(
+					expected_bits(),
+					vec![signed_1, signed_0],
+					DisputedBitfield::zeros(expected_bits()),
+					&core_lookup,
+				)
+				.is_empty());
+
+				assert_eq!(
+					<PendingAvailability<Test>>::get(chain_a)
+						.unwrap()
+						.availability_votes
+						.count_ones(),
+					1
 				);
+
+				PendingAvailability::<Test>::remove_all(None);
 			}
 
 			// non-pending bit set.
@@ -1476,18 +1535,16 @@ pub(crate) mod tests {
 					&signing_context,
 				));
 
-				assert_matches!(
-					ParaInclusion::process_bitfields(
-						expected_bits(),
-						vec![signed.into()],
-						DisputedBitfield::zeros(expected_bits()),
-						&core_lookup,
-					),
-					Ok(_)
-				);
+				assert!(ParaInclusion::process_bitfields(
+					expected_bits(),
+					vec![signed.into()],
+					DisputedBitfield::zeros(expected_bits()),
+					&core_lookup,
+				)
+				.is_empty());
 			}
 
-			// empty bitfield signed: always OK, but kind of useless.
+			// empty bitfield signed: always ok, but kind of useless.
 			{
 				let bare_bitfield = default_bitfield();
 				let signed = block_on(sign_bitfield(
@@ -1498,15 +1555,13 @@ pub(crate) mod tests {
 					&signing_context,
 				));
 
-				assert_matches!(
-					ParaInclusion::process_bitfields(
-						expected_bits(),
-						vec![signed.into()],
-						DisputedBitfield::zeros(expected_bits()),
-						&core_lookup,
-					),
-					Ok(_)
-				);
+				assert!(ParaInclusion::process_bitfields(
+					expected_bits(),
+					vec![signed.into()],
+					DisputedBitfield::zeros(expected_bits()),
+					&core_lookup,
+				)
+				.is_empty());
 			}
 
 			// bitfield signed with pending bit signed.
@@ -1543,15 +1598,13 @@ pub(crate) mod tests {
 					&signing_context,
 				));
 
-				assert_matches!(
-					ParaInclusion::process_bitfields(
-						expected_bits(),
-						vec![signed.into()],
-						DisputedBitfield::zeros(expected_bits()),
-						&core_lookup,
-					),
-					Ok(_)
-				);
+				assert!(ParaInclusion::process_bitfields(
+					expected_bits(),
+					vec![signed.into()],
+					DisputedBitfield::zeros(expected_bits()),
+					&core_lookup,
+				)
+				.is_empty());
 
 				<PendingAvailability<Test>>::remove(chain_a);
 				PendingAvailabilityCommitments::<Test>::remove(chain_a);
@@ -1588,15 +1641,13 @@ pub(crate) mod tests {
 				));
 
 				// no core is freed
-				assert_eq!(
-					ParaInclusion::process_bitfields(
-						expected_bits(),
-						vec![signed.into()],
-						DisputedBitfield::zeros(expected_bits()),
-						&core_lookup,
-					),
-					Ok(vec![])
-				);
+				assert!(ParaInclusion::process_bitfields(
+					expected_bits(),
+					vec![signed.into()],
+					DisputedBitfield::zeros(expected_bits()),
+					&core_lookup,
+				)
+				.is_empty());
 			}
 		});
 	}
@@ -1652,7 +1703,7 @@ pub(crate) mod tests {
 				CandidatePendingAvailability {
 					core: CoreIndex::from(0),
 					hash: candidate_a.hash(),
-					descriptor: candidate_a.descriptor,
+					descriptor: candidate_a.clone().descriptor,
 					availability_votes: default_availability_votes(),
 					relay_parent_number: 0,
 					backed_in_number: 0,
@@ -1660,7 +1711,10 @@ pub(crate) mod tests {
 					backing_group: GroupIndex::from(0),
 				},
 			);
-			PendingAvailabilityCommitments::<Test>::insert(chain_a, candidate_a.commitments);
+			PendingAvailabilityCommitments::<Test>::insert(
+				chain_a,
+				candidate_a.clone().commitments,
+			);
 
 			let candidate_b = TestCandidateBuilder {
 				para_id: chain_b,
@@ -1732,14 +1786,15 @@ pub(crate) mod tests {
 				})
 				.collect();
 
-			assert_matches!(
+			// only chain A's core is freed.
+			assert_eq!(
 				ParaInclusion::process_bitfields(
 					expected_bits(),
 					signed_bitfields,
 					DisputedBitfield::zeros(expected_bits()),
 					&core_lookup,
 				),
-				Ok(_)
+				vec![(CoreIndex(0), candidate_a.hash())]
 			);
 
 			// chain A had 4 signing off, which is >= threshold.
diff --git a/runtime/parachains/src/paras_inherent.rs b/runtime/parachains/src/paras_inherent.rs
index 74a86f7c9db5..2c2946826f05 100644
--- a/runtime/parachains/src/paras_inherent.rs
+++ b/runtime/parachains/src/paras_inherent.rs
@@ -29,7 +29,6 @@ use crate::{
 };
 use bitvec::prelude::BitVec;
 use frame_support::{
-	fail,
 	inherent::{InherentData, InherentIdentifier, MakeFatalError, ProvideInherent},
 	pallet_prelude::*,
 	traits::Randomness,
@@ -55,6 +54,8 @@ use sp_std::{
 mod benchmarking;
 
 const LOG_TARGET: &str = "runtime::inclusion-inherent";
+const SKIP_SIG_VERIFY: bool = false;
+pub(crate) const VERIFY_SIGS: bool = true;
 
 pub trait WeightInfo {
 	/// Variant over `v`, the count of dispute statements in a dispute statement set. This gives the
@@ -325,8 +326,8 @@ pub mod pallet {
 			data: ParachainsInherentData<T::Header>,
 		) -> DispatchResultWithPostInfo {
 			ensure_none(origin)?;
+
 			ensure!(!Included::<T>::exists(), Error::<T>::TooManyInclusionInherents);
-			// Once we are sure we can proceed, mark as included
 			Included::<T>::set(Some(()));
 
 			let ParachainsInherentData {
@@ -357,7 +358,6 @@ pub mod pallet {
 				bitfields_weight = 0;
 			}
 
-
 			let total_weight = if disputes_weight > max_block_weight {
 				let entropy = compute_entropy::<T>(parent_hash);
 				max_block_weight.saturating_sub(limit_disputes::<T>(&mut disputes, entropy))
@@ -376,6 +376,8 @@ pub mod pallet {
 					.map(|s| (s.session, s.candidate_hash))
 					.collect();
 
+				// Note that `provide_multi_dispute_data` will iterate, verify, and import each
+				// dispute; so the input here must be reasonably bounded.
 				let _ = T::DisputesHandler::provide_multi_dispute_data(disputes.clone())?;
 				if T::DisputesHandler::is_frozen() {
 					// The relay chain we are currently on is invalid. Proceed no further on parachains.
@@ -401,9 +403,8 @@ pub mod pallet {
 					Vec::new()
 				};
 
-				// create a bit index from the set of core indicies
-				// where each index corresponds to a core index
-				// that was freed due to a dispute
+				// Create a bit index from the set of core indices where each index corresponds to
+				// a core index that was freed due to a dispute.
 				let disputed_bitfield = create_disputed_bitfield(
 					expected_bits,
 					freed_disputed.iter().map(|(core_index, _)| core_index),
@@ -426,7 +427,7 @@ pub mod pallet {
 				signed_bitfields,
 				disputed_bitfield,
 				<scheduler::Pallet<T>>::core_para,
-			)?;
+			);
 
 			// Inform the disputes module of all included candidates.
 			let now = <frame_system::Pallet<T>>::block_number();
@@ -440,22 +441,14 @@ pub mod pallet {
 			<scheduler::Pallet<T>>::schedule(freed, now);
 
 			let scheduled = <scheduler::Pallet<T>>::scheduled();
-			let backed_candidates = sanitize_backed_candidates::<T, _, true>(
+			let backed_candidates = sanitize_backed_candidates::<T, _>(
 				parent_hash,
 				backed_candidates,
 				move |candidate_hash: CandidateHash| -> bool {
 					<T>::DisputesHandler::concluded_invalid(current_session, candidate_hash)
 				},
 				&scheduled[..],
-			)
-			.unwrap_or_else(|err| {
-				log::error!(
-					target: LOG_TARGET,
-					"dropping all backed candidates due to sanitization error: {:?}",
-					err,
-				);
-				Vec::new()
-			});
+			);
 
 			// Process backed candidates according to scheduled cores.
 			let parent_storage_root = parent_header.state_root().clone();
@@ -577,25 +570,14 @@ impl<T: Config> Pallet<T> {
 
 				// The following 3 calls are equiv to a call to `process_bitfields`
 				// but we can retain access to `bitfields`.
-				let bitfields = match sanitize_bitfields::<T, false>(
+				let bitfields = sanitize_bitfields::<T, SKIP_SIG_VERIFY>(
 					bitfields,
 					disputed_bitfield,
 					expected_bits,
 					parent_hash,
 					current_session,
 					&validator_public[..],
-				) {
-					Ok(bitfields) => bitfields,
-					Err(err) => {
-						// by convention, when called with `ON_CHAIN_USE=false`, will always return `Ok()`
-						log::error!(
-							target: LOG_TARGET,
-							"BUG: convention violation in create_inherent: {:?}",
-							err
-						);
-						vec![]
-					},
-				};
+				);
 
 				let freed_concluded =
 					<inclusion::Pallet<T>>::update_pending_availability_and_get_freed_cores::<
@@ -625,15 +607,14 @@ impl<T: Config> Pallet<T> {
 				))
 			});
 
-		let backed_candidates = sanitize_backed_candidates::<T, _, false>(
+		let backed_candidates = sanitize_backed_candidates::<T, _>(
 			parent_hash,
 			backed_candidates,
 			move |candidate_hash: CandidateHash| -> bool {
 				concluded_invalid_disputes.contains(&candidate_hash)
 			},
 			&scheduled[..],
-		)
-		.ok()?; // by convention, when called with `ON_CHAIN_USE=false`, will always return `Ok()`
+		);
 
 		let entropy = compute_entropy::<T>(parent_hash);
 		let remaining_weight = limit_disputes::<T>(&mut disputes, entropy.clone());
@@ -649,22 +630,6 @@ impl<T: Config> Pallet<T> {
 	}
 }
 
-/// Assures the `$condition` is `true`, and raises
-/// an error if `$action` is `true`.
-/// If `$action` is `false`, executes `$alt` if present.
-macro_rules! ensure2 {
-	($condition:expr, $err:expr, $action:ident $(, $alt:expr)? $(,)?) => {
-		let condition = $condition;
-		if !condition {
-			if $action {
-				ensure!(condition, $err);
-			} else {
-				$($alt)?
-			}
-		}
-	};
-}
-
 /// Derive a bitfield from dispute
 pub(super) fn create_disputed_bitfield<'a, I>(
 	expected_bits: usize,
@@ -792,81 +757,95 @@ fn apply_weight_limit<T: Config + inclusion::Config>(
 /// they were actually checked and filtered to allow using it in both
 /// cases, as `filtering` and `checking` stage.
 ///
-/// `EARLY_RETURN` determines the behavior.
-/// `false` assures that all inputs are filtered, and invalid ones are filtered out.
-/// It also skips signature verification.
-/// `true` returns an `Err(_)` on the first check failing.
-pub(crate) fn sanitize_bitfields<T: crate::inclusion::Config, const EARLY_RETURN: bool>(
+/// `CHECK_SIGS` determines if validator signatures are checked. If true, bitfields that have an
+/// invalid signature will be filtered out.
+pub(crate) fn sanitize_bitfields<T: crate::inclusion::Config, const CHECK_SIGS: bool>(
 	unchecked_bitfields: UncheckedSignedAvailabilityBitfields,
 	disputed_bitfield: DisputedBitfield,
 	expected_bits: usize,
 	parent_hash: T::Hash,
 	session_index: SessionIndex,
 	validators: &[ValidatorId],
-) -> Result<UncheckedSignedAvailabilityBitfields, DispatchError> {
+) -> UncheckedSignedAvailabilityBitfields {
 	let mut bitfields = Vec::with_capacity(unchecked_bitfields.len());
 
 	let mut last_index = None;
 
-	ensure2!(
-		disputed_bitfield.0.len() == expected_bits,
-		crate::inclusion::Error::<T>::WrongBitfieldSize,
-		EARLY_RETURN
-	);
+	if disputed_bitfield.0.len() != expected_bits {
+		// This is a system logic error that should never occur, but we want to handle it gracefully
+		// so we just drop all bitfields
+		log::error!(target: LOG_TARGET, "BUG: disputed_bitfield != expected_bits");
+		return vec![]
+	}
 
 	let all_zeros = BitVec::<bitvec::order::Lsb0, u8>::repeat(false, expected_bits);
 	let signing_context = SigningContext { parent_hash, session_index };
 	for unchecked_bitfield in unchecked_bitfields {
-		ensure2!(
-			unchecked_bitfield.unchecked_payload().0.len() == expected_bits,
-			crate::inclusion::Error::<T>::WrongBitfieldSize,
-			EARLY_RETURN,
+		// Find and skip invalid bitfields.
+		if unchecked_bitfield.unchecked_payload().0.len() != expected_bits {
+			log::trace!(
+				target: LOG_TARGET,
+				"[CHECK_SIGS: {}] bad bitfield length: {} != {:?}",
+				CHECK_SIGS,
+				unchecked_bitfield.unchecked_payload().0.len(),
+				expected_bits,
+			);
 			continue
-		);
+		}
 
-		ensure2!(
-			unchecked_bitfield.unchecked_payload().0.clone() & disputed_bitfield.0.clone() ==
-				all_zeros,
-			crate::inclusion::Error::<T>::BitfieldReferencesFreedCore,
-			EARLY_RETURN,
+		if unchecked_bitfield.unchecked_payload().0.clone() & disputed_bitfield.0.clone() !=
+			all_zeros
+		{
+			log::trace!(
+				target: LOG_TARGET,
+				"[CHECK_SIGS: {}] bitfield contains disputed cores: {:?}",
+				CHECK_SIGS,
+				unchecked_bitfield.unchecked_payload().0.clone() & disputed_bitfield.0.clone()
+			);
 			continue
-		);
+		}
 
-		ensure2!(
-			last_index.map_or(true, |last| last < unchecked_bitfield.unchecked_validator_index()),
-			crate::inclusion::Error::<T>::BitfieldDuplicateOrUnordered,
-			EARLY_RETURN,
+		if !last_index.map_or(true, |last| last < unchecked_bitfield.unchecked_validator_index()) {
+			log::trace!(
+				target: LOG_TARGET,
+				"[CHECK_SIGS: {}] bitfield validator index is not greater than last: !({:?} < {:?})",
+				CHECK_SIGS,
+				last_index,
+				unchecked_bitfield.unchecked_validator_index()
+			);
 			continue
-		);
+		}
 
-		ensure2!(
-			(unchecked_bitfield.unchecked_validator_index().0 as usize) < validators.len(),
-			crate::inclusion::Error::<T>::ValidatorIndexOutOfBounds,
-			EARLY_RETURN,
+		if unchecked_bitfield.unchecked_validator_index().0 as usize >= validators.len() {
+			log::trace!(
+				target: LOG_TARGET,
+				"[CHECK_SIGS: {}] bitfield validator index is out of bounds: {:?} >= {:?}",
+				CHECK_SIGS,
+				unchecked_bitfield.unchecked_validator_index().0,
+				validators.len(),
+			);
 			continue
-		);
+		}
 
 		let validator_index = unchecked_bitfield.unchecked_validator_index();
 
 		let validator_public = &validators[validator_index.0 as usize];
 
-		// only check the signatures when returning early
-		if EARLY_RETURN {
-			let signed_bitfield = if let Ok(signed_bitfield) =
+		if CHECK_SIGS {
+			if let Ok(signed_bitfield) =
 				unchecked_bitfield.try_into_checked(&signing_context, validator_public)
 			{
-				signed_bitfield
+				bitfields.push(signed_bitfield.into_unchecked());
 			} else {
-				fail!(crate::inclusion::Error::<T>::InvalidBitfieldSignature);
+				log::warn!(target: LOG_TARGET, "Invalid bitfield signature");
 			};
-			bitfields.push(signed_bitfield.into_unchecked());
 		} else {
 			bitfields.push(unchecked_bitfield);
 		}
 
 		last_index = Some(validator_index);
 	}
-	Ok(bitfields)
+	bitfields
 }
 
 /// Filter out any candidates that have a concluded invalid dispute.
@@ -878,27 +857,20 @@ pub(crate) fn sanitize_bitfields<T: crate::inclusion::Config, const EARLY_RETURN
 ///
 /// `candidate_has_concluded_invalid_dispute` must return `true` if the candidate
 /// is disputed, false otherwise
-fn sanitize_backed_candidates<
-	T: crate::inclusion::Config,
-	F: Fn(CandidateHash) -> bool,
-	const EARLY_RETURN: bool,
->(
+fn sanitize_backed_candidates<T: crate::inclusion::Config, F: Fn(CandidateHash) -> bool>(
 	relay_parent: T::Hash,
 	mut backed_candidates: Vec<BackedCandidate<T::Hash>>,
 	candidate_has_concluded_invalid_dispute: F,
 	scheduled: &[CoreAssignment],
-) -> Result<Vec<BackedCandidate<T::Hash>>, Error<T>> {
-	let n = backed_candidates.len();
+) -> Vec<BackedCandidate<T::Hash>> {
 	// Remove any candidates that were concluded invalid.
 	backed_candidates.retain(|backed_candidate| {
 		!candidate_has_concluded_invalid_dispute(backed_candidate.candidate.hash())
 	});
-	ensure2!(backed_candidates.len() == n, Error::<T>::CandidateConcludedInvalid, EARLY_RETURN);
 
 	// Assure the backed candidate's `ParaId`'s core is free.
 	// This holds under the assumption that `Scheduler::schedule` is called _before_.
 	// Also checks the candidate references the correct relay parent.
-
 	let scheduled_paras_set = scheduled
 		.into_iter()
 		.map(|core_assignment| core_assignment.para_id)
@@ -907,10 +879,8 @@ fn sanitize_backed_candidates<
 		let desc = backed_candidate.descriptor();
 		desc.relay_parent == relay_parent && scheduled_paras_set.contains(&desc.para_id)
 	});
-	ensure2!(backed_candidates.len() == n, Error::<T>::CandidateConcludedInvalid, EARLY_RETURN);
 
-	// Limit weight, to avoid overweight block.
-	Ok(backed_candidates)
+	backed_candidates
 }
 
 fn compute_entropy<T: Config>(parent_hash: T::Hash) -> [u8; 32] {
@@ -1053,7 +1023,7 @@ mod tests {
 		#[test]
 		// Validate that if we create 2 backed candidates which are assigned to 2 cores that will be freed via
 		// becoming fully available, the backed candidates will not be filtered out in `create_inherent` and
-		// will not cause `enter` to exit early.
+		// will not cause `enter` to early.
 		fn include_backed_candidates() {
 			new_test_ext(MockGenesisConfig::default()).execute_with(|| {
 				let dispute_statements = BTreeMap::new();
@@ -1723,7 +1693,6 @@ mod tests {
 
 	mod sanitizers {
 		use super::*;
-		use assert_matches::assert_matches;
 
 		use crate::inclusion::tests::{
 			back_candidate, collator_sign_candidate, BackingKind, TestCandidateBuilder,
@@ -1803,8 +1772,8 @@ mod tests {
 			let disputed_bitfield = DisputedBitfield::zeros(expected_bits);
 
 			{
-				assert_matches!(
-					sanitize_bitfields::<Test, true>(
+				assert_eq!(
+					sanitize_bitfields::<Test, VERIFY_SIGS>(
 						unchecked_bitfields.clone(),
 						disputed_bitfield.clone(),
 						expected_bits,
@@ -1812,10 +1781,10 @@ mod tests {
 						session_index,
 						&validator_public[..]
 					),
-					Ok(_)
+					unchecked_bitfields.clone()
 				);
 				assert_eq!(
-					sanitize_bitfields::<Test, false>(
+					sanitize_bitfields::<Test, SKIP_SIG_VERIFY>(
 						unchecked_bitfields.clone(),
 						disputed_bitfield.clone(),
 						expected_bits,
@@ -1823,7 +1792,7 @@ mod tests {
 						session_index,
 						&validator_public[..]
 					),
-					Ok(unchecked_bitfields.clone())
+					unchecked_bitfields.clone()
 				);
 			}
 
@@ -1835,125 +1804,141 @@ mod tests {
 				disputed_bitfield.0.set(0, true);
 
 				assert_eq!(
-					sanitize_bitfields::<Test, true>(
+					sanitize_bitfields::<Test, VERIFY_SIGS>(
 						unchecked_bitfields.clone(),
 						disputed_bitfield.clone(),
 						expected_bits,
 						parent_hash,
 						session_index,
 						&validator_public[..]
-					),
-					Err(inclusion::Error::<Test>::BitfieldReferencesFreedCore.into())
+					)
+					.len(),
+					1
 				);
-				assert_matches!(
-					sanitize_bitfields::<Test, false>(
+				assert_eq!(
+					sanitize_bitfields::<Test, SKIP_SIG_VERIFY>(
 						unchecked_bitfields.clone(),
 						disputed_bitfield.clone(),
 						expected_bits,
 						parent_hash,
 						session_index,
 						&validator_public[..]
-					),
-					Ok(unchecked_bitfields) => {
-						assert_eq!(unchecked_bitfields.len(), 1);
-					}
+					)
+					.len(),
+					1
 				);
 			}
 
 			// bitfield size mismatch
 			{
+				assert!(sanitize_bitfields::<Test, VERIFY_SIGS>(
+					unchecked_bitfields.clone(),
+					disputed_bitfield.clone(),
+					expected_bits + 1,
+					parent_hash,
+					session_index,
+					&validator_public[..]
+				)
+				.is_empty());
+				assert!(sanitize_bitfields::<Test, SKIP_SIG_VERIFY>(
+					unchecked_bitfields.clone(),
+					disputed_bitfield.clone(),
+					expected_bits + 1,
+					parent_hash,
+					session_index,
+					&validator_public[..]
+				)
+				.is_empty());
+			}
+
+			// remove the last validator
+			{
+				let shortened = validator_public.len() - 2;
 				assert_eq!(
-					sanitize_bitfields::<Test, true>(
+					&sanitize_bitfields::<Test, VERIFY_SIGS>(
 						unchecked_bitfields.clone(),
 						disputed_bitfield.clone(),
-						expected_bits + 1,
+						expected_bits,
 						parent_hash,
 						session_index,
-						&validator_public[..]
-					),
-					Err(inclusion::Error::<Test>::WrongBitfieldSize.into())
+						&validator_public[..shortened]
+					)[..],
+					&unchecked_bitfields[..shortened]
 				);
-				assert_matches!(
-					sanitize_bitfields::<Test, false>(
+				assert_eq!(
+					&sanitize_bitfields::<Test, SKIP_SIG_VERIFY>(
 						unchecked_bitfields.clone(),
 						disputed_bitfield.clone(),
-						expected_bits + 1,
+						expected_bits,
 						parent_hash,
 						session_index,
-						&validator_public[..]
-					),
-					Ok(unchecked_bitfields) => {
-						assert!(unchecked_bitfields.is_empty());
-					}
+						&validator_public[..shortened]
+					)[..],
+					&unchecked_bitfields[..shortened]
 				);
 			}
 
-			// remove the last validator
+			// switch ordering of bitfields
 			{
-				let shortened = validator_public.len() - 2;
+				let mut unchecked_bitfields = unchecked_bitfields.clone();
+				let x = unchecked_bitfields.swap_remove(0);
+				unchecked_bitfields.push(x);
 				assert_eq!(
-					sanitize_bitfields::<Test, true>(
+					&sanitize_bitfields::<Test, VERIFY_SIGS>(
 						unchecked_bitfields.clone(),
 						disputed_bitfield.clone(),
 						expected_bits,
 						parent_hash,
 						session_index,
-						&validator_public[..shortened]
-					),
-					Err(inclusion::Error::<Test>::ValidatorIndexOutOfBounds.into())
+						&validator_public[..]
+					)[..],
+					&unchecked_bitfields[..(unchecked_bitfields.len() - 2)]
 				);
-				assert_matches!(
-					sanitize_bitfields::<Test, false>(
+				assert_eq!(
+					&sanitize_bitfields::<Test, SKIP_SIG_VERIFY>(
 						unchecked_bitfields.clone(),
 						disputed_bitfield.clone(),
 						expected_bits,
 						parent_hash,
 						session_index,
-						&validator_public[..shortened]
-					),
-					Ok(unchecked_bitfields_filtered) => {
-						assert_eq!(
-							&unchecked_bitfields_filtered[..],
-							&unchecked_bitfields[..shortened]
-						);
-					}
+						&validator_public[..]
+					)[..],
+					&unchecked_bitfields[..(unchecked_bitfields.len() - 2)]
 				);
 			}
 
-			// switch ordering of bitfields, bail
+			// check the validators signature
 			{
+				use primitives::v1::ValidatorSignature;
 				let mut unchecked_bitfields = unchecked_bitfields.clone();
-				let x = unchecked_bitfields.swap_remove(0);
-				unchecked_bitfields.push(x);
+
+				// insert a bad signature for the last bitfield
+				let last_bit_idx = unchecked_bitfields.len() - 1;
+				unchecked_bitfields
+					.get_mut(last_bit_idx)
+					.and_then(|u| Some(u.set_signature(ValidatorSignature::default())))
+					.expect("we are accessing a valid index");
 				assert_eq!(
-					sanitize_bitfields::<Test, true>(
+					&sanitize_bitfields::<Test, VERIFY_SIGS>(
 						unchecked_bitfields.clone(),
 						disputed_bitfield.clone(),
 						expected_bits,
 						parent_hash,
 						session_index,
 						&validator_public[..]
-					),
-					Err(inclusion::Error::<Test>::BitfieldDuplicateOrUnordered.into())
+					)[..],
+					&unchecked_bitfields[..last_bit_idx]
 				);
-				assert_matches!(
-					sanitize_bitfields::<Test, false>(
+				assert_eq!(
+					&sanitize_bitfields::<Test, SKIP_SIG_VERIFY>(
 						unchecked_bitfields.clone(),
 						disputed_bitfield.clone(),
 						expected_bits,
 						parent_hash,
 						session_index,
 						&validator_public[..]
-					),
-					Ok(unchecked_bitfields_filtered) => {
-						// the last element is out of order
-						// hence sanitization will have removed only that
-						// one
-						assert_eq!(
-							&unchecked_bitfields[..(unchecked_bitfields.len()-2)],
-							&unchecked_bitfields_filtered[..]
-							);
-					}
+					)[..],
+					&unchecked_bitfields[..]
 				);
 			}
 		}
@@ -2038,83 +2023,44 @@ mod tests {
 				})
 				.collect::<Vec<_>>();
 
-			{
-				assert_matches!(
-					sanitize_backed_candidates::<Test, _, true>(
-						relay_parent,
-						backed_candidates.clone(),
-						has_concluded_invalid,
-						scheduled
-					),
-					Ok(sanitized_backed_candidates) => {
-						assert_eq!(backed_candidates, sanitized_backed_candidates);
-					}
-				);
-				assert_matches!(
-					sanitize_backed_candidates::<Test, _, false>(
-						relay_parent,
-						backed_candidates.clone(),
-						has_concluded_invalid,
-						scheduled
-					),
-					Ok(sanitized_backed_candidates) => {
-						assert_eq!(backed_candidates, sanitized_backed_candidates);
-					}
-				);
-			}
+			// happy path
+			assert_eq!(
+				sanitize_backed_candidates::<Test, _>(
+					relay_parent,
+					backed_candidates.clone(),
+					has_concluded_invalid,
+					scheduled
+				),
+				backed_candidates
+			);
 
-			// nothing is scheduled, so no paraids match,
-			// hence no backed candidate makes it through
+			// nothing is scheduled, so no paraids match, thus all backed candidates are skipped
 			{
 				let scheduled = &[][..];
-				assert_matches!(
-					sanitize_backed_candidates::<Test, _, true>(
-						relay_parent,
-						backed_candidates.clone(),
-						has_concluded_invalid,
-						scheduled
-					),
-					Err(Error::<Test>::CandidateConcludedInvalid)
-				);
-				assert_matches!(
-					sanitize_backed_candidates::<Test, _, false>(
+				assert!(
+					sanitize_backed_candidates::<Test, _>(
 						relay_parent,
 						backed_candidates.clone(),
 						has_concluded_invalid,
 						scheduled
-					),
-					Ok(sanitized_backed_candidates) => {
-						assert_eq!(0, sanitized_backed_candidates.len());
-					}
+					).is_empty()
 				);
 			}
 
 			// relay parent mismatch
 			{
 				let relay_parent = Hash::repeat_byte(0xFA);
-				assert_matches!(
-					sanitize_backed_candidates::<Test, _, true>(
-						relay_parent,
-						backed_candidates.clone(),
-						has_concluded_invalid,
-						scheduled
-					),
-					Err(Error::<Test>::CandidateConcludedInvalid)
-				);
-				assert_matches!(
-					sanitize_backed_candidates::<Test, _, false>(
+				assert!(
+					sanitize_backed_candidates::<Test, _>(
 						relay_parent,
 						backed_candidates.clone(),
 						has_concluded_invalid,
 						scheduled
-					),
-					Ok(sanitized_backed_candidates) => {
-						assert_eq!(0, sanitized_backed_candidates.len());
-					}
+					).is_empty()
 				);
 			}
 
-			// relay parent mismatch
+			// candidates that have concluded as invalid are filtered out
 			{
 				// mark every second one as concluded invalid
 				let set = {
@@ -2127,25 +2073,15 @@ mod tests {
 					set
 				};
 				let has_concluded_invalid = |candidate: CandidateHash| set.contains(&candidate);
-				assert_matches!(
-					sanitize_backed_candidates::<Test, _, true>(
-						relay_parent,
-						backed_candidates.clone(),
-						has_concluded_invalid,
-						scheduled
-					),
-					Err(Error::<Test>::CandidateConcludedInvalid)
-				);
-				assert_matches!(
-					sanitize_backed_candidates::<Test, _, false>(
+				assert_eq!(
+					sanitize_backed_candidates::<Test, _>(
 						relay_parent,
 						backed_candidates.clone(),
 						has_concluded_invalid,
 						scheduled
-					),
-					Ok(sanitized_backed_candidates) => {
-						assert_eq!(0, sanitized_backed_candidates.len() / 2);
-					}
+					)
+					.len(),
+					backed_candidates.len() / 2
 				);
 			}
 		}

From e5665440c6be74e8f27066111a57d32ba462be9a Mon Sep 17 00:00:00 2001
From: Zeke Mostov <z.mostov@gmail.com>
Date: Mon, 15 Nov 2021 18:40:12 +0100
Subject: [PATCH 097/107] Refactor inherent data weight limiting logic (#4287)

---
 runtime/parachains/src/paras_inherent.rs | 163 ++++++++++++-----------
 1 file changed, 85 insertions(+), 78 deletions(-)

diff --git a/runtime/parachains/src/paras_inherent.rs b/runtime/parachains/src/paras_inherent.rs
index 2c2946826f05..c9325c6fd800 100644
--- a/runtime/parachains/src/paras_inherent.rs
+++ b/runtime/parachains/src/paras_inherent.rs
@@ -41,9 +41,9 @@ use primitives::v1::{
 	SessionIndex, SigningContext, UncheckedSignedAvailabilityBitfield,
 	UncheckedSignedAvailabilityBitfields, ValidatorId, PARACHAINS_INHERENT_IDENTIFIER,
 };
-use rand::Rng;
+use rand::{Rng, SeedableRng};
 use scale_info::TypeInfo;
-use sp_runtime::traits::Header as HeaderT;
+use sp_runtime::traits::{Header as HeaderT, Zero};
 use sp_std::{
 	cmp::Ordering,
 	collections::{btree_map::BTreeMap, btree_set::BTreeSet},
@@ -139,14 +139,13 @@ fn signed_bitfields_weight<T: Config>(bitfields_len: usize) -> Weight {
 fn backed_candidate_weight<T: frame_system::Config + Config>(
 	candidate: &BackedCandidate<T::Hash>,
 ) -> Weight {
-	<<T as Config>::WeightInfo as WeightInfo>::enter_backed_candidates_variable(
-		candidate.validity_votes.len() as u32,
-	)
-	.saturating_add(if candidate.candidate.commitments.new_validation_code.is_some() {
+	if candidate.candidate.commitments.new_validation_code.is_some() {
 		<<T as Config>::WeightInfo as WeightInfo>::enter_backed_candidate_code_upgrade()
 	} else {
-		0
-	})
+		<<T as Config>::WeightInfo as WeightInfo>::enter_backed_candidates_variable(
+			candidate.validity_votes.len() as u32,
+		)
+	}
 }
 
 fn backed_candidates_weight<T: frame_system::Config + Config>(
@@ -154,9 +153,7 @@ fn backed_candidates_weight<T: frame_system::Config + Config>(
 ) -> Weight {
 	candidates
 		.iter()
-		.map(|c| {
-			backed_candidate_weight::<T>(c)
-		})
+		.map(|c| backed_candidate_weight::<T>(c))
 		.fold(0, |acc, x| acc.saturating_add(x))
 }
 
@@ -350,19 +347,36 @@ pub mod pallet {
 
 			let max_block_weight = <T as frame_system::Config>::BlockWeights::get().max_block;
 
-			// Abort if the total weight of the block exceeds the max block weight
-			if candidate_weight.saturating_add(bitfields_weight).saturating_add(disputes_weight) > max_block_weight {
-				backed_candidates.clear();
-				candidate_weight = 0;
-				signed_bitfields.clear();
-				bitfields_weight = 0;
-			}
+			// Potentially trim inherent data to ensure processing will be within weight limits
+			let total_weight = {
+				if candidate_weight
+					.saturating_add(bitfields_weight)
+					.saturating_add(disputes_weight) >
+					max_block_weight
+				{
+					// if the total weight is over the max block weight, first try clearing backed
+					// candidates and bitfields.
+					backed_candidates.clear();
+					candidate_weight = 0;
+					signed_bitfields.clear();
+					bitfields_weight = 0;
+				}
 
-			let total_weight = if disputes_weight > max_block_weight {
-				let entropy = compute_entropy::<T>(parent_hash);
-				max_block_weight.saturating_sub(limit_disputes::<T>(&mut disputes, entropy))
-			} else {
-				candidate_weight.saturating_add(bitfields_weight).saturating_add(disputes_weight)
+				if disputes_weight > max_block_weight {
+					// if disputes are overweight, trim the disputes.
+					debug_assert!(candidate_weight == 0 && bitfields_weight == 0);
+
+					let entropy = compute_entropy::<T>(parent_hash);
+					let mut rng = rand_chacha::ChaChaRng::from_seed(entropy.into());
+
+					let remaining_weight =
+						limit_disputes::<T>(&mut disputes, max_block_weight, &mut rng);
+					max_block_weight.saturating_sub(remaining_weight)
+				} else {
+					candidate_weight
+						.saturating_add(bitfields_weight)
+						.saturating_add(disputes_weight)
+				}
 			};
 
 			let expected_bits = <scheduler::Pallet<T>>::availability_cores().len();
@@ -617,9 +631,11 @@ impl<T: Config> Pallet<T> {
 		);
 
 		let entropy = compute_entropy::<T>(parent_hash);
-		let remaining_weight = limit_disputes::<T>(&mut disputes, entropy.clone());
+		let mut rng = rand_chacha::ChaChaRng::from_seed(entropy.into());
+		let max_block_weight = <T as frame_system::Config>::BlockWeights::get().max_block;
+		let remaining_weight = limit_disputes::<T>(&mut disputes, max_block_weight, &mut rng);
 		let (_backed_candidates_weight, backed_candidates, bitfields) =
-			apply_weight_limit::<T>(backed_candidates, bitfields, entropy, remaining_weight);
+			apply_weight_limit::<T>(backed_candidates, bitfields, remaining_weight, &mut rng);
 
 		Some(ParachainsInherentData::<T::Header> {
 			bitfields,
@@ -694,8 +710,8 @@ fn random_sel<X, F: Fn(&X) -> Weight>(
 fn apply_weight_limit<T: Config + inclusion::Config>(
 	candidates: Vec<BackedCandidate<<T>::Hash>>,
 	bitfields: UncheckedSignedAvailabilityBitfields,
-	entropy: [u8; 32],
 	max_weight: Weight,
+	rng: &mut rand_chacha::ChaChaRng,
 ) -> (Weight, Vec<BackedCandidate<<T>::Hash>>, UncheckedSignedAvailabilityBitfields) {
 	let total_candidates_weight = backed_candidates_weight::<T>(candidates.as_slice());
 	let total_bitfields_weight = signed_bitfields_weight::<T>(bitfields.len());
@@ -707,15 +723,12 @@ fn apply_weight_limit<T: Config + inclusion::Config>(
 		return (total, candidates, bitfields)
 	}
 
-	use rand_chacha::rand_core::SeedableRng;
-	let mut rng = rand_chacha::ChaChaRng::from_seed(entropy.into());
-
 	// There is weight remaining to be consumed by a subset of candidates
 	// which are going to be picked now.
 	if let Some(remaining_weight) = max_weight.checked_sub(total_bitfields_weight) {
 		let (acc_candidate_weight, indices) =
 			random_sel::<BackedCandidate<<T as frame_system::Config>::Hash>, _>(
-				&mut rng,
+				rng,
 				candidates.clone(),
 				|c| backed_candidate_weight::<T>(c),
 				remaining_weight,
@@ -731,7 +744,7 @@ fn apply_weight_limit<T: Config + inclusion::Config>(
 	// insufficient space for even the bitfields alone, so only try to fit as many of those
 	// into the block and skip the candidates entirely
 	let (total, indices) = random_sel::<UncheckedSignedAvailabilityBitfield, _>(
-		&mut rng,
+		rng,
 		bitfields.clone(),
 		|_| <<T as Config>::WeightInfo as WeightInfo>::enter_bitfields(),
 		max_weight,
@@ -898,8 +911,15 @@ fn compute_entropy<T: Config>(parent_hash: T::Hash) -> [u8; 32] {
 	entropy
 }
 
-fn limit_disputes<T: Config>(disputes: &mut MultiDisputeStatementSet, entropy: [u8; 32]) -> Weight {
-	let mut remaining_weight = <T as frame_system::Config>::BlockWeights::get().max_block;
+/// Limit disputes in place.
+///
+/// Returns the unused weight of `remaining_weight`.
+fn limit_disputes<T: Config>(
+	disputes: &mut MultiDisputeStatementSet,
+	remaining_weight: Weight,
+	rng: &mut rand_chacha::ChaChaRng,
+) -> Weight {
+	let mut remaining_weight = remaining_weight;
 	let disputes_weight = dispute_statements_weight::<T>(&disputes);
 	if disputes_weight > remaining_weight {
 		// Sort the dispute statements according to the following prioritization:
@@ -954,12 +974,9 @@ fn limit_disputes<T: Config>(disputes: &mut MultiDisputeStatementSet, entropy: [
 		// Compute the statements length of all remote disputes
 		let d = remote_disputes.iter().map(|d| d.statements.len() as u32).collect::<Vec<u32>>();
 
-		use rand_chacha::rand_core::SeedableRng;
-		let mut rng = rand_chacha::ChaChaRng::from_seed(entropy.into());
-
 		// Select remote disputes at random until the block is full
 		let (acc_remote_disputes_weight, indices) = random_sel::<u32, _>(
-			&mut rng,
+			rng,
 			d,
 			|v| <<T as Config>::WeightInfo as WeightInfo>::enter_variable_disputes(*v),
 			remaining_weight,
@@ -1263,12 +1280,10 @@ mod tests {
 				// The current schedule is empty prior to calling `create_inherent_enter`.
 				assert_eq!(<scheduler::Pallet<Test>>::scheduled(), vec![]);
 
-				assert_ok!(
-					Pallet::<Test>::enter(
-						frame_system::RawOrigin::None.into(),
-						expected_para_inherent_data,
-					),
-				);
+				assert_ok!(Pallet::<Test>::enter(
+					frame_system::RawOrigin::None.into(),
+					expected_para_inherent_data,
+				));
 			});
 		}
 
@@ -1392,12 +1407,10 @@ mod tests {
 				assert_eq!(<scheduler::Pallet<Test>>::scheduled(), vec![]);
 
 				// Ensure that calling enter with 3 disputes and 2 candidates is over weight
-				assert_ok!(
-					Pallet::<Test>::enter(
-						frame_system::RawOrigin::None.into(),
-						expected_para_inherent_data,
-					),
-				);
+				assert_ok!(Pallet::<Test>::enter(
+					frame_system::RawOrigin::None.into(),
+					expected_para_inherent_data,
+				));
 
 				assert_eq!(
 					// The length of this vec is equal to the number of candidates, so we know
@@ -1536,12 +1549,10 @@ mod tests {
 				// The current schedule is empty prior to calling `create_inherent_enter`.
 				assert_eq!(<scheduler::Pallet<Test>>::scheduled(), vec![]);
 
-				assert_ok!(
-					Pallet::<Test>::enter(
-						frame_system::RawOrigin::None.into(),
-						expected_para_inherent_data,
-					),
-				);
+				assert_ok!(Pallet::<Test>::enter(
+					frame_system::RawOrigin::None.into(),
+					expected_para_inherent_data,
+				));
 
 				assert_eq!(
 					// The length of this vec is equal to the number of candidates, so we know
@@ -1612,7 +1623,7 @@ mod tests {
 				assert_ok!(Pallet::<Test>::enter(
 					frame_system::RawOrigin::None.into(),
 					limit_inherent_data,
-				),);
+				));
 
 				assert_eq!(
 					// The length of this vec is equal to the number of candidates, so we know our 2
@@ -1661,12 +1672,10 @@ mod tests {
 				// * 3 disputes.
 				assert_eq!(expected_para_inherent_data.disputes.len(), 3);
 
-				assert_ok!(
-					Pallet::<Test>::enter(
-						frame_system::RawOrigin::None.into(),
-						expected_para_inherent_data,
-					),
-				);
+				assert_ok!(Pallet::<Test>::enter(
+					frame_system::RawOrigin::None.into(),
+					expected_para_inherent_data,
+				));
 
 				assert_eq!(
 					// The length of this vec is equal to the number of candidates, so we know our 2
@@ -2037,27 +2046,25 @@ mod tests {
 			// nothing is scheduled, so no paraids match, thus all backed candidates are skipped
 			{
 				let scheduled = &[][..];
-				assert!(
-					sanitize_backed_candidates::<Test, _>(
-						relay_parent,
-						backed_candidates.clone(),
-						has_concluded_invalid,
-						scheduled
-					).is_empty()
-				);
+				assert!(sanitize_backed_candidates::<Test, _>(
+					relay_parent,
+					backed_candidates.clone(),
+					has_concluded_invalid,
+					scheduled
+				)
+				.is_empty());
 			}
 
 			// relay parent mismatch
 			{
 				let relay_parent = Hash::repeat_byte(0xFA);
-				assert!(
-					sanitize_backed_candidates::<Test, _>(
-						relay_parent,
-						backed_candidates.clone(),
-						has_concluded_invalid,
-						scheduled
-					).is_empty()
-				);
+				assert!(sanitize_backed_candidates::<Test, _>(
+					relay_parent,
+					backed_candidates.clone(),
+					has_concluded_invalid,
+					scheduled
+				)
+				.is_empty());
 			}
 
 			// candidates that have concluded as invalid are filtered out

From 62cda420ef4b8b41239f87cac535269fda25537b Mon Sep 17 00:00:00 2001
From: Zeke Mostov <z.mostov@gmail.com>
Date: Mon, 15 Nov 2021 18:46:55 +0100
Subject: [PATCH 098/107] Apply suggestions from code review

---
 runtime/parachains/src/builder.rs | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/runtime/parachains/src/builder.rs b/runtime/parachains/src/builder.rs
index 67d025a0bc62..2e5f783a297c 100644
--- a/runtime/parachains/src/builder.rs
+++ b/runtime/parachains/src/builder.rs
@@ -119,6 +119,8 @@ impl<T: paras_inherent::Config> BenchBuilder<T> {
 		self
 	}
 
+        /// Maximum number of validators per core (a.k.a. max validators per group). This value is used if none is
+        /// explicitly set on the builder.
 	pub(crate) fn fallback_max_validators_per_core() -> u32 {
 		configuration::Pallet::<T>::config().max_validators_per_core.unwrap_or(5)
 	}

From 933cba7d5d841c8f6e7c1bb060e3d23c4b655dec Mon Sep 17 00:00:00 2001
From: Bernhard Schuster <bernhard@ahoi.io>
Date: Mon, 15 Nov 2021 18:51:09 +0100
Subject: [PATCH 099/107] Update runtime/parachains/src/builder.rs

Co-authored-by: Zeke Mostov <z.mostov@gmail.com>
---
 runtime/parachains/src/builder.rs | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/runtime/parachains/src/builder.rs b/runtime/parachains/src/builder.rs
index 2e5f783a297c..4e9cf3282319 100644
--- a/runtime/parachains/src/builder.rs
+++ b/runtime/parachains/src/builder.rs
@@ -230,12 +230,6 @@ impl<T: paras_inherent::Config> BenchBuilder<T> {
 		}
 	}
 
-	/// Setup para ids.
-	/// * setup `para_ids` traverses each core,
-	/// * creates a ParaId for that CoreIndex,
-	/// * inserts ParaLifeCycle::Onboarding for that ParaId,
-	/// * inserts the upcoming paras genesis,
-	/// * inserts the ParaId into the `ActionsQueue`
 	fn setup_para_ids(cores: u32) {
 		// make sure parachains exist prior to session change.
 		for i in 0..cores {

From 2c858073983b2bbde4cbe7322b76a5590dffd37c Mon Sep 17 00:00:00 2001
From: Zeke Mostov <z.mostov@gmail.com>
Date: Mon, 15 Nov 2021 18:53:23 +0100
Subject: [PATCH 100/107] Update runtime/parachains/src/builder.rs

---
 runtime/parachains/src/builder.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/runtime/parachains/src/builder.rs b/runtime/parachains/src/builder.rs
index 4e9cf3282319..17901cd460c6 100644
--- a/runtime/parachains/src/builder.rs
+++ b/runtime/parachains/src/builder.rs
@@ -569,8 +569,8 @@ impl<T: paras_inherent::Config> BenchBuilder<T> {
 		assert_eq!(inclusion::PendingAvailability::<T>::iter().count(), used_cores as usize,);
 
 		// Mark all the use cores as occupied. We expect that their are `backed_and_concluding_cores`
-		// that are pending availability and that there are `non_spam_dispute_cores` which are about
-		// to be disputed.
+		// that are pending availability and that there are `used_cores - backed_and_concluding_cores `
+		// which are about to be disputed.
 		scheduler::AvailabilityCores::<T>::set(vec![
 			Some(CoreOccupied::Parachain);
 			used_cores as usize

From 168ac98a2cc3497064f86fe6f16523afcf36c4f6 Mon Sep 17 00:00:00 2001
From: Zeke Mostov <z.mostov@gmail.com>
Date: Tue, 16 Nov 2021 09:21:12 +0100
Subject: [PATCH 101/107] Update runtime/parachains/src/paras_inherent.rs

---
 runtime/parachains/src/paras_inherent.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/parachains/src/paras_inherent.rs b/runtime/parachains/src/paras_inherent.rs
index c9325c6fd800..163b312e990e 100644
--- a/runtime/parachains/src/paras_inherent.rs
+++ b/runtime/parachains/src/paras_inherent.rs
@@ -334,8 +334,8 @@ pub mod pallet {
 				mut disputes,
 			} = data;
 
-			let parent_hash = <frame_system::Pallet<T>>::parent_hash();
 			// Check that the submitted parent header indeed corresponds to the previous block hash.
+			let parent_hash = <frame_system::Pallet<T>>::parent_hash();
 			ensure!(
 				parent_header.hash().as_ref() == parent_hash.as_ref(),
 				Error::<T>::InvalidParentHeader,

From 2a0869b020c24aeded6d9141c373fdee5bbbfbc8 Mon Sep 17 00:00:00 2001
From: Bernhard Schuster <bernhard@ahoi.io>
Date: Tue, 16 Nov 2021 16:06:23 +0100
Subject: [PATCH 102/107] final pass

---
 .../src/validator_side/mod.rs                 |  2 +-
 .../statement-distribution/src/tests.rs       |  2 +-
 runtime/parachains/src/builder.rs             |  4 +-
 runtime/parachains/src/paras_inherent.rs      | 93 ++++++++++++-------
 4 files changed, 65 insertions(+), 36 deletions(-)

diff --git a/node/network/collator-protocol/src/validator_side/mod.rs b/node/network/collator-protocol/src/validator_side/mod.rs
index f7672d932dcd..6eb92f5f81e7 100644
--- a/node/network/collator-protocol/src/validator_side/mod.rs
+++ b/node/network/collator-protocol/src/validator_side/mod.rs
@@ -1462,7 +1462,7 @@ async fn poll_collation_response(
 				);
 
 				CollationFetchResult::Error(COST_WRONG_PARA)
-			}
+			},
 			Ok(CollationFetchingResponse::Collation(receipt, pov)) => {
 				tracing::debug!(
 					target: LOG_TARGET,
diff --git a/node/network/statement-distribution/src/tests.rs b/node/network/statement-distribution/src/tests.rs
index 2c41d5e7ddf0..80fdb50a4f26 100644
--- a/node/network/statement-distribution/src/tests.rs
+++ b/node/network/statement-distribution/src/tests.rs
@@ -1814,7 +1814,7 @@ fn peer_cant_flood_with_large_statements() {
 					if p == peer_a && r == COST_APPARENT_FLOOD =>
 				{
 					punished = true;
-				}
+				},
 
 				m => panic!("Unexpected message: {:?}", m),
 			}
diff --git a/runtime/parachains/src/builder.rs b/runtime/parachains/src/builder.rs
index 17901cd460c6..fb58b05f5d56 100644
--- a/runtime/parachains/src/builder.rs
+++ b/runtime/parachains/src/builder.rs
@@ -119,8 +119,8 @@ impl<T: paras_inherent::Config> BenchBuilder<T> {
 		self
 	}
 
-        /// Maximum number of validators per core (a.k.a. max validators per group). This value is used if none is
-        /// explicitly set on the builder.
+	/// Maximum number of validators per core (a.k.a. max validators per group). This value is used if none is
+	/// explicitly set on the builder.
 	pub(crate) fn fallback_max_validators_per_core() -> u32 {
 		configuration::Pallet::<T>::config().max_validators_per_core.unwrap_or(5)
 	}
diff --git a/runtime/parachains/src/paras_inherent.rs b/runtime/parachains/src/paras_inherent.rs
index 163b312e990e..d5655f8203cd 100644
--- a/runtime/parachains/src/paras_inherent.rs
+++ b/runtime/parachains/src/paras_inherent.rs
@@ -39,11 +39,12 @@ use primitives::v1::{
 	BackedCandidate, CandidateHash, CoreIndex, DisputeStatementSet,
 	InherentData as ParachainsInherentData, MultiDisputeStatementSet, ScrapedOnChainVotes,
 	SessionIndex, SigningContext, UncheckedSignedAvailabilityBitfield,
-	UncheckedSignedAvailabilityBitfields, ValidatorId, PARACHAINS_INHERENT_IDENTIFIER,
+	UncheckedSignedAvailabilityBitfields, ValidatorId, ValidatorIndex,
+	PARACHAINS_INHERENT_IDENTIFIER,
 };
 use rand::{Rng, SeedableRng};
 use scale_info::TypeInfo;
-use sp_runtime::traits::{Header as HeaderT, Zero};
+use sp_runtime::traits::Header as HeaderT;
 use sp_std::{
 	cmp::Ordering,
 	collections::{btree_map::BTreeMap, btree_set::BTreeSet},
@@ -363,7 +364,7 @@ pub mod pallet {
 				}
 
 				if disputes_weight > max_block_weight {
-					// if disputes are overweight, trim the disputes.
+					// if disputes are by themselves overweight already, trim the disputes.
 					debug_assert!(candidate_weight == 0 && bitfields_weight == 0);
 
 					let entropy = compute_entropy::<T>(parent_hash);
@@ -530,7 +531,7 @@ impl<T: Config> Pallet<T> {
 
 		T::DisputesHandler::filter_multi_dispute_data(&mut disputes);
 
-		let (concluded_invalid_disputes, bitfields, scheduled) =
+		let (concluded_invalid_disputes, mut bitfields, scheduled) =
 			frame_support::storage::with_transaction(|| {
 				// we don't care about fresh or not disputes
 				// this writes them to storage, so let's query it via those means
@@ -621,7 +622,7 @@ impl<T: Config> Pallet<T> {
 				))
 			});
 
-		let backed_candidates = sanitize_backed_candidates::<T, _>(
+		let mut backed_candidates = sanitize_backed_candidates::<T, _>(
 			parent_hash,
 			backed_candidates,
 			move |candidate_hash: CandidateHash| -> bool {
@@ -633,9 +634,13 @@ impl<T: Config> Pallet<T> {
 		let entropy = compute_entropy::<T>(parent_hash);
 		let mut rng = rand_chacha::ChaChaRng::from_seed(entropy.into());
 		let max_block_weight = <T as frame_system::Config>::BlockWeights::get().max_block;
-		let remaining_weight = limit_disputes::<T>(&mut disputes, max_block_weight, &mut rng);
-		let (_backed_candidates_weight, backed_candidates, bitfields) =
-			apply_weight_limit::<T>(backed_candidates, bitfields, remaining_weight, &mut rng);
+		let _consumed_weight = apply_weight_limit::<T>(
+			&mut backed_candidates,
+			&mut bitfields,
+			&mut disputes,
+			max_block_weight,
+			&mut rng,
+		);
 
 		Some(ParachainsInherentData::<T::Header> {
 			bitfields,
@@ -664,6 +669,13 @@ where
 	DisputedBitfield::from(bitvec)
 }
 
+/// Select a random subset
+///
+/// Adds random items to the set until all candidates
+/// are tried or the remaining weight is depleted.
+///
+/// Returns the weight of all selected items from `selectables`
+/// as well as their indices in ascending order.
 fn random_sel<X, F: Fn(&X) -> Weight>(
 	rng: &mut rand_chacha::ChaChaRng,
 	selectables: Vec<X>,
@@ -708,24 +720,29 @@ fn random_sel<X, F: Fn(&X) -> Weight>(
 /// If even the bitfields are too large to fit into the `max_weight` limit, bitfields are randomly
 /// picked and _no_ candidates will be included.
 fn apply_weight_limit<T: Config + inclusion::Config>(
-	candidates: Vec<BackedCandidate<<T>::Hash>>,
-	bitfields: UncheckedSignedAvailabilityBitfields,
-	max_weight: Weight,
+	candidates: &mut Vec<BackedCandidate<<T>::Hash>>,
+	bitfields: &mut UncheckedSignedAvailabilityBitfields,
+	disputes: &mut MultiDisputeStatementSet,
+	max_block_weight: Weight,
 	rng: &mut rand_chacha::ChaChaRng,
-) -> (Weight, Vec<BackedCandidate<<T>::Hash>>, UncheckedSignedAvailabilityBitfields) {
+) -> Weight {
+	// include as many disputes as possible, always
+	let remaining_weight = limit_disputes::<T>(disputes, max_block_weight, rng);
+
 	let total_candidates_weight = backed_candidates_weight::<T>(candidates.as_slice());
+
 	let total_bitfields_weight = signed_bitfields_weight::<T>(bitfields.len());
 
 	let total = total_bitfields_weight.saturating_add(total_candidates_weight);
 
-	// everything fits into the block
-	if max_weight >= total {
-		return (total, candidates, bitfields)
+	// candidates + bitfields fit into the block
+	if remaining_weight >= total {
+		return total
 	}
 
 	// There is weight remaining to be consumed by a subset of candidates
 	// which are going to be picked now.
-	if let Some(remaining_weight) = max_weight.checked_sub(total_bitfields_weight) {
+	if let Some(remaining_weight) = remaining_weight.checked_sub(total_bitfields_weight) {
 		let (acc_candidate_weight, indices) =
 			random_sel::<BackedCandidate<<T as frame_system::Config>::Hash>, _>(
 				rng,
@@ -733,25 +750,37 @@ fn apply_weight_limit<T: Config + inclusion::Config>(
 				|c| backed_candidate_weight::<T>(c),
 				remaining_weight,
 			);
-		let candidates =
-			indices.into_iter().map(move |idx| candidates[idx].clone()).collect::<Vec<_>>();
+		let mut idx = 0_usize;
+		candidates.retain(|_backed_candidate| {
+			let exists = indices.binary_search(&idx).is_ok();
+			idx += 1;
+			exists
+		});
 		// pick all bitfields, and
 		// fill the remaining space with candidates
 		let total = acc_candidate_weight.saturating_add(total_bitfields_weight);
-		return (total, candidates, bitfields)
+		return total
 	}
 
+	candidates.clear();
+
 	// insufficient space for even the bitfields alone, so only try to fit as many of those
 	// into the block and skip the candidates entirely
 	let (total, indices) = random_sel::<UncheckedSignedAvailabilityBitfield, _>(
 		rng,
 		bitfields.clone(),
 		|_| <<T as Config>::WeightInfo as WeightInfo>::enter_bitfields(),
-		max_weight,
+		remaining_weight,
 	);
 
-	let bitfields = indices.into_iter().map(move |idx| bitfields[idx].clone()).collect::<Vec<_>>();
-	(total, vec![], bitfields)
+	let mut idx = 0_usize;
+	bitfields.retain(|_bitfield| {
+		let exists = indices.binary_search(&idx).is_ok();
+		idx += 1;
+		exists
+	});
+
+	total
 }
 
 /// Filter bitfields based on freed core indices, validity, and other sanity checks.
@@ -782,7 +811,7 @@ pub(crate) fn sanitize_bitfields<T: crate::inclusion::Config, const CHECK_SIGS:
 ) -> UncheckedSignedAvailabilityBitfields {
 	let mut bitfields = Vec::with_capacity(unchecked_bitfields.len());
 
-	let mut last_index = None;
+	let mut last_index: Option<ValidatorIndex> = None;
 
 	if disputed_bitfield.0.len() != expected_bits {
 		// This is a system logic error that should never occur, but we want to handle it gracefully
@@ -818,13 +847,15 @@ pub(crate) fn sanitize_bitfields<T: crate::inclusion::Config, const CHECK_SIGS:
 			continue
 		}
 
-		if !last_index.map_or(true, |last| last < unchecked_bitfield.unchecked_validator_index()) {
+		let validator_index = unchecked_bitfield.unchecked_validator_index();
+
+		if !last_index.map_or(true, |last_index: ValidatorIndex| last_index < validator_index) {
 			log::trace!(
 				target: LOG_TARGET,
-				"[CHECK_SIGS: {}] bitfield validator index is not greater than last: !({:?} < {:?})",
+				"[CHECK_SIGS: {}] bitfield validator index is not greater than last: !({:?} < {})",
 				CHECK_SIGS,
-				last_index,
-				unchecked_bitfield.unchecked_validator_index()
+				last_index.as_ref().map(|x| x.0),
+				validator_index.0
 			);
 			continue
 		}
@@ -832,16 +863,14 @@ pub(crate) fn sanitize_bitfields<T: crate::inclusion::Config, const CHECK_SIGS:
 		if unchecked_bitfield.unchecked_validator_index().0 as usize >= validators.len() {
 			log::trace!(
 				target: LOG_TARGET,
-				"[CHECK_SIGS: {}] bitfield validator index is out of bounds: {:?} >= {:?}",
+				"[CHECK_SIGS: {}] bitfield validator index is out of bounds: {} >= {}",
 				CHECK_SIGS,
-				unchecked_bitfield.unchecked_validator_index().0,
+				validator_index.0,
 				validators.len(),
 			);
 			continue
 		}
 
-		let validator_index = unchecked_bitfield.unchecked_validator_index();
-
 		let validator_public = &validators[validator_index.0 as usize];
 
 		if CHECK_SIGS {
@@ -925,7 +954,7 @@ fn limit_disputes<T: Config>(
 		// Sort the dispute statements according to the following prioritization:
 		//  1. Prioritize local disputes over remote disputes.
 		//  2. Prioritize older disputes over newer disputes.
-		disputes.sort_by(|a, b| {
+		disputes.sort_unstable_by(|a, b| {
 			let a_local_block = T::DisputesHandler::included_state(a.session, a.candidate_hash);
 			let b_local_block = T::DisputesHandler::included_state(b.session, b.candidate_hash);
 			match (a_local_block, b_local_block) {

From 811f168c21f78488a5d1e7703f5a1c17e5546238 Mon Sep 17 00:00:00 2001
From: emostov <32168567+emostov@users.noreply.github.com>
Date: Tue, 16 Nov 2021 16:13:18 +0100
Subject: [PATCH 103/107] Run cargo +nightly-2021-10-29 fmt

---
 node/network/collator-protocol/src/validator_side/mod.rs | 2 +-
 node/network/statement-distribution/src/tests.rs         | 2 +-
 primitives/src/v0.rs                                     | 4 ++--
 runtime/parachains/src/builder.rs                        | 4 ++--
 runtime/parachains/src/paras_inherent.rs                 | 2 +-
 5 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/node/network/collator-protocol/src/validator_side/mod.rs b/node/network/collator-protocol/src/validator_side/mod.rs
index f7672d932dcd..6eb92f5f81e7 100644
--- a/node/network/collator-protocol/src/validator_side/mod.rs
+++ b/node/network/collator-protocol/src/validator_side/mod.rs
@@ -1462,7 +1462,7 @@ async fn poll_collation_response(
 				);
 
 				CollationFetchResult::Error(COST_WRONG_PARA)
-			}
+			},
 			Ok(CollationFetchingResponse::Collation(receipt, pov)) => {
 				tracing::debug!(
 					target: LOG_TARGET,
diff --git a/node/network/statement-distribution/src/tests.rs b/node/network/statement-distribution/src/tests.rs
index 2c41d5e7ddf0..80fdb50a4f26 100644
--- a/node/network/statement-distribution/src/tests.rs
+++ b/node/network/statement-distribution/src/tests.rs
@@ -1814,7 +1814,7 @@ fn peer_cant_flood_with_large_statements() {
 					if p == peer_a && r == COST_APPARENT_FLOOD =>
 				{
 					punished = true;
-				}
+				},
 
 				m => panic!("Unexpected message: {:?}", m),
 			}
diff --git a/primitives/src/v0.rs b/primitives/src/v0.rs
index 2000c173b879..64163d04bdd7 100644
--- a/primitives/src/v0.rs
+++ b/primitives/src/v0.rs
@@ -107,8 +107,8 @@ impl MallocSizeOf for ValidatorId {
 }
 
 /// Index of the validator is used as a lightweight replacement of the `ValidatorId` when appropriate.
-#[derive(Eq, Ord, PartialEq, PartialOrd, Copy, Clone, Encode, Decode, TypeInfo)]
-#[cfg_attr(feature = "std", derive(Serialize, Deserialize, Debug, Hash, MallocSizeOf))]
+#[derive(Eq, Ord, PartialEq, PartialOrd, Copy, Clone, Encode, Decode, TypeInfo, Debug)]
+#[cfg_attr(feature = "std", derive(Serialize, Deserialize, Hash, MallocSizeOf))]
 pub struct ValidatorIndex(pub u32);
 
 // We should really get https://github.com/paritytech/polkadot/issues/2403 going ..
diff --git a/runtime/parachains/src/builder.rs b/runtime/parachains/src/builder.rs
index 17901cd460c6..fb58b05f5d56 100644
--- a/runtime/parachains/src/builder.rs
+++ b/runtime/parachains/src/builder.rs
@@ -119,8 +119,8 @@ impl<T: paras_inherent::Config> BenchBuilder<T> {
 		self
 	}
 
-        /// Maximum number of validators per core (a.k.a. max validators per group). This value is used if none is
-        /// explicitly set on the builder.
+	/// Maximum number of validators per core (a.k.a. max validators per group). This value is used if none is
+	/// explicitly set on the builder.
 	pub(crate) fn fallback_max_validators_per_core() -> u32 {
 		configuration::Pallet::<T>::config().max_validators_per_core.unwrap_or(5)
 	}
diff --git a/runtime/parachains/src/paras_inherent.rs b/runtime/parachains/src/paras_inherent.rs
index 163b312e990e..dfe861deed2d 100644
--- a/runtime/parachains/src/paras_inherent.rs
+++ b/runtime/parachains/src/paras_inherent.rs
@@ -43,7 +43,7 @@ use primitives::v1::{
 };
 use rand::{Rng, SeedableRng};
 use scale_info::TypeInfo;
-use sp_runtime::traits::{Header as HeaderT, Zero};
+use sp_runtime::traits::Header as HeaderT;
 use sp_std::{
 	cmp::Ordering,
 	collections::{btree_map::BTreeMap, btree_set::BTreeSet},

From e12bc3134d5ad891ac21ec6538cffdae9d1fd512 Mon Sep 17 00:00:00 2001
From: Zeke Mostov <z.mostov@gmail.com>
Date: Tue, 16 Nov 2021 16:15:40 +0100
Subject: [PATCH 104/107] Update implementors guide with `sanitize_*` & `enter`
 (#4294)

---
 .../src/runtime/inclusion.md                  | 46 ++++++++--------
 .../src/runtime/parainherent.md               | 53 +++++++++++++------
 .../implementers-guide/src/types/runtime.md   |  4 +-
 runtime/parachains/src/paras_inherent.rs      |  5 +-
 4 files changed, 65 insertions(+), 43 deletions(-)

diff --git a/roadmap/implementers-guide/src/runtime/inclusion.md b/roadmap/implementers-guide/src/runtime/inclusion.md
index 52e09fd60f9f..84b513cb985c 100644
--- a/roadmap/implementers-guide/src/runtime/inclusion.md
+++ b/roadmap/implementers-guide/src/runtime/inclusion.md
@@ -51,31 +51,31 @@ All failed checks should lead to an unrecoverable error making the block invalid
   1. For each applied bit of each availability-bitfield, set the bit for the validator in the `CandidatePendingAvailability`'s `availability_votes` bitfield. Track all candidates that now have >2/3 of bits set in their `availability_votes`. These candidates are now available and can be enacted.
   1. For all now-available candidates, invoke the `enact_candidate` routine with the candidate and relay-parent number.
   1. Return a list of `(CoreIndex, CandidateHash)` from freed cores consisting of the cores where candidates have become available.
-* `sanitize_bitfields<const EARLY_RETURN>(
-    usab: UncheckedSignedAvailabilityBitfields,
-    db: DisputedBitfield,
+* `sanitize_bitfields<T: crate::inclusion::Config, const CHECK_SIGS: bool>(
+    unchecked_bitfields: UncheckedSignedAvailabilityBitfields,
+    disputed_bitfield: DisputedBitfield,
     expected_bits: usize,
-    parent_hash: Hash,
+    parent_hash: T::Hash,
     session_index: SessionIndex,
-    validators: &[ValidatorId]
+    validators: &[ValidatorId],
   )`:
-  1. if `EARLY_RETURN` is `true`, return an error when encountering a bitfield that does not pass the checks, if `false`, drop the bitfield from the set that will be returned.
-  1. check that there is at most 1 bitfield per validator and that the number of bits in each bitfield is equal to `expected_bits`.
-  1. check that there are no duplicates
-  1. check that the validator bit index is not out of bounds
-  1. check all validator signatures, iff `EARLY_RETURN=true`, since in the other case, checking is supposed to be done before the call
-  1. check that there are no bits set that reference a disputed candidate
-
-* `sanitize_backed_candidates<const EARLY_RETURN: bool>(
-	relay_parent: Hash,
-	mut backed_candidates: Vec<BackedCandidate>,
-	candidate_has_concluded_invalid_dispute: Fn(CandidateHash) -> bool,
-	scheduled: &[CoreAssignment],
-)`
-  1. if `EARLY_RETURN` is `true`, return an error when encountering a backed candidates that does not pass the checks, if `false`, drop the backed candidates from the set that will be returned.
-  1. check all backed candidates have no concluded invalid dispute by means of the provided closure `candidate_has_concluded_invalid_dispute`
-  1. check all backed candidates have the matching `relay_parent`
-  1. check all backed candidates paraid was scheduled by means of the provided `scheduled` parameter
+  1. check that `disputed_bitfield` has the same number of bits as the `expected_bits`, iff not return early with an empty vec.
+  1. each of the below checks is for each bitfield. If a check does not pass the bitfield will be skipped.
+  1. check that there are no bits set that reference a disputed candidate.
+  1. check that the number of bits is equal to `expected_bits`.
+  1. check that the validator index is strictly increasing (and thus also unique).
+  1. check that the validator bit index is not out of bounds.
+  1. check the validators signature, iff `CHECK_SIGS=true`.
+
+* `sanitize_backed_candidates<T: crate::inclusion::Config, F: Fn(CandidateHash) -> bool>(
+    relay_parent: T::Hash,
+    mut backed_candidates: Vec<BackedCandidate<T::Hash>>,
+    candidate_has_concluded_invalid_dispute: F,
+    scheduled: &[CoreAssignment],
+  ) `
+  1. filter out any backed candidates that have concluded invalid.
+  1. filter out backed candidates that don't have a matching `relay_parent`.
+  1. filters backed candidates whom's paraid was scheduled by means of the provided `scheduled` parameter.
 
 * `process_candidates(parent_storage_root, BackedCandidates, scheduled: Vec<CoreAssignment>, group_validators: Fn(GroupIndex) -> Option<Vec<ValidatorIndex>>)`:
   1. check that each candidate corresponds to a scheduled core and that they are ordered in the same order the cores appear in assignments in `scheduled`.
@@ -104,8 +104,6 @@ All failed checks should lead to an unrecoverable error making the block invalid
   1. call `Hrmp::queue_outbound_hrmp` with the para id of the candidate and the list of horizontal messages taken from the commitment,
   1. Call `Paras::note_new_head` using the `HeadData` from the receipt and `relay_parent_number`.
 
-* `fn sanitize_bitfields`
-
 * `collect_pending`:
 
   ```rust
diff --git a/roadmap/implementers-guide/src/runtime/parainherent.md b/roadmap/implementers-guide/src/runtime/parainherent.md
index 82804cf1cd95..dd67f9f108f8 100644
--- a/roadmap/implementers-guide/src/runtime/parainherent.md
+++ b/roadmap/implementers-guide/src/runtime/parainherent.md
@@ -29,10 +29,19 @@ OnChainVotes: Option<ScrapedOnChainVotes>,
 
 ## Entry Points
 
-* `enter`: This entry-point accepts three parameters: The relay-chain parent block header, [`Bitfields`](../types/availability.md#signed-availability-bitfield) and [`BackedCandidates`](../types/backing.md#backed-candidate).
-    1. Hash the parent header and make sure that it corresponds to the block hash of the parent (tracked by the `frame_system` FRAME module),
+* `enter`: This entry-point accepts one parameter: [`ParaInherentData`](../types/runtime.md#ParaInherentData).
+    1. Ensure the origin is none.
+    1. Ensure `Included` is set as `None`.
+    1. Set `Included` as `Some`.
+    1. Unpack `ParachainsInherentData` into `signed_bitfields`, `backed_candidates`, `parent_header`, and `disputes`.
+    1. Hash the parent header and make sure that it corresponds to the block hash of the parent (tracked by the `frame_system` FRAME module).
+    1. Calculate the `candidate_weight`, `bitfields_weight`, and `disputes_weight`.
+    1. If the sum of `candidate_weight`, `bitfields_weight`, and `disputes_weight` is greater than the max block weight we do the following with the goal of prioritizing the inclusion of disputes without making it game-able by block authors:
+      1. clear `bitfields` and set `bitfields_weight` equal to 0.
+      1. clear `backed_candidates` and set `candidate_weight` equal to 0.
+      1. invoke `limit_disputes` on the `disputes` with the max block weight iff the disputes weight is greater than the max block weight.
     1. Invoke `Disputes::provide_multi_dispute_data`.
-    1. If `Disputes::is_frozen`, return and set `Included` to `Some(())`.
+    1. If `Disputes::is_frozen`, return.
     1. If there are any concluded disputes from the current session, invoke `Inclusion::collect_disputed` with the disputed candidates. Annotate each returned core with `FreedReason::Concluded`, sort them, and invoke `Scheduler::free_cores` with them.
     1. The `Bitfields` are first forwarded to the `Inclusion::process_bitfields` routine, returning a set included candidates and the respective freed cores. Provide the number of availability cores (`Scheduler::availability_cores().len()`) as the expected number of bits and a `Scheduler::core_para` as a core-lookup to the `process_bitfields` routine. Annotate each of these freed cores with `FreedReason::Concluded`.
     1. For each freed candidate from the `Inclusion::process_bitfields` call, invoke `Disputes::note_included(current_session, candidate)`.
@@ -51,19 +60,29 @@ OnChainVotes: Option<ScrapedOnChainVotes>,
 
 
 * `create_inherent`: This entry-point accepts one parameter: `InherentData`.
-  1. Unpack `InherentData` into its parts, `bitfields`, `backed_candidates`, `disputes` and the `parent_header`.
-  1. Hash the `parent_header` and make sure that it corresponds to the block hash of the parent (tracked by the `frame_system` FRAME module),
+  1. Invoke [`create_inherent_inner(InherentData)`](#routines), the unit testable logic for filtering and sanitzing the inherent data used when invoking `enter`. Save the result as `inherent_data`.
+  1. If the `inherent_data` is an `Err` variant, return the `enter` call signature with all inherent data cleared else return the `enter` call signature with `inherent_data` passed in as the `data` param.
+
+# Routines
+
+* `create_inherent_inner(data: &InherentData) -> Option<ParachainsInherentData<T::Header>>`
+  1. Unpack `InherentData` into its parts, `bitfields`, `backed_candidates`, `disputes` and the `parent_header`. If data cannot be unpacked return `None`.
+  1. Hash the `parent_header` and make sure that it corresponds to the block hash of the parent (tracked by the `frame_system` FRAME module).
   1. Invoke `Disputes::filter_multi_dispute_data` to remove duplicates et al from `disputes`.
-  1. Run the following within a  `with_transaction` closure to avoid side effects:
+  1. Run the following within a  `with_transaction` closure to avoid side effects (we are essentially replicating the logic that would otherwise happen within `enter` so we can get the filtered bitfields and the `concluded_invalid_disputes` + `scheduled` to use in filtering the `backed_candidates`.):
     1. Invoke `Disputes::provide_multi_dispute_data`.
-    1. Collect the newly concluded disputes as `current_concluded_invalid_disputes`.
-    1. If there are any concluded disputes from the current session, invoke `Inclusion::collect_disputed` with the newly disputed candidates. Annotate each returned core with `FreedReason::Concluded`, sort them, and invoke `Scheduler::free_cores` with them.
-    1. Collect the concluded invalid disputes in the current session as `conlcuded_invalid_disputes`.
-    1. Return `TransactionOutcome::Rollback(freed_cores, concluded_invalid_disputes)`.
-  1. Call `sanitize_bitfields<false>` and only use the sanitized set of bitfields afterwards.
-  1. Call `sanitize_backed_candidates<false>`.
-  1. Collect entropy based on `CurrentBlockRandomness::random`.
-  1. Call `apply_weight_limit` to utilize the block as well as possible, with a randomized heuristic.
-  1. Re-create a `InherentData` from the sanitized and weight bounded information.
-  1. Call `Self::enter` which now should not error anymore, but we do this call anyways for now.
-  1. Return `Call::enter { .. }`.
\ No newline at end of file
+    1. Collect `current_concluded_invalid_disputes`, the disputed candidate hashes from the current session that have concluded invalid.
+    1. Collect `concluded_invalid_disputes`, the disputed candidate hashes from the given `backed_candidates`.
+    1. Invoke `Inclusion::collect_disputed` with the newly disputed candidates. Annotate each returned core with `FreedReason::Concluded`, sort them, and invoke `Scheduler::free_cores` with them.
+    1. Collect filtered `bitfields` by invoking [`sanitize_bitfields<false>`](inclusion.md#Routines).
+    1. Collect `freed_concluded` by invoking `update_pending_availability_and_get_freed_cores` on the filtered bitfields.
+    1. Collect all `freed` cores by invoking `collect_all_freed_cores` on `freed_concluding`.
+    1. Invoke `scheduler::Pallet<T>>::clear()`.
+    1. Invoke `scheduler::Pallet<T>>::schedule` with `freed` and the current block number to create the same schedule of the cores that `enter` will create.
+    1. Read the new `<scheduler::Pallet<T>>::scheduled()` into `schedule`.
+    1. From the `with_transaction` closure return `concluded_invalid_disputes`, `bitfields`, and `scheduled`.
+  1. Invoke `sanitize_backed_candidates` using the `scheduled` return from the `with_transaction` and pass the closure `|candidate_hash: CandidateHash| -> bool { DisputesHandler::concluded_invalid(current_session, candidate_hash) }` for the param `candidate_has_concluded_invalid_dispute`.
+  1. create a `rng` from `rand_chacha::ChaChaRng::from_seed(compute_entropy::<T>(parent_hash))`.
+  1. Invoke `limit_disputes` with the max block weight and `rng`, storing the returned weigh in `remaining_weight`.
+  1. Fill up the remaining of the block weight with backed candidates and bitfields by invoking `apply_weight_limit` with `remaining_weigh` and `rng`.
+  1. Return `Some(ParachainsInherentData { bitfields, backed_candidates, disputes, parent_header }`.
diff --git a/roadmap/implementers-guide/src/types/runtime.md b/roadmap/implementers-guide/src/types/runtime.md
index 5749aeca866a..b8eef8f11204 100644
--- a/roadmap/implementers-guide/src/types/runtime.md
+++ b/roadmap/implementers-guide/src/types/runtime.md
@@ -116,15 +116,17 @@ struct HostConfiguration {
 
 Inherent data passed to a runtime entry-point for the advancement of parachain consensus.
 
-This contains 3 pieces of data:
+This contains 4 pieces of data:
 1. [`Bitfields`](availability.md#signed-availability-bitfield)
 2. [`BackedCandidates`](backing.md#backed-candidate)
 3. [`MultiDisputeStatementSet`](disputes.md#multidisputestatementset)
+4. `Header`
 
 ```rust
 struct ParaInherentData {
 	bitfields: Bitfields,
 	backed_candidates: BackedCandidates,
 	dispute_statements: MultiDisputeStatementSet,
+  parent_header: Header
 }
 ```
diff --git a/runtime/parachains/src/paras_inherent.rs b/runtime/parachains/src/paras_inherent.rs
index d5655f8203cd..4c683d037cf0 100644
--- a/runtime/parachains/src/paras_inherent.rs
+++ b/runtime/parachains/src/paras_inherent.rs
@@ -280,7 +280,8 @@ pub mod pallet {
 		}
 	}
 
-	/// Collect all freed cores based on storage data.
+	/// Collect all freed cores based on storage data. (i.e. append cores freed from timeouts to
+	/// the given `freed_concluded`).
 	///
 	/// The parameter `freed_concluded` contains all core indicies that became
 	/// free due to candidate that became available.
@@ -547,6 +548,8 @@ impl<T: Config> Pallet<T> {
 					});
 
 				// current concluded invalid disputes, only including the current block's votes
+				// TODO why does this say "only including the current block's votes"? This can include
+				// remote disputes, right?
 				let current_concluded_invalid_disputes = disputes
 					.iter()
 					.filter(|dss| dss.session == current_session)

From a52ffb7d0d67e7efe7eb45af4dd5605a2d587e07 Mon Sep 17 00:00:00 2001
From: emostov <32168567+emostov@users.noreply.github.com>
Date: Tue, 16 Nov 2021 16:33:57 +0100
Subject: [PATCH 105/107] Make spell check happier

---
 runtime/parachains/src/builder.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/parachains/src/builder.rs b/runtime/parachains/src/builder.rs
index fb58b05f5d56..e38b4ba889da 100644
--- a/runtime/parachains/src/builder.rs
+++ b/runtime/parachains/src/builder.rs
@@ -125,7 +125,7 @@ impl<T: paras_inherent::Config> BenchBuilder<T> {
 		configuration::Pallet::<T>::config().max_validators_per_core.unwrap_or(5)
 	}
 
-	/// Specify a mapping of core_idx/para_id/group_idx seed to the number of dispute statements for the
+	/// Specify a mapping of core index, para id, group index seed to the number of dispute statements for the
 	/// corresponding dispute statement set. Note that if the number of disputes is not specified it fallbacks
 	/// to having a dispute per every validator. Additionally, an entry is not guaranteed to have a dispute - it
 	/// must line up with the cores marked as disputed as defined in `Self::Build`.

From 9e7eed5013ac089524f9f39f49aa5812fa195d2e Mon Sep 17 00:00:00 2001
From: Zeke Mostov <z.mostov@gmail.com>
Date: Tue, 16 Nov 2021 18:45:37 +0100
Subject: [PATCH 106/107] Make wasm runtimes compile with benchmarks enabled
 (#4303)

* comment stuff out, use old toml

* Seems to be working?

* Remove feature gating from builder

* Remove commented out stuff

* Remove generic from digest
---
 Cargo.lock                        |  9 ++++-----
 parachain/src/primitives.rs       | 13 +++++++++++--
 primitives/Cargo.toml             | 10 ++++------
 primitives/src/v1/mod.rs          |  9 +++------
 primitives/src/v1/signed.rs       |  8 ++++++--
 runtime/parachains/Cargo.toml     | 13 +++++--------
 runtime/parachains/src/builder.rs | 14 +++++++-------
 7 files changed, 40 insertions(+), 36 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 1c2211ca5ff7..ca21d2f3bdc8 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2558,9 +2558,9 @@ checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70"
 
 [[package]]
 name = "hex-literal"
-version = "0.3.3"
+version = "0.3.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "21e4590e13640f19f249fe3e4eca5113bc4289f2497710378190e7f4bd96f45b"
+checksum = "7ebdb29d2ea9ed0083cd8cece49bbd968021bd99b0849edb4a9a7ee0fdf6a4e0"
 
 [[package]]
 name = "hex_fmt"
@@ -6683,7 +6683,7 @@ dependencies = [
 
 [[package]]
 name = "polkadot-primitives"
-version = "0.9.12"
+version = "0.9.13"
 dependencies = [
  "bitvec 0.20.1",
  "frame-system",
@@ -6875,7 +6875,7 @@ dependencies = [
 
 [[package]]
 name = "polkadot-runtime-parachains"
-version = "0.9.12"
+version = "0.9.13"
 dependencies = [
  "bitflags",
  "bitvec 0.20.1",
@@ -6904,7 +6904,6 @@ dependencies = [
  "scale-info",
  "serde",
  "sp-api",
- "sp-application-crypto",
  "sp-core",
  "sp-inherents",
  "sp-io",
diff --git a/parachain/src/primitives.rs b/parachain/src/primitives.rs
index c766ca357cb6..39d59dcfe7a4 100644
--- a/parachain/src/primitives.rs
+++ b/parachain/src/primitives.rs
@@ -41,10 +41,19 @@ pub use polkadot_core_primitives::BlockNumber as RelayChainBlockNumber;
 
 /// Parachain head data included in the chain.
 #[derive(
-	PartialEq, Eq, Clone, PartialOrd, Ord, Encode, Decode, RuntimeDebug, derive_more::From, TypeInfo,
+	PartialEq,
+	Eq,
+	Clone,
+	PartialOrd,
+	Ord,
+	Encode,
+	Decode,
+	RuntimeDebug,
+	derive_more::From,
+	TypeInfo,
+	Default,
 )]
 #[cfg_attr(feature = "std", derive(Serialize, Deserialize, Hash, MallocSizeOf))]
-#[cfg_attr(any(feature = "std", feature = "runtime-benchmarks"), derive(Default))]
 pub struct HeadData(#[cfg_attr(feature = "std", serde(with = "bytes"))] pub Vec<u8>);
 
 impl HeadData {
diff --git a/primitives/Cargo.toml b/primitives/Cargo.toml
index 6614f6d17e65..86cf62fffa14 100644
--- a/primitives/Cargo.toml
+++ b/primitives/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "polkadot-primitives"
-version = "0.9.12"
+version = "0.9.13"
 authors = ["Parity Technologies <admin@parity.io>"]
 edition = "2018"
 
@@ -26,9 +26,10 @@ polkadot-core-primitives = { path = "../core-primitives", default-features = fal
 trie = { package = "sp-trie", git = "https://github.com/paritytech/substrate", branch = "master", default-features = false }
 bitvec = { version = "0.20.1", default-features = false, features = ["alloc"] }
 frame-system = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false }
-hex-literal = "0.3.3"
+hex-literal = "0.3.4"
 parity-util-mem = { version = "0.10.0", default-features = false, optional = true }
 
+
 [features]
 default = ["std"]
 std = [
@@ -54,7 +55,4 @@ std = [
 	"polkadot-core-primitives/std",
 	"bitvec/std",
 	"frame-system/std",
-]
-runtime-benchmarks = [
-	"polkadot-parachain/runtime-benchmarks",
-]
+]
\ No newline at end of file
diff --git a/primitives/src/v1/mod.rs b/primitives/src/v1/mod.rs
index 5de50aa2d4d3..b1acf7c11750 100644
--- a/primitives/src/v1/mod.rs
+++ b/primitives/src/v1/mod.rs
@@ -327,9 +327,8 @@ fn check_collator_signature<H: AsRef<[u8]>>(
 }
 
 /// A unique descriptor of the candidate receipt.
-#[derive(PartialEq, Eq, Clone, Encode, Decode, TypeInfo)]
+#[derive(PartialEq, Eq, Clone, Encode, Decode, TypeInfo, Default)]
 #[cfg_attr(feature = "std", derive(Debug, Hash, MallocSizeOf))]
-#[cfg_attr(any(feature = "std", feature = "runtime-benchmarks"), derive(Default))]
 pub struct CandidateDescriptor<H = Hash> {
 	/// The ID of the para this is a candidate for.
 	pub para_id: Id,
@@ -408,9 +407,8 @@ pub struct FullCandidateReceipt<H = Hash, N = BlockNumber> {
 }
 
 /// A candidate-receipt with commitments directly included.
-#[derive(PartialEq, Eq, Clone, Encode, Decode, TypeInfo)]
+#[derive(PartialEq, Eq, Clone, Encode, Decode, TypeInfo, Default)]
 #[cfg_attr(feature = "std", derive(Debug, Hash, MallocSizeOf))]
-#[cfg_attr(any(feature = "std", feature = "runtime-benchmarks"), derive(Default))]
 pub struct CommittedCandidateReceipt<H = Hash> {
 	/// The descriptor of the candidate.
 	pub descriptor: CandidateDescriptor<H>,
@@ -511,9 +509,8 @@ impl<H: Encode, N: Encode> PersistedValidationData<H, N> {
 }
 
 /// Commitments made in a `CandidateReceipt`. Many of these are outputs of validation.
-#[derive(PartialEq, Eq, Clone, Encode, Decode, TypeInfo)]
+#[derive(PartialEq, Eq, Clone, Encode, Decode, TypeInfo, Default)]
 #[cfg_attr(feature = "std", derive(Debug, Hash, MallocSizeOf))]
-#[cfg_attr(any(feature = "std", feature = "runtime-benchmarks"), derive(Default))]
 pub struct CandidateCommitments<N = BlockNumber> {
 	/// Messages destined to be interpreted by the Relay chain itself.
 	pub upward_messages: Vec<UpwardMessage>,
diff --git a/primitives/src/v1/signed.rs b/primitives/src/v1/signed.rs
index cc84448f88f8..d2515b4873aa 100644
--- a/primitives/src/v1/signed.rs
+++ b/primitives/src/v1/signed.rs
@@ -262,7 +262,9 @@ impl<Payload: EncodeAs<RealPayload>, RealPayload: Encode> UncheckedSigned<Payloa
 	}
 
 	/// Sign this payload with the given context and pair.
-	#[cfg(any(feature = "runtime-benchmarks", feature = "std"))]
+	///
+	/// # WARNING
+	/// Only meant for usage in tests and and benchmarks.
 	pub fn benchmark_sign<H: Encode>(
 		public: &crate::v0::ValidatorId,
 		payload: Payload,
@@ -277,7 +279,9 @@ impl<Payload: EncodeAs<RealPayload>, RealPayload: Encode> UncheckedSigned<Payloa
 	}
 
 	/// Immutably access the signature.
-	#[cfg(any(feature = "runtime-benchmarks", feature = "std"))]
+	///
+	/// # WARNING
+	/// Only meant for usage in tests and and benchmarks.
 	pub fn benchmark_signature(&self) -> ValidatorSignature {
 		self.signature.clone()
 	}
diff --git a/runtime/parachains/Cargo.toml b/runtime/parachains/Cargo.toml
index 8c89508ff8d7..705141192093 100644
--- a/runtime/parachains/Cargo.toml
+++ b/runtime/parachains/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "polkadot-runtime-parachains"
-version = "0.9.12"
+version = "0.9.13"
 authors = ["Parity Technologies <admin@parity.io>"]
 edition = "2018"
 
@@ -32,6 +32,7 @@ pallet-session = { git = "https://github.com/paritytech/substrate", branch = "ma
 pallet-staking = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false }
 pallet-timestamp = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false }
 pallet-vesting = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false }
+frame-benchmarking = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false, optional = true }
 frame-support = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false }
 frame-system = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false }
 
@@ -42,16 +43,14 @@ primitives = { package = "polkadot-primitives", path = "../../primitives", defau
 rand = { version = "0.8.3", default-features = false }
 rand_chacha = { version = "0.3.1", default-features = false }
 
-application-crypto = { package = "sp-application-crypto", git = "https://github.com/paritytech/substrate", branch = "master", default-features = false, optional = true }
-frame-benchmarking = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false, optional = true }
-
 [dev-dependencies]
 futures = "0.3.17"
-hex-literal = "0.3.3"
+hex-literal = "0.3.4"
 keyring = { package = "sp-keyring", git = "https://github.com/paritytech/substrate", branch = "master" }
 frame-support-test = { git = "https://github.com/paritytech/substrate", branch = "master" }
 sc-keystore = { git = "https://github.com/paritytech/substrate", branch = "master" }
 
+
 [features]
 default = ["std"]
 no_std = []
@@ -87,8 +86,6 @@ runtime-benchmarks = [
 	"frame-benchmarking",
 	"frame-support/runtime-benchmarks",
 	"frame-system/runtime-benchmarks",
-	"primitives/runtime-benchmarks",
-	"application-crypto/full_crypto",
 ]
 try-runtime = [
 	"frame-support/try-runtime",
@@ -98,4 +95,4 @@ try-runtime = [
 	"pallet-staking/try-runtime",
 	"pallet-timestamp/try-runtime",
 	"pallet-vesting/try-runtime",
-]
+]
\ No newline at end of file
diff --git a/runtime/parachains/src/builder.rs b/runtime/parachains/src/builder.rs
index e38b4ba889da..362d97ecc259 100644
--- a/runtime/parachains/src/builder.rs
+++ b/runtime/parachains/src/builder.rs
@@ -26,7 +26,7 @@ use sp_std::{collections::btree_map::BTreeMap, convert::TryInto, prelude::Vec, v
 /// This is directly from frame-benchmarking. Copy/pasted so we can use it when not compiling with
 /// "features = runtime-benchmarks".
 fn account<AccountId: Decode + Default>(name: &'static str, index: u32, seed: u32) -> AccountId {
-	let entropy = (name, index, seed).using_encoded(sp_core::blake2_256);
+	let entropy = (name, index, seed).using_encoded(sp_io::hashing::blake2_256);
 	AccountId::decode(&mut &entropy[..]).unwrap_or_default()
 }
 
@@ -54,7 +54,7 @@ pub(crate) struct BenchBuilder<T: paras_inherent::Config> {
 }
 
 /// Paras inherent `enter` benchmark scenario.
-#[cfg(any(feature = "runtime-benchmarks", test))]
+#[allow(dead_code)]
 pub(crate) struct Bench<T: paras_inherent::Config> {
 	pub(crate) data: ParachainsInherentData<T::Header>,
 	pub(crate) _session: u32,
@@ -113,7 +113,7 @@ impl<T: paras_inherent::Config> BenchBuilder<T> {
 		self.max_validators.unwrap_or(Self::fallback_max_validators())
 	}
 
-	#[cfg(not(feature = "runtime-benchmarks"))]
+	#[allow(dead_code)]
 	pub(crate) fn set_max_validators(mut self, n: u32) -> Self {
 		self.max_validators = Some(n);
 		self
@@ -129,7 +129,7 @@ impl<T: paras_inherent::Config> BenchBuilder<T> {
 	/// corresponding dispute statement set. Note that if the number of disputes is not specified it fallbacks
 	/// to having a dispute per every validator. Additionally, an entry is not guaranteed to have a dispute - it
 	/// must line up with the cores marked as disputed as defined in `Self::Build`.
-	#[cfg(not(feature = "runtime-benchmarks"))]
+	#[allow(dead_code)]
 	pub(crate) fn set_dispute_statements(mut self, m: BTreeMap<u32, u32>) -> Self {
 		self.dispute_statements = m;
 		self
@@ -140,7 +140,7 @@ impl<T: paras_inherent::Config> BenchBuilder<T> {
 	}
 
 	/// Set maximum number of validators per core.
-	#[cfg(not(feature = "runtime-benchmarks"))]
+	#[allow(dead_code)]
 	pub(crate) fn set_max_validators_per_core(mut self, n: u32) -> Self {
 		self.max_validators_per_core = Some(n);
 		self
@@ -152,7 +152,7 @@ impl<T: paras_inherent::Config> BenchBuilder<T> {
 	}
 
 	/// Minimum number of validity votes in order for a backed candidate to be included.
-	#[cfg(feature = "runtime-benchmarks")]
+	#[allow(dead_code)]
 	pub(crate) fn fallback_min_validity_votes() -> u32 {
 		(Self::fallback_max_validators() / 2) + 1
 	}
@@ -298,7 +298,7 @@ impl<T: paras_inherent::Config> BenchBuilder<T> {
 		frame_system::Pallet::<T>::initialize(
 			&header.number(),
 			&header.hash(),
-			&Digest::<T::Hash> { logs: Vec::new() },
+			&Digest { logs: Vec::new() },
 			Default::default(),
 		);
 

From 3d760b6c1c1bb6c8ffe57d380f62041535d8e212 Mon Sep 17 00:00:00 2001
From: emostov <32168567+emostov@users.noreply.github.com>
Date: Tue, 16 Nov 2021 18:49:41 +0100
Subject: [PATCH 107/107] Update weight files for runtime

---
 .../runtime_parachains_paras_inherent.rs      | 60 +++++++++++++------
 .../runtime_parachains_paras_inherent.rs      | 60 +++++++++++++------
 .../runtime_parachains_paras_inherent.rs      | 60 +++++++++++++------
 3 files changed, 129 insertions(+), 51 deletions(-)

diff --git a/runtime/polkadot/src/weights/runtime_parachains_paras_inherent.rs b/runtime/polkadot/src/weights/runtime_parachains_paras_inherent.rs
index 033af3be5067..e43221b98995 100644
--- a/runtime/polkadot/src/weights/runtime_parachains_paras_inherent.rs
+++ b/runtime/polkadot/src/weights/runtime_parachains_paras_inherent.rs
@@ -16,7 +16,7 @@
 //! Autogenerated weights for `runtime_parachains::paras_inherent`
 //!
 //! THIS FILE WAS AUTO-GENERATED USING THE SUBSTRATE BENCHMARK CLI VERSION 4.0.0-dev
-//! DATE: 2021-11-04, STEPS: `50`, REPEAT: 20, LOW RANGE: `[]`, HIGH RANGE: `[]`
+//! DATE: 2021-11-14, STEPS: `50`, REPEAT: 3, LOW RANGE: `[]`, HIGH RANGE: `[]`
 //! EXECUTION: Some(Wasm), WASM-EXECUTION: Compiled, CHAIN: Some("kusama-dev"), DB CACHE: 128
 
 // Executed Command:
@@ -24,7 +24,7 @@
 // benchmark
 // --chain=kusama-dev
 // --steps=50
-// --repeat=20
+// --repeat=3
 // --pallet=runtime_parachains::paras_inherent
 // --extrinsic=*
 // --execution=wasm
@@ -64,17 +64,16 @@ impl<T: frame_system::Config> runtime_parachains::paras_inherent::WeightInfo for
 	// Storage: ParaScheduler ParathreadQueue (r:1 w:1)
 	// Storage: ParaScheduler Scheduled (r:1 w:1)
 	// Storage: ParaScheduler ValidatorGroups (r:1 w:0)
-	// Storage: System BlockWeight (r:1 w:0)
 	// Storage: Ump NeedsDispatch (r:1 w:1)
 	// Storage: Ump NextDispatchRoundStartWith (r:1 w:1)
 	// Storage: ParaInherent OnChainVotes (r:0 w:1)
 	// Storage: Hrmp HrmpWatermarks (r:0 w:1)
 	// Storage: Paras Heads (r:0 w:1)
 	fn enter_variable_disputes(v: u32, ) -> Weight {
-		(224_853_000 as Weight)
-			// Standard Error: 2_000
-			.saturating_add((229_000 as Weight).saturating_mul(v as Weight))
-			.saturating_add(T::DbWeight::get().reads(24 as Weight))
+		(316_331_000 as Weight)
+			// Standard Error: 112_000
+			.saturating_add((325_000 as Weight).saturating_mul(v as Weight))
+			.saturating_add(T::DbWeight::get().reads(23 as Weight))
 			.saturating_add(T::DbWeight::get().writes(14 as Weight))
 	}
 	// Storage: ParaInherent Included (r:1 w:1)
@@ -97,7 +96,6 @@ impl<T: frame_system::Config> runtime_parachains::paras_inherent::WeightInfo for
 	// Storage: ParaScheduler ParathreadQueue (r:1 w:1)
 	// Storage: ParaScheduler Scheduled (r:1 w:1)
 	// Storage: ParaScheduler ValidatorGroups (r:1 w:0)
-	// Storage: System BlockWeight (r:1 w:0)
 	// Storage: Ump NeedsDispatch (r:1 w:1)
 	// Storage: Ump NextDispatchRoundStartWith (r:1 w:1)
 	// Storage: ParaInclusion AvailabilityBitfields (r:0 w:1)
@@ -105,8 +103,8 @@ impl<T: frame_system::Config> runtime_parachains::paras_inherent::WeightInfo for
 	// Storage: Hrmp HrmpWatermarks (r:0 w:1)
 	// Storage: Paras Heads (r:0 w:1)
 	fn enter_bitfields() -> Weight {
-		(265_872_000 as Weight)
-			.saturating_add(T::DbWeight::get().reads(24 as Weight))
+		(352_749_000 as Weight)
+			.saturating_add(T::DbWeight::get().reads(23 as Weight))
 			.saturating_add(T::DbWeight::get().writes(15 as Weight))
 	}
 	// Storage: ParaInherent Included (r:1 w:1)
@@ -129,7 +127,6 @@ impl<T: frame_system::Config> runtime_parachains::paras_inherent::WeightInfo for
 	// Storage: ParaScheduler ParathreadQueue (r:1 w:1)
 	// Storage: ParaScheduler Scheduled (r:1 w:1)
 	// Storage: ParaScheduler ValidatorGroups (r:1 w:0)
-	// Storage: System BlockWeight (r:1 w:0)
 	// Storage: Paras PastCodeMeta (r:1 w:0)
 	// Storage: Paras CurrentCodeHash (r:1 w:0)
 	// Storage: Ump RelayDispatchQueueSize (r:1 w:0)
@@ -139,14 +136,43 @@ impl<T: frame_system::Config> runtime_parachains::paras_inherent::WeightInfo for
 	// Storage: Hrmp HrmpWatermarks (r:0 w:1)
 	// Storage: Paras Heads (r:0 w:1)
 	fn enter_backed_candidates_variable(v: u32, ) -> Weight {
-		(345_183_000 as Weight)
-			// Standard Error: 23_000
-			.saturating_add((49_234_000 as Weight).saturating_mul(v as Weight))
-			.saturating_add(T::DbWeight::get().reads(27 as Weight))
+		(88_047_000 as Weight)
+			// Standard Error: 3_275_000
+			.saturating_add((68_499_000 as Weight).saturating_mul(v as Weight))
+			.saturating_add(T::DbWeight::get().reads(26 as Weight))
 			.saturating_add(T::DbWeight::get().writes(14 as Weight))
 	}
+	// Storage: ParaInherent Included (r:1 w:1)
+	// Storage: System ParentHash (r:1 w:0)
+	// Storage: ParaScheduler AvailabilityCores (r:1 w:1)
+	// Storage: ParasShared CurrentSessionIndex (r:1 w:0)
+	// Storage: ParasShared ActiveValidatorKeys (r:1 w:0)
+	// Storage: Paras Parachains (r:1 w:0)
+	// Storage: ParaInclusion PendingAvailability (r:2 w:1)
+	// Storage: ParaInclusion PendingAvailabilityCommitments (r:1 w:1)
+	// Storage: Configuration ActiveConfig (r:1 w:0)
+	// Storage: Session Validators (r:1 w:0)
+	// Storage: ParasShared ActiveValidatorIndices (r:1 w:0)
+	// Storage: Staking ActiveEra (r:1 w:0)
+	// Storage: Staking ErasRewardPoints (r:1 w:1)
+	// Storage: Dmp DownwardMessageQueues (r:1 w:1)
+	// Storage: Hrmp HrmpChannelDigests (r:1 w:1)
+	// Storage: Paras FutureCodeUpgrades (r:1 w:0)
+	// Storage: ParaScheduler SessionStartBlock (r:1 w:0)
+	// Storage: ParaScheduler ParathreadQueue (r:1 w:1)
+	// Storage: ParaScheduler Scheduled (r:1 w:1)
+	// Storage: ParaScheduler ValidatorGroups (r:1 w:0)
+	// Storage: Paras PastCodeMeta (r:1 w:0)
+	// Storage: Paras CurrentCodeHash (r:1 w:0)
+	// Storage: Ump RelayDispatchQueueSize (r:1 w:0)
+	// Storage: Ump NeedsDispatch (r:1 w:1)
+	// Storage: Ump NextDispatchRoundStartWith (r:1 w:1)
+	// Storage: ParaInherent OnChainVotes (r:0 w:1)
+	// Storage: Hrmp HrmpWatermarks (r:0 w:1)
+	// Storage: Paras Heads (r:0 w:1)
 	fn enter_backed_candidate_code_upgrade() -> Weight {
-		0
+		(53_728_168_000 as Weight)
+			.saturating_add(T::DbWeight::get().reads(26 as Weight))
+			.saturating_add(T::DbWeight::get().writes(14 as Weight))
 	}
-
 }
diff --git a/runtime/rococo/src/weights/runtime_parachains_paras_inherent.rs b/runtime/rococo/src/weights/runtime_parachains_paras_inherent.rs
index 033af3be5067..e43221b98995 100644
--- a/runtime/rococo/src/weights/runtime_parachains_paras_inherent.rs
+++ b/runtime/rococo/src/weights/runtime_parachains_paras_inherent.rs
@@ -16,7 +16,7 @@
 //! Autogenerated weights for `runtime_parachains::paras_inherent`
 //!
 //! THIS FILE WAS AUTO-GENERATED USING THE SUBSTRATE BENCHMARK CLI VERSION 4.0.0-dev
-//! DATE: 2021-11-04, STEPS: `50`, REPEAT: 20, LOW RANGE: `[]`, HIGH RANGE: `[]`
+//! DATE: 2021-11-14, STEPS: `50`, REPEAT: 3, LOW RANGE: `[]`, HIGH RANGE: `[]`
 //! EXECUTION: Some(Wasm), WASM-EXECUTION: Compiled, CHAIN: Some("kusama-dev"), DB CACHE: 128
 
 // Executed Command:
@@ -24,7 +24,7 @@
 // benchmark
 // --chain=kusama-dev
 // --steps=50
-// --repeat=20
+// --repeat=3
 // --pallet=runtime_parachains::paras_inherent
 // --extrinsic=*
 // --execution=wasm
@@ -64,17 +64,16 @@ impl<T: frame_system::Config> runtime_parachains::paras_inherent::WeightInfo for
 	// Storage: ParaScheduler ParathreadQueue (r:1 w:1)
 	// Storage: ParaScheduler Scheduled (r:1 w:1)
 	// Storage: ParaScheduler ValidatorGroups (r:1 w:0)
-	// Storage: System BlockWeight (r:1 w:0)
 	// Storage: Ump NeedsDispatch (r:1 w:1)
 	// Storage: Ump NextDispatchRoundStartWith (r:1 w:1)
 	// Storage: ParaInherent OnChainVotes (r:0 w:1)
 	// Storage: Hrmp HrmpWatermarks (r:0 w:1)
 	// Storage: Paras Heads (r:0 w:1)
 	fn enter_variable_disputes(v: u32, ) -> Weight {
-		(224_853_000 as Weight)
-			// Standard Error: 2_000
-			.saturating_add((229_000 as Weight).saturating_mul(v as Weight))
-			.saturating_add(T::DbWeight::get().reads(24 as Weight))
+		(316_331_000 as Weight)
+			// Standard Error: 112_000
+			.saturating_add((325_000 as Weight).saturating_mul(v as Weight))
+			.saturating_add(T::DbWeight::get().reads(23 as Weight))
 			.saturating_add(T::DbWeight::get().writes(14 as Weight))
 	}
 	// Storage: ParaInherent Included (r:1 w:1)
@@ -97,7 +96,6 @@ impl<T: frame_system::Config> runtime_parachains::paras_inherent::WeightInfo for
 	// Storage: ParaScheduler ParathreadQueue (r:1 w:1)
 	// Storage: ParaScheduler Scheduled (r:1 w:1)
 	// Storage: ParaScheduler ValidatorGroups (r:1 w:0)
-	// Storage: System BlockWeight (r:1 w:0)
 	// Storage: Ump NeedsDispatch (r:1 w:1)
 	// Storage: Ump NextDispatchRoundStartWith (r:1 w:1)
 	// Storage: ParaInclusion AvailabilityBitfields (r:0 w:1)
@@ -105,8 +103,8 @@ impl<T: frame_system::Config> runtime_parachains::paras_inherent::WeightInfo for
 	// Storage: Hrmp HrmpWatermarks (r:0 w:1)
 	// Storage: Paras Heads (r:0 w:1)
 	fn enter_bitfields() -> Weight {
-		(265_872_000 as Weight)
-			.saturating_add(T::DbWeight::get().reads(24 as Weight))
+		(352_749_000 as Weight)
+			.saturating_add(T::DbWeight::get().reads(23 as Weight))
 			.saturating_add(T::DbWeight::get().writes(15 as Weight))
 	}
 	// Storage: ParaInherent Included (r:1 w:1)
@@ -129,7 +127,6 @@ impl<T: frame_system::Config> runtime_parachains::paras_inherent::WeightInfo for
 	// Storage: ParaScheduler ParathreadQueue (r:1 w:1)
 	// Storage: ParaScheduler Scheduled (r:1 w:1)
 	// Storage: ParaScheduler ValidatorGroups (r:1 w:0)
-	// Storage: System BlockWeight (r:1 w:0)
 	// Storage: Paras PastCodeMeta (r:1 w:0)
 	// Storage: Paras CurrentCodeHash (r:1 w:0)
 	// Storage: Ump RelayDispatchQueueSize (r:1 w:0)
@@ -139,14 +136,43 @@ impl<T: frame_system::Config> runtime_parachains::paras_inherent::WeightInfo for
 	// Storage: Hrmp HrmpWatermarks (r:0 w:1)
 	// Storage: Paras Heads (r:0 w:1)
 	fn enter_backed_candidates_variable(v: u32, ) -> Weight {
-		(345_183_000 as Weight)
-			// Standard Error: 23_000
-			.saturating_add((49_234_000 as Weight).saturating_mul(v as Weight))
-			.saturating_add(T::DbWeight::get().reads(27 as Weight))
+		(88_047_000 as Weight)
+			// Standard Error: 3_275_000
+			.saturating_add((68_499_000 as Weight).saturating_mul(v as Weight))
+			.saturating_add(T::DbWeight::get().reads(26 as Weight))
 			.saturating_add(T::DbWeight::get().writes(14 as Weight))
 	}
+	// Storage: ParaInherent Included (r:1 w:1)
+	// Storage: System ParentHash (r:1 w:0)
+	// Storage: ParaScheduler AvailabilityCores (r:1 w:1)
+	// Storage: ParasShared CurrentSessionIndex (r:1 w:0)
+	// Storage: ParasShared ActiveValidatorKeys (r:1 w:0)
+	// Storage: Paras Parachains (r:1 w:0)
+	// Storage: ParaInclusion PendingAvailability (r:2 w:1)
+	// Storage: ParaInclusion PendingAvailabilityCommitments (r:1 w:1)
+	// Storage: Configuration ActiveConfig (r:1 w:0)
+	// Storage: Session Validators (r:1 w:0)
+	// Storage: ParasShared ActiveValidatorIndices (r:1 w:0)
+	// Storage: Staking ActiveEra (r:1 w:0)
+	// Storage: Staking ErasRewardPoints (r:1 w:1)
+	// Storage: Dmp DownwardMessageQueues (r:1 w:1)
+	// Storage: Hrmp HrmpChannelDigests (r:1 w:1)
+	// Storage: Paras FutureCodeUpgrades (r:1 w:0)
+	// Storage: ParaScheduler SessionStartBlock (r:1 w:0)
+	// Storage: ParaScheduler ParathreadQueue (r:1 w:1)
+	// Storage: ParaScheduler Scheduled (r:1 w:1)
+	// Storage: ParaScheduler ValidatorGroups (r:1 w:0)
+	// Storage: Paras PastCodeMeta (r:1 w:0)
+	// Storage: Paras CurrentCodeHash (r:1 w:0)
+	// Storage: Ump RelayDispatchQueueSize (r:1 w:0)
+	// Storage: Ump NeedsDispatch (r:1 w:1)
+	// Storage: Ump NextDispatchRoundStartWith (r:1 w:1)
+	// Storage: ParaInherent OnChainVotes (r:0 w:1)
+	// Storage: Hrmp HrmpWatermarks (r:0 w:1)
+	// Storage: Paras Heads (r:0 w:1)
 	fn enter_backed_candidate_code_upgrade() -> Weight {
-		0
+		(53_728_168_000 as Weight)
+			.saturating_add(T::DbWeight::get().reads(26 as Weight))
+			.saturating_add(T::DbWeight::get().writes(14 as Weight))
 	}
-
 }
diff --git a/runtime/westend/src/weights/runtime_parachains_paras_inherent.rs b/runtime/westend/src/weights/runtime_parachains_paras_inherent.rs
index 033af3be5067..e43221b98995 100644
--- a/runtime/westend/src/weights/runtime_parachains_paras_inherent.rs
+++ b/runtime/westend/src/weights/runtime_parachains_paras_inherent.rs
@@ -16,7 +16,7 @@
 //! Autogenerated weights for `runtime_parachains::paras_inherent`
 //!
 //! THIS FILE WAS AUTO-GENERATED USING THE SUBSTRATE BENCHMARK CLI VERSION 4.0.0-dev
-//! DATE: 2021-11-04, STEPS: `50`, REPEAT: 20, LOW RANGE: `[]`, HIGH RANGE: `[]`
+//! DATE: 2021-11-14, STEPS: `50`, REPEAT: 3, LOW RANGE: `[]`, HIGH RANGE: `[]`
 //! EXECUTION: Some(Wasm), WASM-EXECUTION: Compiled, CHAIN: Some("kusama-dev"), DB CACHE: 128
 
 // Executed Command:
@@ -24,7 +24,7 @@
 // benchmark
 // --chain=kusama-dev
 // --steps=50
-// --repeat=20
+// --repeat=3
 // --pallet=runtime_parachains::paras_inherent
 // --extrinsic=*
 // --execution=wasm
@@ -64,17 +64,16 @@ impl<T: frame_system::Config> runtime_parachains::paras_inherent::WeightInfo for
 	// Storage: ParaScheduler ParathreadQueue (r:1 w:1)
 	// Storage: ParaScheduler Scheduled (r:1 w:1)
 	// Storage: ParaScheduler ValidatorGroups (r:1 w:0)
-	// Storage: System BlockWeight (r:1 w:0)
 	// Storage: Ump NeedsDispatch (r:1 w:1)
 	// Storage: Ump NextDispatchRoundStartWith (r:1 w:1)
 	// Storage: ParaInherent OnChainVotes (r:0 w:1)
 	// Storage: Hrmp HrmpWatermarks (r:0 w:1)
 	// Storage: Paras Heads (r:0 w:1)
 	fn enter_variable_disputes(v: u32, ) -> Weight {
-		(224_853_000 as Weight)
-			// Standard Error: 2_000
-			.saturating_add((229_000 as Weight).saturating_mul(v as Weight))
-			.saturating_add(T::DbWeight::get().reads(24 as Weight))
+		(316_331_000 as Weight)
+			// Standard Error: 112_000
+			.saturating_add((325_000 as Weight).saturating_mul(v as Weight))
+			.saturating_add(T::DbWeight::get().reads(23 as Weight))
 			.saturating_add(T::DbWeight::get().writes(14 as Weight))
 	}
 	// Storage: ParaInherent Included (r:1 w:1)
@@ -97,7 +96,6 @@ impl<T: frame_system::Config> runtime_parachains::paras_inherent::WeightInfo for
 	// Storage: ParaScheduler ParathreadQueue (r:1 w:1)
 	// Storage: ParaScheduler Scheduled (r:1 w:1)
 	// Storage: ParaScheduler ValidatorGroups (r:1 w:0)
-	// Storage: System BlockWeight (r:1 w:0)
 	// Storage: Ump NeedsDispatch (r:1 w:1)
 	// Storage: Ump NextDispatchRoundStartWith (r:1 w:1)
 	// Storage: ParaInclusion AvailabilityBitfields (r:0 w:1)
@@ -105,8 +103,8 @@ impl<T: frame_system::Config> runtime_parachains::paras_inherent::WeightInfo for
 	// Storage: Hrmp HrmpWatermarks (r:0 w:1)
 	// Storage: Paras Heads (r:0 w:1)
 	fn enter_bitfields() -> Weight {
-		(265_872_000 as Weight)
-			.saturating_add(T::DbWeight::get().reads(24 as Weight))
+		(352_749_000 as Weight)
+			.saturating_add(T::DbWeight::get().reads(23 as Weight))
 			.saturating_add(T::DbWeight::get().writes(15 as Weight))
 	}
 	// Storage: ParaInherent Included (r:1 w:1)
@@ -129,7 +127,6 @@ impl<T: frame_system::Config> runtime_parachains::paras_inherent::WeightInfo for
 	// Storage: ParaScheduler ParathreadQueue (r:1 w:1)
 	// Storage: ParaScheduler Scheduled (r:1 w:1)
 	// Storage: ParaScheduler ValidatorGroups (r:1 w:0)
-	// Storage: System BlockWeight (r:1 w:0)
 	// Storage: Paras PastCodeMeta (r:1 w:0)
 	// Storage: Paras CurrentCodeHash (r:1 w:0)
 	// Storage: Ump RelayDispatchQueueSize (r:1 w:0)
@@ -139,14 +136,43 @@ impl<T: frame_system::Config> runtime_parachains::paras_inherent::WeightInfo for
 	// Storage: Hrmp HrmpWatermarks (r:0 w:1)
 	// Storage: Paras Heads (r:0 w:1)
 	fn enter_backed_candidates_variable(v: u32, ) -> Weight {
-		(345_183_000 as Weight)
-			// Standard Error: 23_000
-			.saturating_add((49_234_000 as Weight).saturating_mul(v as Weight))
-			.saturating_add(T::DbWeight::get().reads(27 as Weight))
+		(88_047_000 as Weight)
+			// Standard Error: 3_275_000
+			.saturating_add((68_499_000 as Weight).saturating_mul(v as Weight))
+			.saturating_add(T::DbWeight::get().reads(26 as Weight))
 			.saturating_add(T::DbWeight::get().writes(14 as Weight))
 	}
+	// Storage: ParaInherent Included (r:1 w:1)
+	// Storage: System ParentHash (r:1 w:0)
+	// Storage: ParaScheduler AvailabilityCores (r:1 w:1)
+	// Storage: ParasShared CurrentSessionIndex (r:1 w:0)
+	// Storage: ParasShared ActiveValidatorKeys (r:1 w:0)
+	// Storage: Paras Parachains (r:1 w:0)
+	// Storage: ParaInclusion PendingAvailability (r:2 w:1)
+	// Storage: ParaInclusion PendingAvailabilityCommitments (r:1 w:1)
+	// Storage: Configuration ActiveConfig (r:1 w:0)
+	// Storage: Session Validators (r:1 w:0)
+	// Storage: ParasShared ActiveValidatorIndices (r:1 w:0)
+	// Storage: Staking ActiveEra (r:1 w:0)
+	// Storage: Staking ErasRewardPoints (r:1 w:1)
+	// Storage: Dmp DownwardMessageQueues (r:1 w:1)
+	// Storage: Hrmp HrmpChannelDigests (r:1 w:1)
+	// Storage: Paras FutureCodeUpgrades (r:1 w:0)
+	// Storage: ParaScheduler SessionStartBlock (r:1 w:0)
+	// Storage: ParaScheduler ParathreadQueue (r:1 w:1)
+	// Storage: ParaScheduler Scheduled (r:1 w:1)
+	// Storage: ParaScheduler ValidatorGroups (r:1 w:0)
+	// Storage: Paras PastCodeMeta (r:1 w:0)
+	// Storage: Paras CurrentCodeHash (r:1 w:0)
+	// Storage: Ump RelayDispatchQueueSize (r:1 w:0)
+	// Storage: Ump NeedsDispatch (r:1 w:1)
+	// Storage: Ump NextDispatchRoundStartWith (r:1 w:1)
+	// Storage: ParaInherent OnChainVotes (r:0 w:1)
+	// Storage: Hrmp HrmpWatermarks (r:0 w:1)
+	// Storage: Paras Heads (r:0 w:1)
 	fn enter_backed_candidate_code_upgrade() -> Weight {
-		0
+		(53_728_168_000 as Weight)
+			.saturating_add(T::DbWeight::get().reads(26 as Weight))
+			.saturating_add(T::DbWeight::get().writes(14 as Weight))
 	}
-
 }