move paras inherent filtering to runtime

node/network/availability-distribution/src/requester/mod.rs

@@ -159,7 +159,7 @@ impl Requester {
 				// Just book keeping - we are already requesting that chunk:
 				{
 					e.get_mut().add_leaf(leaf);
-				}
+				},
 				Entry::Vacant(e) => {
 					let tx = self.tx.clone();
 					let metrics = self.metrics.clone();

parachain/Cargo.toml

@@ -36,3 +36,4 @@ std = [
 	"polkadot-core-primitives/std",
 	"frame-support/std",
 ]
+runtime-benchmarks = []

parachain/src/primitives.rs

@@ -43,7 +43,8 @@ pub use polkadot_core_primitives::BlockNumber as RelayChainBlockNumber;
 #[derive(
 	PartialEq, Eq, Clone, PartialOrd, Ord, Encode, Decode, RuntimeDebug, derive_more::From, TypeInfo,
 )]
-#[cfg_attr(feature = "std", derive(Serialize, Deserialize, Default, Hash, MallocSizeOf))]
+#[cfg_attr(feature = "std", derive(Serialize, Deserialize, Hash, MallocSizeOf))]
+#[cfg_attr(any(feature = "std", feature = "runtime-benchmarks"), derive(Default))]
 pub struct HeadData(#[cfg_attr(feature = "std", serde(with = "bytes"))] pub Vec<u8>);
 
 impl HeadData {

primitives/Cargo.toml

@@ -29,7 +29,6 @@ frame-system = { git = "https://github.com/paritytech/substrate", branch = "mast
 hex-literal = "0.3.3"
 parity-util-mem = { version = "0.10.0", default-features = false, optional = true }
 
-
 [features]
 default = ["std"]
 std = [
@@ -56,3 +55,6 @@ std = [
 	"bitvec/std",
 	"frame-system/std",
 ]
+runtime-benchmarks = [
+	"polkadot-parachain/runtime-benchmarks",
+]

primitives/src/v1/mod.rs

@@ -328,7 +328,8 @@ fn check_collator_signature<H: AsRef<[u8]>>(
 
 /// A unique descriptor of the candidate receipt.
 #[derive(PartialEq, Eq, Clone, Encode, Decode, TypeInfo)]
-#[cfg_attr(feature = "std", derive(Debug, Default, Hash, MallocSizeOf))]
+#[cfg_attr(feature = "std", derive(Debug, Hash, MallocSizeOf))]
+#[cfg_attr(any(feature = "std", feature = "runtime-benchmarks"), derive(Default))]
 pub struct CandidateDescriptor<H = Hash> {
 	/// The ID of the para this is a candidate for.
 	pub para_id: Id,
@@ -408,7 +409,8 @@ pub struct FullCandidateReceipt<H = Hash, N = BlockNumber> {
 
 /// A candidate-receipt with commitments directly included.
 #[derive(PartialEq, Eq, Clone, Encode, Decode, TypeInfo)]
-#[cfg_attr(feature = "std", derive(Debug, Default, Hash, MallocSizeOf))]
+#[cfg_attr(feature = "std", derive(Debug, Hash, MallocSizeOf))]
+#[cfg_attr(any(feature = "std", feature = "runtime-benchmarks"), derive(Default))]
 pub struct CommittedCandidateReceipt<H = Hash> {
 	/// The descriptor of the candidate.
 	pub descriptor: CandidateDescriptor<H>,
@@ -510,7 +512,8 @@ impl<H: Encode, N: Encode> PersistedValidationData<H, N> {
 
 /// Commitments made in a `CandidateReceipt`. Many of these are outputs of validation.
 #[derive(PartialEq, Eq, Clone, Encode, Decode, TypeInfo)]
-#[cfg_attr(feature = "std", derive(Debug, Default, Hash, MallocSizeOf))]
+#[cfg_attr(feature = "std", derive(Debug, Hash, MallocSizeOf))]
+#[cfg_attr(any(feature = "std", feature = "runtime-benchmarks"), derive(Default))]
 pub struct CandidateCommitments<N = BlockNumber> {
 	/// Messages destined to be interpreted by the Relay chain itself.
 	pub upward_messages: Vec<UpwardMessage>,
@@ -534,6 +537,8 @@ impl CandidateCommitments {
 }
 
 /// A bitfield concerning availability of backed candidates.
+///
+/// Every bit refers to an availability core index.
 #[derive(PartialEq, Eq, Clone, Encode, Decode, RuntimeDebug, TypeInfo)]
 pub struct AvailabilityBitfield(pub BitVec<bitvec::order::Lsb0, u8>);
 

primitives/src/v1/signed.rs

@@ -41,6 +41,13 @@ use crate::v0::{SigningContext, ValidatorId, ValidatorIndex, ValidatorSignature}
 #[derive(Clone, PartialEq, Eq, RuntimeDebug)]
 pub struct Signed<Payload, RealPayload = Payload>(UncheckedSigned<Payload, RealPayload>);
 
+impl<Payload, RealPayload> Signed<Payload, RealPayload> {
+	/// Convert back to an unchecked type.
+	pub fn into_unchecked(self) -> UncheckedSigned<Payload, RealPayload> {
+		self.0
+	}
+}
+
 /// Unchecked signed data, can be converted to `Signed` by checking the signature.
 #[derive(Clone, PartialEq, Eq, RuntimeDebug, Encode, Decode, TypeInfo)]
 pub struct UncheckedSigned<Payload, RealPayload = Payload> {
@@ -253,6 +260,33 @@ impl<Payload: EncodeAs<RealPayload>, RealPayload: Encode> UncheckedSigned<Payloa
 			Err(())
 		}
 	}
+
+	/// Sign this payload with the given context and pair.
+	#[cfg(any(feature = "runtime-benchmarks", feature = "std"))]
+	pub fn benchmark_sign<H: Encode>(
+		public: &crate::v0::ValidatorId,
+		payload: Payload,
+		context: &SigningContext<H>,
+		validator_index: ValidatorIndex,
+	) -> Self {
+		use application_crypto::RuntimeAppPublic;
+		let data = Self::payload_data(&payload, context);
+		let signature = public.sign(&data).unwrap();
+
+		Self { payload, validator_index, signature, real_payload: sp_std::marker::PhantomData }
+	}
+
+	/// Immutably access the signature.
+	#[cfg(any(feature = "runtime-benchmarks", feature = "std"))]
+	pub fn benchmark_signature(&self) -> ValidatorSignature {
+		self.signature.clone()
+	}
+
+	/// Set the signature. Only should be used for creating testing mocks.
+	#[cfg(feature = "std")]
+	pub fn set_signature(&mut self, signature: ValidatorSignature) {
+		self.signature = signature
+	}
 }
 
 impl<Payload, RealPayload> From<Signed<Payload, RealPayload>>

roadmap/implementers-guide/src/runtime/inclusion.md

@@ -45,13 +45,38 @@ PendingAvailabilityCommitments: map ParaId => CandidateCommitments;
 All failed checks should lead to an unrecoverable error making the block invalid.
 
 * `process_bitfields(expected_bits, Bitfields, core_lookup: Fn(CoreIndex) -> Option<ParaId>)`:
-  1. check that there is at most 1 bitfield per validator and that the number of bits in each bitfield is equal to `expected_bits`.
-  1. check that there are no duplicates
-  1. check all validator signatures.
+  1. call `sanitize_bitfields<true>` and use the sanitized `signed_bitfields` from now on.
+  1. call `sanitize_backed_candidates<true>` and use the sanitized `backed_candidates` from now on.
   1. apply each bit of bitfield to the corresponding pending candidate. looking up parathread cores using the `core_lookup`. Disregard bitfields that have a `1` bit for any free cores.
   1. For each applied bit of each availability-bitfield, set the bit for the validator in the `CandidatePendingAvailability`'s `availability_votes` bitfield. Track all candidates that now have >2/3 of bits set in their `availability_votes`. These candidates are now available and can be enacted.
   1. For all now-available candidates, invoke the `enact_candidate` routine with the candidate and relay-parent number.
   1. Return a list of `(CoreIndex, CandidateHash)` from freed cores consisting of the cores where candidates have become available.
+* `sanitize_bitfields<const EARLY_RETURN>(
+    usab: UncheckedSignedAvailabilityBitfields,
+    db: DisputedBitfield,
+    expected_bits: usize,
+    parent_hash: Hash,
+    session_index: SessionIndex,
+    validators: &[ValidatorId]
+  )`:
+  1. if `EARLY_RETURN` is `true`, return an error when encountering a bitfield that does not pass the checks, if `false`, drop the bitfield from the set that will be returned.
+  1. check that there is at most 1 bitfield per validator and that the number of bits in each bitfield is equal to `expected_bits`.
+  1. check that there are no duplicates
+  1. check that the validator bit index is not out of bounds
+  1. check all validator signatures, iff `EARLY_RETURN=true`, since in the other case, checking is supposed to be done before the call
+  1. check that there are no bits set that reference a disputed candidate
+
+* `sanitize_backed_candidates<const EARLY_RETURN: bool>(
+	relay_parent: Hash,
+	mut backed_candidates: Vec<BackedCandidate>,
+	candidate_has_concluded_invalid_dispute: Fn(CandidateHash) -> bool,
+	scheduled: &[CoreAssignment],
+)`
+  1. if `EARLY_RETURN` is `true`, return an error when encountering a backed candidates that does not pass the checks, if `false`, drop the backed candidates from the set that will be returned.
+  1. check all backed candidates have no concluded invalid dispute by means of the provided closure `candidate_has_concluded_invalid_dispute`
+  1. check all backed candidates have the matching `relay_parent`
+  1. check all backed candidates paraid was scheduled by means of the provided `scheduled` parameter
+
 * `process_candidates(parent_storage_root, BackedCandidates, scheduled: Vec<CoreAssignment>, group_validators: Fn(GroupIndex) -> Option<Vec<ValidatorIndex>>)`:
   1. check that each candidate corresponds to a scheduled core and that they are ordered in the same order the cores appear in assignments in `scheduled`.
   1. check that `scheduled` is sorted ascending by `CoreIndex`, without duplicates.
@@ -78,6 +103,9 @@ All failed checks should lead to an unrecoverable error making the block invalid
   1. call `Hrmp::prune_hrmp` with the para id of the candiate and the candidate's `hrmp_watermark`.
   1. call `Hrmp::queue_outbound_hrmp` with the para id of the candidate and the list of horizontal messages taken from the commitment,
   1. Call `Paras::note_new_head` using the `HeadData` from the receipt and `relay_parent_number`.
+
+* `fn sanitize_bitfields`
+
 * `collect_pending`:
 
   ```rust

roadmap/implementers-guide/src/runtime/parainherent.md

@@ -48,3 +48,22 @@ OnChainVotes: Option<ScrapedOnChainVotes>,
     1. Call `Scheduler::occupied` using the `occupied` core indices of the returned  above, first sorting the list of assigned core indices.
     1. Call the `Ump::process_pending_upward_messages` routine to execute all messages in upward dispatch queues.
     1. If all of the above succeeds, set `Included` to `Some(())`.
+
+
+* `create_inherent`: This entry-point accepts one parameter: `InherentData`.
+  1. Unpack `InherentData` into its parts, `bitfields`, `backed_candidates`, `disputes` and the `parent_header`.
+  1. Hash the `parent_header` and make sure that it corresponds to the block hash of the parent (tracked by the `frame_system` FRAME module),
+  1. Invoke `Disputes::filter_multi_dispute_data` to remove duplicates et al from `disputes`.
+  1. Run the following within a  `with_transaction` closure to avoid side effects:
+    1. Invoke `Disputes::provide_multi_dispute_data`.
+    1. Collect the newly concluded disputes as `current_concluded_invalid_disputes`.
+    1. If there are any concluded disputes from the current session, invoke `Inclusion::collect_disputed` with the newly disputed candidates. Annotate each returned core with `FreedReason::Concluded`, sort them, and invoke `Scheduler::free_cores` with them.
+    1. Collect the concluded invalid disputes in the current session as `conlcuded_invalid_disputes`.
+    1. Return `TransactionOutcome::Rollback(freed_cores, concluded_invalid_disputes)`.
+  1. Call `sanitize_bitfields<false>` and only use the sanitized set of bitfields afterwards.
+  1. Call `sanitize_backed_candidates<false>`.
+  1. Collect entropy based on `CurrentBlockRandomness::random`.
+  1. Call `apply_weight_limit` to utilize the block as well as possible, with a randomized heuristic.
+  1. Re-create a `InherentData` from the sanitized and weight bounded information.
+  1. Call `Self::enter` which now should not error anymore, but we do this call anyways for now.
+  1. Return `Call::enter { .. }`.

runtime/kusama/Cargo.toml

@@ -214,6 +214,7 @@ runtime-benchmarks = [
 	"xcm-builder/runtime-benchmarks",
 	"frame-election-provider-support/runtime-benchmarks",
 	"pallet-bags-list/runtime-benchmarks",
+	"runtime-parachains/runtime-benchmarks",
 ]
 try-runtime = [
 	"frame-executive/try-runtime",

runtime/kusama/src/lib.rs

@@ -1201,7 +1201,9 @@ impl parachains_hrmp::Config for Runtime {
 	type Currency = Balances;
 }
 
-impl parachains_paras_inherent::Config for Runtime {}
+impl parachains_paras_inherent::Config for Runtime {
+	type WeightInfo = weights::runtime_parachains_paras_inherent::WeightInfo<Runtime>;
+}
 
 impl parachains_scheduler::Config for Runtime {}
 
@@ -1933,6 +1935,7 @@ sp_api::impl_runtime_apis! {
 			list_benchmark!(list, extra, runtime_common::paras_registrar, Registrar);
 			list_benchmark!(list, extra, runtime_parachains::configuration, Configuration);
 			list_benchmark!(list, extra, runtime_parachains::initializer, Initializer);
+			list_benchmark!(list, extra, runtime_parachains::paras_inherent, ParaInherent);
 			list_benchmark!(list, extra, runtime_parachains::paras, Paras);
 			// Substrate
 			list_benchmark!(list, extra, pallet_bags_list, BagsList);
@@ -2010,6 +2013,7 @@ sp_api::impl_runtime_apis! {
 			add_benchmark!(params, batches, runtime_common::paras_registrar, Registrar);
 			add_benchmark!(params, batches, runtime_parachains::configuration, Configuration);
 			add_benchmark!(params, batches, runtime_parachains::initializer, Initializer);
+			add_benchmark!(params, batches, runtime_parachains::paras_inherent, ParaInherent);
 			add_benchmark!(params, batches, runtime_parachains::paras, Paras);
 			// Substrate
 			add_benchmark!(params, batches, pallet_balances, Balances);

runtime/kusama/src/weights/mod.rs

@@ -47,3 +47,4 @@ pub mod runtime_common_slots;
 pub mod runtime_parachains_configuration;
 pub mod runtime_parachains_initializer;
 pub mod runtime_parachains_paras;
+pub mod runtime_parachains_paras_inherent;