Merge branch 'master' into ao-validator-discovery-api

* master:
  Companion PR for #6984 (#1661)
  Update some dependencies. (#1718)
  Add a specific memory requirements (#1716)
  Companion PR for #7039: grandpa-rpc dont share subscription manager, only executor (#1687)
  Update bytes. (#1715)
  Add a note about memory requirements (#1714)
  Update parity-multiaddr. (#1700)
  typo in proxy tests (#1713)
  Companion PR for ` Add a `build-sync-spec` subcommand and remove the CHT roots from the light sync state.` (#1670)
  Forwardport: Validation: don't detect STDIN closing when running in process (#1695) (#1703)

README.md

@@ -100,6 +100,8 @@ git checkout <latest tagged release>
 cargo build --release
 ```
 
+Note that compilation is a memory intensive process. We recommend having 4 GiB of phyiscal RAM or swap available (keep in mind that if a build hits swap it tends to be very slow).
+
 ## Networks
 
 This repo supports runtimes for Polkadot, Kusama, and Westend.

cli/src/cli.rs

@@ -24,6 +24,9 @@ pub enum Subcommand {
 	/// Build a chain specification.
 	BuildSpec(sc_cli::BuildSpecCmd),
 
+	/// Build a chain specification with a light client sync state.
+	BuildSyncSpec(sc_cli::BuildSyncSpecCmd),
+
 	/// Validate blocks.
 	CheckBlock(sc_cli::CheckBlockCmd),
 

cli/src/command.rs

@@ -21,6 +21,7 @@ use service::{IdentifyVariant, self};
 use service_new::{IdentifyVariant, self as service};
 use sc_cli::{SubstrateCli, Result, RuntimeVersion, Role};
 use crate::cli::{Cli, Subcommand};
+use std::sync::Arc;
 
 fn get_exec_name() -> Option<String> {
 	std::env::current_exe()
@@ -154,6 +155,39 @@ pub fn run() -> Result<()> {
 			let runner = cli.create_runner(cmd)?;
 			runner.sync_run(|config| cmd.run(config.chain_spec, config.network))
 		},
+		Some(Subcommand::BuildSyncSpec(cmd)) => {
+			let runner = cli.create_runner(cmd)?;
+			let chain_spec = &runner.config().chain_spec;
+
+			set_default_ss58_version(chain_spec);
+
+			let authority_discovery_enabled = cli.run.authority_discovery_enabled;
+			let grandpa_pause = if cli.run.grandpa_pause.is_empty() {
+				None
+			} else {
+				Some((cli.run.grandpa_pause[0], cli.run.grandpa_pause[1]))
+			};
+
+			if chain_spec.is_kusama() {
+				info!("----------------------------");
+				info!("This chain is not in any way");
+				info!("      endorsed by the       ");
+				info!("     KUSAMA FOUNDATION      ");
+				info!("----------------------------");
+			}
+
+			runner.async_run(|config| {
+				let chain_spec = config.chain_spec.cloned_box();
+				let network_config = config.network.clone();
+				let service::NewFull { task_manager, client, network_status_sinks, .. }
+					= service::new_full_nongeneric(
+						config, None, authority_discovery_enabled, grandpa_pause, false,
+					)?;
+				let client = Arc::new(client);
+
+				Ok((cmd.run(chain_spec, network_config, client, network_status_sinks), task_manager))
+			})
+		},
 		Some(Subcommand::CheckBlock(cmd)) => {
 			let runner = cli.create_runner(cmd)?;
 			let chain_spec = &runner.config().chain_spec;

node/core/candidate-validation/src/lib.rs

@@ -37,7 +37,7 @@ use polkadot_primitives::v1::{
 };
 use polkadot_parachain::wasm_executor::{
 	self, ValidationPool, ExecutionMode, ValidationError,
-	InvalidCandidate as WasmInvalidCandidate, ValidationExecutionMode,
+	InvalidCandidate as WasmInvalidCandidate,
 };
 use polkadot_parachain::primitives::{ValidationResult as WasmValidationResult, ValidationParams};
 
@@ -130,7 +130,7 @@ async fn run(
 )
 	-> SubsystemResult<()>
 {
-	let pool = ValidationPool::new(ValidationExecutionMode::ExternalProcessSelfHost);
+	let execution_mode = ExecutionMode::ExternalProcessSelfHost(ValidationPool::new());
 
 	loop {
 		match ctx.recv().await? {
@@ -145,7 +145,7 @@ async fn run(
 				) => {
 					let res = spawn_validate_from_chain_state(
 						&mut ctx,
-						Some(pool.clone()),
+						execution_mode.clone(),
 						descriptor,
 						pov,
 						spawn.clone(),
@@ -169,7 +169,7 @@ async fn run(
 				) => {
 					let res = spawn_validate_exhaustive(
 						&mut ctx,
-						Some(pool.clone()),
+						execution_mode.clone(),
 						persisted_validation_data,
 						transient_validation_data,
 						validation_code,
@@ -271,7 +271,7 @@ async fn check_assumption_validation_data(
 
 async fn spawn_validate_from_chain_state(
 	ctx: &mut impl SubsystemContext<Message = CandidateValidationMessage>,
-	validation_pool: Option<ValidationPool>,
+	execution_mode: ExecutionMode,
 	descriptor: CandidateDescriptor,
 	pov: Arc<PoV>,
 	spawn: impl SpawnNamed + 'static,
@@ -288,7 +288,7 @@ async fn spawn_validate_from_chain_state(
 		AssumptionCheckOutcome::Matches(validation_data, validation_code) => {
 			return spawn_validate_exhaustive(
 				ctx,
-				validation_pool,
+				execution_mode,
 				validation_data.persisted,
 				Some(validation_data.transient),
 				validation_code,
@@ -309,7 +309,7 @@ async fn spawn_validate_from_chain_state(
 		AssumptionCheckOutcome::Matches(validation_data, validation_code) => {
 			return spawn_validate_exhaustive(
 				ctx,
-				validation_pool,
+				execution_mode,
 				validation_data.persisted,
 				Some(validation_data.transient),
 				validation_code,
@@ -330,7 +330,7 @@ async fn spawn_validate_from_chain_state(
 
 async fn spawn_validate_exhaustive(
 	ctx: &mut impl SubsystemContext<Message = CandidateValidationMessage>,
-	validation_pool: Option<ValidationPool>,
+	execution_mode: ExecutionMode,
 	persisted_validation_data: PersistedValidationData,
 	transient_validation_data: Option<TransientValidationData>,
 	validation_code: ValidationCode,
@@ -341,7 +341,7 @@ async fn spawn_validate_exhaustive(
 	let (tx, rx) = oneshot::channel();
 	let fut = async move {
 		let res = validate_candidate_exhaustive::<RealValidationBackend, _>(
-			validation_pool,
+			execution_mode,
 			persisted_validation_data,
 			transient_validation_data,
 			validation_code,
@@ -422,22 +422,18 @@ trait ValidationBackend {
 struct RealValidationBackend;
 
 impl ValidationBackend for RealValidationBackend {
-	type Arg = Option<ValidationPool>;
+	type Arg = ExecutionMode;
 
 	fn validate<S: SpawnNamed + 'static>(
-		pool: Option<ValidationPool>,
+		execution_mode: ExecutionMode,
 		validation_code: &ValidationCode,
 		params: ValidationParams,
 		spawn: S,
 	) -> Result<WasmValidationResult, ValidationError> {
-		let execution_mode = pool.as_ref()
-			.map(ExecutionMode::Remote)
-			.unwrap_or(ExecutionMode::Local);
-
 		wasm_executor::validate_candidate(
 			&validation_code.0,
 			params,
-			execution_mode,
+			&execution_mode,
 			spawn,
 		)
 	}

node/service/src/lib.rs

@@ -158,7 +158,10 @@ fn new_partial<RuntimeApi, Executor>(config: &mut Configuration) -> Result<
 		consensus_common::DefaultImportQueue<Block, FullClient<RuntimeApi, Executor>>,
 		sc_transaction_pool::FullPool<Block, FullClient<RuntimeApi, Executor>>,
 		(
-			impl Fn(polkadot_rpc::DenyUnsafe, polkadot_rpc::SubscriptionManager) -> polkadot_rpc::RpcExtension,
+			impl Fn(
+				polkadot_rpc::DenyUnsafe,
+				polkadot_rpc::SubscriptionTaskExecutor,
+			) -> polkadot_rpc::RpcExtension,
 			(
 				babe::BabeBlockImport<
 					Block, FullClient<RuntimeApi, Executor>, FullGrandpaBlockImport<RuntimeApi, Executor>
@@ -245,7 +248,7 @@ fn new_partial<RuntimeApi, Executor>(config: &mut Configuration) -> Result<
 		let transaction_pool = transaction_pool.clone();
 		let select_chain = select_chain.clone();
 
-		move |deny_unsafe, subscriptions| -> polkadot_rpc::RpcExtension {
+		move |deny_unsafe, subscription_executor| -> polkadot_rpc::RpcExtension {
 			let deps = polkadot_rpc::FullDeps {
 				client: client.clone(),
 				pool: transaction_pool.clone(),
@@ -260,7 +263,7 @@ fn new_partial<RuntimeApi, Executor>(config: &mut Configuration) -> Result<
 					shared_voter_state: shared_voter_state.clone(),
 					shared_authority_set: shared_authority_set.clone(),
 					justification_stream: justification_stream.clone(),
-					subscriptions,
+					subscription_executor,
 				},
 			};
 

node/test-service/src/lib.rs

@@ -27,7 +27,7 @@ use polkadot_primitives::v0::{
 };
 use polkadot_runtime_common::BlockHashCount;
 use polkadot_service::{
-	new_full, FullNodeHandles, AbstractClient, ClientHandle, ExecuteWithClient,
+	new_full, NewFull, FullNodeHandles, AbstractClient, ClientHandle, ExecuteWithClient,
 };
 use polkadot_test_runtime::{Runtime, SignedExtra, SignedPayload, VERSION};
 use sc_chain_spec::ChainSpec;
@@ -74,7 +74,7 @@ pub fn polkadot_test_new_full(
 	),
 	ServiceError,
 > {
-	let (task_manager, client, handles, network, rpc_handlers) =
+	let NewFull { task_manager, client, node_handles, network, rpc_handlers, .. } =
 		new_full::<polkadot_test_runtime::RuntimeApi, PolkadotTestExecutor>(
 			config,
 			collating_for,
@@ -83,7 +83,7 @@ pub fn polkadot_test_new_full(
 			true,
 		)?;
 
-	Ok((task_manager, client, handles, network, rpc_handlers))
+	Ok((task_manager, client, node_handles, network, rpc_handlers))
 }
 
 /// A wrapper for the test client that implements `ClientHandle`.

parachain/src/wasm_executor/mod.rs

@@ -20,15 +20,15 @@
 //! Assuming the parameters are correct, this module provides a wrapper around
 //! a WASM VM for re-execution of a parachain candidate.
 
-use std::any::{TypeId, Any};
+use std::{any::{TypeId, Any}, path::PathBuf};
 use crate::primitives::{ValidationParams, ValidationResult};
 use codec::{Decode, Encode};
 use sp_core::{storage::{ChildInfo, TrackedStorageKey}, traits::{CallInWasm, SpawnNamed}};
 use sp_externalities::Extensions;
 use sp_wasm_interface::HostFunctions as _;
 
 #[cfg(not(any(target_os = "android", target_os = "unknown")))]
-pub use validation_host::{run_worker, ValidationPool, EXECUTION_TIMEOUT_SEC, ValidationExecutionMode};
+pub use validation_host::{run_worker, ValidationPool, EXECUTION_TIMEOUT_SEC, WORKER_ARGS};
 
 mod validation_host;
 
@@ -58,16 +58,29 @@ pub fn run_worker(_: &str) -> Result<(), String> {
 	Err("Cannot run validation worker on this platform".to_string())
 }
 
-/// WASM code execution mode.
-///
-/// > Note: When compiling for WASM, the `Remote` variants are not available.
-pub enum ExecutionMode<'a> {
-	/// Execute in-process. The execution can not be interrupted or aborted.
-	Local,
-	/// Remote execution in a spawned process.
-	Remote(&'a ValidationPool),
+/// The execution mode for the `ValidationPool`.
+#[derive(Clone)]
+#[cfg_attr(not(any(target_os = "android", target_os = "unknown")), derive(Debug))]
+pub enum ExecutionMode {
+	/// The validation worker is ran in a thread inside the same process.
+	InProcess,
+	/// The validation worker is ran using the process' executable and the subcommand `validation-worker` is passed
+	/// following by the address of the shared memory.
+	ExternalProcessSelfHost(ValidationPool),
+	/// The validation worker is ran using the command provided and the argument provided. The address of the shared
+	/// memory is added at the end of the arguments.
+	ExternalProcessCustomHost {
+		/// Validation pool.
+		pool: ValidationPool,
+		/// Path to the validation worker. The file must exists and be executable.
+		binary: PathBuf,
+		/// List of arguments passed to the validation worker. The address of the shared memory will be automatically
+		/// added after the arguments.
+		args: Vec<String>,
+	},
 }
 
+
 #[derive(Debug, derive_more::Display, derive_more::From)]
 /// Candidate validation error.
 pub enum ValidationError {
@@ -132,19 +145,24 @@ impl std::error::Error for ValidationError {
 pub fn validate_candidate(
 	validation_code: &[u8],
 	params: ValidationParams,
-	options: ExecutionMode<'_>,
+	execution_mode: &ExecutionMode,
 	spawner: impl SpawnNamed + 'static,
 ) -> Result<ValidationResult, ValidationError> {
-	match options {
-		ExecutionMode::Local => {
+	match execution_mode {
+		ExecutionMode::InProcess => {
 			validate_candidate_internal(validation_code, &params.encode(), spawner)
 		},
 		#[cfg(not(any(target_os = "android", target_os = "unknown")))]
-		ExecutionMode::Remote(pool) => {
+		ExecutionMode::ExternalProcessSelfHost(pool) => {
 			pool.validate_candidate(validation_code, params)
 		},
+		#[cfg(not(any(target_os = "android", target_os = "unknown")))]
+		ExecutionMode::ExternalProcessCustomHost { pool, binary, args } => {
+			let args: Vec<&str> = args.iter().map(|x| x.as_str()).collect();
+			pool.validate_candidate_custom(validation_code, params, binary, &args)
+		},
 		#[cfg(any(target_os = "android", target_os = "unknown"))]
-		ExecutionMode::Remote(_pool) =>
+		ExecutionMode::ExternalProcessSelfHost(_) | ExecutionMode::ExternalProcessCustomHost { .. } =>
 			Err(ValidationError::Internal(InternalError::System(
 				Box::<dyn std::error::Error + Send + Sync>::from(
 					"Remote validator not available".to_string()