Build the standard library crates when building the runtimes

substrate/utils/wasm-builder/src/lib.rs

@@ -87,6 +87,9 @@
 //!   required as we walk up from the target directory until we find a `Cargo.toml`. If the target
 //!   directory is changed for the build, this environment variable can be used to point to the
 //!   actual workspace.
+//! - `WASM_BUILD_STD` - Sets whether the Rust's standard library crates will also be built. This is
+//!   necessary to make sure the standard library crates only use the exact WASM feature set that
+//!   our executor supports. Enabled by default.
 //! - `CARGO_NET_OFFLINE` - If `true`, `--offline` will be passed to all processes launched to
 //!   prevent network access. Useful in offline environments.
 //!
@@ -158,6 +161,9 @@ const FORCE_WASM_BUILD_ENV: &str = "FORCE_WASM_BUILD";
 /// Environment variable that hints the workspace we are building.
 const WASM_BUILD_WORKSPACE_HINT: &str = "WASM_BUILD_WORKSPACE_HINT";
 
+/// Environment variable to set whether we'll build `core`/`std`.
+const WASM_BUILD_STD: &str = "WASM_BUILD_STD";
+
 /// Write to the given `file` if the `content` is different.
 fn write_file_if_changed(file: impl AsRef<Path>, content: impl AsRef<str>) {
 	if fs::read_to_string(file.as_ref()).ok().as_deref() != Some(content.as_ref()) {
@@ -282,6 +288,12 @@ impl CargoCommand {
 		self.version
 	}
 
+	/// Returns whether this version of the toolchain supports nightly features.
+	fn supports_nightly_features(&self) -> bool {
+		self.version.map(|version| version.is_nightly).unwrap_or(false) ||
+			env::var("RUSTC_BOOTSTRAP").is_ok()
+	}
+
 	/// Check if the supplied cargo command supports our Substrate wasm environment.
 	///
 	/// This means that either the cargo version is at minimum 1.68.0 or this is a nightly cargo.
@@ -332,3 +344,26 @@ impl std::ops::Deref for CargoCommandVersioned {
 fn color_output_enabled() -> bool {
 	env::var(crate::WASM_BUILD_NO_COLOR).is_err()
 }
+
+/// Fetches a boolean environment variable. Will exit the process if the value is invalid.
+fn get_bool_environment_variable(name: &str) -> Option<bool> {
+	let value = env::var_os(name)?;
+
+	// We're comparing `OsString`s here so we can't use a `match`.
+	if value == "1" {
+		Some(true)
+	} else if value == "0" {
+		Some(false)
+	} else {
+		build_helper::warning!(
+			"the '{}' environment variable has an invalid value; it must be either '1' or '0'",
+			name
+		);
+		std::process::exit(1);
+	}
+}
+
+/// Returns whether we need to also compile the standard library when compiling the runtime.
+fn build_std_required() -> bool {
+	crate::get_bool_environment_variable(crate::WASM_BUILD_STD).unwrap_or(true)
+}

substrate/utils/wasm-builder/src/prerequisites.rs

@@ -47,135 +47,111 @@ pub(crate) fn check() -> Result<CargoCommandVersioned, String> {
 	check_wasm_toolchain_installed(cargo_command)
 }
 
-/// Create the project that will be used to check that the wasm toolchain is installed and to
-/// extract the rustc version.
-fn create_check_toolchain_project(project_dir: &Path) {
-	let lib_rs_file = project_dir.join("src/lib.rs");
-	let main_rs_file = project_dir.join("src/main.rs");
-	let build_rs_file = project_dir.join("build.rs");
-	let manifest_path = project_dir.join("Cargo.toml");
+/// Creates a minimal dummy crate at the given path and returns the manifest path.
+fn create_minimal_crate(project_dir: &Path) -> std::path::PathBuf {
+	fs::create_dir_all(project_dir.join("src")).expect("Creating src dir does not fail; qed");
 
+	let manifest_path = project_dir.join("Cargo.toml");
 	write_file_if_changed(
 		&manifest_path,
 		r#"
 			[package]
 			name = "wasm-test"
 			version = "1.0.0"
 			edition = "2021"
-			build = "build.rs"
-
-			[lib]
-			name = "wasm_test"
-			crate-type = ["cdylib"]
 
 			[workspace]
 		"#,
 	);
-	write_file_if_changed(lib_rs_file, "pub fn test() {}");
-
-	// We want to know the rustc version of the rustc that is being used by our cargo command.
-	// The cargo command is determined by some *very* complex algorithm to find the cargo command
-	// that supports nightly.
-	// The best solution would be if there is a `cargo rustc --version` command, which sadly
-	// doesn't exists. So, the only available way of getting the rustc version is to build a project
-	// and capture the rustc version in this build process. This `build.rs` is exactly doing this.
-	// It gets the rustc version by calling `rustc --version` and exposing it in the `RUSTC_VERSION`
-	// environment variable.
-	write_file_if_changed(
-		build_rs_file,
-		r#"
-			fn main() {
-				let rustc_cmd = std::env::var("RUSTC").ok().unwrap_or_else(|| "rustc".into());
-
-				let rustc_version = std::process::Command::new(rustc_cmd)
-					.arg("--version")
-					.output()
-					.ok()
-					.and_then(|o| String::from_utf8(o.stdout).ok());
-
-				println!(
-					"cargo:rustc-env=RUSTC_VERSION={}",
-					rustc_version.unwrap_or_else(|| "unknown rustc version".into()),
-				);
-			}
-		"#,
-	);
-	// Just prints the `RURSTC_VERSION` environment variable that is being created by the
-	// `build.rs` script.
-	write_file_if_changed(
-		main_rs_file,
-		r#"
-			fn main() {
-				println!("{}", env!("RUSTC_VERSION"));
-			}
-		"#,
-	);
+
+	write_file_if_changed(project_dir.join("src/main.rs"), "fn main() {}");
+	manifest_path
 }
 
 fn check_wasm_toolchain_installed(
 	cargo_command: CargoCommand,
 ) -> Result<CargoCommandVersioned, String> {
 	let temp = tempdir().expect("Creating temp dir does not fail; qed");
-	fs::create_dir_all(temp.path().join("src")).expect("Creating src dir does not fail; qed");
-	create_check_toolchain_project(temp.path());
-
-	let err_msg = print_error_message("Rust WASM toolchain not installed, please install it!");
-	let manifest_path = temp.path().join("Cargo.toml").display().to_string();
-
-	let mut build_cmd = cargo_command.command();
-	// Chdir to temp to avoid including project's .cargo/config.toml
-	// by accident - it can happen in some CI environments.
-	build_cmd.current_dir(&temp);
-	build_cmd.args(&[
-		"build",
-		"--target=wasm32-unknown-unknown",
-		"--manifest-path",
-		&manifest_path,
-	]);
+	let manifest_path = create_minimal_crate(temp.path()).display().to_string();
+
+	let prepare_command = |subcommand| {
+		let mut cmd = cargo_command.command();
+		// Chdir to temp to avoid including project's .cargo/config.toml
+		// by accident - it can happen in some CI environments.
+		cmd.current_dir(&temp);
+		cmd.args(&[
+			subcommand,
+			"--target=wasm32-unknown-unknown",
+			"--manifest-path",
+			&manifest_path,
+		]);
+
+		if super::color_output_enabled() {
+			cmd.arg("--color=always");
+		}
 
-	if super::color_output_enabled() {
-		build_cmd.arg("--color=always");
+		// manually set the `CARGO_TARGET_DIR` to prevent a cargo deadlock
+		let target_dir = temp.path().join("target").display().to_string();
+		cmd.env("CARGO_TARGET_DIR", &target_dir);
+
+		// Make sure the host's flags aren't used here, e.g. if an alternative linker is specified
+		// in the RUSTFLAGS then the check we do here will break unless we clear these.
+		cmd.env_remove("CARGO_ENCODED_RUSTFLAGS");
+		cmd.env_remove("RUSTFLAGS");
+		cmd
+	};
+
+	let err_msg =
+		print_error_message("Rust WASM toolchain is not properly installed; please install it!");
+	let build_result = prepare_command("build").output().map_err(|_| err_msg.clone())?;
+	if !build_result.status.success() {
+		return match String::from_utf8(build_result.stderr) {
+			Ok(ref err) if err.contains("the `wasm32-unknown-unknown` target may not be installed") =>
+				Err(print_error_message("Cannot compile the WASM runtime: the `wasm32-unknown-unknown` target is not installed!\n\
+				                         You can install it with `rustup target add wasm32-unknown-unknown` if you're using `rustup`.")),
+
+			// Apparently this can happen when we're running on a non Tier 1 platform.
+			Ok(ref err) if err.contains("linker `rust-lld` not found") =>
+				Err(print_error_message("Cannot compile the WASM runtime: `rust-lld` not found!")),
+
+			Ok(ref err) => Err(format!(
+				"{}\n\n{}\n{}\n{}{}\n",
+				err_msg,
+				Color::Yellow.bold().paint("Further error information:"),
+				Color::Yellow.bold().paint("-".repeat(60)),
+				err,
+				Color::Yellow.bold().paint("-".repeat(60)),
+			)),
+
+			Err(_) => Err(err_msg),
+		};
 	}
 
-	let mut run_cmd = cargo_command.command();
-	// Chdir to temp to avoid including project's .cargo/config.toml
-	// by accident - it can happen in some CI environments.
-	run_cmd.current_dir(&temp);
-	run_cmd.args(&["run", "--manifest-path", &manifest_path]);
-
-	// manually set the `CARGO_TARGET_DIR` to prevent a cargo deadlock
-	let target_dir = temp.path().join("target").display().to_string();
-	build_cmd.env("CARGO_TARGET_DIR", &target_dir);
-	run_cmd.env("CARGO_TARGET_DIR", &target_dir);
-
-	// Make sure the host's flags aren't used here, e.g. if an alternative linker is specified
-	// in the RUSTFLAGS then the check we do here will break unless we clear these.
-	build_cmd.env_remove("CARGO_ENCODED_RUSTFLAGS");
-	run_cmd.env_remove("CARGO_ENCODED_RUSTFLAGS");
-	build_cmd.env_remove("RUSTFLAGS");
-	run_cmd.env_remove("RUSTFLAGS");
-
-	build_cmd.output().map_err(|_| err_msg.clone()).and_then(|s| {
-		if s.status.success() {
-			let version = run_cmd.output().ok().and_then(|o| String::from_utf8(o.stdout).ok());
-			Ok(CargoCommandVersioned::new(
-				cargo_command,
-				version.unwrap_or_else(|| "unknown rustc version".into()),
-			))
-		} else {
-			match String::from_utf8(s.stderr) {
-				Ok(ref err) if err.contains("linker `rust-lld` not found") =>
-					Err(print_error_message("`rust-lld` not found, please install it!")),
-				Ok(ref err) => Err(format!(
-					"{}\n\n{}\n{}\n{}{}\n",
-					err_msg,
-					Color::Yellow.bold().paint("Further error information:"),
-					Color::Yellow.bold().paint("-".repeat(60)),
-					err,
-					Color::Yellow.bold().paint("-".repeat(60)),
-				)),
-				Err(_) => Err(err_msg),
+	let mut run_cmd = prepare_command("rustc");
+	run_cmd.args(&["-q", "--", "--version"]);
+
+	let version = run_cmd
+		.output()
+		.ok()
+		.and_then(|o| String::from_utf8(o.stdout).ok())
+		.unwrap_or_else(|| "unknown rustc version".into());
+
+	if crate::build_std_required() {
+		let mut sysroot_cmd = prepare_command("rustc");
+		sysroot_cmd.args(&["-q", "--", "--print", "sysroot"]);
+		if let Some(sysroot) =
+			sysroot_cmd.output().ok().and_then(|o| String::from_utf8(o.stdout).ok())
+		{
+			let src_path =
+				Path::new(sysroot.trim()).join("lib").join("rustlib").join("src").join("rust");
+			if !src_path.exists() {
+				return Err(print_error_message(
+					"Cannot compile the WASM runtime: no standard library sources found!\n\
+					 You can install them with `rustup component add rust-src` if you're using `rustup`.",
+				))
 			}
 		}
-	})
+	}
+
+	Ok(CargoCommandVersioned::new(cargo_command, version))
 }

substrate/utils/wasm-builder/src/version.rs

@@ -212,4 +212,21 @@ mod tests {
 			version_1_69_0,
 		);
 	}
+
+	#[test]
+	fn parse_rustc_version() {
+		let version = Version::extract("rustc 1.73.0 (cc66ad468 2023-10-03)").unwrap();
+		assert_eq!(
+			version,
+			Version {
+				major: 1,
+				minor: 73,
+				patch: 0,
+				is_nightly: false,
+				year: Some(2023),
+				month: Some(10),
+				day: Some(03),
+			}
+		);
+	}
 }

substrate/utils/wasm-builder/src/wasm_project.rs

@@ -751,6 +751,25 @@ fn build_bloaty_blob(
 		build_cmd.arg("--offline");
 	}
 
+	// Our executor currently only supports the WASM MVP feature set, however nowadays
+	// when compiling WASM the Rust compiler has more features enabled by default.
+	//
+	// We do set the `-C target-cpu=mvp` flag to make sure that *our* code gets compiled
+	// in a way that is compatible with our executor, however this doesn't affect Rust's
+	// standard library crates (`std`, `core` and `alloc`) which are by default precompiled
+	// and still can make use of these extra features.
+	//
+	// So here we force the compiler to also compile the standard library crates for us
+	// to make sure that they also only use the MVP features.
+	if crate::build_std_required() {
+		// Unfortunately this is still a nightly-only flag, but FWIW it is pretty widely used
+		// so it's unlikely to break without a replacement.
+		build_cmd.arg("-Z").arg("build-std");
+		if !cargo_cmd.supports_nightly_features() {
+			build_cmd.env("RUSTC_BOOTSTRAP", "1");
+		}
+	}
+
 	println!("{}", colorize_info_message("Information that should be included in a bug report."));
 	println!("{} {:?}", colorize_info_message("Executing build command:"), build_cmd);
 	println!("{} {}", colorize_info_message("Using rustc version:"), cargo_cmd.rustc_version());
@@ -948,6 +967,7 @@ fn generate_rerun_if_changed_instructions(
 	println!("cargo:rerun-if-env-changed={}", crate::WASM_BUILD_RUSTFLAGS_ENV);
 	println!("cargo:rerun-if-env-changed={}", crate::WASM_TARGET_DIRECTORY);
 	println!("cargo:rerun-if-env-changed={}", crate::WASM_BUILD_TOOLCHAIN);
+	println!("cargo:rerun-if-env-changed={}", crate::WASM_BUILD_STD);
 }
 
 /// Track files and paths related to the given package to rerun `build.rs` on any relevant change.