Wasm-builder 3.0 (#7532)

* Build every wasm crate in its own project with wasm-builder

Building all wasm crates in one workspace was a nice idea, however it
just introduced problems:

1. We needed to prune old members, but this didn't worked for old git
deps.
2. We locked the whole wasm workspace while building one crate. This
could lead to infinitely locking the workspace on a crash.

Now we just build every crate in its own project, this means we will
build the dependencies multiple times. While building the dependencies
multiple times, we still decrease the build time by around 30 seconds
for Polkadot and Substrate because of the new parallelism ;)

* Remove the requirement on wasm-builder-runner

This removes the requirement on wasm-builder-runner by using the new
`build_dep` feature of cargo. We use nightly anyway and that enables us
to use this feature. This solves the problem of not mixing
build/proc-macro deps with normal deps. By doing this we get rid off
this complicated project structure and can depend directly on
`wasm-builder`. This also removes all the code from wasm-builder-runner
and mentions that it is deprecated.

* Copy the `Cargo.lock` to the correct folder

* Remove wasm-builder-runner

* Update docs

* Fix deterministic check

Modified-by: Bastian Köcher <git@kchr.de>

* Try to make the ui test happy

* Switch to `SKIP_WASM_BUILD`

* Rename `SKIP_WASM_BINARY` to the correct name...

* Update utils/wasm-builder/src/builder.rs

Co-authored-by: André Silva <123550+andresilva@users.noreply.github.com>

* Update utils/wasm-builder/src/builder.rs

Co-authored-by: André Silva <123550+andresilva@users.noreply.github.com>

Co-authored-by: André Silva <123550+andresilva@users.noreply.github.com>

.gitlab-ci.yml

@@ -197,7 +197,7 @@ cargo-check-benches:
   <<:                              *docker-env
   <<:                              *test-refs
   script:
-    - BUILD_DUMMY_WASM_BINARY=1 time cargo +nightly check --benches --all
+    - SKIP_WASM_BUILD=1 time cargo +nightly check --benches --all
     - cargo run --release -p node-bench -- ::node::import::native::sr25519::transfer_keep_alive::paritydb::small
     - cargo run --release -p node-bench -- ::trie::read::small
     - sccache -s
@@ -208,7 +208,7 @@ cargo-check-subkey:
   <<:                              *test-refs
   script:
     - cd ./bin/utils/subkey
-    - BUILD_DUMMY_WASM_BINARY=1 time cargo check --release
+    - SKIP_WASM_BUILD=1 time cargo check --release
     - sccache -s
 
 test-deterministic-wasm:
@@ -222,7 +222,7 @@ test-deterministic-wasm:
     # build runtime
     - cargo build --verbose --release -p node-runtime
     # make checksum
-    - sha256sum target/release/wbuild/target/wasm32-unknown-unknown/release/node_runtime.wasm > checksum.sha256
+    - sha256sum target/release/wbuild/node-runtime/target/wasm32-unknown-unknown/release/node_runtime.wasm > checksum.sha256
     # clean up – FIXME: can we reuse some of the artifacts?
     - cargo clean
     # build again
@@ -343,7 +343,7 @@ cargo-check-macos:
   <<:                              *docker-env
   <<:                              *test-refs
   script:
-    - BUILD_DUMMY_WASM_BINARY=1 time cargo check --release
+    - SKIP_WASM_BUILD=1 time cargo check --release
     - sccache -s
   tags:
     - osx
@@ -451,7 +451,7 @@ build-linux-subkey:                &build-subkey
     - mkdir -p ./artifacts/subkey
   script:
     - cd ./bin/utils/subkey
-    - BUILD_DUMMY_WASM_BINARY=1 time cargo build --release --verbose
+    - SKIP_WASM_BUILD=1 time cargo build --release --verbose
     - cd -
     - mv ./target/release/subkey ./artifacts/subkey/.
     - echo -n "Subkey version = "
@@ -483,7 +483,7 @@ build-rust-doc:
     - ./crate-docs/
   script:
     - rm -f ./crate-docs/index.html # use it as an indicator if the job succeeds
-    - BUILD_DUMMY_WASM_BINARY=1 RUSTDOCFLAGS="--html-in-header $(pwd)/.maintain/rustdoc-header.html"
+    - SKIP_WASM_BUILD=1 RUSTDOCFLAGS="--html-in-header $(pwd)/.maintain/rustdoc-header.html"
         time cargo +nightly doc --no-deps --workspace --all-features --verbose
     - mv ./target/doc ./crate-docs
     - echo "<meta http-equiv=refresh content=0;url=sc_service/index.html>" > ./crate-docs/index.html

Cargo.toml

@@ -59,7 +59,6 @@ members = [
 	"client/transaction-pool",
 	"client/transaction-pool/graph",
 	"utils/prometheus",
-	"utils/wasm-builder-runner",
 	"frame/assets",
 	"frame/aura",
 	"frame/atomic-swap",

bin/node-template/runtime/Cargo.toml

@@ -48,7 +48,7 @@ hex-literal = { version = "0.3.1", optional = true }
 template = { version = "2.0.0", default-features = false, path = "../pallets/template", package = "pallet-template" }
 
 [build-dependencies]
-wasm-builder-runner = { version = "2.0.0", package = "substrate-wasm-builder-runner", path = "../../../utils/wasm-builder-runner" }
+substrate-wasm-builder = { version = "3.0.0", path = "../../../utils/wasm-builder" }
 
 [features]
 default = ["std"]

bin/node-template/runtime/build.rs

@@ -1,9 +1,8 @@
-use wasm_builder_runner::WasmBuilder;
+use substrate_wasm_builder::WasmBuilder;
 
 fn main() {
 	WasmBuilder::new()
 		.with_current_project()
-		.with_wasm_builder_from_crates("2.0.1")
 		.export_heap_base()
 		.import_memory()
 		.build()

bin/node/runtime/Cargo.toml

@@ -79,7 +79,7 @@ pallet-transaction-payment-rpc-runtime-api = { version = "2.0.0", default-featur
 pallet-vesting = { version = "2.0.0", default-features = false, path = "../../../frame/vesting" }
 
 [build-dependencies]
-wasm-builder-runner = { version = "2.0.0", package = "substrate-wasm-builder-runner", path = "../../../utils/wasm-builder-runner" }
+substrate-wasm-builder = { version = "3.0.0", path = "../../../utils/wasm-builder" }
 
 [dev-dependencies]
 sp-io = { version = "2.0.0", path = "../../../primitives/io" }

bin/node/runtime/build.rs

@@ -15,12 +15,11 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use wasm_builder_runner::WasmBuilder;
+use substrate_wasm_builder::WasmBuilder;
 
 fn main() {
 	WasmBuilder::new()
 		.with_current_project()
-		.with_wasm_builder_from_crates_or_path("2.0.1", "../../../utils/wasm-builder")
 		.export_heap_base()
 		.import_memory()
 		.build()

bin/node/runtime/src/lib.rs

@@ -91,11 +91,11 @@ use sp_runtime::generic::Era;
 #[cfg(feature = "std")]
 include!(concat!(env!("OUT_DIR"), "/wasm_binary.rs"));
 
+/// Wasm binary unwrapped. If built with `SKIP_WASM_BUILD`, the function panics.
 #[cfg(feature = "std")]
-/// Wasm binary unwrapped. If built with `BUILD_DUMMY_WASM_BINARY`, the function panics.
 pub fn wasm_binary_unwrap() -> &'static [u8] {
 	WASM_BINARY.expect("Development wasm binary is not available. This means the client is \
-						built with `BUILD_DUMMY_WASM_BINARY` flag and it is only usable for \
+						built with `SKIP_WASM_BUILD` flag and it is only usable for \
 						production chains. Please rebuild with the flag disabled.")
 }
 

client/executor/runtime-test/Cargo.toml

@@ -22,7 +22,7 @@ sp-std = { version = "2.0.0", default-features = false, path = "../../../primiti
 sp-tasks = { version = "2.0.0", default-features = false, path = "../../../primitives/tasks" }
 
 [build-dependencies]
-wasm-builder-runner = { version = "2.0.0", package = "substrate-wasm-builder-runner", path = "../../../utils/wasm-builder-runner" }
+substrate-wasm-builder = { version = "3.0.0", path = "../../../utils/wasm-builder" }
 
 [features]
 default = [ "std" ]

client/executor/runtime-test/build.rs

@@ -14,24 +14,22 @@
 // You should have received a copy of the GNU General Public License
 // along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
 
-use wasm_builder_runner::WasmBuilder;
+use substrate_wasm_builder::WasmBuilder;
 
 fn main() {
 	// regular build
 	WasmBuilder::new()
 		.with_current_project()
-		.with_wasm_builder_from_crates_or_path("2.0.1", "../../../utils/wasm-builder")
 		.export_heap_base()
 		.import_memory()
 		.build();
 
 	// and building with tracing activated
 	WasmBuilder::new()
 		.with_current_project()
-		.with_wasm_builder_from_crates_or_path("2.0.1", "../../../utils/wasm-builder")
 		.export_heap_base()
 		.import_memory()
 		.set_file_name("wasm_binary_with_tracing.rs")
-		.append_to_rust_flags("--cfg feature=\\\"with-tracing\\\"")
+		.append_to_rust_flags(r#"--cfg feature="with-tracing""#)
 		.build();
 }

client/executor/runtime-test/src/lib.rs

@@ -4,8 +4,8 @@
 #[cfg(feature = "std")]
 include!(concat!(env!("OUT_DIR"), "/wasm_binary.rs"));
 
+/// Wasm binary unwrapped. If built with `SKIP_WASM_BUILD`, the function panics.
 #[cfg(feature = "std")]
-/// Wasm binary unwrapped. If built with `BUILD_DUMMY_WASM_BINARY`, the function panics.
 pub fn wasm_binary_unwrap() -> &'static [u8] {
 	WASM_BINARY.expect("Development wasm binary is not available. Testing is only \
 						supported with the flag disabled.")

docs/README.adoc

@@ -308,28 +308,6 @@ cargo run --release \-- \
 
 Additional Substrate CLI usage options are available and may be shown by running `cargo run \-- --help`.
 
-=== WASM binaries
-
-The WASM binaries are built during the normal `cargo build` process. To control the WASM binary building,
-we support multiple environment variables:
-
-* `SKIP_WASM_BUILD` - Skips building any WASM binary. This is useful when only native should be recompiled.
-* `BUILD_DUMMY_WASM_BINARY` - Builds dummy WASM binaries. These dummy binaries are empty and useful
-                              for `cargo check` runs.
-* `WASM_BUILD_TYPE` - Sets the build type for building WASM binaries. Supported values are `release` or `debug`.
-                      By default the build type is equal to the build type used by the main build.
-* `FORCE_WASM_BUILD` - Can be set to force a WASM build. On subsequent calls the value of the variable
-                       needs to change. As WASM builder instructs `cargo` to watch for file changes
-                       this environment variable should only be required in certain circumstances.
-* `WASM_TARGET_DIRECTORY` - Will copy release build WASM binary to the given directory. The path needs
-                            to be absolute.
-* `WASM_BUILD_RUSTFLAGS` - Extend `RUSTFLAGS` given to `cargo build` while building the wasm binary.
-* `WASM_BUILD_NO_COLOR` - Disable color output of the wasm build.
-
-Each project can be skipped individually by using the environment variable `SKIP_PROJECT_NAME_WASM_BUILD`.
-Where `PROJECT_NAME` needs to be replaced by the name of the cargo project, e.g. `node-runtime` will
-be `NODE_RUNTIME`.
-
 [[flaming-fir]]
 === Joining the Flaming Fir Testnet
 

frame/support/test/tests/construct_runtime_ui.rs

@@ -21,7 +21,7 @@ use std::env;
 #[test]
 fn ui() {
 	// As trybuild is using `cargo check`, we don't need the real WASM binaries.
-	env::set_var("BUILD_DUMMY_WASM_BINARY", "1");
+	env::set_var("SKIP_WASM_BUILD", "1");
 
 	let t = trybuild::TestCases::new();
 	t.compile_fail("tests/construct_runtime_ui/*.rs");

frame/support/test/tests/decl_module_ui.rs

@@ -19,7 +19,7 @@
 #[test]
 fn decl_module_ui() {
 	// As trybuild is using `cargo check`, we don't need the real WASM binaries.
-	std::env::set_var("BUILD_DUMMY_WASM_BINARY", "1");
+	std::env::set_var("SKIP_WASM_BUILD", "1");
 
 	let t = trybuild::TestCases::new();
 	t.compile_fail("tests/decl_module_ui/*.rs");

frame/support/test/tests/decl_storage_ui.rs

@@ -19,7 +19,7 @@
 #[test]
 fn decl_storage_ui() {
 	// As trybuild is using `cargo check`, we don't need the real WASM binaries.
-	std::env::set_var("BUILD_DUMMY_WASM_BINARY", "1");
+	std::env::set_var("SKIP_WASM_BUILD", "1");
 
 	let t = trybuild::TestCases::new();
 	t.compile_fail("tests/decl_storage_ui/*.rs");

frame/support/test/tests/derive_no_bound_ui.rs

@@ -19,7 +19,7 @@
 #[test]
 fn derive_no_bound_ui() {
 	// As trybuild is using `cargo check`, we don't need the real WASM binaries.
-	std::env::set_var("BUILD_DUMMY_WASM_BINARY", "1");
+	std::env::set_var("SKIP_WASM_BUILD", "1");
 
 	let t = trybuild::TestCases::new();
 	t.compile_fail("tests/derive_no_bound_ui/*.rs");

frame/support/test/tests/reserved_keyword.rs

@@ -19,7 +19,7 @@
 #[test]
 fn reserved_keyword() {
 	// As trybuild is using `cargo check`, we don't need the real WASM binaries.
-	std::env::set_var("BUILD_DUMMY_WASM_BINARY", "1");
+	std::env::set_var("SKIP_WASM_BUILD", "1");
 
 	let t = trybuild::TestCases::new();
 	t.compile_fail("tests/reserved_keyword/*.rs");