Test Refactor

Cargo.toml

@@ -27,16 +27,20 @@ fern = { version = "0.5", features = ["colored"], optional = true }
 log = "0.4"
 structopt = "0.3"
 wabt = { version = "0.9.1", optional = true }
-wasmer = { path = "lib/api" }
-wasmer-clif-backend = { path = "lib/clif-backend", optional = true }
-wasmer-singlepass-backend = { path = "lib/singlepass-backend", optional = true }
+wasmer = { path = "lib/api", default-features = false }
 wasmer-middleware-common = { path = "lib/middleware-common" }
-wasmer-runtime = { path = "lib/runtime" }
+wasmer-runtime = { path = "lib/runtime", default-features = false }
 wasmer-runtime-core = { path = "lib/runtime-core" }
-wasmer-emscripten = { path = "lib/emscripten" }
+wasmer-kernel-loader = { path = "lib/kernel-loader", optional = true }
+
+# Backends
+wasmer-singlepass-backend = { path = "lib/singlepass-backend", optional = true }
+wasmer-clif-backend = { path = "lib/clif-backend", optional = true }
 wasmer-llvm-backend = { path = "lib/llvm-backend", optional = true }
+
+# Frontends
+wasmer-emscripten = { path = "lib/emscripten" }
 wasmer-wasi = { path = "lib/wasi", optional = true }
-wasmer-kernel-loader = { path = "lib/kernel-loader", optional = true }
 wasmer-wasi-experimental-io-devices = { path = "lib/wasi-experimental-io-devices", optional = true }
 
 [workspace]
@@ -76,21 +80,25 @@ rustc_version = "0.2"
 [dev-dependencies]
 anyhow = "1.0.19"
 wasmer-wast = { path = "tests/wast" }
+lazy_static = "1.4.0"
+# To allow parametrized tests
+test-case = "1.0.0"
 criterion = "0.3"
 glob = "0.3"
 libc = "0.2.60" # for `tests/dev-utils`'s Stdout capturing
 serde = { version = "1", features = ["derive"] } # used by the plugin example
+serde_json = "1"
 typetag = "0.1" # used by the plugin example
 wabt = "0.9.1"
 
 [features]
-default = ["fast-tests", "wasi", "backend-cranelift", "wabt"]
+# Don't add the backend features in default, please add them on the Makefile
+# since we might want to autoconfigure them depending on the availability on the host.
+default = ["wasi", "wabt"]
 "loader-kernel" = ["wasmer-kernel-loader"]
 debug = ["fern", "log/max_level_debug", "log/release_max_level_debug"]
 trace = ["fern", "log/max_level_trace", "log/release_max_level_trace"]
 docs = ["wasmer-runtime/docs"]
-# This feature will allow cargo test to run much faster
-fast-tests = []
 backend-cranelift = [
     "wasmer-clif-backend",
     "wasmer-clif-backend/generate-debug-information",

Makefile

@@ -1,5 +1,45 @@
 .PHONY: spectests emtests clean build install lint precommit docs examples 
 
+ARCH := $(shell uname -m)
+
+backends :=
+
+ifeq ($(ARCH), x86_64)
+  # In X64, Cranelift is enabled
+  backends += cranelift
+  # LLVM is enabled if not in Windows
+  ifneq ($(OS), Windows_NT)
+	LLVM_VERSION := $(shell llvm-config --version)
+	# If findstring is not empty, then it have found the value
+    ifneq (, $(findstring 9,$(LLVM_VERSION)))
+	  backends += llvm
+	endif
+  endif
+endif
+
+backends := $(filter-out ,$(backends))
+
+# Singlepass is enabled
+RUST_VERSION := $(shell rustc -V)
+
+ifneq (, $(findstring nightly,$(RUST_VERSION)))
+	backends += singlepass
+endif
+
+ifneq ($(OS), Windows_NT)
+	bold := $(shell tput bold)
+	green := $(shell tput setaf 2)
+	reset := $(shell tput sgr0)
+endif
+
+
+$(info Available backends: $(bold)$(green)${backends}$(reset))
+
+backend_features_spaced := $(foreach backend,$(backends),backend-$(backend))
+backend_features := --features "$(backend_features_spaced)"
+
+# $(info Cargo features ${backend_features})
+
 # Generate files
 generate-emtests:
 	WASM_EMSCRIPTEN_GENERATE_EMTESTS=1 cargo build --release \
@@ -25,57 +65,53 @@ emtests-generate: generate-emtests
 wasitests-generate: generate-wasitests
 
 wasitests-setup-toolchain: wasitests-setup
-	WASITESTS_SET_UP_TOOLCHAIN=1 cargo build --release -vv
+	WASM_WASI_SET_UP_TOOLCHAIN=1 cargo build --release -vv
 
 wasitests-setup-toolchain-all: wasitests-setup
-	WASI_TEST_GENERATE_ALL=1 WASITESTS_SET_UP_TOOLCHAIN=1 cargo build --release -vv
+	WASI_TEST_GENERATE_ALL=1 WASM_WASI_SET_UP_TOOLCHAIN=1 cargo build --release -vv
 
 generate: generate-emtests generate-wasitests
 
 
 # Spectests
 spectests-singlepass:
-	WASMER_TEST_SINGLEPASS=1 cargo test singlepass::spec --release --no-default-features --features "wasi backend-singlepass" -- --nocapture
+	cargo test singlepass::spec --release $(backend_features)
 
 spectests-cranelift:
-	WASMER_TEST_CRANELFIT=1 cargo test cranelift::spec --release --no-default-features --features "wasi backend-cranelift" -- --nocapture
+	cargo test cranelift::spec --release $(backend_features)
 
 spectests-llvm:
-	WASMER_TEST_LLVM=1 cargo test llvm::spec --release --no-default-features --features "wasi backend-llvm wasmer-llvm-backend/test" -- --nocapture --test-threads=1
+	cargo test llvm::spec --release $(backend_features) -- --test-threads=1
 
-spectests-all:
-	WASMER_TEST_CRANELIFT=1 WASMER_TEST_LLVM=1 WASMER_TEST_SINGLEPASS=1 \
-	cargo test spec --release --no-default-features --features "wasi backend-cranelift backend-singlepass backend-llvm wasmer-llvm-backend/test" 
-
-
-spectests: spectests-singlepass spectests-cranelift spectests-llvm
+spectests:
+	cargo test spec --release $(backend_features) -- --test-threads=1
 
 
 # Emscripten tests
 emtests-singlepass:
-	WASMER_TEST_SINGLEPASS=1 cargo test emtest --release --no-default-features --features "wasi backend-singlepass" -- --test-threads=1
+	cargo test singlepass::emscripten --release $(backend_features)
 
 emtests-cranelift:
-	WASMER_TEST_CRANELIFT=1 cargo test emtest --release --no-default-features --features "wasi backend-cranelift" -- --test-threads=1
+	cargo test cranelift::emscripten --release $(backend_features)
 
 emtests-llvm:
-	WASMER_TEST_LLVM=1 cargo test emtest --release --no-default-features --features "wasi backend-llvm" -- --test-threads=1
+	cargo test llvm::emscripten --release $(backend_features) -- --test-threads=1
 
-emtests-unit:
-	cargo test emscripten --release
+emtests-all:
+	cargo test emscripten --release $(backend_features) -- --test-threads=1
 
-emtests: emtests-unit emtests-singlepass emtests-cranelift emtests-llvm
+emtests: emtests-singlepass emtests-cranelift emtests-llvm
 
 
 # Middleware tests
 middleware-singlepass:
-	cargo test middleware --release --no-default-features --features "wasi backend-singlepass"
+	cargo test middleware::singlepass --release $(backend_features)
 
 middleware-cranelift:
-	cargo test middleware --release --no-default-features --features "wasi backend-cranelift" 
+	cargo test middleware::cranelift --release $(backend_features)
 
 middleware-llvm:
-	cargo test middleware --release --no-default-features --features "wasi backend-llvm" 
+	cargo test middleware::llvm --release $(backend_features)
 
 middleware: middleware-singlepass middleware-cranelift middleware-llvm
 
@@ -86,13 +122,16 @@ wasitests-setup:
 	mkdir -p tests/wasi_test_resources/test_fs/temp
 
 wasitests-singlepass: wasitests-setup
-	WASMER_TEST_SINGLEPASS=1 cargo test wasitest --release --no-default-features --features "wasi backend-singlepass" -- --test-threads=1
+	cargo test wasi::singlepass --release $(backend_features) -- --test-threads=1
 
 wasitests-cranelift: wasitests-setup
-	WASMER_TEST_CRANELIFT=1 cargo test wasitest --release --no-default-features --features "wasi backend-cranelift" -- --test-threads=1 --nocapture
+	cargo test wasi::cranelift --release $(backend_features) -- --test-threads=1 --nocapture
 
 wasitests-llvm: wasitests-setup
-	WASMER_TEST_LLVM=1 cargo test wasitest --release --no-default-features --features "wasi backend-llvm" -- --test-threads=1
+	cargo test wasi::llvm --release $(backend_features) -- --test-threads=1
+
+wasitests-all: wasitests-setup
+	cargo test wasi --release $(backend_features) -- --test-threads=1
 
 wasitests-unit: wasitests-setup
 	cargo test --manifest-path lib/wasi/Cargo.toml --release
@@ -103,15 +142,15 @@ wasitests: wasitests-unit wasitests-singlepass wasitests-cranelift wasitests-llv
 # Backends
 singlepass: wasitests-setup
 	cargo test -p wasmer-singlepass-backend --release
-	WASMER_TEST_SINGLEPASS=1 cargo test --release --no-default-features --features "wasi backend-singlepass" -- --test-threads=1
+	cargo test singlepass --release $(backend_features)
 
 cranelift: wasitests-setup
 	cargo test -p wasmer-clif-backend --release
-	WASMER_TEST_CRANELIFT=1 cargo test --release --no-default-features --features "wasi backend-cranelift" -- --test-threads=1
+	cargo test cranelift --release $(backend_features)
 
 llvm: wasitests-setup
 	cargo test -p wasmer-llvm-backend --release
-	WASMER_TEST_LLVM=1 cargo test --release --no-default-features --features "wasi backend-llvm" -- --test-threads=1
+	cargo test llvm --release $(backend_features) -- --test-threads=1
 
 
 # All tests
@@ -163,7 +202,7 @@ test-rest:
 	cargo test --release -p wasmer-wasi-experimental-io-devices
 	cargo test --release -p wasmer-win-exception-handler
 
-test: spectests emtests middleware wasitests test-rest examples
+test: singlepass cranelift llvm test-rest examples
 
 test-android:
 	ci/run-docker.sh x86_64-linux-android --manifest-path=lib/singlepass-backend/Cargo.toml
@@ -188,20 +227,20 @@ lint:
 precommit: lint test
 
 debug:
-	cargo build --release --features backend-cranelift,backend-singlepass,debug,trace
+	cargo build --release --features "debug trace"
 
 install:
 	cargo install --path .
 
 # Checks
 check-bench-singlepass:
-	cargo check --benches --all --no-default-features --features "backend-singlepass" \
+	cargo check --benches --all singlepass \
 	--exclude wasmer-clif-backend --exclude wasmer-llvm-backend --exclude wasmer-kernel-loader
 check-bench-clif:
-	cargo check --benches --all --no-default-features --features "backend-cranelift" \
+	cargo check --benches --all cranelift \
 	--exclude wasmer-singlepass-backend --exclude wasmer-llvm-backend --exclude wasmer-kernel-loader
 check-bench-llvm:
-	cargo check --benches --all --no-default-features --features "backend-llvm" \
+	cargo check --benches --all llvm \
 	--exclude wasmer-singlepass-backend --exclude wasmer-clif-backend --exclude wasmer-kernel-loader
 
 check-bench: check-bench-singlepass check-bench-llvm
@@ -270,36 +309,36 @@ check: check-bench
 
 # Release
 release:
-	cargo build --release --features backend-singlepass,backend-cranelift,backend-llvm,loader-kernel,experimental-io-devices,log/release_max_level_off
+	cargo build --release $(backend_features) --features loader-kernel,experimental-io-devices,log/release_max_level_off
 
 # Release with musl target
 release-musl:
 	# backend-llvm is not included due to dependency on wabt.
 	# experimental-io-devices is not included due to missing x11-fb.
 	cargo build --release --target x86_64-unknown-linux-musl --features backend-singlepass,backend-cranelift,loader-kernel,log/release_max_level_off,wasi --no-default-features
 
-# Only one backend (cranelift)
-release-clif:
-	# If you are on macOS, you will need mingw-w64 for cross compiling to Windows
-	# brew install mingw-w64
-	cargo build --release --features backend-cranelift
+# This way of releasing is deprecated, since backends are now detected
+# automatically
+release-clif: release
 
-release-singlepass:
-	cargo build --release --features backend-singlepass
+# This way of releasing is deprecated, since backends are now detected
+# automatically
+release-singlepass: release
 
-release-llvm:
-	cargo build --release --features backend-llvm,experimental-io-devices
+# This way of releasing is deprecated, since backends are now detected
+# automatically
+release-llvm: release
 
 bench-singlepass:
 # NOTE this will run some benchmarks using clif; TODO: fix this
-	cargo bench --all --no-default-features --features "backend-singlepass" \
+	cargo bench --all singlepass \
 	--exclude wasmer-clif-backend --exclude wasmer-llvm-backend --exclude wasmer-kernel-loader
 bench-clif:
-	cargo bench --all --no-default-features --features "backend-cranelift" \
+	cargo bench --all cranelift \
 	--exclude wasmer-singlepass-backend --exclude wasmer-llvm-backend --exclude wasmer-kernel-loader
 bench-llvm:
 # NOTE this will run some benchmarks using clif; TODO: fix this
-	cargo bench --all --no-default-features --features "backend-llvm" \
+	cargo bench --all llvm \
 	--exclude wasmer-singlepass-backend --exclude wasmer-clif-backend --exclude wasmer-kernel-loader
 
 build-install-package: