diff --git a/.github/workflows/integration.yaml b/.github/workflows/integration.yaml index ce71ab2fec..39e88186ad 100644 --- a/.github/workflows/integration.yaml +++ b/.github/workflows/integration.yaml @@ -21,6 +21,8 @@ env: # Update this prior to requiring a higher minor version in go.mod GO_VERSION: "1.21" # 1.xx == latest patch of 1.xx TINYGO_VERSION: "0.30.0" ZIG_VERSION: "0.11.0" + BINARYEN_VERSION: "116" + STDLIB_TESTS: "internal/integration_test/stdlibs" concurrency: # https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#example-using-concurrency-to-cancel-any-in-progress-job-or-run @@ -36,6 +38,7 @@ jobs: env: ZIG_INSTALL: ~/zig-install ZIG_SOURCE: ~/zig-source + BINARYEN_INSTALL: ~/binaryen-install steps: - name: Checkout wazero @@ -48,7 +51,7 @@ jobs: enableCrossOsArchive: true key: zig-stdlib-test-binary-${{ env.ZIG_VERSION }} path: - ./zigbin/ + ${{ env.STDLIB_TESTS }}/testdata/zig - name: Install Zig build if: steps.binary-cache.outputs.cache-hit != 'true' @@ -62,16 +65,20 @@ jobs: mkdir -p ${{ env.ZIG_SOURCE }} curl -sSL https://ziglang.org/download/${{ env.ZIG_VERSION }}/zig-${{ env.ZIG_VERSION }}.tar.xz | tar -xJ --strip-components=1 -C ${{ env.ZIG_SOURCE }} + - name: Install Binaryen build + if: steps.binary-cache.outputs.cache-hit != 'true' + run: | + mkdir -p ${{ env.BINARYEN_INSTALL }} + curl -sSL https://github.com/WebAssembly/binaryen/releases/download/version_${{ env.BINARYEN_VERSION }}/binaryen-version_${{ env.BINARYEN_VERSION }}-x86_64-linux.tar.gz | tar -xz --strip-components=1 -C ${{ env.BINARYEN_INSTALL }} + - name: Build Stdlib test binary if: steps.binary-cache.outputs.cache-hit != 'true' # --test-no-exec allows building of the test Wasm binary without executing command. # We use find because the test.wasm will be something like ./zig-cache/o/dd6df1361b2134adc5eee9d027495436/test.wasm run: | - mkdir ${{ github.workspace }}/zigbin - cd ${{ env.ZIG_SOURCE }} - ${{ env.ZIG_INSTALL }}/zig test --test-no-exec -target wasm32-wasi --zig-lib-dir ./lib ./lib/std/std.zig - _ZIG_TEST_BINARY_PATH=$(find . -name test.wasm) - cp ${_ZIG_TEST_BINARY_PATH} ${{ github.workspace }}/zigbin/test.wasm + PATH=${{ env.ZIG_INSTALL }}:${{ env.BINARYEN_INSTALL }}/bin:$PATH + cd ${{ env.STDLIB_TESTS }} + make build.zig zigroot=${{ env.ZIG_SOURCE }} zig: needs: build_zig_test_binary @@ -101,13 +108,14 @@ jobs: fail-on-cache-miss: true key: zig-stdlib-test-binary-${{ env.ZIG_VERSION }} path: - ./zigbin/ + ${{ env.STDLIB_TESTS }}/testdata/zig - uses: actions/setup-go@v4 with: go-version: ${{ env.GO_VERSION }} - - name: Build wazero + - if: ${{ matrix.compiler == 'optimizing' }} + name: Build wazero run: go build -o ./wazerocli ./cmd/wazero env: GOARCH: ${{ matrix.arch }} @@ -136,12 +144,15 @@ jobs: - name: Run built test binaries (container) if: ${{ matrix.compiler == 'optimizing' }} run: | - docker run --platform linux/${{ matrix.arch }} -v $(pwd)/zigbin:/test -v $(pwd)/wazerocli:/wazero -e WAZEROCLI=/wazero --tmpfs /tmp --rm -t wazero:test \ + docker run --platform linux/${{ matrix.arch }} -v $(pwd)/${{ env.STDLIB_TESTS }}/testdata/zig:/test -v $(pwd)/wazerocli:/wazero -e WAZEROCLI=/wazero --tmpfs /tmp --rm -t wazero:test \ /wazero run -optimizing-compiler -mount=:/ ./test/test.wasm - name: Run built test binaries if: ${{ matrix.compiler != 'optimizing' }} - run: ./wazerocli run -mount=:/ ./zigbin/test.wasm + run: | + cd ${{ env.STDLIB_TESTS }} + go test -bench='./zig' -benchtime=1x + build_tinygo_test_binary: name: Build TinyGo test binary @@ -157,7 +168,7 @@ jobs: enableCrossOsArchive: true key: tinygo-test-binaries-${{ env.TINYGO_VERSION }} path: - ./tinygobin/ + ${{ env.STDLIB_TESTS }}/testdata/tinygo - name: Install TinyGo if: steps.binary-cache.outputs.cache-hit != 'true' @@ -181,48 +192,8 @@ jobs: # - compress/zlib is skipped as it depends on the local files https://github.com/golang/go/blob/go1.20/src/compress/zlib/writer_test.go#L16-L30 # - debug/macho is skipped as it depends on the local files https://github.com/golang/go/blob/go1.20/src/debug/macho/file_test.go#L25 run: | - mkdir ./tinygobin - for value in container/heap \ - container/list \ - container/ring \ - crypto/des \ - crypto/md5 \ - crypto/rc4 \ - crypto/sha1 \ - crypto/sha256 \ - crypto/sha512 \ - embed/internal/embedtest \ - encoding \ - encoding/ascii85 \ - encoding/base32 \ - encoding/csv \ - encoding/hex \ - go/scanner \ - hash \ - hash/adler32 \ - hash/crc64 \ - hash/fnv \ - html \ - internal/itoa \ - internal/profile \ - math \ - math/cmplx \ - net \ - net/http/internal/ascii \ - net/mail \ - os \ - path \ - reflect \ - sync \ - testing \ - testing/iotest \ - text/scanner \ - unicode \ - unicode/utf16 \ - unicode/utf8 - do - tinygo test -target wasi -c -o ./tinygobin/${value/\//_}.test $value - done + cd ${{ env.STDLIB_TESTS }} + make build.tinygo tinygo: needs: build_tinygo_test_binary @@ -252,13 +223,14 @@ jobs: fail-on-cache-miss: true key: tinygo-test-binaries-${{ env.TINYGO_VERSION }} path: - ./tinygobin/ + ${{ env.STDLIB_TESTS }}/testdata/tinygo - uses: actions/setup-go@v4 with: go-version: ${{ env.GO_VERSION }} - - name: Build wazero + - if: ${{ matrix.compiler == 'optimizing' }} + name: Build wazero run: go build -o ~/wazerocli ./cmd/wazero env: GOARCH: ${{ matrix.arch }} @@ -286,7 +258,7 @@ jobs: if: ${{ matrix.compiler == 'optimizing' }} # This runs all tests compiled above in sequence. Note: This mounts /tmp to allow t.TempDir() in tests. run: | - cd ./tinygobin + cd ${{ env.STDLIB_TESTS }}/testdata/tinygo for bin in *.test; do echo $bin docker run --platform linux/${{ matrix.arch }} -v $(pwd):/test -v ~/wazerocli:/wazero -e WAZEROCLI=/wazero --tmpfs /tmp --rm -t wazero:test \ @@ -296,11 +268,8 @@ jobs: - name: Run test binaries if: ${{ matrix.compiler != 'optimizing' }} run: | - cd ./tinygobin - for bin in *.test; do - echo $bin - ~/wazerocli run -mount=:/ -mount=:/tmp $bin -- -test.v - done + cd ${{ env.STDLIB_TESTS }} + go test -bench='./tinygo' -benchtime=1x wasi-testsuite: name: wasi-testsuite @@ -413,96 +382,30 @@ jobs: with: go-version: ${{ matrix.go-version }} + - name: Checkout wazero + uses: actions/checkout@v3 + - name: Cache Go test binaries id: cache-go-test-binaries uses: actions/cache@v3 with: - path: ~/tests + path: + ${{ env.STDLIB_TESTS }}/testdata/go # Use precise Go version from setup-go as patch version differences can effect tests. key: go-wasip1-binaries-${{ matrix.os }}-${{ steps.setup-go.outputs.go-version }} - - if: ${{ steps.cache-go-test-binaries.outputs.cache-hit != 'true' }} - name: Build Test Binaries - run: | - mkdir ~/tests - cd $(go env GOROOT) - # Choose important packages to limit execution time. - for value in src/archive/tar \ - src/bufio \ - src/bytes \ - src/context \ - src/encoding/ascii85 \ - src/encoding/asn1 \ - src/encoding/base32 \ - src/encoding/base64 \ - src/encoding/binary \ - src/encoding/csv \ - src/encoding/gob \ - src/encoding/hex \ - src/encoding/json \ - src/encoding/pem \ - src/encoding/xml \ - src/errors \ - src/expvar \ - src/flag \ - src/fmt \ - src/hash \ - src/hash/adler32 \ - src/hash/crc32 \ - src/hash/crc64 \ - src/hash/fnv \ - src/hash/maphash \ - src/io \ - src/io/fs \ - src/io/ioutil \ - src/log \ - src/log/syslog \ - src/maps \ - src/math \ - src/math/big \ - src/math/bits \ - src/math/cmplx \ - src/math/rand \ - src/mime \ - src/mime/multipart \ - src/mime/quotedprintable \ - src/os \ - src/os/exec \ - src/os/signal \ - src/os/user \ - src/path \ - src/reflect \ - src/regexp \ - src/regexp/syntax \ - src/runtime \ - src/runtime/internal/atomic \ - src/runtime/internal/math \ - src/runtime/internal/sys \ - src/slices \ - src/sort \ - src/strconv \ - src/strings \ - src/sync \ - src/sync/atomic \ - src/syscall \ - src/testing \ - src/testing/fstest \ - src/testing/iotest \ - src/testing/quick \ - src/time - do - echo "GOOS=wasip1 GOARCH=wasm go test -v -c -o ~/tests/${value//\//_}.test ./$value" - GOOS=wasip1 GOARCH=wasm go test -v -c -o ~/tests/${value//\//_}.test ./$value - done - - - name: Checkout wazero - uses: actions/checkout@v3 - - - name: Build wazero + - if: ${{ matrix.compiler == 'optimizing' }} + name: Build wazero run: go build -o ~/wazerocli ./cmd/wazero env: GOARCH: ${{ matrix.arch }} + - if: ${{ steps.cache-go-test-binaries.outputs.cache-hit != 'true' }} + name: Build Test Binaries + run: | + cd ${{ env.STDLIB_TESTS }} + make build.gowasip1 + - name: Set up QEMU if: ${{ matrix.compiler == 'optimizing' && matrix.arch == 'arm64' }} uses: docker/setup-qemu-action@v2 @@ -520,12 +423,12 @@ jobs: - if: ${{ matrix.compiler == 'optimizing' }} name: Run test binaries (container) run: | - echo "Running $(find ~/tests -name *.test | wc -l) test binaries" + echo "Running $(find ${{ env.STDLIB_TESTS }}/testdata/go -name '*.test' | wc -l) test binaries" # Skip tests that are hideously slow (mostly compilation time) for running inside QEMU. skip_targets="src_encoding_json|src_runtime|src_os_exec|src_math_big|src_encoding_gob|src_time|src_archive_tar|src_math_rand|src_expvar|src_testing|src_os" # Go tests often look for files relative to the source. Change to the corresponding directory. - for bin in $(find ~/tests -name "*.test" | grep -vE "$skip_targets"); do + for bin in $(find $PWD/${{ env.STDLIB_TESTS }}/testdata/go -name '*.test' | grep -vE "$skip_targets"); do dir=$(basename $bin); dir=${dir%.test}; dir=${dir//_/\/} pushd $(go env GOROOT)/$dir # Mount / as /ROOT in docker and then remount /ROOT as / in wazero; $bin is now in /ROOT/$bin. @@ -534,56 +437,8 @@ jobs: popd done - - if: ${{ matrix.compiler != 'optimizing' && runner.os != 'Windows' }} - name: Run test binaries - run: | - echo "Running $(find ~/tests -name *.test | wc -l) test binaries" - - # Go tests often look for files relative to the source. Change to the corresponding directory. - for bin in ~/tests/*.test; do - dir=$(basename $bin); dir=${dir%.test}; dir=${dir//_/\/} - pushd $(go env GOROOT)/$dir - ~/wazerocli run -mount=/:/ -env PWD=$PWD $bin -- -test.short -test.v - popd - done - - - if: ${{ runner.os == 'Windows' }} - name: Run test binaries (Windows) - # Ack failures on Windows. https://github.com/tetratelabs/wazero/issues/1410 - continue-on-error: true + - name: Run built test binaries + if: ${{ matrix.compiler != 'optimizing' }} run: | - GOOS=$(go env GOOS) - echo "Running $(find ~/tests -name *.test | wc -l) test binaries" - - MOUNT=c:\\:/ - SCRIPT="$HOME/tests.cmd" - # Trim `/c` from the in-Wasm GOROOT. - REAL_GOROOT=$(go env GOROOT) - GOROOT=$(cygpath -u $REAL_GOROOT | cut -c 3-) - # Append early exit on cmd. - POSTFIX="if %errorlevel% neq 0 exit /b %errorlevel%" - RUNNER="cmd //c %USERPROFILE%\tests.cmd" - EXTRAPARAMS="-mount=%TEMP%:/tmp" - - # Go tests often look for files relative to the source. Change to the corresponding directory. - for bin in ~/tests/*.test; do - dir=$(basename $bin); dir=${dir%.test}; dir=${dir//_/\/} - pushd $REAL_GOROOT/$dir - - # Trim `/c` from the in-Wasm pwd. - IN_WASM_PWD=$(pwd | cut -c 3-) - # Convert to a Windows path. - bin=`cygpath -w $bin` - - # Create a script with all the tests (do not run yet). - echo ${MOUNT} ${IN_WASM_PWD} $GOROOT/$dir - COMMAND="~/wazerocli run -mount=${MOUNT} ${EXTRAPARAMS} -hostlogging=filesystem -env PWD=${IN_WASM_PWD} -env GOROOT=${GOROOT} -env GOOS=wasip1 $bin -- -test.short -test.v" - echo $COMMAND >> $SCRIPT - # Uncomment the following line for early exit on error on Windows. - # Otherwise the tests will report are successful evne on failure. - # echo $POSTFIX >> $SCRIPT - popd - done - - # Run all the tests in their own shell. - $RUNNER + cd ${{ env.STDLIB_TESTS }} + go test -bench='./wasip1' -benchtime=1x diff --git a/.gitignore b/.gitignore index f824fdde0b..1d635c3679 100644 --- a/.gitignore +++ b/.gitignore @@ -39,3 +39,6 @@ zig-cache/ zig-out/ .DS_Store + +# Ignore compiled stdlib test cases. +/internal/integration_test/stdlibs/testdata diff --git a/internal/integration_test/stdlibs/Makefile b/internal/integration_test/stdlibs/Makefile new file mode 100644 index 0000000000..5199cccfc0 --- /dev/null +++ b/internal/integration_test/stdlibs/Makefile @@ -0,0 +1,148 @@ +# Note: Some tests might fail if Go has been installed with homebrew because +# the file system layout is different than what the tests expect. +# Easiest fix is to install without using brew. +goroot?=$(shell go env GOROOT) +# Note: The standard binary zig distribution does not ship some testdata. +# You should override with the zig source code path, otherwise some tests will fail. +zigroot?=$(shell dirname $(shell which zig)) + +zig_bin=$(shell pwd)/testdata/zig +tinygo_bin=$(shell pwd)/testdata/tinygo +gowasip1_bin=$(shell pwd)/testdata/go + + +# embed/internal/embedtest deals with some paths that are not available here, ignore. +tinygo_tests=container/heap \ + container/list \ + container/ring \ + crypto/des \ + crypto/md5 \ + crypto/rc4 \ + crypto/sha1 \ + crypto/sha256 \ + crypto/sha512 \ + encoding \ + encoding/ascii85 \ + encoding/base32 \ + encoding/csv \ + encoding/hex \ + go/scanner \ + hash \ + hash/adler32 \ + hash/crc64 \ + hash/fnv \ + html \ + internal/itoa \ + internal/profile \ + math \ + math/cmplx \ + net \ + net/http/internal/ascii \ + net/mail \ + os \ + path \ + reflect \ + sync \ + testing \ + testing/iotest \ + text/scanner \ + unicode \ + unicode/utf16 \ + unicode/utf8 + +gowasip1_tests=src/archive/tar \ + src/bufio \ + src/bytes \ + src/context \ + src/encoding/ascii85 \ + src/encoding/asn1 \ + src/encoding/base32 \ + src/encoding/base64 \ + src/encoding/binary \ + src/encoding/csv \ + src/encoding/gob \ + src/encoding/hex \ + src/encoding/json \ + src/encoding/pem \ + src/encoding/xml \ + src/errors \ + src/expvar \ + src/flag \ + src/fmt \ + src/hash \ + src/hash/adler32 \ + src/hash/crc32 \ + src/hash/crc64 \ + src/hash/fnv \ + src/hash/maphash \ + src/io \ + src/io/fs \ + src/io/ioutil \ + src/log \ + src/log/syslog \ + src/maps \ + src/math \ + src/math/big \ + src/math/bits \ + src/math/cmplx \ + src/math/rand \ + src/mime \ + src/mime/multipart \ + src/mime/quotedprintable \ + src/os \ + src/os/exec \ + src/os/signal \ + src/os/user \ + src/path \ + src/reflect \ + src/regexp \ + src/regexp/syntax \ + src/runtime \ + src/runtime/internal/atomic \ + src/runtime/internal/math \ + src/runtime/internal/sys \ + src/slices \ + src/sort \ + src/strconv \ + src/strings \ + src/sync \ + src/sync/atomic \ + src/syscall \ + src/testing \ + src/testing/fstest \ + src/testing/iotest \ + src/testing/quick \ + src/time + +all: build.zig build.tinygo build.gowasip1 + +run: + go test -bench=. -benchtime=1x + + +.PHONY: build.zig +build.zig: + # --test-no-exec allows building of the test Wasm binary without executing command. + # We use find because the test.wasm will be something like ./zig-cache/o/dd6df1361b2134adc5eee9d027495436/test.wasm + @mkdir -p $(zig_bin) + @cd $(zigroot) && zig test --test-no-exec -target wasm32-wasi --zig-lib-dir $(zigroot)/lib $(zigroot)/lib/std/std.zig + @cp $$(find $(zigroot) -name test.wasm) $(zig_bin)/test.wasm + # The generated test binary is large and produces skewed results in favor of the optimized compiler. + # We also generate a stripped, optimized binary with wasm-opt. + @wasm-opt $(zig_bin)/test.wasm -O --strip-dwarf -o $(zig_bin)/test-opt.wasm + +.PHONY: build.tinygo +build.tinygo: + @mkdir -p $(tinygo_bin) + for value in $(tinygo_tests); do\ + echo Building $${value}... ;\ + tinygo test -target wasi -c -o $(tinygo_bin)/$$(echo $$value | sed -e 's/\//_/g').test $${value} 2>&1 >/dev/null ;\ + done + +.PHONY: build.gowasip1 +build.gowasip1: + @mkdir -p $(gowasip1_bin) + @cd '$(goroot)'; \ + for value in $(gowasip1_tests); do\ + GOOS=wasip1 GOARCH=wasm go test -v -c -o $(gowasip1_bin)/$$(echo $$value | sed -e 's/\//_/g').test ./$${value}; \ + done diff --git a/internal/integration_test/stdlibs/README.md b/internal/integration_test/stdlibs/README.md new file mode 100644 index 0000000000..68fd0cd3bf --- /dev/null +++ b/internal/integration_test/stdlibs/README.md @@ -0,0 +1,32 @@ +# Stdlibs benchmarks + +This directory contains a Makefile to build (a subset of) the stdlibs for Zig, TinyGo and Go (wasip1) +and test them against the baseline compiler and the optimizing compiler. + +## Requirements + +- Zig 0.11.0 in PATH, source code to zig 0.11.0 +- TinyGo in PATH +- Go in PATH + +## Usage + +First, build the test suite (the Zig source root has to be set explicitly): + + make all zigroot=/path/to/zig/source + +Then you can run the test suite against the baseline compiler and the optimizing compiler; e.g.: + + go test -bench=. --benchtime=1x + +If you want to run with the default settings, use `make run`. + +## Caveats + +* The standard binary zig distribution does not ship some testdata. + You should override with the zig source code path, otherwise some tests will fail. + +* Some tests might fail if Go has been installed with homebrew because + the file system layout is different than what the tests expect. + Easiest fix is to install Go without using brew. + diff --git a/internal/integration_test/stdlibs/bench_test.go b/internal/integration_test/stdlibs/bench_test.go new file mode 100644 index 0000000000..268eeb0f0c --- /dev/null +++ b/internal/integration_test/stdlibs/bench_test.go @@ -0,0 +1,189 @@ +package wazevo_test + +import ( + "context" + "crypto/rand" + "os" + "path/filepath" + "runtime" + "strings" + "testing" + + "github.com/tetratelabs/wazero" + "github.com/tetratelabs/wazero/experimental/opt" + "github.com/tetratelabs/wazero/imports/wasi_snapshot_preview1" + "github.com/tetratelabs/wazero/internal/testing/require" + "github.com/tetratelabs/wazero/sys" +) + +func BenchmarkStdlibs(b *testing.B) { + type testConfig struct { + name string + config wazero.RuntimeConfig + } + configs := []testConfig{ + {name: "baseline", config: wazero.NewRuntimeConfigCompiler()}, + } + if runtime.GOARCH == "arm64" { + configs = append(configs, testConfig{name: "optimizing", config: opt.NewRuntimeConfigOptimizingCompiler()}) + } + + cwd, _ := os.Getwd() + defer os.Chdir(cwd) //nolint + ctx := context.Background() + + testCases := []struct { + name, dir string + readTestCase func(fpath string, fname string) ([]byte, wazero.ModuleConfig, error) + }{ + { + name: "zig", + dir: "testdata/zig/", + readTestCase: func(fpath string, fname string) ([]byte, wazero.ModuleConfig, error) { + bin, err := os.ReadFile(fpath) + modCfg := defaultModuleConfig(). + WithFSConfig(wazero.NewFSConfig().WithDirMount(".", "/")). + WithArgs("test.wasm") + + return bin, modCfg, err + }, + }, + { + name: "tinygo", + dir: "testdata/tinygo/", + readTestCase: func(fpath string, fname string) ([]byte, wazero.ModuleConfig, error) { + if !strings.HasSuffix(fname, ".test") { + return nil, nil, nil + } + bin, err := os.ReadFile(fpath) + fsconfig := wazero.NewFSConfig(). + WithDirMount(".", "/"). + WithDirMount(os.TempDir(), "/tmp") + modCfg := defaultModuleConfig(). + WithFSConfig(fsconfig). + WithArgs(fname, "-test.v") + + return bin, modCfg, err + }, + }, + { + name: "wasip1", + dir: "testdata/go/", + readTestCase: func(fpath string, fname string) ([]byte, wazero.ModuleConfig, error) { + if !strings.HasSuffix(fname, ".test") { + return nil, nil, nil + } + bin, err := os.ReadFile(fpath) + if err != nil { + return nil, nil, err + } + fsuffixstripped := strings.ReplaceAll(fname, ".test", "") + inferredpath := strings.ReplaceAll(fsuffixstripped, "_", "/") + testdir := filepath.Join(runtime.GOROOT(), inferredpath) + err = os.Chdir(testdir) + + sysroot := filepath.VolumeName(testdir) + string(os.PathSeparator) + normalizedTestdir := normalizeOsPath(testdir) + + modCfg := defaultModuleConfig(). + WithFSConfig( + wazero.NewFSConfig(). + WithDirMount(sysroot, "/"). + WithDirMount(os.TempDir(), "/tmp")). + WithEnv("PWD", normalizedTestdir) + + args := []string{fname, "-test.short", "-test.v"} + + // Skip tests that are fragile on Windows. + if runtime.GOOS == "windows" { + modCfg = modCfg. + WithEnv("GOROOT", normalizeOsPath(runtime.GOROOT())) + + args = append(args, + "-test.skip=TestRenameCaseDifference/dir|"+ + "TestDirFSPathsValid|TestDirFS|TestDevNullFile|"+ + "TestOpenError|TestSymlinkWithTrailingSlash") + } + modCfg = modCfg.WithArgs(args...) + + return bin, modCfg, err + }, + }, + } + + for _, tc := range testCases { + b.Run(tc.name, func(b *testing.B) { + files, err := os.ReadDir(tc.dir) + require.NoError(b, err) + for _, f := range files { + fname := f.Name() + // Ensure we are on root dir. + err = os.Chdir(cwd) + require.NoError(b, err) + + fpath := filepath.Join(cwd, tc.dir, fname) + bin, modCfg, err := tc.readTestCase(fpath, fname) + require.NoError(b, err) + if bin == nil { + // skip + continue + } + + b.Run(fname, func(b *testing.B) { + for _, cfg := range configs { + r := wazero.NewRuntimeWithConfig(ctx, cfg.config) + wasi_snapshot_preview1.MustInstantiate(ctx, r) + b.Cleanup(func() { r.Close(ctx) }) + + m, err := r.CompileModule(ctx, bin) + require.NoError(b, err) + + b.Run(cfg.name, func(b *testing.B) { + b.Run("Compile", func(b *testing.B) { + _, err := r.CompileModule(ctx, bin) + require.NoError(b, err) + }) + im, err := r.InstantiateModule(ctx, m, modCfg) + require.NoError(b, err) + b.Run("Run", func(b *testing.B) { + _, err := im.ExportedFunction("_start").Call(ctx) + requireZeroExitCode(b, err) + }) + }) + } + }) + } + }) + } +} + +// Normalize an absolute path to a Unix-style path, regardless if it is a Windows path. +func normalizeOsPath(path string) string { + // Remove volume name. This is '/' on *Nix and 'C:' (with C being any letter identifier). + root := filepath.VolumeName(path) + testdirnoprefix := path[len(root):] + // Normalizes all the path separators to a Unix separator. + testdirnormalized := strings.ReplaceAll(testdirnoprefix, string(os.PathSeparator), "/") + return testdirnormalized +} + +func defaultModuleConfig() wazero.ModuleConfig { + return wazero.NewModuleConfig(). + WithSysNanosleep(). + WithSysNanotime(). + WithSysWalltime(). + WithRandSource(rand.Reader). + // Some tests require Stdout and Stderr to be present. + WithStdout(os.Stdout). + WithStderr(os.Stderr). + WithStartFunctions() +} + +func requireZeroExitCode(b *testing.B, err error) { + b.Helper() + if se, ok := err.(*sys.ExitError); ok { + if se.ExitCode() != 0 { // Don't err on success. + require.NoError(b, err) + } + } +}