From 8c71d4d0fe7de6b0df09d989af9d325dce4a1605 Mon Sep 17 00:00:00 2001 From: Takeshi Yoneda Date: Wed, 20 Dec 2023 15:32:15 -0800 Subject: [PATCH] Flattens std lib test cases (#1884) Signed-off-by: Takeshi Yoneda --- .github/workflows/integration.yaml | 7 +- .../integration_test/stdlibs/bench_test.go | 269 ++++++++++-------- 2 files changed, 147 insertions(+), 129 deletions(-) diff --git a/.github/workflows/integration.yaml b/.github/workflows/integration.yaml index 39e88186ad..70e6186079 100644 --- a/.github/workflows/integration.yaml +++ b/.github/workflows/integration.yaml @@ -151,8 +151,7 @@ jobs: if: ${{ matrix.compiler != 'optimizing' }} run: | cd ${{ env.STDLIB_TESTS }} - go test -bench='./zig' -benchtime=1x - + go test -bench='BenchmarkZig' -benchtime=1x build_tinygo_test_binary: name: Build TinyGo test binary @@ -269,7 +268,7 @@ jobs: if: ${{ matrix.compiler != 'optimizing' }} run: | cd ${{ env.STDLIB_TESTS }} - go test -bench='./tinygo' -benchtime=1x + go test -bench='BenchmarkTinyGo' -benchtime=1x wasi-testsuite: name: wasi-testsuite @@ -441,4 +440,4 @@ jobs: if: ${{ matrix.compiler != 'optimizing' }} run: | cd ${{ env.STDLIB_TESTS }} - go test -bench='./wasip1' -benchtime=1x + go test -bench='BenchmarkWasip1' -benchtime=1x diff --git a/internal/integration_test/stdlibs/bench_test.go b/internal/integration_test/stdlibs/bench_test.go index 268eeb0f0c..8533e3d7ca 100644 --- a/internal/integration_test/stdlibs/bench_test.go +++ b/internal/integration_test/stdlibs/bench_test.go @@ -16,143 +16,162 @@ import ( "github.com/tetratelabs/wazero/sys" ) -func BenchmarkStdlibs(b *testing.B) { - type testConfig struct { - name string - config wazero.RuntimeConfig +func BenchmarkZig(b *testing.B) { + if runtime.GOARCH == "arm64" { + b.Run("optimizing", func(b *testing.B) { + c := opt.NewRuntimeConfigOptimizingCompiler() + runtBench(b, context.Background(), c, zigTestCase) + }) } - configs := []testConfig{ - {name: "baseline", config: wazero.NewRuntimeConfigCompiler()}, + b.Run("baseline", func(b *testing.B) { + c := wazero.NewRuntimeConfigCompiler() + runtBench(b, context.Background(), c, zigTestCase) + }) +} + +func BenchmarkTinyGo(b *testing.B) { + if runtime.GOARCH == "arm64" { + b.Run("optimizing", func(b *testing.B) { + c := opt.NewRuntimeConfigOptimizingCompiler() + runtBench(b, context.Background(), c, tinyGoTestCase) + }) } + b.Run("baseline", func(b *testing.B) { + c := wazero.NewRuntimeConfigCompiler() + runtBench(b, context.Background(), c, tinyGoTestCase) + }) +} + +func BenchmarkWasip1(b *testing.B) { if runtime.GOARCH == "arm64" { - configs = append(configs, testConfig{name: "optimizing", config: opt.NewRuntimeConfigOptimizingCompiler()}) + b.Run("optimizing", func(b *testing.B) { + c := opt.NewRuntimeConfigOptimizingCompiler() + runtBench(b, context.Background(), c, wasip1TestCase) + }) } + b.Run("baseline", func(b *testing.B) { + c := wazero.NewRuntimeConfigCompiler() + runtBench(b, context.Background(), c, wasip1TestCase) + }) +} - cwd, _ := os.Getwd() - defer os.Chdir(cwd) //nolint - ctx := context.Background() - - testCases := []struct { - name, dir string - readTestCase func(fpath string, fname string) ([]byte, wazero.ModuleConfig, error) - }{ - { - name: "zig", - dir: "testdata/zig/", - readTestCase: func(fpath string, fname string) ([]byte, wazero.ModuleConfig, error) { - bin, err := os.ReadFile(fpath) - modCfg := defaultModuleConfig(). - WithFSConfig(wazero.NewFSConfig().WithDirMount(".", "/")). - WithArgs("test.wasm") - - return bin, modCfg, err - }, +type testCase struct { + name, dir string + readTestCase func(fpath string, fname string) ([]byte, wazero.ModuleConfig, error) +} + +var ( + zigTestCase = testCase{ + name: "zig", + dir: "testdata/zig/", + readTestCase: func(fpath string, fname string) ([]byte, wazero.ModuleConfig, error) { + bin, err := os.ReadFile(fpath) + modCfg := defaultModuleConfig(). + WithFSConfig(wazero.NewFSConfig().WithDirMount(".", "/")). + WithArgs("test.wasm") + + return bin, modCfg, err }, - { - name: "tinygo", - dir: "testdata/tinygo/", - readTestCase: func(fpath string, fname string) ([]byte, wazero.ModuleConfig, error) { - if !strings.HasSuffix(fname, ".test") { - return nil, nil, nil - } - bin, err := os.ReadFile(fpath) - fsconfig := wazero.NewFSConfig(). - WithDirMount(".", "/"). - WithDirMount(os.TempDir(), "/tmp") - modCfg := defaultModuleConfig(). - WithFSConfig(fsconfig). - WithArgs(fname, "-test.v") - - return bin, modCfg, err - }, + } + tinyGoTestCase = testCase{ + name: "tinygo", + dir: "testdata/tinygo/", + readTestCase: func(fpath string, fname string) ([]byte, wazero.ModuleConfig, error) { + if !strings.HasSuffix(fname, ".test") { + return nil, nil, nil + } + bin, err := os.ReadFile(fpath) + fsconfig := wazero.NewFSConfig(). + WithDirMount(".", "/"). + WithDirMount(os.TempDir(), "/tmp") + modCfg := defaultModuleConfig(). + WithFSConfig(fsconfig). + WithArgs(fname, "-test.v") + + return bin, modCfg, err }, - { - name: "wasip1", - dir: "testdata/go/", - readTestCase: func(fpath string, fname string) ([]byte, wazero.ModuleConfig, error) { - if !strings.HasSuffix(fname, ".test") { - return nil, nil, nil - } - bin, err := os.ReadFile(fpath) - if err != nil { - return nil, nil, err - } - fsuffixstripped := strings.ReplaceAll(fname, ".test", "") - inferredpath := strings.ReplaceAll(fsuffixstripped, "_", "/") - testdir := filepath.Join(runtime.GOROOT(), inferredpath) - err = os.Chdir(testdir) - - sysroot := filepath.VolumeName(testdir) + string(os.PathSeparator) - normalizedTestdir := normalizeOsPath(testdir) - - modCfg := defaultModuleConfig(). - WithFSConfig( - wazero.NewFSConfig(). - WithDirMount(sysroot, "/"). - WithDirMount(os.TempDir(), "/tmp")). - WithEnv("PWD", normalizedTestdir) - - args := []string{fname, "-test.short", "-test.v"} - - // Skip tests that are fragile on Windows. - if runtime.GOOS == "windows" { - modCfg = modCfg. - WithEnv("GOROOT", normalizeOsPath(runtime.GOROOT())) - - args = append(args, - "-test.skip=TestRenameCaseDifference/dir|"+ - "TestDirFSPathsValid|TestDirFS|TestDevNullFile|"+ - "TestOpenError|TestSymlinkWithTrailingSlash") - } - modCfg = modCfg.WithArgs(args...) - - return bin, modCfg, err - }, + } + wasip1TestCase = testCase{ + name: "wasip1", + dir: "testdata/go/", + readTestCase: func(fpath string, fname string) ([]byte, wazero.ModuleConfig, error) { + if !strings.HasSuffix(fname, ".test") { + return nil, nil, nil + } + bin, err := os.ReadFile(fpath) + if err != nil { + return nil, nil, err + } + fsuffixstripped := strings.ReplaceAll(fname, ".test", "") + inferredpath := strings.ReplaceAll(fsuffixstripped, "_", "/") + testdir := filepath.Join(runtime.GOROOT(), inferredpath) + err = os.Chdir(testdir) + + sysroot := filepath.VolumeName(testdir) + string(os.PathSeparator) + normalizedTestdir := normalizeOsPath(testdir) + + modCfg := defaultModuleConfig(). + WithFSConfig( + wazero.NewFSConfig(). + WithDirMount(sysroot, "/"). + WithDirMount(os.TempDir(), "/tmp")). + WithEnv("PWD", normalizedTestdir) + + args := []string{fname, "-test.short", "-test.v"} + + // Skip tests that are fragile on Windows. + if runtime.GOOS == "windows" { + modCfg = modCfg. + WithEnv("GOROOT", normalizeOsPath(runtime.GOROOT())) + + args = append(args, + "-test.skip=TestRenameCaseDifference/dir|"+ + "TestDirFSPathsValid|TestDirFS|TestDevNullFile|"+ + "TestOpenError|TestSymlinkWithTrailingSlash") + } + modCfg = modCfg.WithArgs(args...) + + return bin, modCfg, err }, } +) - for _, tc := range testCases { - b.Run(tc.name, func(b *testing.B) { - files, err := os.ReadDir(tc.dir) - require.NoError(b, err) - for _, f := range files { - fname := f.Name() - // Ensure we are on root dir. - err = os.Chdir(cwd) +func runtBench(b *testing.B, ctx context.Context, rc wazero.RuntimeConfig, tc testCase) { + cwd, _ := os.Getwd() + files, err := os.ReadDir(tc.dir) + require.NoError(b, err) + for _, f := range files { + fname := f.Name() + // Ensure we are on root dir. + err = os.Chdir(cwd) + require.NoError(b, err) + + fpath := filepath.Join(cwd, tc.dir, fname) + bin, modCfg, err := tc.readTestCase(fpath, fname) + require.NoError(b, err) + if bin == nil { + continue + } + b.Run(fname, func(b *testing.B) { + r := wazero.NewRuntimeWithConfig(ctx, rc) + wasi_snapshot_preview1.MustInstantiate(ctx, r) + b.Cleanup(func() { r.Close(ctx) }) + + var cm wazero.CompiledModule + b.Run("Compile", func(b *testing.B) { + b.ResetTimer() + var err error + cm, err = r.CompileModule(ctx, bin) require.NoError(b, err) + }) - fpath := filepath.Join(cwd, tc.dir, fname) - bin, modCfg, err := tc.readTestCase(fpath, fname) - require.NoError(b, err) - if bin == nil { - // skip - continue - } - - b.Run(fname, func(b *testing.B) { - for _, cfg := range configs { - r := wazero.NewRuntimeWithConfig(ctx, cfg.config) - wasi_snapshot_preview1.MustInstantiate(ctx, r) - b.Cleanup(func() { r.Close(ctx) }) - - m, err := r.CompileModule(ctx, bin) - require.NoError(b, err) - - b.Run(cfg.name, func(b *testing.B) { - b.Run("Compile", func(b *testing.B) { - _, err := r.CompileModule(ctx, bin) - require.NoError(b, err) - }) - im, err := r.InstantiateModule(ctx, m, modCfg) - require.NoError(b, err) - b.Run("Run", func(b *testing.B) { - _, err := im.ExportedFunction("_start").Call(ctx) - requireZeroExitCode(b, err) - }) - }) - } - }) - } + im, err := r.InstantiateModule(ctx, cm, modCfg) + require.NoError(b, err) + b.Run("Run", func(b *testing.B) { + b.ResetTimer() + _, err := im.ExportedFunction("_start").Call(ctx) + requireZeroExitCode(b, err) + }) }) } }