Flattens std lib test cases (#1884)

Signed-off-by: Takeshi Yoneda <t.y.mathetake@gmail.com>

.github/workflows/integration.yaml

@@ -151,8 +151,7 @@ jobs:
         if: ${{ matrix.compiler != 'optimizing' }}
         run: |
           cd ${{ env.STDLIB_TESTS }}
-          go test -bench='./zig' -benchtime=1x
+          go test -bench='BenchmarkZig' -benchtime=1x
-
 
   build_tinygo_test_binary:
     name: Build TinyGo test binary
@@ -269,7 +268,7 @@ jobs:
         if: ${{ matrix.compiler != 'optimizing' }}
         run: |
           cd ${{ env.STDLIB_TESTS }}
-          go test -bench='./tinygo' -benchtime=1x
+          go test -bench='BenchmarkTinyGo' -benchtime=1x
 
   wasi-testsuite:
     name: wasi-testsuite
@@ -441,4 +440,4 @@ jobs:
         if: ${{ matrix.compiler != 'optimizing' }}
         run: |
           cd ${{ env.STDLIB_TESTS }}
-          go test -bench='./wasip1' -benchtime=1x
+          go test -bench='BenchmarkWasip1' -benchtime=1x

internal/integration_test/stdlibs/bench_test.go

@@ -16,143 +16,162 @@ import (
 	"github.com/tetratelabs/wazero/sys"
 )
 
-func BenchmarkStdlibs(b *testing.B) {
+func BenchmarkZig(b *testing.B) {
-	type testConfig struct {
-		name   string
-		config wazero.RuntimeConfig
-	}
-	configs := []testConfig{
-		{name: "baseline", config: wazero.NewRuntimeConfigCompiler()},
-	}
 	if runtime.GOARCH == "arm64" {
-		configs = append(configs, testConfig{name: "optimizing", config: opt.NewRuntimeConfigOptimizingCompiler()})
+		b.Run("optimizing", func(b *testing.B) {
+			c := opt.NewRuntimeConfigOptimizingCompiler()
+			runtBench(b, context.Background(), c, zigTestCase)
+		})
 	}
+	b.Run("baseline", func(b *testing.B) {
+		c := wazero.NewRuntimeConfigCompiler()
+		runtBench(b, context.Background(), c, zigTestCase)
+	})
+}
 
-	cwd, _ := os.Getwd()
+func BenchmarkTinyGo(b *testing.B) {
-	defer os.Chdir(cwd) //nolint
+	if runtime.GOARCH == "arm64" {
-	ctx := context.Background()
+		b.Run("optimizing", func(b *testing.B) {
+			c := opt.NewRuntimeConfigOptimizingCompiler()
+			runtBench(b, context.Background(), c, tinyGoTestCase)
+		})
+	}
+	b.Run("baseline", func(b *testing.B) {
+		c := wazero.NewRuntimeConfigCompiler()
+		runtBench(b, context.Background(), c, tinyGoTestCase)
+	})
+}
 
-	testCases := []struct {
+func BenchmarkWasip1(b *testing.B) {
-		name, dir    string
+	if runtime.GOARCH == "arm64" {
-		readTestCase func(fpath string, fname string) ([]byte, wazero.ModuleConfig, error)
+		b.Run("optimizing", func(b *testing.B) {
-	}{
+			c := opt.NewRuntimeConfigOptimizingCompiler()
-		{
+			runtBench(b, context.Background(), c, wasip1TestCase)
-			name: "zig",
+		})
-			dir:  "testdata/zig/",
+	}
-			readTestCase: func(fpath string, fname string) ([]byte, wazero.ModuleConfig, error) {
+	b.Run("baseline", func(b *testing.B) {
-				bin, err := os.ReadFile(fpath)
+		c := wazero.NewRuntimeConfigCompiler()
-				modCfg := defaultModuleConfig().
+		runtBench(b, context.Background(), c, wasip1TestCase)
-					WithFSConfig(wazero.NewFSConfig().WithDirMount(".", "/")).
+	})
-					WithArgs("test.wasm")
+}
 
-				return bin, modCfg, err
+type testCase struct {
-			},
+	name, dir    string
-		},
+	readTestCase func(fpath string, fname string) ([]byte, wazero.ModuleConfig, error)
-		{
+}
-			name: "tinygo",
-			dir:  "testdata/tinygo/",
-			readTestCase: func(fpath string, fname string) ([]byte, wazero.ModuleConfig, error) {
-				if !strings.HasSuffix(fname, ".test") {
-					return nil, nil, nil
-				}
-				bin, err := os.ReadFile(fpath)
-				fsconfig := wazero.NewFSConfig().
-					WithDirMount(".", "/").
-					WithDirMount(os.TempDir(), "/tmp")
-				modCfg := defaultModuleConfig().
-					WithFSConfig(fsconfig).
-					WithArgs(fname, "-test.v")
 
-				return bin, modCfg, err
+var (
-			},
+	zigTestCase = testCase{
-		},
+		name: "zig",
-		{
+		dir:  "testdata/zig/",
-			name: "wasip1",
+		readTestCase: func(fpath string, fname string) ([]byte, wazero.ModuleConfig, error) {
-			dir:  "testdata/go/",
+			bin, err := os.ReadFile(fpath)
-			readTestCase: func(fpath string, fname string) ([]byte, wazero.ModuleConfig, error) {
+			modCfg := defaultModuleConfig().
-				if !strings.HasSuffix(fname, ".test") {
+				WithFSConfig(wazero.NewFSConfig().WithDirMount(".", "/")).
-					return nil, nil, nil
+				WithArgs("test.wasm")
-				}
-				bin, err := os.ReadFile(fpath)
-				if err != nil {
-					return nil, nil, err
-				}
-				fsuffixstripped := strings.ReplaceAll(fname, ".test", "")
-				inferredpath := strings.ReplaceAll(fsuffixstripped, "_", "/")
-				testdir := filepath.Join(runtime.GOROOT(), inferredpath)
-				err = os.Chdir(testdir)
 
-				sysroot := filepath.VolumeName(testdir) + string(os.PathSeparator)
+			return bin, modCfg, err
-				normalizedTestdir := normalizeOsPath(testdir)
-
-				modCfg := defaultModuleConfig().
-					WithFSConfig(
-						wazero.NewFSConfig().
-							WithDirMount(sysroot, "/").
-							WithDirMount(os.TempDir(), "/tmp")).
-					WithEnv("PWD", normalizedTestdir)
-
-				args := []string{fname, "-test.short", "-test.v"}
-
-				// Skip tests that are fragile on Windows.
-				if runtime.GOOS == "windows" {
-					modCfg = modCfg.
-						WithEnv("GOROOT", normalizeOsPath(runtime.GOROOT()))
-
-					args = append(args,
-						"-test.skip=TestRenameCaseDifference/dir|"+
-							"TestDirFSPathsValid|TestDirFS|TestDevNullFile|"+
-							"TestOpenError|TestSymlinkWithTrailingSlash")
-				}
-				modCfg = modCfg.WithArgs(args...)
-
-				return bin, modCfg, err
-			},
 		},
 	}
-
+	tinyGoTestCase = testCase{
-	for _, tc := range testCases {
+		name: "tinygo",
-		b.Run(tc.name, func(b *testing.B) {
+		dir:  "testdata/tinygo/",
-			files, err := os.ReadDir(tc.dir)
+		readTestCase: func(fpath string, fname string) ([]byte, wazero.ModuleConfig, error) {
-			require.NoError(b, err)
+			if !strings.HasSuffix(fname, ".test") {
-			for _, f := range files {
+				return nil, nil, nil
-				fname := f.Name()
-				// Ensure we are on root dir.
-				err = os.Chdir(cwd)
-				require.NoError(b, err)
-
-				fpath := filepath.Join(cwd, tc.dir, fname)
-				bin, modCfg, err := tc.readTestCase(fpath, fname)
-				require.NoError(b, err)
-				if bin == nil {
-					// skip
-					continue
-				}
-
-				b.Run(fname, func(b *testing.B) {
-					for _, cfg := range configs {
-						r := wazero.NewRuntimeWithConfig(ctx, cfg.config)
-						wasi_snapshot_preview1.MustInstantiate(ctx, r)
-						b.Cleanup(func() { r.Close(ctx) })
-
-						m, err := r.CompileModule(ctx, bin)
-						require.NoError(b, err)
-
-						b.Run(cfg.name, func(b *testing.B) {
-							b.Run("Compile", func(b *testing.B) {
-								_, err := r.CompileModule(ctx, bin)
-								require.NoError(b, err)
-							})
-							im, err := r.InstantiateModule(ctx, m, modCfg)
-							require.NoError(b, err)
-							b.Run("Run", func(b *testing.B) {
-								_, err := im.ExportedFunction("_start").Call(ctx)
-								requireZeroExitCode(b, err)
-							})
-						})
-					}
-				})
 			}
+			bin, err := os.ReadFile(fpath)
+			fsconfig := wazero.NewFSConfig().
+				WithDirMount(".", "/").
+				WithDirMount(os.TempDir(), "/tmp")
+			modCfg := defaultModuleConfig().
+				WithFSConfig(fsconfig).
+				WithArgs(fname, "-test.v")
+
+			return bin, modCfg, err
+		},
+	}
+	wasip1TestCase = testCase{
+		name: "wasip1",
+		dir:  "testdata/go/",
+		readTestCase: func(fpath string, fname string) ([]byte, wazero.ModuleConfig, error) {
+			if !strings.HasSuffix(fname, ".test") {
+				return nil, nil, nil
+			}
+			bin, err := os.ReadFile(fpath)
+			if err != nil {
+				return nil, nil, err
+			}
+			fsuffixstripped := strings.ReplaceAll(fname, ".test", "")
+			inferredpath := strings.ReplaceAll(fsuffixstripped, "_", "/")
+			testdir := filepath.Join(runtime.GOROOT(), inferredpath)
+			err = os.Chdir(testdir)
+
+			sysroot := filepath.VolumeName(testdir) + string(os.PathSeparator)
+			normalizedTestdir := normalizeOsPath(testdir)
+
+			modCfg := defaultModuleConfig().
+				WithFSConfig(
+					wazero.NewFSConfig().
+						WithDirMount(sysroot, "/").
+						WithDirMount(os.TempDir(), "/tmp")).
+				WithEnv("PWD", normalizedTestdir)
+
+			args := []string{fname, "-test.short", "-test.v"}
+
+			// Skip tests that are fragile on Windows.
+			if runtime.GOOS == "windows" {
+				modCfg = modCfg.
+					WithEnv("GOROOT", normalizeOsPath(runtime.GOROOT()))
+
+				args = append(args,
+					"-test.skip=TestRenameCaseDifference/dir|"+
+						"TestDirFSPathsValid|TestDirFS|TestDevNullFile|"+
+						"TestOpenError|TestSymlinkWithTrailingSlash")
+			}
+			modCfg = modCfg.WithArgs(args...)
+
+			return bin, modCfg, err
+		},
+	}
+)
+
+func runtBench(b *testing.B, ctx context.Context, rc wazero.RuntimeConfig, tc testCase) {
+	cwd, _ := os.Getwd()
+	files, err := os.ReadDir(tc.dir)
+	require.NoError(b, err)
+	for _, f := range files {
+		fname := f.Name()
+		// Ensure we are on root dir.
+		err = os.Chdir(cwd)
+		require.NoError(b, err)
+
+		fpath := filepath.Join(cwd, tc.dir, fname)
+		bin, modCfg, err := tc.readTestCase(fpath, fname)
+		require.NoError(b, err)
+		if bin == nil {
+			continue
+		}
+		b.Run(fname, func(b *testing.B) {
+			r := wazero.NewRuntimeWithConfig(ctx, rc)
+			wasi_snapshot_preview1.MustInstantiate(ctx, r)
+			b.Cleanup(func() { r.Close(ctx) })
+
+			var cm wazero.CompiledModule
+			b.Run("Compile", func(b *testing.B) {
+				b.ResetTimer()
+				var err error
+				cm, err = r.CompileModule(ctx, bin)
+				require.NoError(b, err)
+			})
+
+			im, err := r.InstantiateModule(ctx, cm, modCfg)
+			require.NoError(b, err)
+			b.Run("Run", func(b *testing.B) {
+				b.ResetTimer()
+				_, err := im.ExportedFunction("_start").Call(ctx)
+				requireZeroExitCode(b, err)
+			})
 		})
 	}
 }