Flattens std lib test cases (#1884)
Signed-off-by: Takeshi Yoneda <t.y.mathetake@gmail.com>
This commit is contained in:
Takeshi Yoneda
GitHub
7
.github/workflows/integration.yaml
vendored
7
.github/workflows/integration.yaml
vendored
@@ -151,8 +151,7 @@ jobs:
|
||||
if: ${{ matrix.compiler != 'optimizing' }}
|
||||
run: |
|
||||
cd ${{ env.STDLIB_TESTS }}
|
||||
go test -bench='./zig' -benchtime=1x
|
||||
|
||||
go test -bench='BenchmarkZig' -benchtime=1x
|
||||
|
||||
build_tinygo_test_binary:
|
||||
name: Build TinyGo test binary
|
||||
@@ -269,7 +268,7 @@ jobs:
|
||||
if: ${{ matrix.compiler != 'optimizing' }}
|
||||
run: |
|
||||
cd ${{ env.STDLIB_TESTS }}
|
||||
go test -bench='./tinygo' -benchtime=1x
|
||||
go test -bench='BenchmarkTinyGo' -benchtime=1x
|
||||
|
||||
wasi-testsuite:
|
||||
name: wasi-testsuite
|
||||
@@ -441,4 +440,4 @@ jobs:
|
||||
if: ${{ matrix.compiler != 'optimizing' }}
|
||||
run: |
|
||||
cd ${{ env.STDLIB_TESTS }}
|
||||
go test -bench='./wasip1' -benchtime=1x
|
||||
go test -bench='BenchmarkWasip1' -benchtime=1x
|
||||
|
||||
@@ -16,143 +16,162 @@ import (
|
||||
"github.com/tetratelabs/wazero/sys"
|
||||
)
|
||||
|
||||
func BenchmarkStdlibs(b *testing.B) {
|
||||
type testConfig struct {
|
||||
name string
|
||||
config wazero.RuntimeConfig
|
||||
}
|
||||
configs := []testConfig{
|
||||
{name: "baseline", config: wazero.NewRuntimeConfigCompiler()},
|
||||
}
|
||||
func BenchmarkZig(b *testing.B) {
|
||||
if runtime.GOARCH == "arm64" {
|
||||
configs = append(configs, testConfig{name: "optimizing", config: opt.NewRuntimeConfigOptimizingCompiler()})
|
||||
b.Run("optimizing", func(b *testing.B) {
|
||||
c := opt.NewRuntimeConfigOptimizingCompiler()
|
||||
runtBench(b, context.Background(), c, zigTestCase)
|
||||
})
|
||||
}
|
||||
b.Run("baseline", func(b *testing.B) {
|
||||
c := wazero.NewRuntimeConfigCompiler()
|
||||
runtBench(b, context.Background(), c, zigTestCase)
|
||||
})
|
||||
}
|
||||
|
||||
cwd, _ := os.Getwd()
|
||||
defer os.Chdir(cwd) //nolint
|
||||
ctx := context.Background()
|
||||
func BenchmarkTinyGo(b *testing.B) {
|
||||
if runtime.GOARCH == "arm64" {
|
||||
b.Run("optimizing", func(b *testing.B) {
|
||||
c := opt.NewRuntimeConfigOptimizingCompiler()
|
||||
runtBench(b, context.Background(), c, tinyGoTestCase)
|
||||
})
|
||||
}
|
||||
b.Run("baseline", func(b *testing.B) {
|
||||
c := wazero.NewRuntimeConfigCompiler()
|
||||
runtBench(b, context.Background(), c, tinyGoTestCase)
|
||||
})
|
||||
}
|
||||
|
||||
testCases := []struct {
|
||||
name, dir string
|
||||
readTestCase func(fpath string, fname string) ([]byte, wazero.ModuleConfig, error)
|
||||
}{
|
||||
{
|
||||
name: "zig",
|
||||
dir: "testdata/zig/",
|
||||
readTestCase: func(fpath string, fname string) ([]byte, wazero.ModuleConfig, error) {
|
||||
bin, err := os.ReadFile(fpath)
|
||||
modCfg := defaultModuleConfig().
|
||||
WithFSConfig(wazero.NewFSConfig().WithDirMount(".", "/")).
|
||||
WithArgs("test.wasm")
|
||||
func BenchmarkWasip1(b *testing.B) {
|
||||
if runtime.GOARCH == "arm64" {
|
||||
b.Run("optimizing", func(b *testing.B) {
|
||||
c := opt.NewRuntimeConfigOptimizingCompiler()
|
||||
runtBench(b, context.Background(), c, wasip1TestCase)
|
||||
})
|
||||
}
|
||||
b.Run("baseline", func(b *testing.B) {
|
||||
c := wazero.NewRuntimeConfigCompiler()
|
||||
runtBench(b, context.Background(), c, wasip1TestCase)
|
||||
})
|
||||
}
|
||||
|
||||
return bin, modCfg, err
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "tinygo",
|
||||
dir: "testdata/tinygo/",
|
||||
readTestCase: func(fpath string, fname string) ([]byte, wazero.ModuleConfig, error) {
|
||||
if !strings.HasSuffix(fname, ".test") {
|
||||
return nil, nil, nil
|
||||
}
|
||||
bin, err := os.ReadFile(fpath)
|
||||
fsconfig := wazero.NewFSConfig().
|
||||
WithDirMount(".", "/").
|
||||
WithDirMount(os.TempDir(), "/tmp")
|
||||
modCfg := defaultModuleConfig().
|
||||
WithFSConfig(fsconfig).
|
||||
WithArgs(fname, "-test.v")
|
||||
type testCase struct {
|
||||
name, dir string
|
||||
readTestCase func(fpath string, fname string) ([]byte, wazero.ModuleConfig, error)
|
||||
}
|
||||
|
||||
return bin, modCfg, err
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "wasip1",
|
||||
dir: "testdata/go/",
|
||||
readTestCase: func(fpath string, fname string) ([]byte, wazero.ModuleConfig, error) {
|
||||
if !strings.HasSuffix(fname, ".test") {
|
||||
return nil, nil, nil
|
||||
}
|
||||
bin, err := os.ReadFile(fpath)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
fsuffixstripped := strings.ReplaceAll(fname, ".test", "")
|
||||
inferredpath := strings.ReplaceAll(fsuffixstripped, "_", "/")
|
||||
testdir := filepath.Join(runtime.GOROOT(), inferredpath)
|
||||
err = os.Chdir(testdir)
|
||||
var (
|
||||
zigTestCase = testCase{
|
||||
name: "zig",
|
||||
dir: "testdata/zig/",
|
||||
readTestCase: func(fpath string, fname string) ([]byte, wazero.ModuleConfig, error) {
|
||||
bin, err := os.ReadFile(fpath)
|
||||
modCfg := defaultModuleConfig().
|
||||
WithFSConfig(wazero.NewFSConfig().WithDirMount(".", "/")).
|
||||
WithArgs("test.wasm")
|
||||
|
||||
sysroot := filepath.VolumeName(testdir) + string(os.PathSeparator)
|
||||
normalizedTestdir := normalizeOsPath(testdir)
|
||||
|
||||
modCfg := defaultModuleConfig().
|
||||
WithFSConfig(
|
||||
wazero.NewFSConfig().
|
||||
WithDirMount(sysroot, "/").
|
||||
WithDirMount(os.TempDir(), "/tmp")).
|
||||
WithEnv("PWD", normalizedTestdir)
|
||||
|
||||
args := []string{fname, "-test.short", "-test.v"}
|
||||
|
||||
// Skip tests that are fragile on Windows.
|
||||
if runtime.GOOS == "windows" {
|
||||
modCfg = modCfg.
|
||||
WithEnv("GOROOT", normalizeOsPath(runtime.GOROOT()))
|
||||
|
||||
args = append(args,
|
||||
"-test.skip=TestRenameCaseDifference/dir|"+
|
||||
"TestDirFSPathsValid|TestDirFS|TestDevNullFile|"+
|
||||
"TestOpenError|TestSymlinkWithTrailingSlash")
|
||||
}
|
||||
modCfg = modCfg.WithArgs(args...)
|
||||
|
||||
return bin, modCfg, err
|
||||
},
|
||||
return bin, modCfg, err
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range testCases {
|
||||
b.Run(tc.name, func(b *testing.B) {
|
||||
files, err := os.ReadDir(tc.dir)
|
||||
require.NoError(b, err)
|
||||
for _, f := range files {
|
||||
fname := f.Name()
|
||||
// Ensure we are on root dir.
|
||||
err = os.Chdir(cwd)
|
||||
require.NoError(b, err)
|
||||
|
||||
fpath := filepath.Join(cwd, tc.dir, fname)
|
||||
bin, modCfg, err := tc.readTestCase(fpath, fname)
|
||||
require.NoError(b, err)
|
||||
if bin == nil {
|
||||
// skip
|
||||
continue
|
||||
}
|
||||
|
||||
b.Run(fname, func(b *testing.B) {
|
||||
for _, cfg := range configs {
|
||||
r := wazero.NewRuntimeWithConfig(ctx, cfg.config)
|
||||
wasi_snapshot_preview1.MustInstantiate(ctx, r)
|
||||
b.Cleanup(func() { r.Close(ctx) })
|
||||
|
||||
m, err := r.CompileModule(ctx, bin)
|
||||
require.NoError(b, err)
|
||||
|
||||
b.Run(cfg.name, func(b *testing.B) {
|
||||
b.Run("Compile", func(b *testing.B) {
|
||||
_, err := r.CompileModule(ctx, bin)
|
||||
require.NoError(b, err)
|
||||
})
|
||||
im, err := r.InstantiateModule(ctx, m, modCfg)
|
||||
require.NoError(b, err)
|
||||
b.Run("Run", func(b *testing.B) {
|
||||
_, err := im.ExportedFunction("_start").Call(ctx)
|
||||
requireZeroExitCode(b, err)
|
||||
})
|
||||
})
|
||||
}
|
||||
})
|
||||
tinyGoTestCase = testCase{
|
||||
name: "tinygo",
|
||||
dir: "testdata/tinygo/",
|
||||
readTestCase: func(fpath string, fname string) ([]byte, wazero.ModuleConfig, error) {
|
||||
if !strings.HasSuffix(fname, ".test") {
|
||||
return nil, nil, nil
|
||||
}
|
||||
bin, err := os.ReadFile(fpath)
|
||||
fsconfig := wazero.NewFSConfig().
|
||||
WithDirMount(".", "/").
|
||||
WithDirMount(os.TempDir(), "/tmp")
|
||||
modCfg := defaultModuleConfig().
|
||||
WithFSConfig(fsconfig).
|
||||
WithArgs(fname, "-test.v")
|
||||
|
||||
return bin, modCfg, err
|
||||
},
|
||||
}
|
||||
wasip1TestCase = testCase{
|
||||
name: "wasip1",
|
||||
dir: "testdata/go/",
|
||||
readTestCase: func(fpath string, fname string) ([]byte, wazero.ModuleConfig, error) {
|
||||
if !strings.HasSuffix(fname, ".test") {
|
||||
return nil, nil, nil
|
||||
}
|
||||
bin, err := os.ReadFile(fpath)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
fsuffixstripped := strings.ReplaceAll(fname, ".test", "")
|
||||
inferredpath := strings.ReplaceAll(fsuffixstripped, "_", "/")
|
||||
testdir := filepath.Join(runtime.GOROOT(), inferredpath)
|
||||
err = os.Chdir(testdir)
|
||||
|
||||
sysroot := filepath.VolumeName(testdir) + string(os.PathSeparator)
|
||||
normalizedTestdir := normalizeOsPath(testdir)
|
||||
|
||||
modCfg := defaultModuleConfig().
|
||||
WithFSConfig(
|
||||
wazero.NewFSConfig().
|
||||
WithDirMount(sysroot, "/").
|
||||
WithDirMount(os.TempDir(), "/tmp")).
|
||||
WithEnv("PWD", normalizedTestdir)
|
||||
|
||||
args := []string{fname, "-test.short", "-test.v"}
|
||||
|
||||
// Skip tests that are fragile on Windows.
|
||||
if runtime.GOOS == "windows" {
|
||||
modCfg = modCfg.
|
||||
WithEnv("GOROOT", normalizeOsPath(runtime.GOROOT()))
|
||||
|
||||
args = append(args,
|
||||
"-test.skip=TestRenameCaseDifference/dir|"+
|
||||
"TestDirFSPathsValid|TestDirFS|TestDevNullFile|"+
|
||||
"TestOpenError|TestSymlinkWithTrailingSlash")
|
||||
}
|
||||
modCfg = modCfg.WithArgs(args...)
|
||||
|
||||
return bin, modCfg, err
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
func runtBench(b *testing.B, ctx context.Context, rc wazero.RuntimeConfig, tc testCase) {
|
||||
cwd, _ := os.Getwd()
|
||||
files, err := os.ReadDir(tc.dir)
|
||||
require.NoError(b, err)
|
||||
for _, f := range files {
|
||||
fname := f.Name()
|
||||
// Ensure we are on root dir.
|
||||
err = os.Chdir(cwd)
|
||||
require.NoError(b, err)
|
||||
|
||||
fpath := filepath.Join(cwd, tc.dir, fname)
|
||||
bin, modCfg, err := tc.readTestCase(fpath, fname)
|
||||
require.NoError(b, err)
|
||||
if bin == nil {
|
||||
continue
|
||||
}
|
||||
b.Run(fname, func(b *testing.B) {
|
||||
r := wazero.NewRuntimeWithConfig(ctx, rc)
|
||||
wasi_snapshot_preview1.MustInstantiate(ctx, r)
|
||||
b.Cleanup(func() { r.Close(ctx) })
|
||||
|
||||
var cm wazero.CompiledModule
|
||||
b.Run("Compile", func(b *testing.B) {
|
||||
b.ResetTimer()
|
||||
var err error
|
||||
cm, err = r.CompileModule(ctx, bin)
|
||||
require.NoError(b, err)
|
||||
})
|
||||
|
||||
im, err := r.InstantiateModule(ctx, cm, modCfg)
|
||||
require.NoError(b, err)
|
||||
b.Run("Run", func(b *testing.B) {
|
||||
b.ResetTimer()
|
||||
_, err := im.ExportedFunction("_start").Call(ctx)
|
||||
requireZeroExitCode(b, err)
|
||||
})
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user