Adds benchmarks from lib sodium (#948)

This adds benchmarks from lib sodium, but the results are hard to interpret. We may need to recompile them using zig so that we can export a function instead of using WASI to do it. Right now, we can't really see performance of functions because other runtimes aren't designed to re-instantiate like ours.

This only compares against wasmtime as that's the only runtime besides
ours that is safe to re-instantiate.

Signed-off-by: Adrian Cole <adrian@tetrate.io>

internal/integration_test/vs/bench.go

@@ -83,7 +83,7 @@ func benchmark(b *testing.B, runtime func() Runtime, rtCfg *RuntimeConfig, call
 	})
 
 	// Don't burn CPU when this is already going to be called in runTestBenchmark_Call_CompilerFastest
-	if ensureCompilerFastest != "true" || rt.Name() == compilerRuntime {
+	if call != nil && (ensureCompilerFastest != "true" || rt.Name() == compilerRuntime) {
 		b.Run("Call", func(b *testing.B) {
 			b.ReportAllocs()
 			benchmarkCall(b, rt, rtCfg, call)
@@ -161,6 +161,20 @@ func testCall(t *testing.T, runtime func() Runtime, rtCfg *RuntimeConfig, testCa
 	}
 }
 
+func testInstantiate(t *testing.T, runtime func() Runtime, rtCfg *RuntimeConfig) {
+	rt := runtime()
+	err := rt.Compile(testCtx, rtCfg)
+	require.NoError(t, err)
+	defer rt.Close(testCtx)
+
+	// Ensure the module can be re-instantiated times, even if not all runtimes allow renaming.
+	for i := 0; i < 10; i++ {
+		m, err := rt.Instantiate(testCtx, rtCfg)
+		require.NoError(t, err)
+		require.NoError(t, m.Close(testCtx))
+	}
+}
+
 func readRelativeFile(relativePath string) []byte {
 	// We can't resolve relative paths as init() is called from each of its subdirs
 	_, source, _, _ := runtime.Caller(1) // 1 as this utility is in a different source than the caller.

internal/integration_test/vs/bench_shorthash.go

@@ -0,0 +1,33 @@
+package vs
+
+import (
+	_ "embed"
+	"testing"
+)
+
+var (
+	// shorthashWasm is a wasi binary which runs the same wasm function inside
+	// a loop. See https://github.com/tetratelabs/wazero/issues/947
+	//
+	// Taken from https://github.com/jedisct1/webassembly-benchmarks/tree/master/2022-12/wasm
+	//go:embed testdata/shorthash.wasm
+	shorthashWasm   []byte
+	shorthashConfig *RuntimeConfig
+)
+
+func init() {
+	shorthashConfig = &RuntimeConfig{
+		ModuleName: "shorthash",
+		ModuleWasm: shorthashWasm,
+		NeedsWASI:  true, // runs as a _start function
+	}
+}
+
+func RunTestShorthash(t *testing.T, runtime func() Runtime) {
+	// not testCall as there are no exported functions except _start
+	testInstantiate(t, runtime, shorthashConfig)
+}
+
+func RunBenchmarkShorthash(b *testing.B, runtime func() Runtime) {
+	benchmark(b, runtime, shorthashConfig, nil)
+}

internal/integration_test/vs/compiler/compiler_test.go

@@ -39,3 +39,11 @@ func TestMemory(t *testing.T) {
 func BenchmarkMemory(b *testing.B) {
 	vs.RunBenchmarkMemory(b, runtime)
 }
+
+func TestShorthash(t *testing.T) {
+	vs.RunTestShorthash(t, runtime)
+}
+
+func BenchmarkShorthash(b *testing.B) {
+	vs.RunBenchmarkShorthash(b, runtime)
+}

internal/integration_test/vs/wasmtime/wasmtime.go

@@ -17,6 +17,7 @@ func newWasmtimeRuntime() vs.Runtime {
 
 type wasmtimeRuntime struct {
 	engine *wasmtime.Engine
+	module *wasmtime.Module
 }
 
 type wasmtimeModule struct {
@@ -42,21 +43,20 @@ func (m *wasmtimeModule) log(_ *wasmtime.Caller, args []wasmtime.Val) ([]wasmtim
 	return []wasmtime.Val{}, nil
 }
 
-func (r *wasmtimeRuntime) Compile(_ context.Context, _ *vs.RuntimeConfig) (err error) {
+func (r *wasmtimeRuntime) Compile(_ context.Context, cfg *vs.RuntimeConfig) (err error) {
 	r.engine = wasmtime.NewEngine()
-	// We can't reuse a store because even if we call close, re-instantiating too many times leads to:
-	// >> resource limit exceeded: instance count too high at 10001
+	if r.module, err = wasmtime.NewModule(r.engine, cfg.ModuleWasm); err != nil {
+		return
+	}
 	return
 }
 
 func (r *wasmtimeRuntime) Instantiate(_ context.Context, cfg *vs.RuntimeConfig) (mod vs.Module, err error) {
 	wm := &wasmtimeModule{funcs: map[string]*wasmtime.Func{}}
-	wm.store = wasmtime.NewStore(r.engine)
-	var m *wasmtime.Module
-	if m, err = wasmtime.NewModule(wm.store.Engine, cfg.ModuleWasm); err != nil {
-		return
-	}
 
+	// We can't reuse a store because even if we call close, re-instantiating too many times leads to:
+	// >> resource limit exceeded: instance count too high at 10001
+	wm.store = wasmtime.NewStore(r.engine)
 	linker := wasmtime.NewLinker(wm.store.Engine)
 
 	// Instantiate WASI, if configured.
@@ -64,7 +64,9 @@ func (r *wasmtimeRuntime) Instantiate(_ context.Context, cfg *vs.RuntimeConfig)
 		if err = linker.DefineWasi(); err != nil {
 			return
 		}
-		wm.store.SetWasi(wasmtime.NewWasiConfig())
+		config := wasmtime.NewWasiConfig() // defaults to toss stdout
+		config.InheritStderr()             // see errors
+		wm.store.SetWasi(config)
 	}
 
 	// Instantiate the host module, "env", if configured.
@@ -102,13 +104,14 @@ func (r *wasmtimeRuntime) Instantiate(_ context.Context, cfg *vs.RuntimeConfig)
 	}
 
 	// Set the module name.
-	if err = linker.DefineModule(wm.store, cfg.ModuleName, m); err != nil {
+	if err = linker.DefineModule(wm.store, cfg.ModuleName, r.module); err != nil {
 		return
 	}
 
 	// Instantiate the module.
-	instance, err := linker.Instantiate(wm.store, m)
-	if err != nil {
+	instance, instantiateErr := linker.Instantiate(wm.store, r.module)
+	if instantiateErr != nil {
+		err = instantiateErr
 		return
 	}
 
@@ -143,6 +146,7 @@ func (r *wasmtimeRuntime) Instantiate(_ context.Context, cfg *vs.RuntimeConfig)
 }
 
 func (r *wasmtimeRuntime) Close(context.Context) error {
+	r.module = nil
 	r.engine = nil
 	return nil // wasmtime only closes via finalizer
 }

internal/integration_test/vs/wasmtime/wasmtime_test.go

@@ -57,3 +57,11 @@ func BenchmarkMemory(b *testing.B) {
 func TestBenchmarkMemory_CompilerFastest(t *testing.T) {
 	vs.RunTestBenchmarkMemory_CompilerFastest(t, runtime())
 }
+
+func TestShorthash(t *testing.T) {
+	vs.RunTestShorthash(t, runtime)
+}
+
+func BenchmarkShorthash(b *testing.B) {
+	vs.RunBenchmarkShorthash(b, runtime)
+}