vs: adds the benchmark to see the cost of host function calls (#756)

This adds the new vs target to measure the cost of host function calls.
Notably, I can see that wazero is roughly 2x to 4x times faster than CGO-based
runtimes in terms of host call boundary crossing. One implication here is that
we can just focus on the native code generation rather than how to organize the
Go function calls. For example, it's not prioritized to call Go functions directly
from the native code.

Signed-off-by: Takeshi Yoneda <takeshi@tetrate.io>

internal/integration_test/vs/bench_hostcall.go

@@ -0,0 +1,48 @@
+package vs
+
+import (
+	_ "embed"
+	"testing"
+
+	"github.com/tetratelabs/wazero/internal/testing/require"
+)
+
+var (
+	//go:embed testdata/hostcall.wasm
+	hostCallWasm     []byte
+	hostCallConfig   *RuntimeConfig
+	hostCallFunction = "call_host_func"
+	hostCallParam    = uint64(12345)
+)
+
+func init() {
+	hostCallConfig = &RuntimeConfig{
+		ModuleName:      "hostcall",
+		ModuleWasm:      hostCallWasm,
+		FuncNames:       []string{hostCallFunction},
+		EnvFReturnValue: 0xffff,
+	}
+}
+
+func RunTestHostCall(t *testing.T, runtime func() Runtime) {
+	testCall(t, runtime, hostCallConfig, testHostCall)
+}
+
+func testHostCall(t *testing.T, m Module, instantiation, iteration int) {
+	res, err := m.CallI64_I64(testCtx, hostCallFunction, hostCallParam)
+	require.NoError(t, err, "instantiation[%d] iteration[%d] failed", instantiation, iteration)
+	require.Equal(t, hostCallConfig.EnvFReturnValue, res)
+}
+
+func RunTestBenchmarkHostCall_CompilerFastest(t *testing.T, vsRuntime Runtime) {
+	runTestBenchmark_Call_CompilerFastest(t, hostCallConfig, "HostCall_CrossBoundary", hostCall, vsRuntime)
+}
+
+func RunBenchmarkHostCall(b *testing.B, runtime func() Runtime) {
+	benchmark(b, runtime, hostCallConfig, hostCall)
+}
+
+func hostCall(m Module) error {
+	_, err := m.CallI64_I64(testCtx, hostCallFunction, hostCallParam)
+	return err
+}

internal/integration_test/vs/interpreter/interpreter_test.go

@@ -31,3 +31,15 @@ func BenchmarkFactorial(b *testing.B) {
 func TestBenchmarkFactorial_Call_CompilerFastest(t *testing.T) {
 	vs.RunTestBenchmarkFactorial_Call_CompilerFastest(t, runtime())
 }
+
+func TestHostCall(t *testing.T) {
+	vs.RunTestHostCall(t, runtime)
+}
+
+func BenchmarkHostCall(b *testing.B) {
+	vs.RunBenchmarkHostCall(b, runtime)
+}
+
+func TestBenchmarkHostCall_CompilerFastest(t *testing.T) {
+	vs.RunTestBenchmarkHostCall_CompilerFastest(t, runtime())
+}

internal/integration_test/vs/jit/jit_test.go

@@ -23,3 +23,15 @@ func TestFactorial(t *testing.T) {
 func BenchmarkFactorial(b *testing.B) {
 	vs.RunBenchmarkFactorial(b, runtime)
 }
+
+func TestHostCall(t *testing.T) {
+	vs.RunTestHostCall(t, runtime)
+}
+
+func BenchmarkHostCall(b *testing.B) {
+	vs.RunBenchmarkHostCall(b, runtime)
+}
+
+func TestBenchmarkHostCall_CompilerFastest(t *testing.T) {
+	vs.RunTestBenchmarkHostCall_CompilerFastest(t, runtime())
+}

internal/integration_test/vs/runtime.go

@@ -20,6 +20,9 @@ type RuntimeConfig struct {
 	// The implementation invoke this with a byte slice allocated from the offset, length pair.
 	// This function simulates a host function that logs a message.
 	LogFn func([]byte) error
+	// EnvFReturnValue is set to non-zero if we want the runtime to instantiate "env" module with the function "f"
+	// which accepts one i64 value and returns the EnvFReturnValue as i64. This is mutually exclusive to LogFn.
+	EnvFReturnValue uint64
 }
 
 type Runtime interface {
@@ -86,6 +89,16 @@ func (r *wazeroRuntime) Compile(ctx context.Context, cfg *RuntimeConfig) (err er
 			ExportFunction("log", r.log).Compile(ctx, wazero.NewCompileConfig()); err != nil {
 			return err
 		}
+	} else if cfg.EnvFReturnValue != 0 {
+		if r.env, err = r.runtime.NewModuleBuilder("env").
+			ExportFunction("f",
+				// Note: accepting (context.Context, api.Module) is the slowest type of host function with wazero.
+				func(context.Context, api.Module, uint64) uint64 {
+					return cfg.EnvFReturnValue
+				},
+			).Compile(ctx, wazero.NewCompileConfig()); err != nil {
+			return err
+		}
 	}
 	r.compiled, err = r.runtime.CompileModule(ctx, cfg.ModuleWasm, wazero.NewCompileConfig())
 	return

internal/integration_test/vs/testdata/hostcall.wat

@@ -0,0 +1,9 @@
+(module
+  ;; env.f must be a host function for benchmarks on the cost of host calls which cross the Wasm<>Go boundary.
+  (func $host_func (import "env" "f") (param i64) (result i64))
+  ;; call_host_func calls "env.f" and returns the resut as-is.
+  (func (export "call_host_func") (param i64) (result i64)
+    local.get 0
+    call $host_func
+  )
+)

internal/integration_test/vs/wasmedge/wasmedge.go

@@ -79,6 +79,15 @@ func (r *wasmedgeRuntime) Instantiate(_ context.Context, cfg *vs.RuntimeConfig)
 		if err = m.vm.RegisterImport(m.env); err != nil {
 			return nil, err
 		}
+	} else if cfg.EnvFReturnValue != 0 {
+		m.env = wasmedge.NewImportObject("env")
+		fType := wasmedge.NewFunctionType([]wasmedge.ValType{wasmedge.ValType_I64}, []wasmedge.ValType{wasmedge.ValType_I64})
+		m.env.AddFunction("f", wasmedge.NewFunction(fType, func(data interface{}, mem *wasmedge.Memory, params []interface{}) ([]interface{}, wasmedge.Result) {
+			return []interface{}{int64(cfg.EnvFReturnValue)}, wasmedge.Result_Success
+		}, nil, 0))
+		if err = m.vm.RegisterImport(m.env); err != nil {
+			return nil, err
+		}
 	}
 
 	// Instantiate the module.

internal/integration_test/vs/wasmedge/wasmedge_test.go

@@ -35,3 +35,15 @@ func BenchmarkFactorial(b *testing.B) {
 func TestBenchmarkFactorial_Call_CompilerFastest(t *testing.T) {
 	vs.RunTestBenchmarkFactorial_Call_CompilerFastest(t, runtime())
 }
+
+func TestHostCall(t *testing.T) {
+	vs.RunTestHostCall(t, runtime)
+}
+
+func BenchmarkHostCall(b *testing.B) {
+	vs.RunBenchmarkHostCall(b, runtime)
+}
+
+func TestBenchmarkHostCall_CompilerFastest(t *testing.T) {
+	vs.RunTestBenchmarkHostCall_CompilerFastest(t, runtime())
+}

internal/integration_test/vs/wasmer/wasmer.go

@@ -89,6 +89,20 @@ func (r *wasmerRuntime) Instantiate(_ context.Context, cfg *vs.RuntimeConfig) (m
 				),
 			},
 		)
+	} else if cfg.EnvFReturnValue != 0 {
+		ret := []wasmer.Value{wasmer.NewValue(int64(cfg.EnvFReturnValue), wasmer.I64)}
+		importObject.Register(
+			"env",
+			map[string]wasmer.IntoExtern{
+				"f": wasmer.NewFunction(
+					wm.store,
+					wasmer.NewFunctionType(wasmer.NewValueTypes(wasmer.I64), wasmer.NewValueTypes(wasmer.I64)),
+					func(args []wasmer.Value) ([]wasmer.Value, error) {
+						return ret, nil
+					},
+				),
+			},
+		)
 	}
 
 	// TODO: wasmer_module_set_name is not exposed in wasmer-go

internal/integration_test/vs/wasmer/wasmer_test.go

@@ -33,3 +33,15 @@ func BenchmarkFactorial(b *testing.B) {
 func TestBenchmarkFactorial_Call_CompilerFastest(t *testing.T) {
 	vs.RunTestBenchmarkFactorial_Call_CompilerFastest(t, runtime())
 }
+
+func TestHostCall(t *testing.T) {
+	vs.RunTestHostCall(t, runtime)
+}
+
+func BenchmarkHostCall(b *testing.B) {
+	vs.RunBenchmarkHostCall(b, runtime)
+}
+
+func TestBenchmarkHostCall_CompilerFastest(t *testing.T) {
+	vs.RunTestBenchmarkHostCall_CompilerFastest(t, runtime())
+}

internal/integration_test/vs/wasmtime/wasmtime.go

@@ -83,6 +83,22 @@ func (r *wasmtimeRuntime) Instantiate(_ context.Context, cfg *vs.RuntimeConfig)
 		)); err != nil {
 			return
 		}
+	} else if cfg.EnvFReturnValue != 0 {
+		ret := []wasmtime.Val{wasmtime.ValI64(int64(cfg.EnvFReturnValue))}
+		if err = linker.Define("env", "f", wasmtime.NewFunc(
+			wm.store,
+			wasmtime.NewFuncType(
+				[]*wasmtime.ValType{
+					wasmtime.NewValType(wasmtime.KindI64),
+				},
+				[]*wasmtime.ValType{wasmtime.NewValType(wasmtime.KindI64)},
+			),
+			func(_ *wasmtime.Caller, args []wasmtime.Val) ([]wasmtime.Val, *wasmtime.Trap) {
+				return ret, nil
+			},
+		)); err != nil {
+			return
+		}
 	}
 
 	// Set the module name.

internal/integration_test/vs/wasmtime/wasmtime_test.go

@@ -33,3 +33,15 @@ func BenchmarkFactorial(b *testing.B) {
 func TestBenchmarkFactorial_Call_CompilerFastest(t *testing.T) {
 	vs.RunTestBenchmarkFactorial_Call_CompilerFastest(t, runtime())
 }
+
+func TestHostCall(t *testing.T) {
+	vs.RunTestHostCall(t, runtime)
+}
+
+func BenchmarkHostCall(b *testing.B) {
+	vs.RunBenchmarkHostCall(b, runtime)
+}
+
+func TestBenchmarkHostCall_CompilerFastest(t *testing.T) {
+	vs.RunTestBenchmarkHostCall_CompilerFastest(t, runtime())
+}