Move e2e tests in JIT under tests dir (#181)

This PR generalizes e2e tests in jit/engine_test.go and moves them
tests/adhoc directory to run these tests against interpreter as well.

Notably this removes all the testdata directory from wasm/*
and helps isolates the usage of raw binary or texts into examples,
bench and tests directory.

During the course of generalization, I found the inconsistency on the
panic handling between JIT and Interpreter which seems a bug of latter
so I fixed it.

examples/README.md

@@ -1,4 +0,0 @@
-
-This is where we put e2e tests of virtual machine. Please place examples so that `examples/foo_test.go` is testing 
-`examples/testdata/foo.wasm` binary which is generated by compiling
-`examples/testdata/foo.go` with latest version of TinyGo.

examples/testdata/trap.go

@@ -1,20 +0,0 @@
-package main
-
-func main() {}
-
-//export cause_panic
-func causePanic() {
-	one()
-}
-
-func one() {
-	two()
-}
-
-func two() {
-	three()
-}
-
-func three() {
-	panic("causing panic!!!!!!!!!!")
-}

examples/trap_test.go

@@ -1,41 +0,0 @@
-package examples
-
-import (
-	"os"
-	"testing"
-
-	"github.com/stretchr/testify/require"
-
-	"github.com/tetratelabs/wazero/wasi"
-	"github.com/tetratelabs/wazero/wasm"
-	"github.com/tetratelabs/wazero/wasm/binary"
-	"github.com/tetratelabs/wazero/wasm/interpreter"
-)
-
-func Test_trap(t *testing.T) {
-	buf, err := os.ReadFile("testdata/trap.wasm")
-	require.NoError(t, err)
-
-	mod, err := binary.DecodeModule((buf))
-	require.NoError(t, err)
-
-	store := wasm.NewStore(interpreter.NewEngine())
-
-	err = wasi.NewEnvironment().Register(store)
-	require.NoError(t, err)
-
-	err = store.Instantiate(mod, "test")
-	require.NoError(t, err)
-
-	_, _, err = store.CallFunction("test", "cause_panic")
-	require.Error(t, err)
-
-	const expErrMsg = `wasm runtime error: unreachable
-wasm backtrace:
-	0: runtime._panic
-	1: main.three
-	2: main.two
-	3: main.one
-	4: cause_panic`
-	require.Equal(t, expErrMsg, err.Error())
-}

tests/README.md

@@ -1,3 +1,4 @@
 This directory contains tests which use multiple packages. For example:
+- `engine` contains variety of e2e tests, mainly to ensure the consistency in the behavior between engines.
 - `codec` contains a test and benchmark on text and binary decoders.
 - `spectest` contains end-to-end tests with the [WebAssembly specification tests](https://github.com/WebAssembly/spec/tree/wg-1.0/test/core).

tests/engine/adhoc_test.go

@@ -0,0 +1,172 @@
+package adhoc
+
+import (
+	"os"
+	"reflect"
+	"runtime"
+	"sync"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/tetratelabs/wazero/wasm"
+	"github.com/tetratelabs/wazero/wasm/binary"
+	"github.com/tetratelabs/wazero/wasm/interpreter"
+	"github.com/tetratelabs/wazero/wasm/jit"
+)
+
+func TestJIT(t *testing.T) {
+	if runtime.GOARCH != "amd64" {
+		t.Skip()
+	}
+	runTests(t, jit.NewEngine)
+}
+
+func TestInterpreter(t *testing.T) {
+	runTests(t, interpreter.NewEngine)
+}
+
+func runTests(t *testing.T, newEngine func() wasm.Engine) {
+	fibonacci(t, newEngine)
+	fac(t, newEngine)
+	unreachable(t, newEngine)
+	memory(t, newEngine)
+	recursiveEntry(t, newEngine)
+}
+
+func fibonacci(t *testing.T, newEngine func() wasm.Engine) {
+	buf, err := os.ReadFile("testdata/fib.wasm")
+	require.NoError(t, err)
+	mod, err := binary.DecodeModule(buf)
+	require.NoError(t, err)
+
+	// We execute 1000 times in order to ensure the JIT engine is stable under high concurrency
+	// and we have no conflict with Go's runtime.
+	const goroutines = 1000
+	var wg sync.WaitGroup
+	wg.Add(goroutines)
+	for i := 0; i < goroutines; i++ {
+		go func() {
+			defer wg.Done()
+			store := wasm.NewStore(newEngine())
+			require.NoError(t, err)
+			err = store.Instantiate(mod, "test")
+			require.NoError(t, err)
+			out, _, err := store.CallFunction("test", "fib", 20)
+			require.NoError(t, err)
+			require.Equal(t, uint64(10946), out[0])
+		}()
+	}
+	wg.Wait()
+}
+
+func fac(t *testing.T, newEngine func() wasm.Engine) {
+	buf, err := os.ReadFile("testdata/fac.wasm")
+	require.NoError(t, err)
+	mod, err := binary.DecodeModule(buf)
+	require.NoError(t, err)
+	store := wasm.NewStore(newEngine())
+	require.NoError(t, err)
+	err = store.Instantiate(mod, "test")
+	require.NoError(t, err)
+	for _, name := range []string{
+		"fac-rec",
+		"fac-iter",
+		"fac-rec-named",
+		"fac-iter-named",
+		"fac-opt",
+	} {
+		name := name
+		t.Run(name, func(t *testing.T) {
+			out, _, err := store.CallFunction("test", name, 25)
+			require.NoError(t, err)
+			require.Equal(t, uint64(7034535277573963776), out[0])
+		})
+	}
+
+	_, _, err = store.CallFunction("test", "fac-rec", 1073741824)
+	require.ErrorIs(t, err, wasm.ErrRuntimeCallStackOverflow)
+}
+
+func unreachable(t *testing.T, newEngine func() wasm.Engine) {
+	buf, err := os.ReadFile("testdata/unreachable.wasm")
+	require.NoError(t, err)
+	mod, err := binary.DecodeModule(buf)
+	require.NoError(t, err)
+	store := wasm.NewStore(newEngine())
+	require.NoError(t, err)
+
+	const moduleName = "test"
+
+	callUnreachable := func(ctx *wasm.HostFunctionCallContext) {
+		_, _, err := store.CallFunction(moduleName, "unreachable_func")
+		require.NoError(t, err)
+	}
+	err = store.AddHostFunction("host", "cause_unreachable", reflect.ValueOf(callUnreachable))
+	require.NoError(t, err)
+
+	err = store.Instantiate(mod, moduleName)
+	require.NoError(t, err)
+
+	_, _, err = store.CallFunction(moduleName, "main")
+	exp := `wasm runtime error: unreachable
+wasm backtrace:
+	0: unreachable_func
+	1: host.cause_unreachable
+	2: two
+	3: one
+	4: main`
+	require.ErrorIs(t, err, wasm.ErrRuntimeUnreachable)
+	require.Equal(t, exp, err.Error())
+}
+
+func memory(t *testing.T, newEngine func() wasm.Engine) {
+	buf, err := os.ReadFile("testdata/memory.wasm")
+	require.NoError(t, err)
+	mod, err := binary.DecodeModule(buf)
+	require.NoError(t, err)
+	store := wasm.NewStore(newEngine())
+	require.NoError(t, err)
+	err = store.Instantiate(mod, "test")
+	require.NoError(t, err)
+	// First, we have zero-length memory instance.
+	out, _, err := store.CallFunction("test", "size")
+	require.NoError(t, err)
+	require.Equal(t, uint64(0), out[0])
+	// Then grow the memory.
+	const newPages uint64 = 10
+	out, _, err = store.CallFunction("test", "grow", newPages)
+	require.NoError(t, err)
+	// Grow returns the previous number of memory pages, namely zero.
+	require.Equal(t, uint64(0), out[0])
+	// Now size should return the new pages -- 10.
+	out, _, err = store.CallFunction("test", "size")
+	require.NoError(t, err)
+	require.Equal(t, newPages, out[0])
+	// Growing memory with zero pages is valid but should be noop.
+	out, _, err = store.CallFunction("test", "grow", 0)
+	require.NoError(t, err)
+	require.Equal(t, newPages, out[0])
+}
+
+func recursiveEntry(t *testing.T, newEngine func() wasm.Engine) {
+	buf, err := os.ReadFile("testdata/recursive.wasm")
+	require.NoError(t, err)
+	mod, err := binary.DecodeModule(buf)
+	require.NoError(t, err)
+
+	store := wasm.NewStore(newEngine())
+
+	hostfunc := func(ctx *wasm.HostFunctionCallContext) {
+		_, _, err := store.CallFunction("test", "called_by_host_func")
+		require.NoError(t, err)
+	}
+	err = store.AddHostFunction("env", "host_func", reflect.ValueOf(hostfunc))
+	require.NoError(t, err)
+
+	err = store.Instantiate(mod, "test")
+	require.NoError(t, err)
+
+	_, _, err = store.CallFunction("test", "main", uint64(1))
+	require.NoError(t, err)
+}

tests/spectest/spec_test.go

@@ -7,6 +7,7 @@ import (
 	"os"
 	"path/filepath"
 	"reflect"
+	"runtime"
 	"strconv"
 	"strings"
 	"testing"
@@ -213,6 +214,9 @@ func addSpectestModule(t *testing.T, store *wasm.Store) {
 }
 
 func TestJIT(t *testing.T) {
+	if runtime.GOARCH != "amd64" {
+		t.Skip()
+	}
 	runTest(t, jit.NewEngine)
 }
 

wasm/interpreter/interpreter.go

@@ -427,34 +427,46 @@ func (it *interpreter) lowerIROps(f *wasm.FunctionInstance,
 // Call implements an interpreted wasm.Engine.
 func (it *interpreter) Call(f *wasm.FunctionInstance, params ...uint64) (results []uint64, err error) {
 	prevFrameLen := len(it.frames)
+
+	// shouldRecover is true when a panic at the origin of callstack should be recovered
+	//
+	// If this is the recursive call into Wasm (prevFrameLen != 0), we do not recover, and delegate the
+	// recovery to the first interpreter.Call.
+	//
+	// For example, given the call stack:
+	//	 "original host function" --(interpreter.Call)--> Wasm func A --> Host func --(interpreter.Call)--> Wasm function B,
+	// if the top Wasm function panics, we go back to the "original host function".
+	shouldRecover := prevFrameLen == 0
 	defer func() {
-		if v := recover(); v != nil {
-			if buildoptions.IsDebugMode {
-				debug.PrintStack()
-			}
-			traceNum := len(it.frames) - prevFrameLen
-			traces := make([]string, 0, traceNum)
-			for i := 0; i < traceNum; i++ {
-				frame := it.popFrame()
-				name := frame.f.funcInstance.Name
-				// TODO: include the original instruction which corresponds
-				// to frame.f.body[frame.pc].
-				traces = append(traces, fmt.Sprintf("\t%d: %s", i, name))
-			}
-
-			it.frames = it.frames[:prevFrameLen]
-			err2, ok := v.(error)
-			if ok {
-				if err2.Error() == "runtime error: integer divide by zero" {
-					err2 = wasm.ErrRuntimeIntegerDivideByZero
+		if shouldRecover {
+			if v := recover(); v != nil {
+				if buildoptions.IsDebugMode {
+					debug.PrintStack()
+				}
+				traceNum := len(it.frames) - prevFrameLen
+				traces := make([]string, 0, traceNum)
+				for i := 0; i < traceNum; i++ {
+					frame := it.popFrame()
+					name := frame.f.funcInstance.Name
+					// TODO: include the original instruction which corresponds
+					// to frame.f.body[frame.pc].
+					traces = append(traces, fmt.Sprintf("\t%d: %s", i, name))
 				}
-				err = fmt.Errorf("wasm runtime error: %w", err2)
-			} else {
-				err = fmt.Errorf("wasm runtime error: %v", v)
-			}
 
-			if len(traces) > 0 {
-				err = fmt.Errorf("%w\nwasm backtrace:\n%s", err, strings.Join(traces, "\n"))
+				it.frames = it.frames[:prevFrameLen]
+				err2, ok := v.(error)
+				if ok {
+					if err2.Error() == "runtime error: integer divide by zero" {
+						err2 = wasm.ErrRuntimeIntegerDivideByZero
+					}
+					err = fmt.Errorf("wasm runtime error: %w", err2)
+				} else {
+					err = fmt.Errorf("wasm runtime error: %v", v)
+				}
+
+				if len(traces) > 0 {
+					err = fmt.Errorf("%w\nwasm backtrace:\n%s", err, strings.Join(traces, "\n"))
+				}
 			}
 		}
 	}()

wasm/jit/engine_test.go

@@ -2,17 +2,12 @@ package jit
 
 import (
 	"math"
-	"os"
-	"reflect"
-	"sync"
 	"testing"
 	"unsafe"
 
 	"github.com/stretchr/testify/require"
 
 	"github.com/tetratelabs/wazero/wasm"
-	"github.com/tetratelabs/wazero/wasm/binary"
-	"github.com/tetratelabs/wazero/wasm/text"
 )
 
 // Ensures that the offset consts do not drift when we manipulate the target structs.
@@ -81,169 +76,3 @@ func TestVerifyOffsetValue(t *testing.T) {
 	var globalInstance wasm.GlobalInstance
 	require.Equal(t, int(unsafe.Offsetof(globalInstance.Val)), globalInstanceValueOffset)
 }
-
-func Test_Simple(t *testing.T) {
-	mod, err := text.DecodeModule([]byte(`(module
-	(import "" "hello" (func $hello))
-	(start $hello)
-)`))
-	require.NoError(t, err)
-
-	engine := newEngine()
-	store := wasm.NewStore(engine)
-
-	msg := "hello!"
-	hostFunction := func(ctx *wasm.HostFunctionCallContext) {
-		require.NotNil(t, ctx.Memory)
-		copy(ctx.Memory.Buffer, msg)
-	}
-	require.NoError(t, store.AddHostFunction("", "hello", reflect.ValueOf(hostFunction)))
-
-	memoryInstance := &wasm.MemoryInstance{Buffer: make([]byte, len(msg))}
-	engine.compiledFunctions[0].source.ModuleInstance.Memory = memoryInstance
-
-	moduleName := "simple"
-	require.NoError(t, store.Instantiate(mod, moduleName))
-
-	// The "hello" function was imported as $hello in Wasm. Since it was marked as the start
-	// function, it is invoked on instantiation. Ensure that worked: "hello" was called!
-	require.Equal(t, msg, string(memoryInstance.Buffer))
-}
-
-func TestEngine_fibonacci(t *testing.T) {
-	buf, err := os.ReadFile("testdata/fib.wasm")
-	require.NoError(t, err)
-	mod, err := binary.DecodeModule(buf)
-	require.NoError(t, err)
-
-	// We execute 1000 times in order to ensure the JIT engine is stable under high concurrency
-	// and we have no conflict with Go's runtime.
-	const goroutines = 1000
-	var wg sync.WaitGroup
-	wg.Add(goroutines)
-	for i := 0; i < goroutines; i++ {
-		go func() {
-			defer wg.Done()
-			store := wasm.NewStore(NewEngine())
-			require.NoError(t, err)
-			err = store.Instantiate(mod, "test")
-			require.NoError(t, err)
-			out, _, err := store.CallFunction("test", "fib", 20)
-			require.NoError(t, err)
-			require.Equal(t, uint64(10946), out[0])
-		}()
-	}
-	wg.Wait()
-}
-
-func TestEngine_fac(t *testing.T) {
-	buf, err := os.ReadFile("testdata/fac.wasm")
-	require.NoError(t, err)
-	mod, err := binary.DecodeModule(buf)
-	require.NoError(t, err)
-	store := wasm.NewStore(NewEngine())
-	require.NoError(t, err)
-	err = store.Instantiate(mod, "test")
-	require.NoError(t, err)
-	for _, name := range []string{
-		"fac-rec",
-		"fac-iter",
-		"fac-rec-named",
-		"fac-iter-named",
-		"fac-opt",
-	} {
-		name := name
-		t.Run(name, func(t *testing.T) {
-			out, _, err := store.CallFunction("test", name, 25)
-			require.NoError(t, err)
-			require.Equal(t, uint64(7034535277573963776), out[0])
-		})
-	}
-
-	_, _, err = store.CallFunction("test", "fac-rec", 1073741824)
-	require.ErrorIs(t, err, wasm.ErrRuntimeCallStackOverflow)
-}
-
-func TestEngine_unreachable(t *testing.T) {
-	buf, err := os.ReadFile("testdata/unreachable.wasm")
-	require.NoError(t, err)
-	mod, err := binary.DecodeModule(buf)
-	require.NoError(t, err)
-	store := wasm.NewStore(NewEngine())
-	require.NoError(t, err)
-
-	const moduleName = "test"
-
-	callUnreachable := func(ctx *wasm.HostFunctionCallContext) {
-		_, _, err := store.CallFunction(moduleName, "unreachable_func")
-		require.NoError(t, err)
-	}
-	err = store.AddHostFunction("host", "cause_unreachable", reflect.ValueOf(callUnreachable))
-	require.NoError(t, err)
-
-	err = store.Instantiate(mod, moduleName)
-	require.NoError(t, err)
-
-	_, _, err = store.CallFunction(moduleName, "main")
-	exp := `wasm runtime error: unreachable
-wasm backtrace:
-	0: unreachable_func
-	1: host.cause_unreachable
-	2: two
-	3: one
-	4: main`
-	require.ErrorIs(t, err, wasm.ErrRuntimeUnreachable)
-	require.Equal(t, exp, err.Error())
-}
-
-func TestEngine_memory(t *testing.T) {
-	buf, err := os.ReadFile("testdata/memory.wasm")
-	require.NoError(t, err)
-	mod, err := binary.DecodeModule(buf)
-	require.NoError(t, err)
-	store := wasm.NewStore(NewEngine())
-	require.NoError(t, err)
-	err = store.Instantiate(mod, "test")
-	require.NoError(t, err)
-	// First, we have zero-length memory instance.
-	out, _, err := store.CallFunction("test", "size")
-	require.NoError(t, err)
-	require.Equal(t, uint64(0), out[0])
-	// Then grow the memory.
-	const newPages uint64 = 10
-	out, _, err = store.CallFunction("test", "grow", newPages)
-	require.NoError(t, err)
-	// Grow returns the previous number of memory pages, namely zero.
-	require.Equal(t, uint64(0), out[0])
-	// Now size should return the new pages -- 10.
-	out, _, err = store.CallFunction("test", "size")
-	require.NoError(t, err)
-	require.Equal(t, newPages, out[0])
-	// Growing memory with zero pages is valid but should be noop.
-	out, _, err = store.CallFunction("test", "grow", 0)
-	require.NoError(t, err)
-	require.Equal(t, newPages, out[0])
-}
-
-func TestEngine_RecursiveEntry(t *testing.T) {
-	buf, err := os.ReadFile("testdata/recursive.wasm")
-	require.NoError(t, err)
-	mod, err := binary.DecodeModule(buf)
-	require.NoError(t, err)
-
-	eng := newEngine()
-	store := wasm.NewStore(eng)
-
-	hostfunc := func(ctx *wasm.HostFunctionCallContext) {
-		_, _, err := store.CallFunction("test", "called_by_host_func")
-		require.NoError(t, err)
-	}
-	err = store.AddHostFunction("env", "host_func", reflect.ValueOf(hostfunc))
-	require.NoError(t, err)
-
-	err = store.Instantiate(mod, "test")
-	require.NoError(t, err)
-
-	_, _, err = store.CallFunction("test", "main", uint64(1))
-	require.NoError(t, err)
-}