package wazevo import ( "context" "encoding/binary" "fmt" "reflect" "unsafe" "github.com/tetratelabs/wazero/api" "github.com/tetratelabs/wazero/experimental" "github.com/tetratelabs/wazero/internal/engine/wazevo/wazevoapi" "github.com/tetratelabs/wazero/internal/internalapi" "github.com/tetratelabs/wazero/internal/wasm" "github.com/tetratelabs/wazero/internal/wasmdebug" "github.com/tetratelabs/wazero/internal/wasmruntime" ) type ( // callEngine implements api.Function. callEngine struct { internalapi.WazeroOnly stack []byte // stackTop is the pointer to the *aligned* top of the stack. This must be updated // whenever the stack is changed. This is passed to the assembly function // at the very beginning of api.Function Call/CallWithStack. stackTop uintptr // executable is the pointer to the executable code for this function. executable *byte preambleExecutable *byte // parent is the *moduleEngine from which this callEngine is created. parent *moduleEngine // indexInModule is the index of the function in the module. indexInModule wasm.Index // sizeOfParamResultSlice is the size of the parameter/result slice. sizeOfParamResultSlice int requiredParams int // execCtx holds various information to be read/written by assembly functions. execCtx executionContext // execCtxPtr holds the pointer to the executionContext which doesn't change after callEngine is created. execCtxPtr uintptr numberOfResults int stackIteratorImpl stackIterator } // executionContext is the struct to be read/written by assembly functions. executionContext struct { // exitCode holds the wazevoapi.ExitCode describing the state of the function execution. exitCode wazevoapi.ExitCode // callerModuleContextPtr holds the moduleContextOpaque for Go function calls. callerModuleContextPtr *byte // originalFramePointer holds the original frame pointer of the caller of the assembly function. originalFramePointer uintptr // originalStackPointer holds the original stack pointer of the caller of the assembly function. originalStackPointer uintptr // goReturnAddress holds the return address to go back to the caller of the assembly function. goReturnAddress uintptr // stackBottomPtr holds the pointer to the bottom of the stack. stackBottomPtr *byte // goCallReturnAddress holds the return address to go back to the caller of the Go function. goCallReturnAddress *byte // stackPointerBeforeGoCall holds the stack pointer before calling a Go function. stackPointerBeforeGoCall *uint64 // stackGrowRequiredSize holds the required size of stack grow. stackGrowRequiredSize uintptr // memoryGrowTrampolineAddress holds the address of memory grow trampoline function. memoryGrowTrampolineAddress *byte // stackGrowCallTrampolineAddress holds the address of stack grow trampoline function. stackGrowCallTrampolineAddress *byte // checkModuleExitCodeTrampolineAddress holds the address of check-module-exit-code function. checkModuleExitCodeTrampolineAddress *byte // savedRegisters is the opaque spaces for save/restore registers. // We want to align 16 bytes for each register, so we use [64][2]uint64. savedRegisters [64][2]uint64 // goFunctionCallCalleeModuleContextOpaque is the pointer to the target Go function's moduleContextOpaque. goFunctionCallCalleeModuleContextOpaque uintptr // tableGrowTrampolineAddress holds the address of table grow trampoline function. tableGrowTrampolineAddress *byte // refFuncTrampolineAddress holds the address of ref-func trampoline function. refFuncTrampolineAddress *byte // memmoveAddress holds the address of memmove function implemented by Go runtime. See memmove.go. memmoveAddress uintptr } ) func (c *callEngine) requiredInitialStackSize() int { const initialStackSizeDefault = 512 stackSize := initialStackSizeDefault paramResultInBytes := c.sizeOfParamResultSlice * 8 * 2 // * 8 because uint64 is 8 bytes, and *2 because we need both separated param/result slots. required := paramResultInBytes + 32 + 16 // 32 is enough to accommodate the call frame info, and 16 exists just in case when []byte is not aligned to 16 bytes. if required > stackSize { stackSize = required } return stackSize } func (c *callEngine) init() { stackSize := c.requiredInitialStackSize() if wazevoapi.StackGuardCheckEnabled { stackSize += wazevoapi.StackGuardCheckGuardPageSize } c.stack = make([]byte, stackSize) c.stackTop = alignedStackTop(c.stack) if wazevoapi.StackGuardCheckEnabled { c.execCtx.stackBottomPtr = &c.stack[wazevoapi.StackGuardCheckGuardPageSize] } else { c.execCtx.stackBottomPtr = &c.stack[0] } c.execCtxPtr = uintptr(unsafe.Pointer(&c.execCtx)) } // alignedStackTop returns 16-bytes aligned stack top of given stack. // 16 bytes should be good for all platform (arm64/amd64). func alignedStackTop(s []byte) uintptr { stackAddr := uintptr(unsafe.Pointer(&s[len(s)-1])) return stackAddr - (stackAddr & (16 - 1)) } // Definition implements api.Function. func (c *callEngine) Definition() api.FunctionDefinition { return c.parent.module.Source.FunctionDefinition(c.indexInModule) } // Call implements api.Function. func (c *callEngine) Call(ctx context.Context, params ...uint64) ([]uint64, error) { if c.requiredParams != len(params) { return nil, fmt.Errorf("expected %d params, but passed %d", c.requiredParams, len(params)) } paramResultSlice := make([]uint64, c.sizeOfParamResultSlice) copy(paramResultSlice, params) if err := c.callWithStack(ctx, paramResultSlice); err != nil { return nil, err } return paramResultSlice[:c.numberOfResults], nil } func (c *callEngine) addFrame(builder wasmdebug.ErrorBuilder, addr uintptr) (def api.FunctionDefinition, listener experimental.FunctionListener) { eng := c.parent.parent.parent cm := eng.compiledModuleOfAddr(addr) if cm != nil { index := cm.functionIndexOf(addr) def = cm.module.FunctionDefinition(cm.module.ImportFunctionCount + index) var sources []string if dw := cm.module.DWARFLines; dw != nil { sourceOffset := cm.getSourceOffset(addr) sources = dw.Line(sourceOffset) } builder.AddFrame(def.DebugName(), def.ParamTypes(), def.ResultTypes(), sources) if len(cm.listeners) > 0 { listener = cm.listeners[index] } } return } // CallWithStack implements api.Function. func (c *callEngine) CallWithStack(ctx context.Context, paramResultStack []uint64) (err error) { if c.sizeOfParamResultSlice > len(paramResultStack) { return fmt.Errorf("need %d params, but stack size is %d", c.sizeOfParamResultSlice, len(paramResultStack)) } return c.callWithStack(ctx, paramResultStack) } // CallWithStack implements api.Function. func (c *callEngine) callWithStack(ctx context.Context, paramResultStack []uint64) (err error) { if wazevoapi.StackGuardCheckEnabled { defer func() { wazevoapi.CheckStackGuardPage(c.stack) }() } p := c.parent ensureTermination := p.parent.ensureTermination m := p.module if ensureTermination { select { case <-ctx.Done(): // If the provided context is already done, close the module and return the error. m.CloseWithCtxErr(ctx) return m.FailIfClosed() default: } } var paramResultPtr *uint64 if len(paramResultStack) > 0 { paramResultPtr = ¶mResultStack[0] } defer func() { if r := recover(); r != nil { type listenerForAbort struct { def api.FunctionDefinition lsn experimental.FunctionListener } var listeners []listenerForAbort builder := wasmdebug.NewErrorBuilder() def, lsn := c.addFrame(builder, uintptr(unsafe.Pointer(c.execCtx.goCallReturnAddress))) if lsn != nil { listeners = append(listeners, listenerForAbort{def, lsn}) } returnAddrs := unwindStack(uintptr(unsafe.Pointer(c.execCtx.stackPointerBeforeGoCall)), c.stackTop, nil) for _, retAddr := range returnAddrs[:len(returnAddrs)-1] { // the last return addr is the trampoline, so we skip it. def, lsn = c.addFrame(builder, retAddr) if lsn != nil { listeners = append(listeners, listenerForAbort{def, lsn}) } } err = builder.FromRecovered(r) for _, lsn := range listeners { lsn.lsn.Abort(ctx, m, lsn.def, err) } } else { if err != wasmruntime.ErrRuntimeStackOverflow { // Stackoverflow case shouldn't be panic (to avoid extreme stack unwinding). err = c.parent.module.FailIfClosed() } } if err != nil { // Ensures that we can reuse this callEngine even after an error. c.execCtx.exitCode = wazevoapi.ExitCodeOK } }() if ensureTermination { done := m.CloseModuleOnCanceledOrTimeout(ctx) defer done() } entrypoint(c.preambleExecutable, c.executable, c.execCtxPtr, c.parent.opaquePtr, paramResultPtr, c.stackTop) for { switch ec := c.execCtx.exitCode; ec & wazevoapi.ExitCodeMask { case wazevoapi.ExitCodeOK: return nil case wazevoapi.ExitCodeGrowStack: var newsp uintptr if wazevoapi.StackGuardCheckEnabled { newsp, err = c.growStackWithGuarded() } else { newsp, err = c.growStack() } if err != nil { return err } c.execCtx.exitCode = wazevoapi.ExitCodeOK afterGoFunctionCallEntrypoint(c.execCtx.goCallReturnAddress, c.execCtxPtr, newsp) case wazevoapi.ExitCodeGrowMemory: mod := c.callerModuleInstance() mem := mod.MemoryInstance s := goCallStackView(c.execCtx.stackPointerBeforeGoCall) argRes := &s[0] if res, ok := mem.Grow(uint32(*argRes)); !ok { *argRes = uint64(0xffffffff) // = -1 in signed 32-bit integer. } else { *argRes = uint64(res) calleeOpaque := opaqueViewFromPtr(uintptr(unsafe.Pointer(c.execCtx.callerModuleContextPtr))) if mod.Source.MemorySection != nil { // Local memory. putLocalMemory(calleeOpaque, 8 /* local memory begins at 8 */, mem) } else { // Imported memory's owner at offset 16 of the callerModuleContextPtr. opaquePtr := uintptr(binary.LittleEndian.Uint64(calleeOpaque[16:])) importedMemOwner := opaqueViewFromPtr(opaquePtr) putLocalMemory(importedMemOwner, 8 /* local memory begins at 8 */, mem) } } c.execCtx.exitCode = wazevoapi.ExitCodeOK afterGoFunctionCallEntrypoint(c.execCtx.goCallReturnAddress, c.execCtxPtr, uintptr(unsafe.Pointer(c.execCtx.stackPointerBeforeGoCall))) case wazevoapi.ExitCodeTableGrow: mod := c.callerModuleInstance() s := goCallStackView(c.execCtx.stackPointerBeforeGoCall) tableIndex, num, ref := s[0], uint32(s[1]), uintptr(s[2]) table := mod.Tables[tableIndex] s[0] = uint64(uint32(int32(table.Grow(num, ref)))) c.execCtx.exitCode = wazevoapi.ExitCodeOK afterGoFunctionCallEntrypoint(c.execCtx.goCallReturnAddress, c.execCtxPtr, uintptr(unsafe.Pointer(c.execCtx.stackPointerBeforeGoCall))) case wazevoapi.ExitCodeCallGoFunction: index := wazevoapi.GoFunctionIndexFromExitCode(ec) f := hostModuleGoFuncFromOpaque[api.GoFunction](index, c.execCtx.goFunctionCallCalleeModuleContextOpaque) f.Call(ctx, goCallStackView(c.execCtx.stackPointerBeforeGoCall)) // Back to the native code. c.execCtx.exitCode = wazevoapi.ExitCodeOK afterGoFunctionCallEntrypoint(c.execCtx.goCallReturnAddress, c.execCtxPtr, uintptr(unsafe.Pointer(c.execCtx.stackPointerBeforeGoCall))) case wazevoapi.ExitCodeCallGoFunctionWithListener: index := wazevoapi.GoFunctionIndexFromExitCode(ec) f := hostModuleGoFuncFromOpaque[api.GoFunction](index, c.execCtx.goFunctionCallCalleeModuleContextOpaque) listeners := hostModuleListenersSliceFromOpaque(c.execCtx.goFunctionCallCalleeModuleContextOpaque) s := goCallStackView(c.execCtx.stackPointerBeforeGoCall) // Call Listener.Before. callerModule := c.callerModuleInstance() listener := listeners[index] hostModule := hostModuleFromOpaque(c.execCtx.goFunctionCallCalleeModuleContextOpaque) def := hostModule.FunctionDefinition(wasm.Index(index)) listener.Before(ctx, callerModule, def, s, c.stackIterator(true)) // Call into the Go function. f.Call(ctx, s) // Call Listener.After. listener.After(ctx, callerModule, def, s) // Back to the native code. c.execCtx.exitCode = wazevoapi.ExitCodeOK afterGoFunctionCallEntrypoint(c.execCtx.goCallReturnAddress, c.execCtxPtr, uintptr(unsafe.Pointer(c.execCtx.stackPointerBeforeGoCall))) case wazevoapi.ExitCodeCallGoModuleFunction: index := wazevoapi.GoFunctionIndexFromExitCode(ec) f := hostModuleGoFuncFromOpaque[api.GoModuleFunction](index, c.execCtx.goFunctionCallCalleeModuleContextOpaque) mod := c.callerModuleInstance() f.Call(ctx, mod, goCallStackView(c.execCtx.stackPointerBeforeGoCall)) // Back to the native code. c.execCtx.exitCode = wazevoapi.ExitCodeOK afterGoFunctionCallEntrypoint(c.execCtx.goCallReturnAddress, c.execCtxPtr, uintptr(unsafe.Pointer(c.execCtx.stackPointerBeforeGoCall))) case wazevoapi.ExitCodeCallGoModuleFunctionWithListener: index := wazevoapi.GoFunctionIndexFromExitCode(ec) f := hostModuleGoFuncFromOpaque[api.GoModuleFunction](index, c.execCtx.goFunctionCallCalleeModuleContextOpaque) listeners := hostModuleListenersSliceFromOpaque(c.execCtx.goFunctionCallCalleeModuleContextOpaque) s := goCallStackView(c.execCtx.stackPointerBeforeGoCall) // Call Listener.Before. callerModule := c.callerModuleInstance() listener := listeners[index] hostModule := hostModuleFromOpaque(c.execCtx.goFunctionCallCalleeModuleContextOpaque) def := hostModule.FunctionDefinition(wasm.Index(index)) listener.Before(ctx, callerModule, def, s, c.stackIterator(true)) // Call into the Go function. f.Call(ctx, callerModule, s) // Call Listener.After. listener.After(ctx, callerModule, def, s) // Back to the native code. c.execCtx.exitCode = wazevoapi.ExitCodeOK afterGoFunctionCallEntrypoint(c.execCtx.goCallReturnAddress, c.execCtxPtr, uintptr(unsafe.Pointer(c.execCtx.stackPointerBeforeGoCall))) case wazevoapi.ExitCodeCallListenerBefore: stack := goCallStackView(c.execCtx.stackPointerBeforeGoCall) index := stack[0] mod := c.callerModuleInstance() listener := mod.Engine.(*moduleEngine).listeners[index] def := mod.Source.FunctionDefinition(wasm.Index(index)) listener.Before(ctx, mod, def, stack[1:], c.stackIterator(false)) c.execCtx.exitCode = wazevoapi.ExitCodeOK afterGoFunctionCallEntrypoint(c.execCtx.goCallReturnAddress, c.execCtxPtr, uintptr(unsafe.Pointer(c.execCtx.stackPointerBeforeGoCall))) case wazevoapi.ExitCodeCallListenerAfter: stack := goCallStackView(c.execCtx.stackPointerBeforeGoCall) index := stack[0] mod := c.callerModuleInstance() listener := mod.Engine.(*moduleEngine).listeners[index] def := mod.Source.FunctionDefinition(wasm.Index(index)) listener.After(ctx, mod, def, stack[1:]) c.execCtx.exitCode = wazevoapi.ExitCodeOK afterGoFunctionCallEntrypoint(c.execCtx.goCallReturnAddress, c.execCtxPtr, uintptr(unsafe.Pointer(c.execCtx.stackPointerBeforeGoCall))) case wazevoapi.ExitCodeCheckModuleExitCode: // Note: this operation must be done in Go, not native code. The reason is that // native code cannot be preempted and that means it can block forever if there are not // enough OS threads (which we don't have control over). if err := m.FailIfClosed(); err != nil { panic(err) } c.execCtx.exitCode = wazevoapi.ExitCodeOK afterGoFunctionCallEntrypoint(c.execCtx.goCallReturnAddress, c.execCtxPtr, uintptr(unsafe.Pointer(c.execCtx.stackPointerBeforeGoCall))) case wazevoapi.ExitCodeRefFunc: mod := c.callerModuleInstance() s := goCallStackView(c.execCtx.stackPointerBeforeGoCall) funcIndex := s[0] ref := mod.Engine.FunctionInstanceReference(wasm.Index(funcIndex)) s[0] = uint64(ref) c.execCtx.exitCode = wazevoapi.ExitCodeOK afterGoFunctionCallEntrypoint(c.execCtx.goCallReturnAddress, c.execCtxPtr, uintptr(unsafe.Pointer(c.execCtx.stackPointerBeforeGoCall))) case wazevoapi.ExitCodeUnreachable: panic(wasmruntime.ErrRuntimeUnreachable) case wazevoapi.ExitCodeMemoryOutOfBounds: panic(wasmruntime.ErrRuntimeOutOfBoundsMemoryAccess) case wazevoapi.ExitCodeTableOutOfBounds: panic(wasmruntime.ErrRuntimeInvalidTableAccess) case wazevoapi.ExitCodeIndirectCallNullPointer: panic(wasmruntime.ErrRuntimeInvalidTableAccess) case wazevoapi.ExitCodeIndirectCallTypeMismatch: panic(wasmruntime.ErrRuntimeIndirectCallTypeMismatch) case wazevoapi.ExitCodeIntegerOverflow: panic(wasmruntime.ErrRuntimeIntegerOverflow) case wazevoapi.ExitCodeIntegerDivisionByZero: panic(wasmruntime.ErrRuntimeIntegerDivideByZero) case wazevoapi.ExitCodeInvalidConversionToInteger: panic(wasmruntime.ErrRuntimeInvalidConversionToInteger) default: panic("BUG") } } } func (c *callEngine) callerModuleInstance() *wasm.ModuleInstance { return moduleInstanceFromOpaquePtr(c.execCtx.callerModuleContextPtr) } func opaqueViewFromPtr(ptr uintptr) []byte { var opaque []byte sh := (*reflect.SliceHeader)(unsafe.Pointer(&opaque)) sh.Data = ptr sh.Len = 24 sh.Cap = 24 return opaque } const callStackCeiling = uintptr(50000000) // in uint64 (8 bytes) == 400000000 bytes in total == 400mb. func (c *callEngine) growStackWithGuarded() (newSP uintptr, err error) { if wazevoapi.StackGuardCheckEnabled { wazevoapi.CheckStackGuardPage(c.stack) } newSP, err = c.growStack() if err != nil { return } if wazevoapi.StackGuardCheckEnabled { c.execCtx.stackBottomPtr = &c.stack[wazevoapi.StackGuardCheckGuardPageSize] } return } // growStack grows the stack, and returns the new stack pointer. func (c *callEngine) growStack() (newSP uintptr, err error) { currentLen := uintptr(len(c.stack)) if callStackCeiling < currentLen { err = wasmruntime.ErrRuntimeStackOverflow return } newLen := 2*currentLen + c.execCtx.stackGrowRequiredSize + 16 // Stack might be aligned to 16 bytes, so add 16 bytes just in case. newStack := make([]byte, newLen) relSp := c.stackTop - uintptr(unsafe.Pointer(c.execCtx.stackPointerBeforeGoCall)) // Copy the existing contents in the previous Go-allocated stack into the new one. var prevStackAligned, newStackAligned []byte { sh := (*reflect.SliceHeader)(unsafe.Pointer(&prevStackAligned)) sh.Data = c.stackTop - relSp sh.Len = int(relSp) sh.Cap = int(relSp) } newTop := alignedStackTop(newStack) { newSP = newTop - relSp sh := (*reflect.SliceHeader)(unsafe.Pointer(&newStackAligned)) sh.Data = newSP sh.Len = int(relSp) sh.Cap = int(relSp) } copy(newStackAligned, prevStackAligned) c.stack = newStack c.stackTop = newTop c.execCtx.stackBottomPtr = &newStack[0] return } func (c *callEngine) stackIterator(onHostCall bool) experimental.StackIterator { c.stackIteratorImpl.reset(c, onHostCall) return &c.stackIteratorImpl } // stackIterator implements experimental.StackIterator. type stackIterator struct { retAddrs []uintptr retAddrCursor int eng *engine pc uint64 currentDef *wasm.FunctionDefinition } func (si *stackIterator) reset(c *callEngine, onHostCall bool) { if onHostCall { si.retAddrs = append(si.retAddrs[:0], uintptr(unsafe.Pointer(c.execCtx.goCallReturnAddress))) } else { si.retAddrs = si.retAddrs[:0] } si.retAddrs = unwindStack(uintptr(unsafe.Pointer(c.execCtx.stackPointerBeforeGoCall)), c.stackTop, si.retAddrs) si.retAddrs = si.retAddrs[:len(si.retAddrs)-1] // the last return addr is the trampoline, so we skip it. si.retAddrCursor = 0 si.eng = c.parent.parent.parent } // Next implements the same method as documented on experimental.StackIterator. func (si *stackIterator) Next() bool { if si.retAddrCursor >= len(si.retAddrs) { return false } addr := si.retAddrs[si.retAddrCursor] cm := si.eng.compiledModuleOfAddr(addr) if cm != nil { index := cm.functionIndexOf(addr) def := cm.module.FunctionDefinition(cm.module.ImportFunctionCount + index) si.currentDef = def si.retAddrCursor++ si.pc = uint64(addr) return true } return false } // ProgramCounter implements the same method as documented on experimental.StackIterator. func (si *stackIterator) ProgramCounter() experimental.ProgramCounter { return experimental.ProgramCounter(si.pc) } // Function implements the same method as documented on experimental.StackIterator. func (si *stackIterator) Function() experimental.InternalFunction { return si } // Definition implements the same method as documented on experimental.InternalFunction. func (si *stackIterator) Definition() api.FunctionDefinition { return si.currentDef } // SourceOffsetForPC implements the same method as documented on experimental.InternalFunction. func (si *stackIterator) SourceOffsetForPC(pc experimental.ProgramCounter) uint64 { upc := uintptr(pc) cm := si.eng.compiledModuleOfAddr(upc) return cm.getSourceOffset(upc) }