experimental: allow custom memory allocator Reallocate to fail (#2315)

Signed-off-by: Nuno Cruces <ncruces@users.noreply.github.com>

experimental/memory.go

@@ -35,6 +35,8 @@ type LinearMemory interface {
 	// Notes:
 	//   - To back a shared memory, Reallocate can't change the address of the
 	//     backing []byte (only its length/capacity may change).
+	//   - Reallocate may return nil if fails to grow the LinearMemory. This
+	//     condition may or may not be handled gracefully by the Wasm module.
 	Reallocate(size uint64) []byte
 	// Free the backing memory buffer.
 	Free()

internal/wasm/memory.go

@@ -77,6 +77,7 @@ func NewMemoryInstance(memSec *Memory, allocator experimental.MemoryAllocator, m
 	if allocator != nil {
 		expBuffer = allocator.Allocate(capBytes, maxBytes)
 		buffer = expBuffer.Reallocate(minBytes)
+		_ = buffer[:minBytes] // Bounds check that the minimum was allocated.
 	} else if memSec.IsShared {
 		// Shared memory needs a fixed buffer, so allocate with the maximum size.
 		//
@@ -238,12 +239,15 @@ func (m *MemoryInstance) Grow(delta uint32) (result uint32, ok bool) {
 		return currentPages, true
 	}
 
-	// If exceeds the max of memory size, we push -1 according to the spec.
 	newPages := currentPages + delta
 	if newPages > m.Max || int32(delta) < 0 {
 		return 0, false
 	} else if m.expBuffer != nil {
 		buffer := m.expBuffer.Reallocate(MemoryPagesToBytesNum(newPages))
+		if buffer == nil {
+			// Allocator failed to grow.
+			return 0, false
+		}
 		if m.Shared {
 			if unsafe.SliceData(buffer) != unsafe.SliceData(m.Buffer) {
 				panic("shared memory cannot move, this is a bug in the memory allocator")

internal/wasm/memory_test.go

@@ -33,13 +33,15 @@ func TestMemoryBytesNumToPages(t *testing.T) {
 
 func TestMemoryInstance_Grow_Size(t *testing.T) {
 	tests := []struct {
-		name         string
-		capEqualsMax bool
-		expAllocator bool
+		name          string
+		capEqualsMax  bool
+		expAllocator  bool
+		failAllocator bool
 	}{
 		{name: ""},
 		{name: "capEqualsMax", capEqualsMax: true},
 		{name: "expAllocator", expAllocator: true},
+		{name: "failAllocator", failAllocator: true},
 	}
 
 	for _, tt := range tests {
@@ -58,6 +60,9 @@ func TestMemoryInstance_Grow_Size(t *testing.T) {
 			case tc.expAllocator:
 				expBuffer := sliceAllocator(0, maxBytes)
 				m = &MemoryInstance{Max: max, Buffer: expBuffer.Reallocate(0), expBuffer: expBuffer}
+			case tc.failAllocator:
+				expBuffer := sliceAllocator(0, maxBytes)
+				m = &MemoryInstance{Max: max * 2, Buffer: expBuffer.Reallocate(0), expBuffer: expBuffer}
 			}
 			m.ownerModuleEngine = me
 
@@ -1023,6 +1028,9 @@ type sliceBuffer struct {
 func (b *sliceBuffer) Free() {}
 
 func (b *sliceBuffer) Reallocate(size uint64) []byte {
+	if size > b.max {
+		return nil
+	}
 	if cap := uint64(cap(b.buf)); size > cap {
 		b.buf = append(b.buf[:cap], make([]byte, size-cap)...)
 	} else {