wasi: adds fd_readdir (#865)

This adds an implementation of `fd_readdir` for WASI, which ensures a
very large directory is not kept in host memory until its directory is
closed.

Original implementation and test data are with thanks from @jerbob92.

Signed-off-by: Adrian Cole <adrian@tetrate.io>
Co-authored-by: Takeshi Yoneda <takeshi@tetrate.io>
Co-authored-by: jerbob92 <jerbob92@users.noreply.github.com>

Makefile

@@ -68,7 +68,7 @@ build.examples.tinygo: $(tinygo_sources)
 	done
 
 # We use zig to build C as it is easy to install and embeds a copy of zig-cc.
-c_sources := imports/wasi_snapshot_preview1/example/testdata/zig-cc/cat.c
+c_sources := imports/wasi_snapshot_preview1/example/testdata/zig-cc/cat.c imports/wasi_snapshot_preview1/testdata/zig-cc/ls.c
 .PHONY: build.examples.zig-cc
 build.examples.zig-cc: $(c_sources)
 	@for f in $^; do \
@@ -103,9 +103,10 @@ build.examples.emscripten: $(emscripten_sources)
 
 %/greet.wasm : cargo_target := wasm32-unknown-unknown
 %/cat.wasm : cargo_target := wasm32-wasi
+%/ls.wasm : cargo_target := wasm32-wasi
 
 .PHONY: build.examples.rust
-build.examples.rust: examples/allocation/rust/testdata/greet.wasm imports/wasi_snapshot_preview1/example/testdata/cargo-wasi/cat.wasm
+build.examples.rust: examples/allocation/rust/testdata/greet.wasm imports/wasi_snapshot_preview1/example/testdata/cargo-wasi/cat.wasm imports/wasi_snapshot_preview1/testdata/cargo-wasi/ls.wasm
 
 # Builds rust using cargo normally, or cargo-wasi.
 %.wasm: %.rs

imports/wasi_snapshot_preview1/fs.go

@@ -6,6 +6,7 @@ import (
 	"errors"
 	"io"
 	"io/fs"
+	"math"
 	"path"
 	"syscall"
 
@@ -643,11 +644,268 @@ func fdRead_shouldContinueRead(n, l uint32, err error) (bool, Errno) {
 // entries from a directory.
 //
 // See https://github.com/WebAssembly/WASI/blob/snapshot-01/phases/snapshot/docs.md#-fd_readdirfd-fd-buf-pointeru8-buf_len-size-cookie-dircookie---errno-size
-var fdReaddir = stubFunction(
-	functionFdReaddir,
-	[]wasm.ValueType{i32, i32, i32, i64, i32},
-	[]string{"fd", "buf", "buf_len", "cookie", "result.bufused"},
-)
+var fdReaddir = &wasm.HostFunc{
+	ExportNames: []string{functionFdReaddir},
+	Name:        functionFdReaddir,
+	ParamTypes:  []wasm.ValueType{i32, i32, i32, i64, i32},
+	ParamNames:  []string{"fd", "buf", "buf_len", "cookie", "result.bufused"},
+	ResultTypes: []api.ValueType{i32},
+	Code: &wasm.Code{
+		IsHostFunction: true,
+		GoFunc:         wasiFunc(fdReaddirFn),
+	},
+}
+
+func fdReaddirFn(ctx context.Context, mod api.Module, params []uint64) Errno {
+	fd := uint32(params[0])
+	buf := uint32(params[1])
+	bufLen := uint32(params[2])
+	// We control the value of the cookie, and it should never be negative.
+	// However, we coerce it to signed to ensure the caller doesn't manipulate
+	// it in such a way that becomes negative.
+	cookie := int64(params[3])
+	resultBufused := uint32(params[4])
+
+	// Validate the FD is a directory
+	rd, dir, errno := openedDir(ctx, mod, fd)
+	if errno != ErrnoSuccess {
+		return errno
+	}
+
+	// expect a cookie only if we are continuing a read.
+	if cookie == 0 && dir.CountRead > 0 {
+		return ErrnoInval // invalid as a cookie is minimally one.
+	}
+
+	// First, determine the maximum directory entries that can be encoded as
+	// dirents. The total size is direntSize(24) + nameSize, for each file.
+	// Since a zero-length file name is invalid, the minimum size entry is
+	// 25 (direntSize + 1 character).
+	maxDirEntries := int(bufLen/direntSize + 1)
+
+	// While unlikely maxDirEntries will fit into bufLen, add one more just in
+	// case, as we need to know if we hit the end of the directory or not to
+	// write the correct bufused (e.g. == bufLen unless EOF).
+	//	>> If less than the size of the read buffer, the end of the
+	//	>> directory has been reached.
+	maxDirEntries += 1
+
+	// The host keeps state for any unread entries from the prior call because
+	// we cannot seek to a previous directory position. Collect these entries.
+	entries, errno := lastDirEntries(dir, cookie)
+	if errno != ErrnoSuccess {
+		return errno
+	}
+
+	// Check if we have maxDirEntries, and read more from the FS as needed.
+	if entryCount := len(entries); entryCount < maxDirEntries {
+		if l, err := rd.ReadDir(maxDirEntries - entryCount); err != io.EOF {
+			if err != nil {
+				return ErrnoIo
+			}
+			dir.CountRead += uint64(len(l))
+			entries = append(entries, l...)
+			// Replace the cache with up to maxDirEntries, starting at cookie.
+			dir.Entries = entries
+		}
+	}
+
+	mem := mod.Memory()
+
+	// Determine how many dirents we can write, excluding a potentially
+	// truncated entry.
+	bufused, direntCount, writeTruncatedEntry := maxDirents(entries, bufLen)
+
+	// Now, write entries to the underlying buffer.
+	if bufused > 0 {
+
+		// d_next is the index of the next file in the list, so it should
+		// always be one higher than the requested cookie.
+		d_next := uint64(cookie + 1)
+		// ^^ yes this can overflow to negative, which means our implementation
+		// doesn't support writing greater than max int64 entries.
+
+		dirents, ok := mem.Read(ctx, buf, bufused)
+		if !ok {
+			return ErrnoFault
+		}
+
+		writeDirents(entries, direntCount, writeTruncatedEntry, dirents, d_next)
+	}
+
+	if !mem.WriteUint32Le(ctx, resultBufused, bufused) {
+		return ErrnoFault
+	}
+	return ErrnoSuccess
+}
+
+const largestDirent = int64(math.MaxUint32 - direntSize)
+
+// lastDirEntries is broken out from fdReaddirFn for testability.
+func lastDirEntries(dir *internalsys.ReadDir, cookie int64) (entries []fs.DirEntry, errno Errno) {
+	if cookie < 0 {
+		errno = ErrnoInval // invalid as we will never send a negative cookie.
+		return
+	}
+
+	entryCount := int64(len(dir.Entries))
+	if entryCount == 0 { // there was no prior call
+		if cookie != 0 {
+			errno = ErrnoInval // invalid as we haven't sent that cookie
+		}
+		return
+	}
+
+	// Get the first absolute position in our window of results
+	firstPos := int64(dir.CountRead) - entryCount
+	cookiePos := cookie - firstPos
+
+	switch {
+	case cookiePos < 0: // cookie is asking for results outside our window.
+		errno = ErrnoNosys // we can't implement directory seeking backwards.
+	case cookiePos == 0: // cookie is asking for the next page.
+	case cookiePos > entryCount:
+		errno = ErrnoInval // invalid as we read that far, yet.
+	case cookiePos > 0: // truncate so to avoid large lists.
+		entries = dir.Entries[cookiePos:]
+	default:
+		entries = dir.Entries
+	}
+	if len(entries) == 0 {
+		entries = nil
+	}
+	return
+}
+
+// direntSize is the size of the dirent struct, which should be followed by the
+// length of a file name.
+const direntSize = uint32(24)
+
+// maxDirents returns the maximum count and total entries that can fit in
+// maxLen bytes.
+//
+// truncatedEntryLen is the amount of bytes past bufLen needed to write the
+// next entry. We have to return bufused == bufLen unless the directory is
+// exhausted.
+//
+// See https://github.com/WebAssembly/WASI/blob/snapshot-01/phases/snapshot/docs.md#fd_readdir
+// See https://github.com/WebAssembly/wasi-libc/blob/659ff414560721b1660a19685110e484a081c3d4/libc-bottom-half/cloudlibc/src/libc/dirent/readdir.c#L44
+func maxDirents(entries []fs.DirEntry, bufLen uint32) (bufused, direntCount uint32, writeTruncatedEntry bool) {
+	lenRemaining := bufLen
+	for _, e := range entries {
+		if lenRemaining < direntSize {
+			// We don't have enough space in bufLen for another struct,
+			// entry. A caller who wants more will retry.
+
+			// bufused == bufLen means more entries exist, which is the case
+			// when the dirent is larger than bytes remaining.
+			bufused = bufLen
+			break
+		}
+
+		// use int64 to guard against huge filenames
+		nameLen := int64(len(e.Name()))
+		var entryLen uint32
+
+		// Check to see if direntSize + nameLen overflows, or if it would be
+		// larger than possible to encode.
+		if el := int64(direntSize) + nameLen; el < 0 || el > largestDirent {
+			// panic, as testing is difficult. ex we would have to extract a
+			// function to get size of a string or allocate a 2^32 size one!
+			panic("invalid filename: too large")
+		} else { // we know this can fit into a uint32
+			entryLen = uint32(el)
+		}
+
+		if entryLen > lenRemaining {
+			// We haven't room to write the entry, and docs say to write the
+			// header. This helps especially when there is an entry with a very
+			// long filename. Ex if bufLen is 4096 and the filename is 4096,
+			// we need to write direntSize(24) + 4096 bytes to write the entry.
+			// In this case, we only write up to direntSize(24) to allow the
+			// caller to resize.
+
+			// bufused == bufLen means more entries exist, which is the case
+			// when the next entry is larger than bytes remaining.
+			bufused = bufLen
+
+			// We do have enough space to write the header, this value will be
+			// passed on to writeDirents to only write the header for this entry.
+			writeTruncatedEntry = true
+			break
+		}
+
+		// This won't go negative because we checked entryLen <= lenRemaining.
+		lenRemaining -= entryLen
+		bufused += entryLen
+		direntCount++
+	}
+	return
+}
+
+// writeDirents writes the directory entries to the buffer, which is pre-sized
+// based on maxDirents.	truncatedEntryLen means write one past entryCount,
+// without its name. See maxDirents for why
+func writeDirents(
+	entries []fs.DirEntry,
+	entryCount uint32,
+	writeTruncatedEntry bool,
+	dirents []byte,
+	d_next uint64,
+) {
+	pos, i := uint32(0), uint32(0)
+	for ; i < entryCount; i++ {
+		e := entries[i]
+		nameLen := uint32(len(e.Name()))
+
+		writeDirent(dirents[pos:], d_next, nameLen, e.IsDir())
+		pos += direntSize
+
+		copy(dirents[pos:], e.Name())
+		pos += nameLen
+		d_next++
+	}
+
+	if !writeTruncatedEntry {
+		return
+	}
+
+	// Write a dirent without its name
+	dirent := make([]byte, direntSize)
+	e := entries[i]
+	writeDirent(dirent, d_next, uint32(len(e.Name())), e.IsDir())
+
+	// Potentially truncate it
+	copy(dirents[pos:], dirent)
+}
+
+// writeDirent writes direntSize bytes
+func writeDirent(buf []byte, dNext uint64, dNamlen uint32, dType bool) {
+	binary.LittleEndian.PutUint64(buf, dNext)        // d_next
+	binary.LittleEndian.PutUint64(buf[8:], 0)        // no d_ino
+	binary.LittleEndian.PutUint32(buf[16:], dNamlen) // d_namlen
+
+	filetype := wasiFiletypeRegularFile
+	if dType {
+		filetype = wasiFiletypeDirectory
+	}
+	binary.LittleEndian.PutUint32(buf[20:], uint32(filetype)) //  d_type
+}
+
+// openedDir returns the directory and ErrnoSuccess if the fd points to a readable directory.
+func openedDir(ctx context.Context, mod api.Module, fd uint32) (fs.ReadDirFile, *internalsys.ReadDir, Errno) {
+	fsc := mod.(*wasm.CallContext).Sys.FS(ctx)
+	if f, ok := fsc.OpenedFile(ctx, fd); !ok {
+		return nil, nil, ErrnoBadf
+	} else if d, ok := f.File.(fs.ReadDirFile); !ok {
+		return nil, nil, ErrnoNotdir
+	} else {
+		if f.ReadDir == nil {
+			f.ReadDir = &internalsys.ReadDir{}
+		}
+		return d, f.ReadDir, ErrnoSuccess
+	}
+}
 
 // fdRenumber is the WASI function named functionFdRenumber which atomically
 // replaces a file descriptor by renumbering another file descriptor.

imports/wasi_snapshot_preview1/fs_test.go

@@ -2,6 +2,7 @@ package wasi_snapshot_preview1
 
 import (
 	"bytes"
+	_ "embed"
 	"io"
 	"io/fs"
 	"math"
@@ -13,6 +14,7 @@ import (
 
 	"github.com/tetratelabs/wazero"
 	"github.com/tetratelabs/wazero/api"
+	internalsys "github.com/tetratelabs/wazero/internal/sys"
 	"github.com/tetratelabs/wazero/internal/testing/require"
 	"github.com/tetratelabs/wazero/internal/wasm"
 )
@@ -1004,15 +1006,712 @@ func Test_fdRead_shouldContinueRead(t *testing.T) {
 	}
 }
 
-// Test_fdReaddir only tests it is stubbed for GrainLang per #271
+var (
+	fdReadDirFs = fstest.MapFS{
+		"notdir":   {},
+		"emptydir": {Mode: fs.ModeDir},
+		"dir":      {Mode: fs.ModeDir},
+		"dir/-":    {},                 // len = 24+1 = 25
+		"dir/a-":   {Mode: fs.ModeDir}, // len = 24+2 = 26
+		"dir/ab-":  {},                 // len = 24+3 = 27
+	}
+
+	testDirEntries = func() []fs.DirEntry {
+		entries, err := fdReadDirFs.ReadDir("dir")
+		if err != nil {
+			panic(err)
+		}
+		return entries
+	}()
+
+	dirent1 = []byte{
+		1, 0, 0, 0, 0, 0, 0, 0, // d_next = 1
+		0, 0, 0, 0, 0, 0, 0, 0, // d_ino = 0
+		1, 0, 0, 0, // d_namlen = 1 character
+		4, 0, 0, 0, // d_type = regular_file
+		'-', // name
+	}
+	dirent2 = []byte{
+		2, 0, 0, 0, 0, 0, 0, 0, // d_next = 2
+		0, 0, 0, 0, 0, 0, 0, 0, // d_ino = 0
+		2, 0, 0, 0, // d_namlen = 1 character
+		3, 0, 0, 0, // d_type =  directory
+		'a', '-', // name
+	}
+	dirent3 = []byte{
+		3, 0, 0, 0, 0, 0, 0, 0, // d_next = 3
+		0, 0, 0, 0, 0, 0, 0, 0, // d_ino = 0
+		3, 0, 0, 0, // d_namlen = 3 characters
+		4, 0, 0, 0, // d_type = regular_file
+		'a', 'b', '-', // name
+	}
+)
+
 func Test_fdReaddir(t *testing.T) {
-	log := requireErrnoNosys(t, functionFdReaddir, 0, 0, 0, 0, 0)
-	require.Equal(t, `
---> proxy.fd_readdir(fd=0,buf=0,buf_len=0,cookie=0,result.bufused=0)
-	--> wasi_snapshot_preview1.fd_readdir(fd=0,buf=0,buf_len=0,cookie=0,result.bufused=0)
-	<-- ENOSYS
-<-- (52)
-`, log)
+	mod, r, log := requireProxyModule(t, wazero.NewModuleConfig().WithFS(fdReadDirFs))
+	defer r.Close(testCtx)
+
+	fsc := mod.(*wasm.CallContext).Sys.FS(testCtx)
+
+	fd, err := fsc.OpenFile(testCtx, "dir")
+	require.NoError(t, err)
+
+	tests := []struct {
+		name            string
+		dir             func() *internalsys.FileEntry
+		buf, bufLen     uint32
+		cookie          int64
+		expectedMem     []byte
+		expectedMemSize int
+		expectedBufused uint32
+		expectedReadDir *internalsys.ReadDir
+	}{
+		{
+			name: "empty dir",
+			dir: func() *internalsys.FileEntry {
+				dir, err := fdReadDirFs.Open("emptydir")
+				require.NoError(t, err)
+
+				return &internalsys.FileEntry{File: dir}
+			},
+			buf: 0, bufLen: 1,
+			cookie:          0,
+			expectedBufused: 0,
+			expectedMem:     []byte{},
+			expectedReadDir: &internalsys.ReadDir{},
+		},
+		{
+			name: "full read",
+			dir: func() *internalsys.FileEntry {
+				dir, err := fdReadDirFs.Open("dir")
+				require.NoError(t, err)
+
+				return &internalsys.FileEntry{File: dir}
+			},
+			buf: 0, bufLen: 4096,
+			cookie:          0,
+			expectedBufused: 78, // length of all entries
+			expectedMem:     append(append(dirent1, dirent2...), dirent3...),
+			expectedReadDir: &internalsys.ReadDir{
+				CountRead: 3,
+				Entries:   testDirEntries,
+			},
+		},
+		{
+			name: "can't read",
+			dir: func() *internalsys.FileEntry {
+				dir, err := fdReadDirFs.Open("dir")
+				require.NoError(t, err)
+
+				return &internalsys.FileEntry{File: dir}
+			},
+			buf: 0, bufLen: 23, // length is too short for header
+			cookie:          0,
+			expectedBufused: 23, // == bufLen which is the size of the dirent
+			expectedMem:     nil,
+			expectedReadDir: &internalsys.ReadDir{
+				CountRead: 2,
+				Entries:   testDirEntries[:2],
+			},
+		},
+		{
+			name: "can't read name",
+			dir: func() *internalsys.FileEntry {
+				dir, err := fdReadDirFs.Open("dir")
+				require.NoError(t, err)
+
+				return &internalsys.FileEntry{File: dir}
+			},
+			buf: 0, bufLen: 24, // length is long enough for first, but not the name.
+			cookie:          0,
+			expectedBufused: 24,           // == bufLen which is the size of the dirent
+			expectedMem:     dirent1[:24], // header without name
+			expectedReadDir: &internalsys.ReadDir{
+				CountRead: 3,
+				Entries:   testDirEntries,
+			},
+		},
+		{
+			name: "read exactly first",
+			dir: func() *internalsys.FileEntry {
+				dir, err := fdReadDirFs.Open("dir")
+				require.NoError(t, err)
+
+				return &internalsys.FileEntry{File: dir}
+			},
+			buf: 0, bufLen: 25, // length is long enough for first + the name, but not more.
+			cookie:          0,
+			expectedBufused: 25, // length to read exactly first.
+			expectedMem:     dirent1,
+			expectedReadDir: &internalsys.ReadDir{
+				CountRead: 3,
+				Entries:   testDirEntries,
+			},
+		},
+		{
+			name: "read exactly second",
+			dir: func() *internalsys.FileEntry {
+				dir, err := fdReadDirFs.Open("dir")
+				require.NoError(t, err)
+				entry, err := dir.(fs.ReadDirFile).ReadDir(1)
+				require.NoError(t, err)
+
+				return &internalsys.FileEntry{
+					File: dir,
+					ReadDir: &internalsys.ReadDir{
+						CountRead: 1,
+						Entries:   entry,
+					},
+				}
+			},
+			buf: 0, bufLen: 26, // length is long enough for exactly second.
+			cookie:          1,  // d_next of first
+			expectedBufused: 26, // length to read exactly second.
+			expectedMem:     dirent2,
+			expectedReadDir: &internalsys.ReadDir{
+				CountRead: 3,
+				Entries:   testDirEntries[1:],
+			},
+		},
+		{
+			name: "read second and a little more",
+			dir: func() *internalsys.FileEntry {
+				dir, err := fdReadDirFs.Open("dir")
+				require.NoError(t, err)
+				entry, err := dir.(fs.ReadDirFile).ReadDir(1)
+				require.NoError(t, err)
+
+				return &internalsys.FileEntry{
+					File: dir,
+					ReadDir: &internalsys.ReadDir{
+						CountRead: 1,
+						Entries:   entry,
+					},
+				}
+			},
+			buf: 0, bufLen: 30, // length is longer than the second entry, but not long enough for a header.
+			cookie:          1,  // d_next of first
+			expectedBufused: 30, // length to read some more, but not enough for a header, so buf was exhausted.
+			expectedMem:     dirent2,
+			expectedMemSize: len(dirent2), // we do not want to compare the full buffer since we don't know what the leftover 4 bytes will contain.
+			expectedReadDir: &internalsys.ReadDir{
+				CountRead: 3,
+				Entries:   testDirEntries[1:],
+			},
+		},
+		{
+			name: "read second and header of third",
+			dir: func() *internalsys.FileEntry {
+				dir, err := fdReadDirFs.Open("dir")
+				require.NoError(t, err)
+				entry, err := dir.(fs.ReadDirFile).ReadDir(1)
+				require.NoError(t, err)
+
+				return &internalsys.FileEntry{
+					File: dir,
+					ReadDir: &internalsys.ReadDir{
+						CountRead: 1,
+						Entries:   entry,
+					},
+				}
+			},
+			buf: 0, bufLen: 50, // length is longer than the second entry + enough for the header of third.
+			cookie:          1,  // d_next of first
+			expectedBufused: 50, // length to read exactly second and the header of third.
+			expectedMem:     append(dirent2, dirent3[0:24]...),
+			expectedReadDir: &internalsys.ReadDir{
+				CountRead: 3,
+				Entries:   testDirEntries[1:],
+			},
+		},
+		{
+			name: "read second and third",
+			dir: func() *internalsys.FileEntry {
+				dir, err := fdReadDirFs.Open("dir")
+				require.NoError(t, err)
+				entry, err := dir.(fs.ReadDirFile).ReadDir(1)
+				require.NoError(t, err)
+
+				return &internalsys.FileEntry{
+					File: dir,
+					ReadDir: &internalsys.ReadDir{
+						CountRead: 1,
+						Entries:   entry,
+					},
+				}
+			},
+			buf: 0, bufLen: 53, // length is long enough for second and third.
+			cookie:          1,  // d_next of first
+			expectedBufused: 53, // length to read exactly one second and third.
+			expectedMem:     append(dirent2, dirent3...),
+			expectedReadDir: &internalsys.ReadDir{
+				CountRead: 3,
+				Entries:   testDirEntries[1:],
+			},
+		},
+		{
+			name: "read exactly third",
+			dir: func() *internalsys.FileEntry {
+				dir, err := fdReadDirFs.Open("dir")
+				require.NoError(t, err)
+				two, err := dir.(fs.ReadDirFile).ReadDir(2)
+				require.NoError(t, err)
+
+				return &internalsys.FileEntry{
+					File: dir,
+					ReadDir: &internalsys.ReadDir{
+						CountRead: 2,
+						Entries:   two[1:],
+					},
+				}
+			},
+			buf: 0, bufLen: 27, // length is long enough for exactly third.
+			cookie:          2,  // d_next of second.
+			expectedBufused: 27, // length to read exactly third.
+			expectedMem:     dirent3,
+			expectedReadDir: &internalsys.ReadDir{
+				CountRead: 3,
+				Entries:   testDirEntries[2:],
+			},
+		},
+		{
+			name: "read third and beyond",
+			dir: func() *internalsys.FileEntry {
+				dir, err := fdReadDirFs.Open("dir")
+				require.NoError(t, err)
+				two, err := dir.(fs.ReadDirFile).ReadDir(2)
+				require.NoError(t, err)
+
+				return &internalsys.FileEntry{
+					File: dir,
+					ReadDir: &internalsys.ReadDir{
+						CountRead: 2,
+						Entries:   two[1:],
+					},
+				}
+			},
+			buf: 0, bufLen: 100, // length is long enough for third and more, but there is nothing more.
+			cookie:          2,  // d_next of second.
+			expectedBufused: 27, // length to read exactly third.
+			expectedMem:     dirent3,
+			expectedReadDir: &internalsys.ReadDir{
+				CountRead: 3,
+				Entries:   testDirEntries[2:],
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		tc := tt
+		t.Run(tc.name, func(t *testing.T) {
+			defer log.Reset()
+
+			// Assign the state we are testing
+			file, ok := fsc.OpenedFile(testCtx, fd)
+			require.True(t, ok)
+			dir := tc.dir()
+			defer dir.File.Close()
+
+			file.File = dir.File
+			file.ReadDir = dir.ReadDir
+
+			maskMemory(t, testCtx, mod, int(tc.bufLen))
+
+			// use an arbitrarily high value for the buf used position.
+			resultBufused := uint32(16192)
+			requireErrno(t, ErrnoSuccess, mod, functionFdReaddir,
+				uint64(fd), uint64(tc.buf), uint64(tc.bufLen), uint64(tc.cookie), uint64(resultBufused))
+
+			// read back the bufused and compare memory against it
+			bufUsed, ok := mod.Memory().ReadUint32Le(testCtx, resultBufused)
+			require.True(t, ok)
+			require.Equal(t, tc.expectedBufused, bufUsed)
+
+			mem, ok := mod.Memory().Read(testCtx, tc.buf, bufUsed)
+			require.True(t, ok)
+
+			if tc.expectedMem != nil {
+				if tc.expectedMemSize == 0 {
+					tc.expectedMemSize = len(tc.expectedMem)
+				}
+				require.Equal(t, tc.expectedMem, mem[:tc.expectedMemSize])
+			}
+
+			require.Equal(t, tc.expectedReadDir, file.ReadDir)
+		})
+	}
+}
+
+func Test_fdReaddir_Errors(t *testing.T) {
+	mod, r, log := requireProxyModule(t, wazero.NewModuleConfig().WithFS(fdReadDirFs))
+	defer r.Close(testCtx)
+	memLen := mod.Memory().Size(testCtx)
+
+	fsc := mod.(*wasm.CallContext).Sys.FS(testCtx)
+
+	dirFD, err := fsc.OpenFile(testCtx, "dir")
+	require.NoError(t, err)
+
+	fileFD, err := fsc.OpenFile(testCtx, "notdir")
+	require.NoError(t, err)
+
+	tests := []struct {
+		name                           string
+		dir                            func() *internalsys.FileEntry
+		fd, buf, bufLen, resultBufused uint32
+		cookie                         int64
+		readDir                        *internalsys.ReadDir
+		expectedErrno                  Errno
+		expectedLog                    string
+	}{
+		{
+			name:          "out-of-memory reading buf",
+			fd:            dirFD,
+			buf:           memLen,
+			bufLen:        1000,
+			expectedErrno: ErrnoFault,
+			expectedLog: `
+--> proxy.fd_readdir(fd=4,buf=65536,buf_len=1000,cookie=0,result.bufused=0)
+	==> wasi_snapshot_preview1.fd_readdir(fd=4,buf=65536,buf_len=1000,cookie=0,result.bufused=0)
+	<== EFAULT
+<-- (21)
+`,
+		},
+		{
+			name:          "invalid fd",
+			fd:            42, // arbitrary invalid fd
+			expectedErrno: ErrnoBadf,
+			expectedLog: `
+--> proxy.fd_readdir(fd=42,buf=0,buf_len=0,cookie=0,result.bufused=0)
+	==> wasi_snapshot_preview1.fd_readdir(fd=42,buf=0,buf_len=0,cookie=0,result.bufused=0)
+	<== EBADF
+<-- (8)
+`,
+		},
+		{
+			name:          "not a dir",
+			fd:            fileFD,
+			expectedErrno: ErrnoNotdir,
+			expectedLog: `
+--> proxy.fd_readdir(fd=5,buf=0,buf_len=0,cookie=0,result.bufused=0)
+	==> wasi_snapshot_preview1.fd_readdir(fd=5,buf=0,buf_len=0,cookie=0,result.bufused=0)
+	<== ENOTDIR
+<-- (54)
+`,
+		},
+		{
+			name:          "out-of-memory reading buf",
+			fd:            dirFD,
+			buf:           memLen,
+			bufLen:        1000,
+			expectedErrno: ErrnoFault,
+			expectedLog: `
+--> proxy.fd_readdir(fd=4,buf=65536,buf_len=1000,cookie=0,result.bufused=0)
+	==> wasi_snapshot_preview1.fd_readdir(fd=4,buf=65536,buf_len=1000,cookie=0,result.bufused=0)
+	<== EFAULT
+<-- (21)
+`,
+		},
+		{
+			name:          "out-of-memory reading bufLen",
+			fd:            dirFD,
+			buf:           memLen - 1,
+			bufLen:        1000,
+			expectedErrno: ErrnoFault,
+			expectedLog: `
+--> proxy.fd_readdir(fd=4,buf=65535,buf_len=1000,cookie=0,result.bufused=0)
+	==> wasi_snapshot_preview1.fd_readdir(fd=4,buf=65535,buf_len=1000,cookie=0,result.bufused=0)
+	<== EFAULT
+<-- (21)
+`,
+		},
+		{
+			name: "resultBufused is outside memory",
+			fd:   dirFD,
+			buf:  0, bufLen: 1,
+			resultBufused: memLen,
+			expectedErrno: ErrnoFault,
+			expectedLog: `
+--> proxy.fd_readdir(fd=4,buf=0,buf_len=1,cookie=0,result.bufused=65536)
+	==> wasi_snapshot_preview1.fd_readdir(fd=4,buf=0,buf_len=1,cookie=0,result.bufused=65536)
+	<== EFAULT
+<-- (21)
+`,
+		},
+		{
+			name: "cookie invalid when no prior state",
+			fd:   dirFD,
+			buf:  0, bufLen: 1000,
+			cookie:        1,
+			resultBufused: 2000,
+			expectedErrno: ErrnoInval,
+			expectedLog: `
+--> proxy.fd_readdir(fd=4,buf=0,buf_len=1000,cookie=1,result.bufused=2000)
+	==> wasi_snapshot_preview1.fd_readdir(fd=4,buf=0,buf_len=1000,cookie=1,result.bufused=2000)
+	<== EINVAL
+<-- (28)
+`,
+		},
+		{
+			name: "negative cookie invalid",
+			fd:   dirFD,
+			buf:  0, bufLen: 1000,
+			cookie:        -1,
+			readDir:       &internalsys.ReadDir{CountRead: 1},
+			resultBufused: 2000,
+			expectedErrno: ErrnoInval,
+			expectedLog: `
+--> proxy.fd_readdir(fd=4,buf=0,buf_len=1000,cookie=18446744073709551615,result.bufused=2000)
+	==> wasi_snapshot_preview1.fd_readdir(fd=4,buf=0,buf_len=1000,cookie=18446744073709551615,result.bufused=2000)
+	<== EINVAL
+<-- (28)
+`,
+		},
+	}
+
+	for _, tt := range tests {
+		tc := tt
+		t.Run(tc.name, func(t *testing.T) {
+			defer log.Reset()
+
+			// Reset the directory so that tests don't taint each other.
+			if file, ok := fsc.OpenedFile(testCtx, tc.fd); ok && tc.fd == dirFD {
+				dir, err := fdReadDirFs.Open("dir")
+				require.NoError(t, err)
+				defer dir.Close()
+
+				file.File = dir
+				file.ReadDir = nil
+			}
+
+			requireErrno(t, tc.expectedErrno, mod, functionFdReaddir,
+				uint64(tc.fd), uint64(tc.buf), uint64(tc.bufLen), uint64(tc.cookie), uint64(tc.resultBufused))
+			require.Equal(t, tc.expectedLog, "\n"+log.String())
+		})
+	}
+}
+
+func Test_lastDirEntries(t *testing.T) {
+	tests := []struct {
+		name            string
+		f               *internalsys.ReadDir
+		cookie          int64
+		expectedEntries []fs.DirEntry
+		expectedErrno   Errno
+	}{
+		{
+			name: "no prior call",
+		},
+		{
+			name:          "no prior call, but passed a cookie",
+			cookie:        1,
+			expectedErrno: ErrnoInval,
+		},
+		{
+			name: "cookie is negative",
+			f: &internalsys.ReadDir{
+				CountRead: 3,
+				Entries:   testDirEntries,
+			},
+			cookie:        -1,
+			expectedErrno: ErrnoInval,
+		},
+		{
+			name: "cookie is greater than last d_next",
+			f: &internalsys.ReadDir{
+				CountRead: 3,
+				Entries:   testDirEntries,
+			},
+			cookie:        5,
+			expectedErrno: ErrnoInval,
+		},
+		{
+			name: "cookie is last pos",
+			f: &internalsys.ReadDir{
+				CountRead: 3,
+				Entries:   testDirEntries,
+			},
+			cookie:          3,
+			expectedEntries: nil,
+		},
+		{
+			name: "cookie is one before last pos",
+			f: &internalsys.ReadDir{
+				CountRead: 3,
+				Entries:   testDirEntries,
+			},
+			cookie:          2,
+			expectedEntries: testDirEntries[2:],
+		},
+		{
+			name: "cookie is before current entries",
+			f: &internalsys.ReadDir{
+				CountRead: 5,
+				Entries:   testDirEntries,
+			},
+			cookie:        1,
+			expectedErrno: ErrnoNosys, // not implemented
+		},
+	}
+
+	for _, tt := range tests {
+		tc := tt
+
+		t.Run(tc.name, func(t *testing.T) {
+			f := tc.f
+			if f == nil {
+				f = &internalsys.ReadDir{}
+			}
+			entries, errno := lastDirEntries(f, tc.cookie)
+			require.Equal(t, tc.expectedErrno, errno)
+			require.Equal(t, tc.expectedEntries, entries)
+		})
+	}
+}
+
+func Test_maxDirents(t *testing.T) {
+	tests := []struct {
+		name                        string
+		entries                     []fs.DirEntry
+		maxLen                      uint32
+		expectedCount               uint32
+		expectedwriteTruncatedEntry bool
+		expectedBufused             uint32
+	}{
+		{
+			name: "no entries",
+		},
+		{
+			name:                        "can't fit one",
+			entries:                     testDirEntries,
+			maxLen:                      23,
+			expectedBufused:             23,
+			expectedwriteTruncatedEntry: false,
+		},
+		{
+			name:                        "only fits header",
+			entries:                     testDirEntries,
+			maxLen:                      24,
+			expectedBufused:             24,
+			expectedwriteTruncatedEntry: true,
+		},
+		{
+			name:            "one",
+			entries:         testDirEntries,
+			maxLen:          25,
+			expectedCount:   1,
+			expectedBufused: 25,
+		},
+		{
+			name:                        "one but not room for two's name",
+			entries:                     testDirEntries,
+			maxLen:                      25 + 25,
+			expectedCount:               1,
+			expectedwriteTruncatedEntry: true, // can write direntSize
+			expectedBufused:             25 + 25,
+		},
+		{
+			name:            "two",
+			entries:         testDirEntries,
+			maxLen:          25 + 26,
+			expectedCount:   2,
+			expectedBufused: 25 + 26,
+		},
+		{
+			name:                        "two but not three's dirent",
+			entries:                     testDirEntries,
+			maxLen:                      25 + 26 + 20,
+			expectedCount:               2,
+			expectedwriteTruncatedEntry: false, // 20 + 4 == direntSize
+			expectedBufused:             25 + 26 + 20,
+		},
+		{
+			name:                        "two but not three's name",
+			entries:                     testDirEntries,
+			maxLen:                      25 + 26 + 26,
+			expectedCount:               2,
+			expectedwriteTruncatedEntry: true, // can write direntSize
+			expectedBufused:             25 + 26 + 26,
+		},
+		{
+			name:                        "three",
+			entries:                     testDirEntries,
+			maxLen:                      25 + 26 + 27,
+			expectedCount:               3,
+			expectedwriteTruncatedEntry: false, // end of dir
+			expectedBufused:             25 + 26 + 27,
+		},
+		{
+			name:                        "max",
+			entries:                     testDirEntries,
+			maxLen:                      100,
+			expectedCount:               3,
+			expectedwriteTruncatedEntry: false, // end of dir
+			expectedBufused:             25 + 26 + 27,
+		},
+	}
+
+	for _, tt := range tests {
+		tc := tt
+
+		t.Run(tc.name, func(t *testing.T) {
+			bufused, direntCount, writeTruncatedEntry := maxDirents(tc.entries, tc.maxLen)
+			require.Equal(t, tc.expectedCount, direntCount)
+			require.Equal(t, tc.expectedwriteTruncatedEntry, writeTruncatedEntry)
+			require.Equal(t, tc.expectedBufused, bufused)
+		})
+	}
+}
+
+func Test_writeDirents(t *testing.T) {
+	tests := []struct {
+		name                string
+		entries             []fs.DirEntry
+		entryCount          uint32
+		writeTruncatedEntry bool
+		expectedEntriesBuf  []byte
+	}{
+		{
+			name:    "none",
+			entries: testDirEntries,
+		},
+		{
+			name:               "one",
+			entries:            testDirEntries,
+			entryCount:         1,
+			expectedEntriesBuf: dirent1,
+		},
+		{
+			name:               "two",
+			entries:            testDirEntries,
+			entryCount:         2,
+			expectedEntriesBuf: append(dirent1, dirent2...),
+		},
+		{
+			name:                "two with truncated",
+			entries:             testDirEntries,
+			entryCount:          2,
+			writeTruncatedEntry: true,
+			expectedEntriesBuf:  append(append(dirent1, dirent2...), dirent3[0:10]...),
+		},
+		{
+			name:               "three",
+			entries:            testDirEntries,
+			entryCount:         3,
+			expectedEntriesBuf: append(append(dirent1, dirent2...), dirent3...),
+		},
+	}
+
+	for _, tt := range tests {
+		tc := tt
+
+		t.Run(tc.name, func(t *testing.T) {
+			cookie := uint64(1)
+			entriesBuf := make([]byte, len(tc.expectedEntriesBuf))
+			writeDirents(tc.entries, tc.entryCount, tc.writeTruncatedEntry, entriesBuf, cookie)
+			require.Equal(t, tc.expectedEntriesBuf, entriesBuf)
+		})
+	}
 }
 
 // Test_fdRenumber only tests it is stubbed for GrainLang per #271

imports/wasi_snapshot_preview1/testdata/cargo-wasi/.gitignore

@@ -0,0 +1,2 @@
+/target
+Cargo.lock

imports/wasi_snapshot_preview1/testdata/cargo-wasi/Cargo.toml

@@ -0,0 +1,8 @@
+[package]
+name = "ls"
+version = "0.1.0"
+edition = "2021"
+
+[[bin]]
+name = "ls"
+path = "ls.rs"

imports/wasi_snapshot_preview1/testdata/cargo-wasi/ls.rs

@@ -0,0 +1,7 @@
+use std::fs;
+
+fn main() {
+    for path in fs::read_dir(".").unwrap() {
+        println!("{}", path.unwrap().path().display())
+    }
+}

imports/wasi_snapshot_preview1/testdata/zig-cc/ls.c

@@ -0,0 +1,15 @@
+#include <dirent.h>
+#include <stdio.h>
+
+int main(void) {
+  DIR *d;
+  struct dirent *dir;
+  d = opendir(".");
+  if (d) {
+    while ((dir = readdir(d)) != NULL) {
+      printf("./%s\n", dir->d_name);
+    }
+    closedir(d);
+  }
+  return(0);
+}

imports/wasi_snapshot_preview1/wasi_stdlib_test.go

@@ -0,0 +1,103 @@
+package wasi_snapshot_preview1
+
+import (
+	"bytes"
+	_ "embed"
+	"io/fs"
+	"strconv"
+	"strings"
+	"testing"
+	"testing/fstest"
+
+	"github.com/tetratelabs/wazero"
+	"github.com/tetratelabs/wazero/internal/testing/require"
+	"github.com/tetratelabs/wazero/sys"
+)
+
+// lsWasmCargoWasi was compiled from testdata/cargo-wasi/ls.rs
+//
+//go:embed testdata/cargo-wasi/ls.wasm
+var lsWasmCargoWasi []byte
+
+// lsZigCc was compiled from testdata/zig-cc/ls.c
+//
+//go:embed testdata/zig-cc/ls.wasm
+var lsZigCc []byte
+
+// Test_fdReaddir_ls ensures that the behavior we've implemented not only
+// matches the wasi spec, but also at least two compilers use of sdks.
+func Test_fdReaddir_ls(t *testing.T) {
+	for toolchain, bin := range map[string][]byte{
+		"cargo-wasi": lsWasmCargoWasi,
+		"zig-cc":     lsZigCc,
+	} {
+		toolchain := toolchain
+		bin := bin
+		t.Run(toolchain, func(t *testing.T) {
+			testFdReaddirLs(t, bin)
+		})
+	}
+}
+
+func testFdReaddirLs(t *testing.T, bin []byte) {
+	t.Run("empty directory", func(t *testing.T) {
+		stdout, stderr := compileAndRun(t, wazero.NewModuleConfig().
+			WithFS(fstest.MapFS{}), bin)
+
+		require.Zero(t, stderr)
+		require.Zero(t, stdout)
+	})
+
+	t.Run("directory with entries", func(t *testing.T) {
+		stdout, stderr := compileAndRun(t, wazero.NewModuleConfig().
+			WithFS(fstest.MapFS{
+				"-":   {},
+				"a-":  {Mode: fs.ModeDir},
+				"ab-": {},
+			}), bin)
+
+		require.Zero(t, stderr)
+		require.Equal(t, `./-
+./a-
+./ab-
+`, stdout)
+	})
+
+	t.Run("directory with tons of entries", func(t *testing.T) {
+		testFS := fstest.MapFS{}
+		count := 8096
+		for i := 0; i < count; i++ {
+			testFS[strconv.Itoa(i)] = &fstest.MapFile{}
+		}
+		stdout, stderr := compileAndRun(t, wazero.NewModuleConfig().
+			WithFS(testFS), bin)
+
+		require.Zero(t, stderr)
+		lines := strings.Split(stdout, "\n")
+		require.Equal(t, count+1 /* trailing newline */, len(lines))
+	})
+}
+
+func compileAndRun(t *testing.T, config wazero.ModuleConfig, bin []byte) (stdout, stderr string) {
+	var stdoutBuf, stderrBuf bytes.Buffer
+
+	r := wazero.NewRuntime(testCtx)
+	defer r.Close(testCtx)
+
+	_, err := Instantiate(testCtx, r)
+	require.NoError(t, err)
+
+	compiled, err := r.CompileModule(testCtx, bin)
+	require.NoError(t, err)
+
+	_, err = r.InstantiateModule(testCtx, compiled, config.WithStdout(&stdoutBuf).WithStderr(&stderrBuf))
+	if exitErr, ok := err.(*sys.ExitError); ok {
+		require.Zero(t, exitErr.ExitCode())
+	} else {
+		require.NoError(t, err)
+	}
+
+	stdout = stdoutBuf.String()
+	stderr = stderrBuf.String()
+	return
+}

internal/sys/fs.go

@@ -39,9 +39,26 @@ func (f *emptyFS) Open(name string) (fs.File, error) {
 
 // FileEntry maps a path to an open file in a file system.
 type FileEntry struct {
+	// Path was the argument to FSContext.OpenFile
 	Path string
+
 	// File when nil this is the root "/" (fd=3)
 	File fs.File
+
+	// ReadDir is present when this File is a fs.ReadDirFile and `ReadDir`
+	// was called.
+	ReadDir *ReadDir
+}
+
+// ReadDir is the status of a prior fs.ReadDirFile call.
+type ReadDir struct {
+	// CountRead is the total count of files read including Entries.
+	CountRead uint64
+
+	// Entries is the contents of the last fs.ReadDirFile call. Notably,
+	// directory listing are not rewindable, so we keep entries around in case
+	// the caller mis-estimated their buffer and needs a few still cached.
+	Entries []fs.DirEntry
 }
 
 type FSContext struct {