wasi: fix nonblocking sockets on *NIX (gotip net/http) (#1503)

Signed-off-by: Adrian Cole <adrian@tetrate.io>
Co-authored-by: Edoardo Vacchi <evacchi@users.noreply.github.com>

internal/sysfs/sock.go

@@ -6,149 +6,32 @@ import (
 	"syscall"
 
 	"github.com/tetratelabs/wazero/internal/fsapi"
-	"github.com/tetratelabs/wazero/internal/platform"
 	socketapi "github.com/tetratelabs/wazero/internal/sock"
 )
 
+// NewTCPListenerFile creates a socketapi.TCPSock for a given *net.TCPListener.
 func NewTCPListenerFile(tl *net.TCPListener) socketapi.TCPSock {
-	return &tcpListenerFile{tl: tl}
+	return newTCPListenerFile(tl)
 }
 
-var _ socketapi.TCPSock = (*tcpListenerFile)(nil)
-
-type tcpListenerFile struct {
+// baseSockFile implements base behavior for all TCPSock, TCPConn files,
+// regardless the platform.
+type baseSockFile struct {
 	fsapi.UnimplementedFile
-
-	tl *net.TCPListener
 }
 
-// Accept implements the same method as documented on socketapi.TCPSock
-func (f *tcpListenerFile) Accept() (socketapi.TCPConn, syscall.Errno) {
-	conn, err := f.tl.Accept()
-	if err != nil {
-		return nil, platform.UnwrapOSError(err)
-	}
-	return &tcpConnFile{tc: conn.(*net.TCPConn)}, 0
-}
+var _ fsapi.File = (*baseSockFile)(nil)
 
 // IsDir implements the same method as documented on File.IsDir
-func (*tcpListenerFile) IsDir() (bool, syscall.Errno) {
+func (*baseSockFile) IsDir() (bool, syscall.Errno) {
 	// We need to override this method because WASI-libc prestats the FD
 	// and the default impl returns ENOSYS otherwise.
 	return false, 0
 }
 
 // Stat implements the same method as documented on File.Stat
-func (f *tcpListenerFile) Stat() (fs fsapi.Stat_t, errno syscall.Errno) {
+func (f *baseSockFile) Stat() (fs fsapi.Stat_t, errno syscall.Errno) {
 	// The mode is not really important, but it should be neither a regular file nor a directory.
 	fs.Mode = os.ModeIrregular
 	return
 }
-
-// Close implements the same method as documented on fsapi.File
-func (f *tcpListenerFile) Close() syscall.Errno {
-	return platform.UnwrapOSError(f.tl.Close())
-}
-
-// Addr is exposed for testing.
-func (f *tcpListenerFile) Addr() *net.TCPAddr {
-	return f.tl.Addr().(*net.TCPAddr)
-}
-
-var _ socketapi.TCPConn = (*tcpConnFile)(nil)
-
-type tcpConnFile struct {
-	fsapi.UnimplementedFile
-
-	tc *net.TCPConn
-
-	// closed is true when closed was called. This ensures proper syscall.EBADF
-	closed bool
-}
-
-// IsDir implements the same method as documented on File.IsDir
-func (*tcpConnFile) IsDir() (bool, syscall.Errno) {
-	// We need to override this method because WASI-libc prestats the FD
-	// and the default impl returns ENOSYS otherwise.
-	return false, 0
-}
-
-// Stat implements the same method as documented on File.Stat
-func (f *tcpConnFile) Stat() (fs fsapi.Stat_t, errno syscall.Errno) {
-	// The mode is not really important, but it should be neither a regular file nor a directory.
-	fs.Mode = os.ModeIrregular
-	return
-}
-
-// SetNonblock implements the same method as documented on fsapi.File
-func (f *tcpConnFile) SetNonblock(enabled bool) (errno syscall.Errno) {
-	syscallConn, err := f.tc.SyscallConn()
-	if err != nil {
-		return platform.UnwrapOSError(err)
-	}
-
-	// Prioritize the error from setNonblock over Control
-	if controlErr := syscallConn.Control(func(fd uintptr) {
-		errno = platform.UnwrapOSError(setNonblock(fd, enabled))
-	}); errno == 0 {
-		errno = platform.UnwrapOSError(controlErr)
-	}
-	return
-}
-
-// Read implements the same method as documented on fsapi.File
-func (f *tcpConnFile) Read(buf []byte) (n int, errno syscall.Errno) {
-	if n, errno = read(f.tc, buf); errno != 0 {
-		// Defer validation overhead until we've already had an error.
-		errno = fileError(f, f.closed, errno)
-	}
-	return
-}
-
-// Write implements the same method as documented on fsapi.File
-func (f *tcpConnFile) Write(buf []byte) (n int, errno syscall.Errno) {
-	if n, errno = write(f.tc, buf); errno != 0 {
-		// Defer validation overhead until we've alwritey had an error.
-		errno = fileError(f, f.closed, errno)
-	}
-	return
-}
-
-// Recvfrom implements the same method as documented on socketapi.TCPConn
-func (f *tcpConnFile) Recvfrom(p []byte, flags int) (n int, errno syscall.Errno) {
-	if flags != MSG_PEEK {
-		errno = syscall.EINVAL
-		return
-	}
-	return recvfromPeek(f.tc, p)
-}
-
-// Shutdown implements the same method as documented on fsapi.Conn
-func (f *tcpConnFile) Shutdown(how int) syscall.Errno {
-	// FIXME: can userland shutdown listeners?
-	var err error
-	switch how {
-	case syscall.SHUT_RD:
-		err = f.tc.CloseRead()
-	case syscall.SHUT_WR:
-		err = f.tc.CloseWrite()
-	case syscall.SHUT_RDWR:
-		return f.close()
-	default:
-		return syscall.EINVAL
-	}
-	return platform.UnwrapOSError(err)
-}
-
-// Close implements the same method as documented on fsapi.File
-func (f *tcpConnFile) Close() syscall.Errno {
-	return f.close()
-}
-
-func (f *tcpConnFile) close() syscall.Errno {
-	if f.closed {
-		return 0
-	}
-	f.closed = true
-	return f.Shutdown(syscall.SHUT_RDWR)
-}

internal/sysfs/sock_test.go

@@ -2,7 +2,9 @@ package sysfs
 
 import (
 	"net"
+	"syscall"
 	"testing"
+	"time"
 
 	"github.com/tetratelabs/wazero/internal/testing/require"
 )
@@ -18,10 +20,18 @@ func TestTcpConnFile_Write(t *testing.T) {
 	require.NoError(t, err)
 	defer tcp.Close() //nolint
 
-	file := tcpConnFile{tc: tcp}
-	n, errno := file.Write([]byte("wazero"))
+	file := newTcpConn(tcp)
+	errno := syscall.Errno(0)
+	// Ensure we don't interrupt until we get a non-zero errno,
+	// and we retry on EAGAIN (i.e. when nonblocking is true).
+	for {
+		_, errno = file.Write([]byte("wazero"))
+		if errno != syscall.EAGAIN {
+			break
+		}
+		time.Sleep(100 * time.Millisecond)
+	}
 	require.Zero(t, errno)
-	require.NotEqual(t, 0, n)
 
 	conn, err := listen.Accept()
 	require.NoError(t, err)
@@ -29,7 +39,7 @@ func TestTcpConnFile_Write(t *testing.T) {
 
 	bytes := make([]byte, 4)
 
-	n, err = conn.Read(bytes)
+	n, err := conn.Read(bytes)
 	require.NoError(t, err)
 	require.NotEqual(t, 0, n)
 
@@ -57,11 +67,20 @@ func TestTcpConnFile_Read(t *testing.T) {
 
 	bytes := make([]byte, 4)
 
-	file := tcpConnFile{tc: conn.(*net.TCPConn)}
-	n, errno := file.Read(bytes)
+	require.NoError(t, err)
+	errno := syscall.Errno(0)
+	file := newTcpConn(conn.(*net.TCPConn))
+	// Ensure we don't interrupt until we get a non-zero errno,
+	// and we retry on EAGAIN (i.e. when nonblocking is true).
+	for {
+		_, errno = file.Read(bytes)
+		if errno != syscall.EAGAIN {
+			break
+		}
+		time.Sleep(100 * time.Millisecond)
+	}
 	require.Zero(t, errno)
-	require.NotEqual(t, 0, n)
-
+	require.NoError(t, err)
 	require.Equal(t, "waze", string(bytes))
 }
 
@@ -80,7 +99,7 @@ func TestTcpConnFile_Stat(t *testing.T) {
 	require.NoError(t, err)
 	defer conn.Close()
 
-	file := tcpConnFile{tc: conn.(*net.TCPConn)}
+	file := newTcpConn(tcp)
 	_, errno := file.Stat()
 	require.Zero(t, errno, "Stat should not fail")
 }

internal/sysfs/sock_unix.go

@@ -7,24 +7,149 @@ import (
 	"syscall"
 
 	"github.com/tetratelabs/wazero/internal/platform"
+	socketapi "github.com/tetratelabs/wazero/internal/sock"
 )
 
+// MSG_PEEK is the constant syscall.MSG_PEEK
 const MSG_PEEK = syscall.MSG_PEEK
 
-// recvfromPeek exposes syscall.Recvfrom with flag MSG_PEEK on POSIX systems.
-func recvfromPeek(conn *net.TCPConn, p []byte) (n int, errno syscall.Errno) {
-	syscallConn, err := conn.SyscallConn()
+// newTCPListenerFile is a constructor for a socketapi.TCPSock.
+//
+// Note: the implementation of socketapi.TCPSock goes straight
+// to the syscall layer, bypassing most of the Go library.
+// For an alternative approach, consider winTcpListenerFile
+// where most APIs are implemented with regular Go std-lib calls.
+func newTCPListenerFile(tl *net.TCPListener) socketapi.TCPSock {
+	conn, err := tl.File()
 	if err != nil {
-		return 0, platform.UnwrapOSError(err)
+		panic(err)
 	}
-
-	// Prioritize the error from Recvfrom over Control
-	if controlErr := syscallConn.Control(func(fd uintptr) {
-		var recvfromErr error
-		n, _, recvfromErr = syscall.Recvfrom(int(fd), p, MSG_PEEK)
-		errno = platform.UnwrapOSError(recvfromErr)
-	}); errno == 0 {
-		errno = platform.UnwrapOSError(controlErr)
+	fd := conn.Fd()
+	// We need to duplicate this file handle, or the lifecycle will be tied
+	// to the TCPListener. We rely on the TCPListener only to set up
+	// the connection correctly and parse/resolve the TCP Address
+	// (notice we actually rely on the listener in the Windows implementation).
+	sysfd, err := syscall.Dup(int(fd))
+	if err != nil {
+		panic(err)
 	}
-	return
+	return &tcpListenerFile{fd: uintptr(sysfd), addr: tl.Addr().(*net.TCPAddr)}
+}
+
+var _ socketapi.TCPSock = (*tcpListenerFile)(nil)
+
+type tcpListenerFile struct {
+	baseSockFile
+
+	fd   uintptr
+	addr *net.TCPAddr
+}
+
+// Accept implements the same method as documented on socketapi.TCPSock
+func (f *tcpListenerFile) Accept() (socketapi.TCPConn, syscall.Errno) {
+	nfd, _, err := syscall.Accept(int(f.fd))
+	errno := platform.UnwrapOSError(err)
+	if errno != 0 {
+		return nil, errno
+	}
+	return &tcpConnFile{fd: uintptr(nfd)}, 0
+}
+
+// SetNonblock implements the same method as documented on fsapi.File
+func (f *tcpListenerFile) SetNonblock(enabled bool) syscall.Errno {
+	return platform.UnwrapOSError(setNonblock(f.fd, enabled))
+}
+
+// Close implements the same method as documented on fsapi.File
+func (f *tcpListenerFile) Close() syscall.Errno {
+	return platform.UnwrapOSError(syscall.Close(int(f.fd)))
+}
+
+// Addr is exposed for testing.
+func (f *tcpListenerFile) Addr() *net.TCPAddr {
+	return f.addr
+}
+
+var _ socketapi.TCPConn = (*tcpConnFile)(nil)
+
+type tcpConnFile struct {
+	baseSockFile
+
+	fd uintptr
+
+	// closed is true when closed was called. This ensures proper syscall.EBADF
+	closed bool
+}
+
+func newTcpConn(tc *net.TCPConn) socketapi.TCPConn {
+	f, err := tc.File()
+	if err != nil {
+		panic(err)
+	}
+	return &tcpConnFile{fd: f.Fd()}
+}
+
+// SetNonblock implements the same method as documented on fsapi.File
+func (f *tcpConnFile) SetNonblock(enabled bool) (errno syscall.Errno) {
+	return platform.UnwrapOSError(setNonblock(f.fd, enabled))
+}
+
+// Read implements the same method as documented on fsapi.File
+func (f *tcpConnFile) Read(buf []byte) (n int, errno syscall.Errno) {
+	n, err := syscall.Read(int(f.fd), buf)
+	if err != nil {
+		// Defer validation overhead until we've already had an error.
+		errno = platform.UnwrapOSError(err)
+		errno = fileError(f, f.closed, errno)
+	}
+	return n, errno
+}
+
+// Write implements the same method as documented on fsapi.File
+func (f *tcpConnFile) Write(buf []byte) (n int, errno syscall.Errno) {
+	n, err := syscall.Write(int(f.fd), buf)
+	if err != nil {
+		// Defer validation overhead until we've already had an error.
+		errno = platform.UnwrapOSError(err)
+		errno = fileError(f, f.closed, errno)
+	}
+	return n, errno
+}
+
+// Recvfrom implements the same method as documented on socketapi.TCPConn
+func (f *tcpConnFile) Recvfrom(p []byte, flags int) (n int, errno syscall.Errno) {
+	if flags != MSG_PEEK {
+		errno = syscall.EINVAL
+		return
+	}
+	n, _, recvfromErr := syscall.Recvfrom(int(f.fd), p, MSG_PEEK)
+	errno = platform.UnwrapOSError(recvfromErr)
+	return n, errno
+}
+
+// Shutdown implements the same method as documented on fsapi.Conn
+func (f *tcpConnFile) Shutdown(how int) syscall.Errno {
+	var err error
+	switch how {
+	case syscall.SHUT_RD, syscall.SHUT_WR:
+		err = syscall.Shutdown(int(f.fd), how)
+	case syscall.SHUT_RDWR:
+		return f.close()
+	default:
+		return syscall.EINVAL
+	}
+	return platform.UnwrapOSError(err)
+}
+
+// Close implements the same method as documented on fsapi.File
+func (f *tcpConnFile) Close() syscall.Errno {
+	return f.close()
+}
+
+func (f *tcpConnFile) close() syscall.Errno {
+	if f.closed {
+		return 0
+	}
+	f.closed = true
+	return platform.UnwrapOSError(syscall.Shutdown(int(f.fd), syscall.SHUT_RDWR))
 }

internal/sysfs/sock_unsupported.go

@@ -5,11 +5,22 @@ package sysfs
 import (
 	"net"
 	"syscall"
+
+	socketapi "github.com/tetratelabs/wazero/internal/sock"
 )
 
-// MSG_PEEK is a filler value
+// MSG_PEEK is a filler value.
 const MSG_PEEK = 0x2
 
-func recvfromPeek(conn *net.TCPConn, p []byte) (n int, errno syscall.Errno) {
-	return 0, syscall.ENOSYS
+func newTCPListenerFile(tl *net.TCPListener) socketapi.TCPSock {
+	return &unsupportedSockFile{}
+}
+
+type unsupportedSockFile struct {
+	baseSockFile
+}
+
+// Accept implements the same method as documented on socketapi.TCPSock
+func (f *unsupportedSockFile) Accept() (socketapi.TCPConn, syscall.Errno) {
+	return nil, syscall.ENOSYS
 }

internal/sysfs/sock_windows.go

@@ -8,31 +8,13 @@ import (
 	"unsafe"
 
 	"github.com/tetratelabs/wazero/internal/platform"
+	socketapi "github.com/tetratelabs/wazero/internal/sock"
 )
 
 // MSG_PEEK is the flag PEEK for syscall.Recvfrom on Windows.
 // This constant is not exported on this platform.
 const MSG_PEEK = 0x2
 
-// recvfromPeek exposes syscall.Recvfrom with flag MSG_PEEK on Windows.
-func recvfromPeek(conn *net.TCPConn, p []byte) (n int, errno syscall.Errno) {
-	syscallConn, err := conn.SyscallConn()
-	if err != nil {
-		errno = platform.UnwrapOSError(err)
-		return
-	}
-
-	// Prioritize the error from recvfrom over Control
-	if controlErr := syscallConn.Control(func(fd uintptr) {
-		var recvfromErr error
-		n, recvfromErr = recvfrom(syscall.Handle(fd), p, MSG_PEEK)
-		errno = platform.UnwrapOSError(recvfromErr)
-	}); errno == 0 {
-		errno = platform.UnwrapOSError(controlErr)
-	}
-	return
-}
-
 var (
 	// modws2_32 is WinSock.
 	modws2_32 = syscall.NewLazyDLL("ws2_32.dll")
@@ -61,3 +43,150 @@ func recvfrom(s syscall.Handle, buf []byte, flags int32) (n int, errno syscall.E
 		0) // fromlen *int (optional)
 	return int(r0), e1
 }
+
+// newTCPListenerFile is a constructor for a socketapi.TCPSock.
+//
+// Note: currently the Windows implementation of socketapi.TCPSock
+// returns a winTcpListenerFile, which is a specialized TCPSock
+// that delegates to a .net.TCPListener.
+// The current strategy is to delegate most behavior to the Go
+// standard library, instead of invoke syscalls/Win32 APIs
+// because they are sensibly different from Unix's.
+func newTCPListenerFile(tl *net.TCPListener) socketapi.TCPSock {
+	return &winTcpListenerFile{tl: tl}
+}
+
+var _ socketapi.TCPSock = (*winTcpListenerFile)(nil)
+
+type winTcpListenerFile struct {
+	baseSockFile
+
+	tl *net.TCPListener
+}
+
+// Accept implements the same method as documented on socketapi.TCPSock
+func (f *winTcpListenerFile) Accept() (socketapi.TCPConn, syscall.Errno) {
+	conn, err := f.tl.Accept()
+	if err != nil {
+		return nil, platform.UnwrapOSError(err)
+	}
+	return &winTcpConnFile{tc: conn.(*net.TCPConn)}, 0
+}
+
+// SetNonblock implements the same method as documented on fsapi.File
+func (f *winTcpListenerFile) SetNonblock(enabled bool) syscall.Errno {
+	return 0 // setNonblock() is a no-op on Windows
+}
+
+// Close implements the same method as documented on fsapi.File
+func (f *winTcpListenerFile) Close() syscall.Errno {
+	return platform.UnwrapOSError(f.tl.Close())
+}
+
+// Addr is exposed for testing.
+func (f *winTcpListenerFile) Addr() *net.TCPAddr {
+	return f.tl.Addr().(*net.TCPAddr)
+}
+
+var _ socketapi.TCPConn = (*winTcpConnFile)(nil)
+
+type winTcpConnFile struct {
+	baseSockFile
+
+	tc *net.TCPConn
+
+	// closed is true when closed was called. This ensures proper syscall.EBADF
+	closed bool
+}
+
+func newTcpConn(tc *net.TCPConn) socketapi.TCPConn {
+	return &winTcpConnFile{tc: tc}
+}
+
+// SetNonblock implements the same method as documented on fsapi.File
+func (f *winTcpConnFile) SetNonblock(enabled bool) (errno syscall.Errno) {
+	syscallConn, err := f.tc.SyscallConn()
+	if err != nil {
+		return platform.UnwrapOSError(err)
+	}
+
+	// Prioritize the error from setNonblock over Control
+	if controlErr := syscallConn.Control(func(fd uintptr) {
+		errno = platform.UnwrapOSError(setNonblock(fd, enabled))
+	}); errno == 0 {
+		errno = platform.UnwrapOSError(controlErr)
+	}
+	return
+}
+
+// Read implements the same method as documented on fsapi.File
+func (f *winTcpConnFile) Read(buf []byte) (n int, errno syscall.Errno) {
+	if n, errno = read(f.tc, buf); errno != 0 {
+		// Defer validation overhead until we've already had an error.
+		errno = fileError(f, f.closed, errno)
+	}
+	return
+}
+
+// Write implements the same method as documented on fsapi.File
+func (f *winTcpConnFile) Write(buf []byte) (n int, errno syscall.Errno) {
+	if n, errno = write(f.tc, buf); errno != 0 {
+		// Defer validation overhead until we've already had an error.
+		errno = fileError(f, f.closed, errno)
+	}
+	return
+}
+
+// Recvfrom implements the same method as documented on socketapi.TCPConn
+func (f *winTcpConnFile) Recvfrom(p []byte, flags int) (n int, errno syscall.Errno) {
+	if flags != MSG_PEEK {
+		errno = syscall.EINVAL
+		return
+	}
+	conn := f.tc
+	syscallConn, err := conn.SyscallConn()
+	if err != nil {
+		errno = platform.UnwrapOSError(err)
+		return
+	}
+
+	// Prioritize the error from recvfrom over Control
+	if controlErr := syscallConn.Control(func(fd uintptr) {
+		var recvfromErr error
+		n, recvfromErr = recvfrom(syscall.Handle(fd), p, MSG_PEEK)
+		errno = platform.UnwrapOSError(recvfromErr)
+	}); errno == 0 {
+		errno = platform.UnwrapOSError(controlErr)
+	}
+	return
+}
+
+// Shutdown implements the same method as documented on fsapi.Conn
+func (f *winTcpConnFile) Shutdown(how int) syscall.Errno {
+	// FIXME: can userland shutdown listeners?
+	var err error
+	switch how {
+	case syscall.SHUT_RD:
+		err = f.tc.CloseRead()
+	case syscall.SHUT_WR:
+		err = f.tc.CloseWrite()
+	case syscall.SHUT_RDWR:
+		return f.close()
+	default:
+		return syscall.EINVAL
+	}
+	return platform.UnwrapOSError(err)
+}
+
+// Close implements the same method as documented on fsapi.File
+func (f *winTcpConnFile) Close() syscall.Errno {
+	return f.close()
+}
+
+func (f *winTcpConnFile) close() syscall.Errno {
+	if f.closed {
+		return 0
+	}
+	f.closed = true
+	return f.Shutdown(syscall.SHUT_RDWR)
+}