cli: adds compile command and -cachedir option (#945)

This adds a `compile` command and a `-cachedir` option to expose our
compilation cache to end users. This allows substantial speedup
especially for large wasm.

Signed-off-by: Adrian Cole <adrian@tetrate.io>
Co-authored-by: Anuraag Agrawal <anuraaga@gmail.com>

cmd/wazero/compositefs.go

@@ -1,37 +0,0 @@
-package main
-
-import (
-	"io/fs"
-	"strings"
-)
-
-type compositeFS struct {
-	paths map[string]fs.FS
-}
-
-func (c *compositeFS) Open(name string) (fs.File, error) {
-	if !fs.ValidPath(name) {
-		return nil, &fs.PathError{Op: "open", Path: name, Err: fs.ErrInvalid}
-	}
-	for path, f := range c.paths {
-		if !strings.HasPrefix(name, path) {
-			continue
-		}
-		rest := name[len(path):]
-		if len(rest) == 0 {
-			// Special case reading directory
-			rest = "."
-		} else {
-			// fs.Open requires a relative path
-			if rest[0] == '/' {
-				rest = rest[1:]
-			}
-		}
-		file, err := f.Open(rest)
-		if err == nil {
-			return file, err
-		}
-	}
-
-	return nil, &fs.PathError{Op: "open", Path: name, Err: fs.ErrNotExist}
-}

cmd/wazero/wazero.go

@@ -12,12 +12,11 @@ import (
 	"strings"
 
 	"github.com/tetratelabs/wazero"
+	"github.com/tetratelabs/wazero/experimental"
 	"github.com/tetratelabs/wazero/imports/wasi_snapshot_preview1"
 	"github.com/tetratelabs/wazero/sys"
 )
 
-var ctx = context.Background()
-
 func main() {
 	doMain(os.Stdout, os.Stderr, os.Exit)
 }
@@ -44,6 +43,8 @@ func doMain(stdOut io.Writer, stdErr io.Writer, exit func(code int)) {
 
 	subCmd := flag.Arg(0)
 	switch subCmd {
+	case "compile":
+		doCompile(flag.Args()[1:], stdErr, exit)
 	case "run":
 		doRun(flag.Args()[1:], stdOut, stdErr, exit)
 	default:
@@ -53,6 +54,48 @@ func doMain(stdOut io.Writer, stdErr io.Writer, exit func(code int)) {
 	}
 }
 
+func doCompile(args []string, stdErr io.Writer, exit func(code int)) {
+	flags := flag.NewFlagSet("compile", flag.ExitOnError)
+	flags.SetOutput(stdErr)
+
+	var help bool
+	flags.BoolVar(&help, "h", false, "print usage")
+
+	cacheDir := cacheDirFlag(flags)
+
+	_ = flags.Parse(args)
+
+	if help {
+		printCompileUsage(stdErr, flags)
+		exit(0)
+	}
+
+	if flags.NArg() < 1 {
+		fmt.Fprintln(stdErr, "missing path to wasm file")
+		printCompileUsage(stdErr, flags)
+		exit(1)
+	}
+	wasmPath := flags.Arg(0)
+
+	wasm, err := os.ReadFile(wasmPath)
+	if err != nil {
+		fmt.Fprintf(stdErr, "error reading wasm binary: %v\n", err)
+		exit(1)
+	}
+
+	ctx := maybeUseCacheDir(context.Background(), cacheDir, stdErr, exit)
+
+	rt := wazero.NewRuntime(ctx)
+	defer rt.Close(ctx)
+
+	if _, err = rt.CompileModule(ctx, wasm); err != nil {
+		fmt.Fprintf(stdErr, "error compiling wasm binary: %v\n", err)
+		exit(1)
+	} else {
+		exit(0)
+	}
+}
+
 func doRun(args []string, stdOut io.Writer, stdErr io.Writer, exit func(code int)) {
 	flags := flag.NewFlagSet("run", flag.ExitOnError)
 	flags.SetOutput(stdErr)
@@ -69,6 +112,8 @@ func doRun(args []string, stdOut io.Writer, stdErr io.Writer, exit func(code int
 		"filesystem path to expose to the binary in the form of <host path>[:<wasm path>]. If wasm path is not "+
 			"provided, the host path will be used. Can be specified multiple times.")
 
+	cacheDir := cacheDirFlag(flags)
+
 	_ = flags.Parse(args)
 
 	if help {
@@ -144,6 +189,8 @@ func doRun(args []string, stdOut io.Writer, stdErr io.Writer, exit func(code int
 
 	wasmExe := filepath.Base(wasmPath)
 
+	ctx := maybeUseCacheDir(context.Background(), cacheDir, stdErr, exit)
+
 	rt := wazero.NewRuntime(ctx)
 	defer rt.Close(ctx)
 
@@ -188,15 +235,42 @@ func doRun(args []string, stdOut io.Writer, stdErr io.Writer, exit func(code int
 	exit(0)
 }
 
+func cacheDirFlag(flags *flag.FlagSet) *string {
+	return flags.String("cachedir", "", "Writeable directory for native code compiled from wasm. "+
+		"Contents are re-used for the same version of wazero.")
+}
+
+func maybeUseCacheDir(ctx context.Context, cacheDir *string, stdErr io.Writer, exit func(code int)) context.Context {
+	if dir := *cacheDir; dir != "" {
+		if ctx, err := experimental.WithCompilationCacheDirName(ctx, dir); err != nil {
+			fmt.Fprintf(stdErr, "invalid cachedir: %v\n", err)
+			exit(1)
+		} else {
+			return ctx
+		}
+	}
+	return ctx
+}
+
 func printUsage(stdErr io.Writer) {
 	fmt.Fprintln(stdErr, "wazero CLI")
 	fmt.Fprintln(stdErr)
 	fmt.Fprintln(stdErr, "Usage:\n  wazero <command>")
 	fmt.Fprintln(stdErr)
 	fmt.Fprintln(stdErr, "Commands:")
+	fmt.Fprintln(stdErr, "  compile\tPre-compiles a WebAssembly binary")
 	fmt.Fprintln(stdErr, "  run\t\tRuns a WebAssembly binary")
 }
 
+func printCompileUsage(stdErr io.Writer, flags *flag.FlagSet) {
+	fmt.Fprintln(stdErr, "wazero CLI")
+	fmt.Fprintln(stdErr)
+	fmt.Fprintln(stdErr, "Usage:\n  wazero compile <options> <path to wasm file>")
+	fmt.Fprintln(stdErr)
+	fmt.Fprintln(stdErr, "Options:")
+	flags.PrintDefaults()
+}
+
 func printRunUsage(stdErr io.Writer, flags *flag.FlagSet) {
 	fmt.Fprintln(stdErr, "wazero CLI")
 	fmt.Fprintln(stdErr)
@@ -216,3 +290,35 @@ func (f *sliceFlag) Set(s string) error {
 	*f = append(*f, s)
 	return nil
 }
+
+// compositeFS is defined in wazero.go to allow debugging in GoLand.
+type compositeFS struct {
+	paths map[string]fs.FS
+}
+
+func (c *compositeFS) Open(name string) (fs.File, error) {
+	if !fs.ValidPath(name) {
+		return nil, &fs.PathError{Op: "open", Path: name, Err: fs.ErrInvalid}
+	}
+	for path, f := range c.paths {
+		if !strings.HasPrefix(name, path) {
+			continue
+		}
+		rest := name[len(path):]
+		if len(rest) == 0 {
+			// Special case reading directory
+			rest = "."
+		} else {
+			// fs.Open requires a relative path
+			if rest[0] == '/' {
+				rest = rest[1:]
+			}
+		}
+		file, err := f.Open(rest)
+		if err == nil {
+			return file, err
+		}
+	}
+
+	return nil, &fs.PathError{Op: "open", Path: name, Err: fs.ErrNotExist}
+}

cmd/wazero/wazero_test.go

@@ -5,8 +5,11 @@ import (
 	_ "embed"
 	"flag"
 	"fmt"
+	"log"
 	"os"
+	"path"
 	"path/filepath"
+	"runtime"
 	"testing"
 
 	"github.com/tetratelabs/wazero/internal/testing/require"
@@ -24,9 +27,150 @@ var wasmWasiFd []byte
 //go:embed testdata/fs/bear.txt
 var bearTxt []byte
 
+func TestMain(m *testing.M) {
+	// For some reason, riscv64 fails to see directory listings.
+	if a := runtime.GOARCH; a == "riscv64" {
+		log.Println("gojs: skipping due to not yet supported GOARCH:", a)
+		os.Exit(0)
+	}
+	os.Exit(m.Run())
+}
+
+func TestCompile(t *testing.T) {
+	tmpDir, oldwd := requireChdirToTemp(t)
+	defer os.Chdir(oldwd) //nolint
+
+	wasmPath := filepath.Join(tmpDir, "test.wasm")
+	require.NoError(t, os.WriteFile(wasmPath, wasmWasiArg, 0o600))
+
+	existingDir1 := filepath.Join(tmpDir, "existing1")
+	require.NoError(t, os.Mkdir(existingDir1, 0o700))
+	existingDir2 := filepath.Join(tmpDir, "existing2")
+	require.NoError(t, os.Mkdir(existingDir2, 0o700))
+
+	tests := []struct {
+		name       string
+		wazeroOpts []string
+		test       func(t *testing.T)
+	}{
+		{
+			name: "no opts",
+		},
+		{
+			name:       "cachedir existing absolute",
+			wazeroOpts: []string{"--cachedir=" + existingDir1},
+			test: func(t *testing.T) {
+				entries, err := os.ReadDir(existingDir1)
+				require.NoError(t, err)
+				require.True(t, len(entries) > 0)
+			},
+		},
+		{
+			name:       "cachedir existing relative",
+			wazeroOpts: []string{"--cachedir=existing2"},
+			test: func(t *testing.T) {
+				entries, err := os.ReadDir(existingDir2)
+				require.NoError(t, err)
+				require.True(t, len(entries) > 0)
+			},
+		},
+		{
+			name:       "cachedir new absolute",
+			wazeroOpts: []string{"--cachedir=" + path.Join(tmpDir, "new1")},
+			test: func(t *testing.T) {
+				entries, err := os.ReadDir("new1")
+				require.NoError(t, err)
+				require.True(t, len(entries) > 0)
+			},
+		},
+		{
+			name:       "cachedir new relative",
+			wazeroOpts: []string{"--cachedir=new2"},
+			test: func(t *testing.T) {
+				entries, err := os.ReadDir("new2")
+				require.NoError(t, err)
+				require.True(t, len(entries) > 0)
+			},
+		},
+	}
+
+	for _, tc := range tests {
+		tt := tc
+		t.Run(tt.name, func(t *testing.T) {
+			args := append([]string{"compile"}, tt.wazeroOpts...)
+			args = append(args, wasmPath)
+			exitCode, stdOut, stdErr := runMain(t, args)
+			require.Zero(t, stdErr)
+			require.Equal(t, 0, exitCode, stdErr)
+			require.Zero(t, stdOut)
+			if test := tt.test; test != nil {
+				test(t)
+			}
+		})
+	}
+}
+
+func requireChdirToTemp(t *testing.T) (string, string) {
+	tmpDir := t.TempDir()
+	oldwd, err := os.Getwd()
+	require.NoError(t, err)
+	require.NoError(t, os.Chdir(tmpDir))
+	return tmpDir, oldwd
+}
+
+func TestCompile_Errors(t *testing.T) {
+	tmpDir := t.TempDir()
+
+	wasmPath := filepath.Join(tmpDir, "test.wasm")
+	require.NoError(t, os.WriteFile(wasmPath, wasmWasiArg, 0o600))
+
+	notWasmPath := filepath.Join(tmpDir, "bears.wasm")
+	require.NoError(t, os.WriteFile(notWasmPath, []byte("pooh"), 0o600))
+
+	tests := []struct {
+		message string
+		args    []string
+	}{
+		{
+			message: "missing path to wasm file",
+			args:    []string{},
+		},
+		{
+			message: "error reading wasm binary",
+			args:    []string{"non-existent.wasm"},
+		},
+		{
+			message: "error compiling wasm binary",
+			args:    []string{notWasmPath},
+		},
+		{
+			message: "invalid cachedir",
+			args:    []string{"--cachedir", notWasmPath, wasmPath},
+		},
+	}
+
+	for _, tc := range tests {
+		tt := tc
+		t.Run(tt.message, func(t *testing.T) {
+			exitCode, _, stdErr := runMain(t, append([]string{"compile"}, tt.args...))
+
+			require.Equal(t, 1, exitCode)
+			require.Contains(t, stdErr, tt.message)
+		})
+	}
+}
+
 func TestRun(t *testing.T) {
-	bearPath := filepath.Join(t.TempDir(), "bear.txt")
-	require.NoError(t, os.WriteFile(bearPath, bearTxt, 0o755))
+	tmpDir, oldwd := requireChdirToTemp(t)
+	defer os.Chdir(oldwd) //nolint
+
+	bearPath := filepath.Join(tmpDir, "bear.txt")
+	require.NoError(t, os.WriteFile(bearPath, bearTxt, 0o600))
+
+	existingDir1 := filepath.Join(tmpDir, "existing1")
+	require.NoError(t, os.Mkdir(existingDir1, 0o700))
+	existingDir2 := filepath.Join(tmpDir, "existing2")
+	require.NoError(t, os.Mkdir(existingDir2, 0o700))
 
 	tests := []struct {
 		name       string
@@ -35,6 +179,7 @@ func TestRun(t *testing.T) {
 		wasmArgs   []string
 		stdOut     string
 		stdErr     string
+		test       func(t *testing.T)
 	}{
 		{
 			name:     "args",
@@ -62,13 +207,65 @@ func TestRun(t *testing.T) {
 			wazeroOpts: []string{fmt.Sprintf("--mount=%s:/", filepath.Dir(bearPath))},
 			stdOut:     "pooh\n",
 		},
+		{
+			name:       "cachedir existing absolute",
+			wazeroOpts: []string{"--cachedir=" + existingDir1},
+			wasm:       wasmWasiArg,
+			wasmArgs:   []string{"hello world"},
+			// Executable name is first arg so is printed.
+			stdOut: "test.wasm\x00hello world\x00",
+			test: func(t *testing.T) {
+				entries, err := os.ReadDir(existingDir1)
+				require.NoError(t, err)
+				require.True(t, len(entries) > 0)
+			},
+		},
+		{
+			name:       "cachedir existing relative",
+			wazeroOpts: []string{"--cachedir=existing2"},
+			wasm:       wasmWasiArg,
+			wasmArgs:   []string{"hello world"},
+			// Executable name is first arg so is printed.
+			stdOut: "test.wasm\x00hello world\x00",
+			test: func(t *testing.T) {
+				entries, err := os.ReadDir(existingDir2)
+				require.NoError(t, err)
+				require.True(t, len(entries) > 0)
+			},
+		},
+		{
+			name:       "cachedir new absolute",
+			wazeroOpts: []string{"--cachedir=" + path.Join(tmpDir, "new1")},
+			wasm:       wasmWasiArg,
+			wasmArgs:   []string{"hello world"},
+			// Executable name is first arg so is printed.
+			stdOut: "test.wasm\x00hello world\x00",
+			test: func(t *testing.T) {
+				entries, err := os.ReadDir("new1")
+				require.NoError(t, err)
+				require.True(t, len(entries) > 0)
+			},
+		},
+		{
+			name:       "cachedir new relative",
+			wazeroOpts: []string{"--cachedir=new2"},
+			wasm:       wasmWasiArg,
+			wasmArgs:   []string{"hello world"},
+			// Executable name is first arg so is printed.
+			stdOut: "test.wasm\x00hello world\x00",
+			test: func(t *testing.T) {
+				entries, err := os.ReadDir("new2")
+				require.NoError(t, err)
+				require.True(t, len(entries) > 0)
+			},
+		},
 	}
 
 	for _, tc := range tests {
 		tt := tc
 		t.Run(tt.name, func(t *testing.T) {
-			wasmPath := filepath.Join(t.TempDir(), "test.wasm")
-			require.NoError(t, os.WriteFile(wasmPath, tt.wasm, 0o755))
+			wasmPath := filepath.Join(tmpDir, "test.wasm")
+			require.NoError(t, os.WriteFile(wasmPath, tt.wasm, 0o700))
 
 			args := append([]string{"run"}, tt.wazeroOpts...)
 			args = append(args, wasmPath)
@@ -77,19 +274,19 @@ func TestRun(t *testing.T) {
 			require.Equal(t, 0, exitCode, stdErr)
 			require.Equal(t, tt.stdOut, stdOut)
 			require.Equal(t, tt.stdErr, stdErr)
+			if test := tt.test; test != nil {
+				test(t)
+			}
 		})
 	}
 }
 
-func TestHelp(t *testing.T) {
-	exitCode, _, stdErr := runMain(t, []string{"-h"})
-	require.Equal(t, 0, exitCode)
-	require.Contains(t, stdErr, "wazero CLI\n\nUsage:")
-}
+func TestRun_Errors(t *testing.T) {
+	wasmPath := filepath.Join(t.TempDir(), "test.wasm")
+	require.NoError(t, os.WriteFile(wasmPath, wasmWasiArg, 0o700))
 
-func TestErrors(t *testing.T) {
 	notWasmPath := filepath.Join(t.TempDir(), "bears.wasm")
-	require.NoError(t, os.WriteFile(notWasmPath, []byte("pooh"), 0o755))
+	require.NoError(t, os.WriteFile(notWasmPath, []byte("pooh"), 0o700))
 
 	tests := []struct {
 		message string
@@ -115,6 +312,10 @@ func TestErrors(t *testing.T) {
 			message: "invalid mount",
 			args:    []string{"--mount=.", "testdata/wasi_env.wasm"},
 		},
+		{
+			message: "invalid cachedir",
+			args:    []string{"--cachedir", notWasmPath, wasmPath},
+		},
 	}
 
 	for _, tc := range tests {
@@ -128,6 +329,12 @@ func TestErrors(t *testing.T) {
 	}
 }
 
+func TestHelp(t *testing.T) {
+	exitCode, _, stdErr := runMain(t, []string{"-h"})
+	require.Equal(t, 0, exitCode)
+	require.Contains(t, stdErr, "wazero CLI\n\nUsage:")
+}
+
 func runMain(t *testing.T, args []string) (int, string, string) {
 	t.Helper()
 	oldArgs := os.Args

experimental/compilation_cache.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 	"fmt"
 	"os"
+	"path/filepath"
 
 	"github.com/tetratelabs/wazero/internal/compilationcache"
 )
@@ -29,6 +30,13 @@ import (
 //	ctx, _ := experimental.WithCompilationCacheDirName(context.Background(), "/home/me/.cache/wazero")
 //	r := wazero.NewRuntimeWithConfig(ctx, wazero.NewRuntimeConfigCompiler())
 func WithCompilationCacheDirName(ctx context.Context, dirname string) (context.Context, error) {
+	// Resolve a potentially relative directory into an absolute one.
+	var err error
+	dirname, err = filepath.Abs(dirname)
+	if err != nil {
+		return nil, err
+	}
+
 	if st, err := os.Stat(dirname); errors.Is(err, os.ErrNotExist) {
 		// If the directory not found, create the cache dir.
 		if err = os.MkdirAll(dirname, 0o700); err != nil {

experimental/compilation_cache_test.go

@@ -2,9 +2,9 @@ package experimental
 
 import (
 	"context"
-	"fmt"
 	"os"
 	"path"
+	"path/filepath"
 	"testing"
 
 	"github.com/tetratelabs/wazero/internal/compilationcache"
@@ -26,13 +26,30 @@ func TestWithCompilationCacheDirName(t *testing.T) {
 		require.Equal(t, 0, len(entries))
 	})
 	t.Run("create dir", func(t *testing.T) {
-		dir := path.Join(t.TempDir(), "1", "2", "3", t.Name()) // Non-existent directory.
-		fmt.Println(dir)
+		tmpDir := path.Join(t.TempDir(), "1", "2", "3")
+		dir := path.Join(tmpDir, "foo") // Non-existent directory.
+		absDir, err := filepath.Abs(dir)
+		require.NoError(t, err)
+
 		ctx, err := WithCompilationCacheDirName(context.Background(), dir)
 		require.NoError(t, err)
 		actual, ok := ctx.Value(compilationcache.FileCachePathKey{}).(string)
 		require.True(t, ok)
-		require.Equal(t, dir, actual)
+
+		requireContainsDir(t, tmpDir, "foo", actual)
+
+		require.Equal(t, absDir, actual)
+	})
+	t.Run("create relative dir", func(t *testing.T) {
+		tmpDir, oldwd := requireChdirToTemp(t)
+		defer os.Chdir(oldwd) //nolint
+
+		ctx, err := WithCompilationCacheDirName(context.Background(), "foo")
+		require.NoError(t, err)
+		actual, ok := ctx.Value(compilationcache.FileCachePathKey{}).(string)
+		require.True(t, ok)
+
+		requireContainsDir(t, tmpDir, "foo", actual)
 	})
 	t.Run("non dir", func(t *testing.T) {
 		f, err := os.CreateTemp(t.TempDir(), "nondir")
@@ -43,3 +60,24 @@ func TestWithCompilationCacheDirName(t *testing.T) {
 		require.Contains(t, err.Error(), "is not dir")
 	})
 }
+
+// requireContainsDir ensures the directory was created in the correct path,
+// as file.Abs can return slightly different answers for a temp directory. For
+// example, /var/folders/... vs /private/var/folders/...
+func requireContainsDir(t *testing.T, parent, dir string, actual string) {
+	require.True(t, filepath.IsAbs(actual))
+
+	entries, err := os.ReadDir(parent)
+	require.NoError(t, err)
+	require.Equal(t, 1, len(entries))
+	require.Equal(t, dir, entries[0].Name())
+	require.True(t, entries[0].IsDir())
+}
+
+func requireChdirToTemp(t *testing.T) (string, string) {
+	tmpDir := t.TempDir()
+	oldwd, err := os.Getwd()
+	require.NoError(t, err)
+	require.NoError(t, os.Chdir(tmpDir))
+	return tmpDir, oldwd
+}

internal/asm/arm64/impl_4_test.go

@@ -716,7 +716,6 @@ func TestAssemblerImpl_encodeReadInstructionAddress(t *testing.T) {
 				pos += 4
 				require.Equal(t, []byte{0xa, 0x7d, 0x80, 0xd2},
 					actual[pos:pos+4], hex.EncodeToString(actual))
-				fmt.Println(hex.EncodeToString(actual))
 
 				require.Equal(t, uint64(4+tc.numDummyInstructions*4+4),
 					target.offsetInBinaryField-adrInst.offsetInBinaryField)

internal/wasmdebug/dwarf_test.go

@@ -3,7 +3,6 @@ package wasmdebug_test
 import (
 	"fmt"
 	"math"
-	"strings"
 	"sync"
 	"testing"
 
@@ -247,8 +246,6 @@ func TestDWARFLines_Line_Rust(t *testing.T) {
 		t.Run(fmt.Sprintf("%#x/%s", tc.offset, tc.exp), func(t *testing.T) {
 			actual := mod.DWARFLines.Line(tc.offset)
 
-			fmt.Println(strings.Join(actual, "\n"))
-
 			require.Equal(t, len(tc.exp), len(actual))
 			for i := range tc.exp {
 				require.Contains(t, actual[i], tc.exp[i])