Moves host function imports into their own directory (#784)

Our root directory is getting crowded and it is also difficult to
organize the "host imports" concept due to this.

This moves common and language-specific imports into their own
directory. This will break go import signatures on the next release, but
is more sustainable overall.

Signed-off-by: Adrian Cole <adrian@tetrate.io>

.github/workflows/examples.yaml

@@ -5,12 +5,14 @@ on:
     paths:
       - '.github/workflows/examples.yaml'
       - 'examples/**'
+      - 'imports/**/example/**'
       - 'Makefile'
   push:
     branches: [main]
     paths:
       - '.github/workflows/examples.yaml'
       - 'examples/**'
+      - 'imports/**/example/**'
       - 'Makefile'
 
 env:
@@ -103,5 +105,4 @@ jobs:
         run: make build.bench
 
       - name: Run example tests
-        # Also runs emscripten as its source is outside the examples directory.
-        run: go test ./examples/... ./emscripten/...
+        run: make test.examples

Makefile

@@ -13,7 +13,7 @@ hugo          := github.com/gohugoio/hugo@v0.101.0
 all_sources   := $(wildcard *.go */*.go */*/*.go */*/*/*.go */*/*/*.go */*/*/*/*.go)
 all_testdata  := $(wildcard testdata/* */testdata/* */*/testdata/* */*/testdata/*/* */*/*/testdata/*)
 all_testing   := $(wildcard internal/testing/* internal/testing/*/* internal/testing/*/*/*)
-all_examples  := $(wildcard examples/* examples/*/* examples/*/*/*)
+all_examples  := $(wildcard examples/* examples/*/* examples/*/*/* */*/example/* */*/example/*/* */*/example/*/*/*)
 all_it        := $(wildcard internal/integration_test/* internal/integration_test/*/* internal/integration_test/*/*/*)
 # main_sources exclude any test or example related code
 main_sources  := $(wildcard $(filter-out %_test.go $(all_testdata) $(all_testing) $(all_examples) $(all_it), $(all_sources)))
@@ -42,27 +42,19 @@ bench_testdata_dir := internal/integration_test/bench/testdata
 build.bench:
 	@tinygo build -o $(bench_testdata_dir)/case.wasm -scheduler=none --no-debug -target=wasi $(bench_testdata_dir)/case.go
 
+.PHONY: test.examples
+test.examples:
+	@go test ./examples/... ./imports/assemblyscript/example/... ./imports/go/example/... ./imports/wasi_snapshot_preview1/example/...
+
 .PHONY: build.examples.as
 build.examples.as:
-	@cd ./examples/assemblyscript/testdata && npm install && npm run build
+	@cd ./imports/assemblyscript/example/testdata && npm install && npm run build
 
 .PHONY: build.examples.zig
-build.examples.zig: examples/allocation/zig/testdata/greet.wasm
+build.examples.zig:
+	@cd examples/allocation/zig/testdata/ && zig build -Drelease-small=true && mv zig-out/lib/greet.wasm .
 
-%.wasm: %.zig
-	@(cd $(@D); zig build)
-	@mv $(@D)/zig-out/lib/$(@F) $(@D)
-
-go_sources := examples/wasm_exec/testdata/cat.go
-.PHONY: build.examples.go
-build.examples.go: $(go_sources)
-	@for f in $^; do \
-	    cd $$(dirname $$f); \
-	    GOARCH=wasm GOOS=js go build -o $$(basename $$f | sed -e 's/\.go/\.wasm/') .; \
-	    cd -; \
-	done
-
-tinygo_sources := $(filter-out $(go_sources), $(wildcard examples/*/testdata/*.go examples/*/*/testdata/*.go examples/*/testdata/*/*.go))
+tinygo_sources := examples/allocation/tinygo/testdata/greet.go imports/wasi_snapshot_preview1/example/testdata/tinygo/cat.go
 .PHONY: build.examples.tinygo
 build.examples.tinygo: $(tinygo_sources)
 	@for f in $^; do \
@@ -70,11 +62,11 @@ build.examples.tinygo: $(tinygo_sources)
 	done
 
 # We use zig to build C as it is easy to install and embeds a copy of zig-cc.
-c_sources := $(wildcard examples/*/testdata/*.c examples/*/*/testdata/*.c examples/*/testdata/*/*.c)
+c_sources := imports/wasi_snapshot_preview1/example/testdata/zig-cc/cat.c
 .PHONY: build.examples.zig-cc
 build.examples.zig-cc: $(c_sources)
 	@for f in $^; do \
-	    zig cc --target=wasm32-wasi -O3 -o $$(echo $$f | sed -e 's/\.c/\.wasm/') $$f; \
+	    zig cc --target=wasm32-wasi -Oz -o $$(echo $$f | sed -e 's/\.c/\.wasm/') $$f; \
 	done
 
 # Here are the emcc args we use:
@@ -90,7 +82,7 @@ build.examples.zig-cc: $(c_sources)
 #   to one page (64KB). To do this, we have to reduce the stack size.
 # * `-s ALLOW_MEMORY_GROWTH` - allows "memory.grow" instructions to succeed, but
 #   requires a function import "emscripten_notify_memory_growth".
-emscripten_sources := $(wildcard emscripten/testdata/*.cc)
+emscripten_sources := $(wildcard imports/emscripten/testdata/*.cc)
 .PHONY: build.examples.emscripten
 build.examples.emscripten: $(emscripten_sources)
 	@for f in $^; do \
@@ -107,7 +99,7 @@ build.examples.emscripten: $(emscripten_sources)
 %/cat.wasm : cargo_target := wasm32-wasi
 
 .PHONY: build.examples.rust
-build.examples.rust: examples/allocation/rust/testdata/greet.wasm examples/wasi/testdata/cargo-wasi/cat.wasm
+build.examples.rust: examples/allocation/rust/testdata/greet.wasm imports/wasi_snapshot_preview1/example/testdata/cargo-wasi/cat.wasm
 
 # Builds rust using cargo normally, or cargo-wasi.
 %.wasm: %.rs

README.md

@@ -66,8 +66,9 @@ func main() {
 
 Notes:
 
-* The Wasm binary is often called the "guest" in WebAssembly.
 * The embedding application is often called the "host" in WebAssembly.
+* The Wasm binary is often called the "guest" in WebAssembly. Sometimes they
+  need [imports][imports] to implement features such as console output.
 * Many languages compile to (target) Wasm including AssemblyScript, C, C++,
   Rust, TinyGo and Zig!
 
@@ -83,7 +84,8 @@ it has no scope to specify how system resources like files are accessed.
 Instead, WebAssembly defines "host functions" and the signatures they can use.
 In wazero, "host functions" are written in Go, and let you do anything
 including access files. The main constraint is that WebAssembly only allows
-numeric types.
+numeric types. wazero includes [imports][imports] for common languages and
+compiler toolchains.
 
 For example, you can grant WebAssembly code access to your console by exporting
 a function written in Go. The below function can be imported into standard

examples/README.md

@@ -2,13 +2,21 @@
 
 The following example projects can help you practice WebAssembly with wazero:
 
-* [allocation](allocation) - how to pass strings in and out of WebAssembly functions defined in Rust or TinyGo.
-* [assemblyscript](assemblyscript) - how to configure special imports needed by AssemblyScript when not using WASI.
+* [allocation](allocation) - how to pass strings in and out of WebAssembly
+  functions defined in Rust or TinyGo.
+* [assemblyscript](../imports/assemblyscript/example) - how to configure
+  special imports needed by AssemblyScript when not using WASI.
 * [basic](basic) - how to use both WebAssembly and Go-defined functions.
-* [import-go](import-go) - how to define, import and call a Go-defined function from a WebAssembly-defined function.
-* [multiple-results](multiple-results) - how to return more than one result from WebAssembly or Go-defined functions.
-* [namespace](namespace) - how WebAssembly modules can import their own host module, such as "env".
-* [replace-import](replace-import) - how to override a module name hard-coded in a WebAssembly module.
-* [wasi](wasi) - how to use I/O in your WebAssembly modules using WASI (WebAssembly System Interface).
+* [import-go](import-go) - how to define, import and call a Go-defined function
+  from a WebAssembly-defined function.
+* [multiple-results](multiple-results) - how to return more than one result
+  from WebAssembly or Go-defined functions.
+* [namespace](namespace) - how WebAssembly modules can import their own host
+  module, such as "env".
+* [replace-import](replace-import) - how to override a module name hard-coded
+  in a WebAssembly module.
+* [wasi](../imports/wasi_snapshot_preview1/example) - how to use I/O in your
+  WebAssembly modules using WASI (WebAssembly System Interface).
 
-Please [open an issue](https://github.com/tetratelabs/wazero/issues/new) if you would like to see another example.
+Please [open an issue](https://github.com/tetratelabs/wazero/issues/new) if you
+would like to see another example.

examples/allocation/tinygo/greet.go

@@ -9,7 +9,7 @@ import (
 
 	"github.com/tetratelabs/wazero"
 	"github.com/tetratelabs/wazero/api"
-	"github.com/tetratelabs/wazero/wasi_snapshot_preview1"
+	"github.com/tetratelabs/wazero/imports/wasi_snapshot_preview1"
 )
 
 // greetWasm was compiled using `tinygo build -o greet.wasm -scheduler=none --no-debug -target=wasi greet.go`

experimental/fs_example_test.go

@@ -9,7 +9,7 @@ import (
 
 	"github.com/tetratelabs/wazero"
 	"github.com/tetratelabs/wazero/experimental"
-	"github.com/tetratelabs/wazero/wasi_snapshot_preview1"
+	"github.com/tetratelabs/wazero/imports/wasi_snapshot_preview1"
 )
 
 // fsWasm was generated by the following:

experimental/listener_example_test.go

@@ -10,7 +10,7 @@ import (
 	"github.com/tetratelabs/wazero"
 	"github.com/tetratelabs/wazero/api"
 	. "github.com/tetratelabs/wazero/experimental"
-	"github.com/tetratelabs/wazero/wasi_snapshot_preview1"
+	"github.com/tetratelabs/wazero/imports/wasi_snapshot_preview1"
 )
 
 // listenerWasm was generated by the following:

experimental/logging/log_listener_example_test.go

@@ -9,7 +9,7 @@ import (
 	"github.com/tetratelabs/wazero"
 	"github.com/tetratelabs/wazero/experimental"
 	"github.com/tetratelabs/wazero/experimental/logging"
-	"github.com/tetratelabs/wazero/wasi_snapshot_preview1"
+	"github.com/tetratelabs/wazero/imports/wasi_snapshot_preview1"
 )
 
 // listenerWasm was generated by the following:

experimental/logging/log_listener_test.go

@@ -9,9 +9,9 @@ import (
 
 	"github.com/tetratelabs/wazero/api"
 	"github.com/tetratelabs/wazero/experimental/logging"
+	"github.com/tetratelabs/wazero/imports/wasi_snapshot_preview1"
 	"github.com/tetratelabs/wazero/internal/testing/require"
 	"github.com/tetratelabs/wazero/internal/wasm"
-	"github.com/tetratelabs/wazero/wasi_snapshot_preview1"
 )
 
 // testCtx is an arbitrary, non-default context. Non-nil also prevents linter errors.

imports/README.md

@@ -0,0 +1,40 @@
+## wazero imports
+
+Packages in this directory implement the *host* imports needed for specific
+languages or shared compiler toolchains.
+
+* [AssemblyScript](assemblyscript) Ex. `asc X.ts --debug -b none -o X.wasm`
+* [Emscripten](emscripten) Ex. `em++ ... -s STANDALONE_WASM -o X.wasm X.cc`
+* [Go](go) Ex. `GOARCH=wasm GOOS=js go build -o X.wasm X.go`
+* [WASI](wasi_snapshot_preview1) Ex. `tinygo build -o X.wasm -target=wasi X.go`
+
+Note: You may not see a language listed here because it either works without
+host imports, or it uses WASI. Refer to https://wazero.io/languages/ for more.
+
+Please [open an issue](https://github.com/tetratelabs/wazero/issues/new) if you
+would like to see support for another compiled language or toolchain.
+
+## Overview
+
+WebAssembly has a virtual machine architecture where the *host* is the process
+embedding wazero and the *guest* is a program compiled into the WebAssembly
+Binary Format, also known as Wasm (`%.wasm`).
+
+The only features that work by default are computational in nature, and the
+only way to communicate is via functions, memory or global variables.
+
+When a compiler targets Wasm, it often needs to import functions from the host
+to satisfy system calls needed for functionality like printing to the console,
+getting the time, or generating random values. The technical term for this
+bridge is Application Binary Interface (ABI), but we'll call them simply host
+imports.
+
+Packages in this directory are sometimes well re-used, such as the case in
+[WASI](https://wazero.io/specs/#wasi). For example, Rust, TinyGo, and Zig can
+all target WebAssembly in a way that imports the same "wasi_snapshot_preview1"
+module in the compiled `%.wasm` file. To support any of these, wazero users can
+invoke `wasi_snapshot_preview1.Instantiate` on their `wazero.Runtime`.
+
+Other times, host imports are either completely compiler-specific, such as the
+case with `GOARCH=wasm GOOS=js`, or coexist alongside WASI, such as the case
+with Emscripten.

imports/assemblyscript/assemblyscript_example_test.go

@@ -6,7 +6,7 @@ import (
 	"log"
 
 	"github.com/tetratelabs/wazero"
-	"github.com/tetratelabs/wazero/assemblyscript"
+	"github.com/tetratelabs/wazero/imports/assemblyscript"
 )
 
 // This shows how to instantiate AssemblyScript's special imports.

imports/assemblyscript/example/assemblyscript.go

@@ -9,7 +9,7 @@ import (
 	"strconv"
 
 	"github.com/tetratelabs/wazero"
-	"github.com/tetratelabs/wazero/assemblyscript"
+	"github.com/tetratelabs/wazero/imports/assemblyscript"
 )
 
 // asWasm compiled using `npm install && npm run build`

imports/emscripten/emscripten_example_test.go

@@ -6,8 +6,8 @@ import (
 	"log"
 
 	"github.com/tetratelabs/wazero"
-	"github.com/tetratelabs/wazero/emscripten"
-	"github.com/tetratelabs/wazero/wasi_snapshot_preview1"
+	"github.com/tetratelabs/wazero/imports/emscripten"
+	"github.com/tetratelabs/wazero/imports/wasi_snapshot_preview1"
 )
 
 // This shows how to instantiate Emscripten function imports.

imports/emscripten/emscripten_test.go

@@ -9,9 +9,9 @@ import (
 	"github.com/tetratelabs/wazero"
 	. "github.com/tetratelabs/wazero/experimental"
 	"github.com/tetratelabs/wazero/experimental/logging"
+	"github.com/tetratelabs/wazero/imports/wasi_snapshot_preview1"
 	"github.com/tetratelabs/wazero/internal/testing/require"
 	"github.com/tetratelabs/wazero/sys"
-	"github.com/tetratelabs/wazero/wasi_snapshot_preview1"
 )
 
 // growWasm was compiled from testdata/grow.cc

imports/go/README.md

@@ -0,0 +1,28 @@
+# Overview
+
+Wazero's "github.com/tetratelabs/wazero/imports/go" package allows you to run
+a `%.wasm` file compiled by Go. See https://wazero.io/languages/go/ for more.
+
+## Usage
+
+When `GOOS=js` and `GOARCH=wasm`, Go's compiler targets WebAssembly 1.0
+Binary format (%.wasm).
+
+Ex.
+```bash
+GOOS=js GOARCH=wasm go build -o cat.wasm .
+```
+
+After compiling `cat.wasm` with wazero.Runtime's `CompileModule`, Run it.
+
+Under the scenes, the compiled Wasm calls host functions that support the
+runtime.GOOS. This is similar to what is implemented in [wasm_exec.js][1].
+
+## Experimental
+
+Go defines js "EXPERIMENTAL... exempt from the Go compatibility promise."
+Accordingly, wazero cannot guarantee this will work from release to release,
+or that usage will be relatively free of bugs. Due to this and the
+relatively high implementation overhead, most will choose TinyGo instead.
+
+[1]: https://github.com/golang/go/blob/go1.19/misc/wasm/wasm_exec.js

imports/go/example/stars.go

@@ -13,7 +13,7 @@ import (
 
 	"github.com/tetratelabs/wazero"
 	"github.com/tetratelabs/wazero/experimental"
-	"github.com/tetratelabs/wazero/experimental/gojs"
+	gojs "github.com/tetratelabs/wazero/imports/go"
 	"github.com/tetratelabs/wazero/sys"
 )
 

imports/wasi_snapshot_preview1/example/cat.go

@@ -9,8 +9,8 @@ import (
 	"os"
 
 	"github.com/tetratelabs/wazero"
+	"github.com/tetratelabs/wazero/imports/wasi_snapshot_preview1"
 	"github.com/tetratelabs/wazero/sys"
-	"github.com/tetratelabs/wazero/wasi_snapshot_preview1"
 )
 
 // catFS is an embedded filesystem limited to test.txt

internal/gojs/compiler_test.go

@@ -18,7 +18,7 @@ import (
 
 	"github.com/tetratelabs/wazero"
 	"github.com/tetratelabs/wazero/experimental"
-	"github.com/tetratelabs/wazero/experimental/gojs"
+	gojs "github.com/tetratelabs/wazero/imports/go"
 )
 
 func compileAndRun(ctx context.Context, arg string, config wazero.ModuleConfig) (stdout, stderr string, err error) {

internal/gojs/http_test.go

@@ -8,7 +8,7 @@ import (
 	"testing"
 
 	"github.com/tetratelabs/wazero"
-	"github.com/tetratelabs/wazero/experimental/gojs"
+	gojs "github.com/tetratelabs/wazero/imports/go"
 	"github.com/tetratelabs/wazero/internal/testing/require"
 )
 

internal/integration_test/bench/bench_test.go

@@ -11,8 +11,8 @@ import (
 	"github.com/tetratelabs/wazero"
 	"github.com/tetratelabs/wazero/api"
 	"github.com/tetratelabs/wazero/experimental"
+	"github.com/tetratelabs/wazero/imports/wasi_snapshot_preview1"
 	"github.com/tetratelabs/wazero/internal/platform"
-	"github.com/tetratelabs/wazero/wasi_snapshot_preview1"
 )
 
 // testCtx is an arbitrary, non-default context. Non-nil also prevents linter errors.

internal/integration_test/bench/codec_test.go

@@ -4,10 +4,10 @@ import (
 	"testing"
 
 	"github.com/tetratelabs/wazero"
+	"github.com/tetratelabs/wazero/imports/wasi_snapshot_preview1"
 	"github.com/tetratelabs/wazero/internal/testing/require"
 	"github.com/tetratelabs/wazero/internal/wasm"
 	"github.com/tetratelabs/wazero/internal/wasm/binary"
-	"github.com/tetratelabs/wazero/wasi_snapshot_preview1"
 )
 
 // example holds the latest supported features as described in the comments of exampleText

internal/integration_test/fs/fs_test.go

@@ -12,8 +12,8 @@ import (
 
 	"github.com/tetratelabs/wazero"
 	"github.com/tetratelabs/wazero/api"
+	"github.com/tetratelabs/wazero/imports/wasi_snapshot_preview1"
 	"github.com/tetratelabs/wazero/internal/testing/require"
-	"github.com/tetratelabs/wazero/wasi_snapshot_preview1"
 )
 
 var testCtx = context.Background()

internal/integration_test/vs/runtime.go

@@ -7,8 +7,8 @@ import (
 
 	"github.com/tetratelabs/wazero"
 	"github.com/tetratelabs/wazero/api"
+	"github.com/tetratelabs/wazero/imports/wasi_snapshot_preview1"
 	"github.com/tetratelabs/wazero/internal/wasm"
-	"github.com/tetratelabs/wazero/wasi_snapshot_preview1"
 )
 
 type RuntimeConfig struct {

site/content/languages/go.md

@@ -242,4 +242,4 @@ the Go stack.
 [11]: https://github.com/WebAssembly/proposals
 [12]: https://github.com/golang/go/blob/go1.19/src/cmd/link/internal/ld/data.go#L2457
 [13]: https://github.com/golang/go/blob/go1.19/src/syscall/tables_js.go#L371-L494
-[14]: https://github.com/tetratelabs/wazero/tree/main/examples/gojs
+[14]: https://github.com/tetratelabs/wazero/tree/main/imports/go/example

site/content/languages/rust.md

@@ -222,8 +222,8 @@ source code may reduce binary size further.
 [7]: https://github.com/tetratelabs/wazero/stargazers
 [8]: https://rustwasm.github.io/docs/book/reference/which-crates-work-with-wasm.html
 [9]: https://github.com/tetratelabs/wazero/tree/main/examples/allocation/rust
-[10]: https://github.com/tetratelabs/wazero/tree/main/examples/wasi
-[11]: https://github.com/tetratelabs/wazero/tree/main/examples/wasi/testdata/cargo-wasi
+[10]: https://github.com/tetratelabs/wazero/tree/main/imports/wasi_snapshot_preview1/example
+[11]: https://github.com/tetratelabs/wazero/tree/main/imports/wasi_snapshot_preview1/example/testdata/cargo-wasi
 [12]: https://github.com/rustwasm/wee_alloc
 [13]: https://doc.rust-lang.org/cargo/reference/profiles.html#profile-settings
 [14]: https://github.com/bytecodealliance/cargo-wasi

site/content/languages/tinygo.md

@@ -378,8 +378,8 @@ functions, such as `fmt.Println`, which can require 100KB of wasm.
 [18]: https://github.com/tinygo-org/tinygo/issues/447
 [19]: https://github.com/tinygo-org/tinygo/issues/3068
 [20]: https://github.com/tinygo-org/tinygo/blob/v0.25.0/src/runtime/arch_tinygowasm.go#L47-L62
-[21]: https://github.com/tetratelabs/wazero/tree/main/examples/wasi
-[22]: https://github.com/tetratelabs/wazero/tree/main/examples/wasi/testdata/tinygo
+[21]: https://github.com/tetratelabs/wazero/tree/main/imports/wasi_snapshot_preview1/example
+[22]: https://github.com/tetratelabs/wazero/tree/main/imports/wasi_snapshot_preview1/example/testdata/tinygo
 [23]: https://github.com/WebAssembly/binaryen/blob/main/src/passes/Asyncify.cpp
 [24]: http://tleyden.github.io/blog/2014/10/30/goroutines-vs-threads/
 [25]: https://github.com/tinygo-org/tinygo/issues/3095