examples: adds rust build and WASI example (#676)

This fixes where our pull requests didn't check the rust source in
examples were valid. It also adds an example of wasi with rust.

This uses cargo-wasi because the default target creates almost 2MB of
wasm for a simple cat program.

Signed-off-by: Adrian Cole <adrian@tetrate.io>

.github/workflows/examples.yaml

@@ -4,27 +4,27 @@ on:
     branches: [main]
     paths:
       - '.github/workflows/examples.yaml'
-      - 'examples/*/testdata/*.go'
-      - 'examples/*/*/testdata/*.go'
-      - 'examples/*/testdata/*/*.go'
-      - 'examples/*/testdata/*/*.c'
-      - 'examples/*/testdata/*.ts'
+      - 'examples/**'
       - 'Makefile'
   push:
     branches: [main]
     paths:
       - '.github/workflows/examples.yaml'
-      - 'examples/*/testdata/*.go'
-      - 'examples/*/*/testdata/*.go'
-      - 'examples/*/testdata/*/*.go'
-      - 'examples/*/testdata/*/*.c'
-      - 'examples/*/testdata/*.ts'
+      - 'examples/**'
       - 'Makefile'
 
 jobs:
+  # Not all toolchains are idempotent when generating wasm, so we don't check
+  # in %.wasm as a part of this job.
   examples:
     name: Build examples
     runs-on: ubuntu-20.04
+    strategy:
+      matrix: # use latest available versions and be consistent on all workflows!
+        go-version:
+          - "1.17" # == ${{ env.GO_VERSION }} because matrix cannot expand env variables
+          - "1.18"
+
     steps:
       - name: Install latest TinyGo
         run: |  # installing via curl so commands are similar on OS/x
@@ -38,21 +38,37 @@ jobs:
         with:  # on laptop, use `brew install zig`
           version: 0.9.1
 
+      - name: Install wasm32-wasi target
+        uses: actions-rs/toolchain@v1
+        with:
+          toolchain: stable
+          target: wasm32-wasi
+
       - name: Checkout
         uses: actions/checkout@v3
 
-      # TinyGo -> Wasm is not idempotent, so we only check things build.
+      - uses: actions/cache@v3
+        with:
+          path: |
+            ~/.cache/go-build
+            ~/go/pkg/mod
+            ~/go/bin
+          key: check-${{ runner.os }}-go-${{ matrix.go-version }}-${{ hashFiles('**/go.sum', 'Makefile') }}
+
       - name: Build TinyGO examples
         run: make build.examples.tinygo
 
-      # AssemblyScript -> Wasm is not idempotent, so we only check things build.
       - name: Build AssemblyScript examples
         run: make build.examples.as
 
-      # zig-cc -> Wasm is not idempotent, so we only check things build.
       - name: Build zig-cc examples
         run: make build.examples.zig-cc
 
-      # TinyGo -> Wasm is not idempotent, so we only check things build.
+      - name: Build Rust examples
+        run: make build.examples.rust
+
       - name: Build bench cases
         run: make build.bench
+
+      - name: Run example tests
+        run: go test ./examples/...

Makefile

@@ -61,6 +61,17 @@ build.examples.zig-cc: $(c_sources)
 	    zig cc $$f -o $$(echo $$f | sed -e 's/\.c/\.wasm/') --target=wasm32-wasi -O3; \
 	done
 
+%/greet.wasm : cargo_target := wasm32-unknown-unknown
+%/cat.wasm : cargo_target := wasm32-wasi
+
+.PHONY: build.examples.rust
+build.examples.rust: examples/allocation/rust/testdata/greet.wasm examples/wasi/testdata/cargo-wasi/cat.wasm
+
+# Builds rust using cargo normally, or cargo-wasi.
+%.wasm: %.rs
+	@(cd $(@D); cargo $(if $(findstring wasi,$(cargo_target)),wasi build,build --target $(cargo_target)) --release)
+	@mv $(@D)/target/$(cargo_target)/release/$(@F) $(@D)
+
 spectest_base_dir := internal/integration_test/spectest
 spectest_v1_dir := $(spectest_base_dir)/v1
 spectest_v1_testdata_dir := $(spectest_v1_dir)/testdata

examples/allocation/rust/testdata/Cargo.toml

@@ -6,6 +6,8 @@ edition = "2021"
 [lib]
 # cdylib builds a a %.wasm file with `cargo build --release --target wasm32-unknown-unknown`
 crate-type = ["cdylib"]
+name = "greet"
+path = "greet.rs"
 
 [dependencies]
 # wee_aloc is a WebAssembly optimized allocator, which is needed to use non-numeric types like strings.

examples/wasi/README.md

@@ -8,20 +8,53 @@ $ go run cat.go /test.txt
 greet filesystem
 ```
 
-If you do not set the environment variable `WASM_COMPILER`, main defaults
+If you do not set the environment variable `TOOLCHAIN`, main defaults
 to use Wasm built with "tinygo". Here are the included examples:
 
-* [tjnygo](testdata/tinygo) - Built via `tinygo build -o cat.wasm -scheduler=none --no-debug -target=wasi cat.go`
+* [cargo-wasi](testdata/cargo-wasi) - Built via `cargo wasi build --release; mv ./target/wasm32-wasi/release/cat.wasm .`
+* [tinygo](testdata/tinygo) - Built via `tinygo build -o cat.wasm -scheduler=none --no-debug -target=wasi cat.go`
 * [zig-cc](testdata/zig-cc) - Built via `zig cc cat.c -o cat.wasm --target=wasm32-wasi -O3`
 
 Ex. To run the same example with zig-cc:
 ```bash
-$ WASM_COMPILER=zig-cc go run cat.go /test.txt
+$ TOOLCHAIN=zig-cc go run cat.go /test.txt
 greet filesystem
 ```
 
-Note: While WASI attempts to be portable, there are no specification tests and
-some compilers only partially implement features.
+### Toolchain notes
 
-For example, Emscripten only supports WASI I/O on stdin/stdout/stderr, so
-cannot be used for this example. See emscripten-core/emscripten#17167
+Examples here check in the resulting binary, as doing so removes a potentially
+expensive toolchain setup. This means we have to be careful how wasm is built,
+so that the repository doesn't bloat (ex more bandwidth for `git clone`).
+
+While WASI attempts to be portable, there are no specification tests and
+some compilers partially implement features. Notes about portability follow.
+
+### cargo-wasi (Rustlang)
+
+The [Rustlang source][testdata/cargo-wasi] uses [cargo-wasi][1] because the
+normal release target `wasm32-wasi` results in almost 2MB, which is too large
+to check into our source tree.
+
+Concretely, if you use `cargo build --target wasm32-wasi --release`, the binary
+`./target/wasm32-wasi/release/cat.wasm` is over 1.9MB. One explanation for this
+bloat is [`wasm32-wasi` target is not pure rust][2]. As that issue is over two
+years old, it is unlikely to change. This means the only way to create smaller
+wasm is via optimization.
+
+The [cargo-wasi][3] crate includes many optimizations in its release target,
+including `wasm-opt`, a part of [binaryen][3]. `cargo wasi build --release`
+generates 82KB of wasm, which is small enough to check in.
+
+### emscripten
+
+Emscripten is not included as we cannot create a cat program without using
+custom filesystem code. Emscripten only supports WASI I/O for
+[stdin/stdout/stderr][4] and suggest using wasi-libc instead. This is used in
+the [zig-cc](testdata/zig-cc) example.
+
+[1]: https://github.com/bytecodealliance/cargo-wasi
+[2]: https://github.com/rust-lang/rust/issues/73432
+[3]: https://github.com/bytecodealliance/cargo-wasi
+[4]: https://github.com/WebAssembly/binaryen
+[5]: https://github.com/emscripten-core/emscripten/issues/17167#issuecomment-1150252755

examples/wasi/cat.go

@@ -18,6 +18,10 @@ import (
 //go:embed testdata/test.txt
 var catFS embed.FS
 
+// catWasmCargoWasi was compiled from testdata/cargo-wasi/cat.rs
+//go:embed testdata/cargo-wasi/cat.wasm
+var catWasmCargoWasi []byte
+
 // catWasmTinyGo was compiled from testdata/tinygo/cat.go
 //go:embed testdata/tinygo/cat.wasm
 var catWasmTinyGo []byte
@@ -59,16 +63,18 @@ func main() {
 	// Choose the binary we want to test. Most compilers that implement WASI
 	// are portable enough to use binaries interchangeably.
 	var catWasm []byte
-	compiler := os.Getenv("WASM_COMPILER")
-	switch compiler {
+	toolchain := os.Getenv("TOOLCHAIN")
+	switch toolchain {
 	case "":
 		fallthrough // default to TinyGo
+	case "cargo-wasi":
+		catWasm = catWasmCargoWasi
 	case "tinygo":
 		catWasm = catWasmTinyGo
 	case "zig-cc":
 		catWasm = catWasmZigCc
 	default:
-		log.Panicln("unknown compiler", compiler)
+		log.Panicln("unknown toolchain", toolchain)
 	}
 
 	// Compile the WebAssembly module using the default configuration.

examples/wasi/cat_test.go

@@ -11,10 +11,10 @@ import (
 //
 //	go run cat.go /test.txt
 func Test_main(t *testing.T) {
-	for _, compiler := range []string{"tinygo", "zig-cc"} {
-		compiler := compiler
-		t.Run(compiler, func(t *testing.T) {
-			t.Setenv("WASM_COMPILER", compiler)
+	for _, toolchain := range []string{"cargo-wasi", "tinygo", "zig-cc"} {
+		toolchain := toolchain
+		t.Run(toolchain, func(t *testing.T) {
+			t.Setenv("TOOLCHAIN", toolchain)
 			stdout, stderr := maintester.TestMain(t, main, "cat", "/test.txt")
 			require.Equal(t, "", stderr)
 			require.Equal(t, "greet filesystem\n", stdout)

examples/wasi/testdata/cargo-wasi/.gitignore

@@ -0,0 +1,2 @@
+/target
+Cargo.lock

examples/wasi/testdata/cargo-wasi/Cargo.toml

@@ -0,0 +1,8 @@
+[package]
+name = "cat"
+version = "0.1.0"
+edition = "2021"
+
+[[bin]]
+name = "cat"
+path = "cat.rs"

examples/wasi/testdata/cargo-wasi/cat.rs

@@ -0,0 +1,17 @@
+use std::env;
+use std::fs::File;
+use std::io;
+use std::io::Write;
+use std::process::exit;
+
+fn main() {
+    // Start at arg[1] because args[0] is the program name.
+    for path in env::args().skip(1) {
+        if let Ok(mut file) = File::open(&path) {
+            io::copy(&mut file, &mut io::stdout()).unwrap();
+        } else {
+            writeln!(io::stderr(), "error opening: {}: No such file or directory", path).unwrap();
+            exit(1);
+        }
+    }
+}