fuzz: make it possible to fuzz wazevo (#1689)

Signed-off-by: Takeshi Yoneda <t.y.mathetake@gmail.com>

Makefile

@@ -256,7 +256,7 @@ clean: ## Ensure a clean build
 fuzz_timeout_seconds ?= 10
 .PHONY: fuzz
 fuzz:
-	@cd internal/integration_test/fuzz && cargo fuzz run basic -- -rss_limit_mb=8192 -max_total_time=$(fuzz_timeout_seconds)
+	@cd internal/integration_test/fuzz && cargo fuzz run no_diff -- -rss_limit_mb=8192 -max_total_time=$(fuzz_timeout_seconds)
 	@cd internal/integration_test/fuzz && cargo fuzz run memory_no_diff -- -rss_limit_mb=8192 -max_total_time=$(fuzz_timeout_seconds)
 	@cd internal/integration_test/fuzz && cargo fuzz run validation -- -rss_limit_mb=8192 -max_total_time=$(fuzz_timeout_seconds)
 

internal/engine/wazevo/config.go

@@ -0,0 +1,42 @@
+package wazevo
+
+import (
+	"context"
+	"unsafe"
+
+	"github.com/tetratelabs/wazero"
+	"github.com/tetratelabs/wazero/api"
+	"github.com/tetratelabs/wazero/internal/filecache"
+	"github.com/tetratelabs/wazero/internal/wasm"
+)
+
+// ConfigureWazevo modifies wazero.RuntimeConfig and sets the wazevo implementation.
+// This is a hack to avoid modifying outside the wazevo package while testing it end-to-end.
+//
+// Until we expose it in the experimental public API, use this for internal testing.
+func ConfigureWazevo(config wazero.RuntimeConfig) {
+	// This is the internal representation of interface in Go.
+	// https://research.swtch.com/interfaces
+	type iface struct {
+		_    *byte
+		data unsafe.Pointer
+	}
+
+	configInterface := (*iface)(unsafe.Pointer(&config))
+
+	// This corresponds to the unexported wazero.runtimeConfig, and the target field newEngine exists
+	// in the middle of the implementation.
+	type newEngine func(context.Context, api.CoreFeatures, filecache.Cache) wasm.Engine
+	type runtimeConfig struct {
+		enabledFeatures       api.CoreFeatures
+		memoryLimitPages      uint32
+		memoryCapacityFromMax bool
+		engineKind            int
+		dwarfDisabled         bool
+		newEngine
+		// Other fields follow, but we don't care.
+	}
+	cm := (*runtimeConfig)(configInterface.data)
+	// Insert the wazevo implementation.
+	cm.newEngine = NewEngine
+}

internal/engine/wazevo/e2e_test.go

@@ -6,14 +6,12 @@ import (
 	"fmt"
 	"math"
 	"testing"
-	"unsafe"
 
 	"github.com/tetratelabs/wazero"
 	"github.com/tetratelabs/wazero/api"
 	"github.com/tetratelabs/wazero/internal/engine/wazevo"
 	"github.com/tetratelabs/wazero/internal/engine/wazevo/testcases"
 	"github.com/tetratelabs/wazero/internal/engine/wazevo/wazevoapi"
-	"github.com/tetratelabs/wazero/internal/filecache"
 	"github.com/tetratelabs/wazero/internal/integration_test/spectest"
 	v1 "github.com/tetratelabs/wazero/internal/integration_test/spectest/v1"
 	v2 "github.com/tetratelabs/wazero/internal/integration_test/spectest/v2"
@@ -32,7 +30,7 @@ const (
 func TestSpectestV1(t *testing.T) {
 	config := wazero.NewRuntimeConfigCompiler().WithCoreFeatures(api.CoreFeaturesV1)
 	// Configure the new optimizing backend!
-	configureWazevo(config)
+	wazevo.ConfigureWazevo(config)
 
 	// TODO: migrate to integration_test/spectest/v1/spec_test.go by the time when closing https://github.com/tetratelabs/wazero/issues/1496
 	for _, tc := range []struct {
@@ -125,7 +123,7 @@ func TestSpectestV1(t *testing.T) {
 func TestSpectestV2(t *testing.T) {
 	config := wazero.NewRuntimeConfigCompiler().WithCoreFeatures(api.CoreFeaturesV2)
 	// Configure the new optimizing backend!
-	configureWazevo(config)
+	wazevo.ConfigureWazevo(config)
 
 	for _, tc := range []struct {
 		name string
@@ -357,7 +355,7 @@ func TestE2E(t *testing.T) {
 			config := wazero.NewRuntimeConfigCompiler()
 
 			// Configure the new optimizing backend!
-			configureWazevo(config)
+			wazevo.ConfigureWazevo(config)
 
 			ctx := context.Background()
 			r := wazero.NewRuntimeWithConfig(ctx, config)
@@ -406,40 +404,11 @@ func TestE2E(t *testing.T) {
 	}
 }
 
-// configureWazevo modifies wazero.RuntimeConfig and sets the wazevo implementation.
-// This is a hack to avoid modifying outside the wazevo package while testing it end-to-end.
-func configureWazevo(config wazero.RuntimeConfig) {
-	// This is the internal representation of interface in Go.
-	// https://research.swtch.com/interfaces
-	type iface struct {
-		_    *byte
-		data unsafe.Pointer
-	}
-
-	configInterface := (*iface)(unsafe.Pointer(&config))
-
-	// This corresponds to the unexported wazero.runtimeConfig, and the target field newEngine exists
-	// in the middle of the implementation.
-	type newEngine func(context.Context, api.CoreFeatures, filecache.Cache) wasm.Engine
-	type runtimeConfig struct {
-		enabledFeatures       api.CoreFeatures
-		memoryLimitPages      uint32
-		memoryCapacityFromMax bool
-		engineKind            int
-		dwarfDisabled         bool
-		newEngine
-		// Other fields follow, but we don't care.
-	}
-	cm := (*runtimeConfig)(configInterface.data)
-	// Insert the wazevo implementation.
-	cm.newEngine = wazevo.NewEngine
-}
-
 func TestE2E_host_functions(t *testing.T) {
 	config := wazero.NewRuntimeConfigCompiler()
 
 	// Configure the new optimizing backend!
-	configureWazevo(config)
+	wazevo.ConfigureWazevo(config)
 
 	ctx := context.Background()
 	r := wazero.NewRuntimeWithConfig(ctx, config)
@@ -514,7 +483,7 @@ func TestE2E_stores(t *testing.T) {
 	config := wazero.NewRuntimeConfigCompiler()
 
 	// Configure the new optimizing backend!
-	configureWazevo(config)
+	wazevo.ConfigureWazevo(config)
 
 	ctx := context.Background()
 	r := wazero.NewRuntimeWithConfig(ctx, config)
@@ -604,7 +573,7 @@ func TestE2E_reexported_memory(t *testing.T) {
 	config := wazero.NewRuntimeConfigCompiler()
 
 	// Configure the new optimizing backend!
-	configureWazevo(config)
+	wazevo.ConfigureWazevo(config)
 
 	ctx := context.Background()
 	r := wazero.NewRuntimeWithConfig(ctx, config)

internal/integration_test/fuzz/README.md

@@ -11,9 +11,9 @@ Fuzzing infrastructure for wazero engines via [wasm-tools](https://github.com/by
 
 Currently, we have the following fuzzing targets:
 
-- `basic`: compares the results from the compiler and interpreter engines, and see if there's a diff in them.
-- `memory_no_diff`: same as `basic` except that in addition to the results, it also compares the entire memory buffer between engines to ensure the consistency around memory access.
-  Therefore, this takes much longer than `basic`.
+- `no_diff`: compares the results from the compiler and interpreter engines, and see if there's a diff in them.
+- `memory_no_diff`: same as `no_diff` except that in addition to the results, it also compares the entire memory buffer between engines to ensure the consistency around memory access.
+  Therefore, this takes much longer than `no_diff`.
 - `validation`: try compiling maybe-invalid Wasm module binaries. This is to ensure that our validation phase works correctly as well as the engines do not panic during compilation.
 
 
@@ -22,9 +22,6 @@ To run the fuzzer on a target, execute the following command:
 ```
 # Running on the host archictecture.
 cargo fuzz run <target>
-
-# Running on the specified architecture which is handy when developping on M1 Mac.
-cargo fuzz run <target>-x86_64-apple-darwin
 ```
 
 where you replace `<target>` with one of the targets described above.
@@ -32,17 +29,17 @@ where you replace `<target>` with one of the targets described above.
 See `cargo fuzz run --help` for the options. Especially, the following flags are useful:
 
 - `-jobs=N`: `cargo fuzz run` by default only spawns one worker, so this flag helps do the parallel fuzzing.
-  - usage: `cargo fuzz run basic -- -jobs=5` will run 5 parallel workers to run fuzzing jobs.
+  - usage: `cargo fuzz run no_diff -- -jobs=5` will run 5 parallel workers to run fuzzing jobs.
 - `-max_total_time`: the maximum total time in seconds to run the fuzzer.
-  - usage: `cargo fuzz run basic -- -max_total_time=100` will run fuzzing for 100 seconds.
+  - usage: `cargo fuzz run no_diff -- -max_total_time=100` will run fuzzing for 100 seconds.
 - `-timeout` sets the timeout seconds _per fuzzing run_, not the entire job.
 - `-rss_limit_mb` sets the memory usage limit which is 2GB by default. Usually 2GB is not enough for some large Wasm binary.
 
 #### Example commands
 
 ```
-# Running the `basic` target with 15 concurrent jobs with total runnig time with 2hrs and 8GB memory limit.
-$ cargo fuzz run basic -- -rss_limit_mb=8192 -max_len=5000000 -max_total_time=7200 -jobs=15
+# Running the `no_diff` target with 15 concurrent jobs with total runnig time with 2hrs and 8GB memory limit.
+$ cargo fuzz run no_diff -- -rss_limit_mb=8192 -max_len=5000000 -max_total_time=7200 -jobs=15
 
 # Running the `memory_no_diff` target with 15 concurrent jobs with timeout 2hrs and setting timeout per fuzz case to 30s.
 $ cargo fuzz run memory_no_diff -- -timeout=30 -max_total_time=7200 -jobs=15
@@ -73,7 +70,7 @@ Also, in the bottom of the output, you can find the message as
 
 Minimize test case with:
 
-        cargo fuzz tmin basic fuzz/artifacts/basic/crash-d2c1f5307fde6f057454606bcc21d5653be9be8d
+        cargo fuzz tmin no_diff fuzz/artifacts/no_diff/crash-d2c1f5307fde6f057454606bcc21d5653be9be8d
 
 ────────────────────────────────────────────────────────────────────────────────
 ```

internal/integration_test/fuzz/fuzz/Cargo.toml

@@ -26,37 +26,7 @@ test = false
 doc = false
 
 [[bin]]
-name = "basic"
-path = "fuzz_targets/basic.rs"
+name = "no_diff"
+path = "fuzz_targets/no_diff.rs"
 test = false
 doc = false
-
-# Note: having different bin target for each architecture in order to have separate corpus and artifacts and
-# to run both fuzzing on M1 Mac.
-[[bin]]
-name = "basic-x86_64-apple-darwin"
-path = "fuzz_targets/basic.rs"
-test = false
-doc = false
-target = "x86_64-apple-darwin"
-
-[[bin]]
-name = "basic-aarch64-apple-darwin"
-path = "fuzz_targets/basic.rs"
-test = false
-doc = false
-target = "aarch64-apple-darwin"
-
-[[bin]]
-name = "basic-x86_64-unknown-linux-gnu"
-path = "fuzz_targets/basic.rs"
-test = false
-doc = false
-target = "x86_64-unknown-linux-gnu"
-
-[[bin]]
-name = "basic-aarch64-unknown-linux-gnu"
-path = "fuzz_targets/basic.rs"
-test = false
-doc = false
-target = "aarch64-unknown-linux-gnu"

internal/integration_test/fuzz/fuzz/fuzz_targets/memory_no_diff.rs

@@ -9,7 +9,7 @@ use wasm_smith::SwarmConfig;
 mod wazero_abi;
 
 fuzz_target!(|data: &[u8]| {
-    drop(run(data));
+    let _ = run(data);
 });
 
 fn run(data: &[u8]) -> Result<()> {
@@ -45,6 +45,8 @@ fn run(data: &[u8]) -> Result<()> {
     config.min_funcs = 1;
     config.max_funcs = config.max_funcs.max(1);
 
+    wazero_abi::maybe_disable_v2(&mut config);
+
     // Generate the random module via wasm-smith.
     let mut module = wasm_smith::Module::new(config.clone(), &mut u)?;
     module.ensure_termination(1000);

internal/integration_test/fuzz/fuzz/fuzz_targets/no_diff.rs

@@ -7,7 +7,7 @@ use wasm_smith::SwarmConfig;
 mod wazero_abi;
 
 fuzz_target!(|data: &[u8]| {
-    drop(run(data));
+    let _ = run(data);
 });
 
 fn run(data: &[u8]) -> Result<()> {
@@ -43,6 +43,8 @@ fn run(data: &[u8]) -> Result<()> {
     config.min_funcs = 1;
     config.max_funcs = config.max_funcs.max(1);
 
+    wazero_abi::maybe_disable_v2(&mut config);
+
     // Generate the random module via wasm-smith.
     let mut module = wasm_smith::Module::new(config.clone(), &mut u)?;
     module.ensure_termination(1000);

internal/integration_test/fuzz/fuzz/fuzz_targets/validation.rs

@@ -6,7 +6,7 @@ use libfuzzer_sys::fuzz_target;
 mod wazero_abi;
 
 fuzz_target!(|data: &[u8]| {
-    drop(run(data));
+    let _ = run(data);
 });
 
 fn run(data: &[u8]) -> Result<()> {

internal/integration_test/fuzz/fuzz/fuzz_targets/wazero_abi.rs

@@ -1,7 +1,10 @@
 //! This module provides the functions implemented by wazero via CGo.
 
+use wasm_smith::SwarmConfig;
+
 extern "C" {
     // require_no_diff is implemented in Go, and accepts the pointer to the binary and its size.
+    #[allow(dead_code)]
     pub fn require_no_diff(
         binary_ptr: *const u8,
         binary_size: usize,
@@ -11,5 +14,17 @@ extern "C" {
     );
 
     // validate is implemented in Go, and accepts the pointer to the binary and its size.
+    #[allow(dead_code)]
     pub fn validate(binary_ptr: *const u8, binary_size: usize);
 }
+
+pub fn maybe_disable_v2(config: &mut SwarmConfig) {
+    if std::env::var("WAZERO_FUZZ_WAZEVO").is_ok() {
+        config.simd_enabled = false;
+        config.multi_value_enabled = false;
+        config.bulk_memory_enabled = false;
+        config.reference_types_enabled = false;
+        config.saturating_float_to_int_enabled = false;
+        config.max_tables = 1;
+    }
+}

internal/integration_test/fuzz/wazerolib/lib.go

@@ -7,6 +7,10 @@ import (
 	"fmt"
 	"os"
 	"path"
+
+	"github.com/tetratelabs/wazero"
+	"github.com/tetratelabs/wazero/api"
+	"github.com/tetratelabs/wazero/internal/engine/wazevo"
 )
 
 func main() {}
@@ -69,3 +73,13 @@ To reproduce the failure, execute: WASM_BINARY_PATH=%s go test -run=%s ./wazerol
 `, hex.EncodeToString(bin), binaryPath, binaryPath, reproduceTestName)
 	}
 }
+
+// This returns a wazevo.RuntimeConfigure whose compiler is either wazevo or the default.
+func newCompilerConfig() wazero.RuntimeConfig {
+	c := wazero.NewRuntimeConfigCompiler()
+	if os.Getenv("WAZERO_FUZZ_WAZEVO") != "" {
+		c = c.WithCoreFeatures(api.CoreFeaturesV1) // Currently only V1 is supported.
+		wazevo.ConfigureWazevo(c)
+	}
+	return c
+}

internal/integration_test/fuzz/wazerolib/nodiff.go

@@ -80,7 +80,7 @@ func requireNoDiff(wasmBin []byte, checkMemory bool, requireNoError func(err err
 	// Choose the context to use for function calls.
 	ctx := context.Background()
 
-	compiler := wazero.NewRuntimeWithConfig(ctx, wazero.NewRuntimeConfigCompiler())
+	compiler := wazero.NewRuntimeWithConfig(ctx, newCompilerConfig())
 	interpreter := wazero.NewRuntimeWithConfig(ctx, wazero.NewRuntimeConfigInterpreter())
 	defer compiler.Close(ctx)
 	defer interpreter.Close(ctx)

internal/integration_test/fuzz/wazerolib/validate.go

@@ -36,6 +36,6 @@ func validate(binaryPtr uintptr, binarySize int) {
 // Ensure that validation and compilation do not panic!
 func tryCompile(wasmBin []byte) {
 	ctx := context.Background()
-	compiler := wazero.NewRuntimeWithConfig(ctx, wazero.NewRuntimeConfigCompiler())
+	compiler := wazero.NewRuntimeWithConfig(ctx, newCompilerConfig())
 	_, _ = compiler.CompileModule(ctx, wasmBin)
 }