fuzz: adds target on checking memory state (#1054)

Signed-off-by: Takeshi Yoneda <takeshi@tetrate.io>

Makefile

@@ -254,4 +254,5 @@ fuzz_timeout_seconds ?= 10
 .PHONY: fuzz
 fuzz:
 	@cd internal/integration_test/fuzz && cargo fuzz run basic -- -max_total_time=$(fuzz_timeout_seconds)
+	@cd internal/integration_test/fuzz && cargo fuzz run memory_no_diff -- -max_total_time=$(fuzz_timeout_seconds)
 	@cd internal/integration_test/fuzz && cargo fuzz run validation -- -max_total_time=$(fuzz_timeout_seconds)

internal/integration_test/fuzz/fuzz/Cargo.toml

@@ -13,6 +13,12 @@ libfuzzer-sys = "0.4.3"
 wasm-smith = "0.11.4"
 wasmprinter = "0.2.39"
 
+[[bin]]
+name = "memory_no_diff"
+path = "fuzz_targets/memory_no_diff.rs"
+test = false
+doc = false
+
 [[bin]]
 name = "validation"
 path = "fuzz_targets/validation.rs"

internal/integration_test/fuzz/fuzz/fuzz_targets/basic.rs

@@ -4,6 +4,8 @@ use libfuzzer_sys::arbitrary::{Result, Unstructured};
 use libfuzzer_sys::fuzz_target;
 use wasm_smith::SwarmConfig;
 
+mod wazero_abi;
+
 fuzz_target!(|data: &[u8]| {
     drop(run(data));
 });
@@ -50,24 +52,15 @@ fn run(data: &[u8]) -> Result<()> {
 
     // Pass the randomly generated module to the wazero library.
     unsafe {
-        require_no_diff(
+        wazero_abi::require_no_diff(
             module_bytes.as_ptr(),
             module_bytes.len(),
             wat_bytes.as_ptr(),
             wat_bytes.len(),
+            false,
         );
     }
 
     // We always return Ok as inside require_no_diff, we cause panic if the binary is interesting.
     Ok(())
 }
-
-extern "C" {
-    // require_no_diff is implemented in Go, and accepts the pointer to the binary and its size.
-    fn require_no_diff(
-        binary_ptr: *const u8,
-        binary_size: usize,
-        wat_ptr: *const u8,
-        wat_size: usize,
-    );
-}

internal/integration_test/fuzz/fuzz/fuzz_targets/memory_no_diff.rs

@@ -0,0 +1,68 @@
+//! memory_diff fuzzing test is almost the same as basic.rs
+//! but this also ensure that there's no difference in the memory buffer
+//! state after the execution.
+
+#![no_main]
+use libfuzzer_sys::arbitrary::{Result, Unstructured};
+use libfuzzer_sys::fuzz_target;
+use wasm_smith::SwarmConfig;
+mod wazero_abi;
+
+fuzz_target!(|data: &[u8]| {
+    drop(run(data));
+});
+
+fn run(data: &[u8]) -> Result<()> {
+    // Create the random source.
+    let mut u = Unstructured::new(data);
+
+    // Generate the configuration.
+    let mut config: SwarmConfig = u.arbitrary()?;
+
+    // 64-bit memory won't be supported by wazero.
+    config.memory64_enabled = false;
+    // For exactly one memory exists.
+    config.max_memories = 1;
+    config.min_memories = 1;
+    // If we don't set the limit, we will soon reach the OOM and the fuzzing will be killed by OS.
+    config.max_memory_pages = 10;
+    config.memory_max_size_required = true;
+    // Don't test too large tables.
+    config.max_tables = 2;
+    config.max_table_elements = 1_000;
+    config.table_max_size_required = true;
+
+    // max_instructions is set to 100 by default which seems a little bit smaller.
+    config.max_instructions = 5000;
+
+    // Without canonicalization of NaNs, the results cannot be matched among engines.
+    config.canonicalize_nans = true;
+
+    // Export all the things so that we can invoke them.
+    config.export_everything = true;
+
+    // Ensures that at least one function exists.
+    config.min_funcs = 1;
+    config.max_funcs = config.max_funcs.max(1);
+
+    // Generate the random module via wasm-smith.
+    let mut module = wasm_smith::Module::new(config.clone(), &mut u)?;
+    module.ensure_termination(1000);
+    let module_bytes = module.to_bytes();
+
+    let wat_bytes = wasmprinter::print_bytes(&module_bytes).unwrap();
+
+    // Pass the randomly generated module to the wazero library.
+    unsafe {
+        wazero_abi::require_no_diff(
+            module_bytes.as_ptr(),
+            module_bytes.len(),
+            wat_bytes.as_ptr(),
+            wat_bytes.len(),
+            true,
+        );
+    }
+
+    // We always return Ok as inside require_no_diff, we cause panic if the binary is interesting.
+    Ok(())
+}

internal/integration_test/fuzz/fuzz/fuzz_targets/validation.rs

@@ -3,6 +3,8 @@
 use libfuzzer_sys::arbitrary::{Result, Unstructured};
 use libfuzzer_sys::fuzz_target;
 
+mod wazero_abi;
+
 fuzz_target!(|data: &[u8]| {
     drop(run(data));
 });
@@ -16,14 +18,9 @@ fn run(data: &[u8]) -> Result<()> {
     let module_bytes = module.to_bytes();
 
     unsafe {
-        validate(module_bytes.as_ptr(), module_bytes.len());
+        wazero_abi::validate(module_bytes.as_ptr(), module_bytes.len());
     }
 
     // We always return Ok as inside validate, we cause panic if the binary is interesting.
     Ok(())
 }
-
-extern "C" {
-    // validate is implemented in Go, and accepts the pointer to the binary and its size.
-    fn validate(binary_ptr: *const u8, binary_size: usize);
-}

internal/integration_test/fuzz/fuzz/fuzz_targets/wazero_abi.rs

@@ -0,0 +1,15 @@
+//! This module provides the functions implemented by wazero via CGo.
+
+extern "C" {
+    // require_no_diff is implemented in Go, and accepts the pointer to the binary and its size.
+    pub fn require_no_diff(
+        binary_ptr: *const u8,
+        binary_size: usize,
+        wat_ptr: *const u8,
+        wat_size: usize,
+        check_memory: bool,
+    );
+
+    // validate is implemented in Go, and accepts the pointer to the binary and its size.
+    pub fn validate(binary_ptr: *const u8, binary_size: usize);
+}

internal/integration_test/fuzz/wazerolib/nodiff.go

@@ -20,7 +20,7 @@ import (
 // And if there's diff, this also saves the problematic binary and wat into testdata directory.
 //
 //export require_no_diff
-func require_no_diff(binaryPtr uintptr, binarySize int, watPtr uintptr, watSize int) {
+func require_no_diff(binaryPtr uintptr, binarySize int, watPtr uintptr, watSize int, checkMemory bool) {
 	wasmBin := *(*[]byte)(unsafe.Pointer(&reflect.SliceHeader{
 		Data: binaryPtr,
 		Len:  binarySize,
@@ -41,7 +41,7 @@ func require_no_diff(binaryPtr uintptr, binarySize int, watPtr uintptr, watSize
 		}
 	}()
 
-	requireNoDiff(wasmBin, func(err error) {
+	requireNoDiff(wasmBin, checkMemory, func(err error) {
 		if err != nil {
 			panic(err)
 		}
@@ -76,7 +76,7 @@ func extractInternalWasmModuleFromCompiledModule(c wazero.CompiledModule) (*wasm
 }
 
 // requireNoDiff ensures that the behavior is the same between the compiler and the interpreter for any given binary.
-func requireNoDiff(wasmBin []byte, requireNoError func(err error)) {
+func requireNoDiff(wasmBin []byte, checkMemory bool, requireNoError func(err error)) {
 	// Choose the context to use for function calls.
 	ctx := context.Background()
 
@@ -111,6 +111,15 @@ func requireNoDiff(wasmBin []byte, requireNoError func(err error)) {
 	if okToInvoke {
 		err = ensureInvocationResultMatch(compilerMod, interpreterMod, interpreterCompiled.ExportedFunctions())
 		requireNoError(err)
+
+		compilerMem, _ := compilerMod.Memory().(*wasm.MemoryInstance)
+		interpreterMem, _ := interpreterMod.Memory().(*wasm.MemoryInstance)
+		if checkMemory && compilerMem != nil && interpreterMem != nil {
+			if !bytes.Equal(compilerMem.Buffer, interpreterMem.Buffer) {
+				requireNoError(fmt.Errorf("memory state mimsmatch\ncompiler: %v\ninterpreter: %v",
+					compilerMem.Buffer, interpreterMem.Buffer))
+			}
+		}
 	}
 }
 

internal/integration_test/fuzz/wazerolib/nodiff_test.go

@@ -16,5 +16,5 @@ func TestReRunFailedRequireNoDiffCase(t *testing.T) {
 		t.Skip(err)
 	}
 
-	requireNoDiff(wasmBin, func(err error) { require.NoError(t, err) })
+	requireNoDiff(wasmBin, true, func(err error) { require.NoError(t, err) })
 }