From 81e96c047da8de6139bb060fd67f80dcd7f69aee Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Mon, 5 Apr 2021 07:24:35 -0700 Subject: [PATCH] Fully support multiple returns in Wasmtime For quite some time now Wasmtime has "supported" multiple return values, but only in the mose bare bones ways. Up until recently you couldn't get a typed version of functions with multiple return values, and never have you been able to use `Func::wrap` with functions that return multiple values. Even recently where `Func::typed` can call functions that return multiple values it uses a double-indirection by calling a trampoline which calls the real function. The underlying reason for this lack of support is that cranelift's ABI for returning multiple values is not possible to write in Rust. For example if a wasm function returns two `i32` values there is no Rust (or C!) function you can write to correspond to that. This commit, however fixes that. This commit adds two new ABIs to Cranelift: `WasmtimeSystemV` and `WasmtimeFastcall`. The intention is that these Wasmtime-specific ABIs match their corresponding ABI (e.g. `SystemV` or `WindowsFastcall`) for everything *except* how multiple values are returned. For multiple return values we simply define our own version of the ABI which Wasmtime implements, which is that for N return values the first is returned as if the function only returned that and the latter N-1 return values are returned via an out-ptr that's the last parameter to the function. These custom ABIs provides the ability for Wasmtime to bind these in Rust meaning that `Func::wrap` can now wrap functions that return multiple values and `Func::typed` no longer uses trampolines when calling functions that return multiple values. Although there's lots of internal changes there's no actual changes in the API surface area of Wasmtime, just a few more impls of more public traits which means that more types are supported in more places! Another change made with this PR is a consolidation of how the ABI of each function in a wasm module is selected. The native `SystemV` ABI, for example, is more efficient at returning multiple values than the wasmtime version of the ABI (since more things are in more registers). To continue to take advantage of this Wasmtime will now classify some functions in a wasm module with the "fast" ABI. Only functions that are not reachable externally from the module are classified with the fast ABI (e.g. those not exported, used in tables, or used with `ref.func`). This should enable purely internal functions of modules to have a faster calling convention than those which might be exposed to Wasmtime itself. Closes #1178 --- Cargo.lock | 1 + cranelift/codegen/src/context.rs | 8 +- cranelift/codegen/src/isa/aarch64/abi.rs | 54 +-- cranelift/codegen/src/isa/call_conv.rs | 32 +- cranelift/codegen/src/isa/x64/abi.rs | 50 +-- cranelift/codegen/src/isa/x86/abi.rs | 15 +- .../codegen/src/isa/x86/unwind/systemv.rs | 6 +- cranelift/codegen/src/machinst/abi_impl.rs | 13 +- cranelift/wasm/src/environ/dummy.rs | 5 +- cranelift/wasm/src/environ/spec.rs | 6 +- cranelift/wasm/src/sections_translator.rs | 14 +- cranelift/wasm/src/translation_utils.rs | 1 + crates/cranelift/Cargo.toml | 1 + crates/cranelift/src/func_environ.rs | 47 ++- crates/cranelift/src/lib.rs | 98 +++++- crates/environ/src/module.rs | 8 +- crates/environ/src/module_environ.rs | 48 +-- crates/jit/src/lib.rs | 1 + crates/jit/src/native.rs | 5 - crates/jit/src/object.rs | 16 +- crates/jit/src/trampoline.rs | 14 +- crates/wasmtime/src/config.rs | 4 +- crates/wasmtime/src/func.rs | 311 ++++++++++++------ crates/wasmtime/src/func/typed.rs | 199 +++++------ crates/wasmtime/src/sig_registry.rs | 19 +- crates/wasmtime/src/store.rs | 10 +- crates/wasmtime/src/trampoline/func.rs | 25 +- crates/wasmtime/src/types.rs | 22 -- tests/all/func.rs | 159 +++++++++ 29 files changed, 748 insertions(+), 444 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 148263def6a7..e034571ae584 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3310,6 +3310,7 @@ dependencies = [ "cranelift-entity", "cranelift-frontend", "cranelift-wasm", + "target-lexicon", "wasmparser", "wasmtime-environ", ] diff --git a/cranelift/codegen/src/context.rs b/cranelift/codegen/src/context.rs index b831f9966a31..f668fe596a9d 100644 --- a/cranelift/codegen/src/context.rs +++ b/cranelift/codegen/src/context.rs @@ -267,13 +267,7 @@ impl Context { isa: &dyn TargetIsa, ) -> CodegenResult> { if let Some(backend) = isa.get_mach_backend() { - use crate::isa::CallConv; - use crate::machinst::UnwindInfoKind; - let unwind_info_kind = match self.func.signature.call_conv { - CallConv::Fast | CallConv::Cold | CallConv::SystemV => UnwindInfoKind::SystemV, - CallConv::WindowsFastcall => UnwindInfoKind::Windows, - _ => UnwindInfoKind::None, - }; + let unwind_info_kind = self.func.signature.call_conv.unwind_info_kind(); let result = self.mach_compile_result.as_ref().unwrap(); return backend.emit_unwind_info(result, unwind_info_kind); } diff --git a/cranelift/codegen/src/isa/aarch64/abi.rs b/cranelift/codegen/src/isa/aarch64/abi.rs index 8f3b2e6f1eb1..b2a95e314be9 100644 --- a/cranelift/codegen/src/isa/aarch64/abi.rs +++ b/cranelift/codegen/src/isa/aarch64/abi.rs @@ -197,18 +197,26 @@ impl ABIMachineSpec for AArch64MachineDeps { next_stack = 16; } - // Note on return values: on the regular non-baldrdash ABI, we may return values in 8 - // registers for V128 and I64 registers independently of the number of register values - // returned in the other class. That is, we can return values in up to 8 integer and 8 - // vector registers at once. - // In Baldrdash, we can only use one register for return value for all the register - // classes. That is, we can't return values in both one integer and one vector register; - // only one return value may be in a register. - - let (max_per_class_reg_vals, mut remaining_reg_vals) = match (args_or_rets, is_baldrdash) { - (ArgsOrRets::Args, _) => (8, 16), // x0-x7 and v0-v7 - (ArgsOrRets::Rets, false) => (8, 16), // x0-x7 and v0-v7 - (ArgsOrRets::Rets, true) => (1, 1), // x0 or v0, but not both + let (max_per_class_reg_vals, mut remaining_reg_vals) = match args_or_rets { + ArgsOrRets::Args => (8, 16), // x0-x7 and v0-v7 + + // Note on return values: on the regular ABI, we may return values + // in 8 registers for V128 and I64 registers independently of the + // number of register values returned in the other class. That is, + // we can return values in up to 8 integer and + // 8 vector registers at once. + // + // In Baldrdash and Wasmtime, we can only use one register for + // return value for all the register classes. That is, we can't + // return values in both one integer and one vector register; only + // one return value may be in a register. + ArgsOrRets::Rets => { + if is_baldrdash || call_conv.extends_wasmtime() { + (1, 1) // x0 or v0, but not both + } else { + (8, 16) // x0-x7 and v0-v7 + } + } }; for i in 0..params.len() { @@ -282,16 +290,18 @@ impl ABIMachineSpec for AArch64MachineDeps { // Compute the stack slot's size. let size = (ty_bits(param.value_type) / 8) as u64; - let size = if call_conv != isa::CallConv::AppleAarch64 { - // Every arg takes a minimum slot of 8 bytes. (16-byte stack - // alignment happens separately after all args.) - std::cmp::max(size, 8) - } else { - // MacOS aarch64 allows stack slots with sizes less than 8 - // bytes. They still need to be properly aligned on their - // natural data alignment, though. - size - }; + let size = + if call_conv == isa::CallConv::AppleAarch64 || call_conv.extends_wasmtime() { + // MacOS aarch64 and Wasmtime allow stack slots with + // sizes less than 8 bytes. They still need to be + // properly aligned on their natural data alignment, + // though. + size + } else { + // Every arg takes a minimum slot of 8 bytes. (16-byte stack + // alignment happens separately after all args.) + std::cmp::max(size, 8) + }; // Align the stack slot. debug_assert!(size.is_power_of_two()); diff --git a/cranelift/codegen/src/isa/call_conv.rs b/cranelift/codegen/src/isa/call_conv.rs index e764f33c6e37..fb0a2ca2635c 100644 --- a/cranelift/codegen/src/isa/call_conv.rs +++ b/cranelift/codegen/src/isa/call_conv.rs @@ -1,3 +1,4 @@ +use crate::machinst::UnwindInfoKind; use crate::settings::{self, LibcallCallConv}; use core::fmt; use core::str; @@ -29,6 +30,10 @@ pub enum CallConv { Baldrdash2020, /// Specialized convention for the probestack function. Probestack, + /// Wasmtime equivalent of SystemV, except the multi-return ABI is tweaked. + WasmtimeSystemV, + /// Wasmtime equivalent of WindowsFastcall, except the multi-return ABI is tweaked. + WasmtimeFastcall, } impl CallConv { @@ -63,7 +68,7 @@ impl CallConv { /// Is the calling convention extending the Windows Fastcall ABI? pub fn extends_windows_fastcall(self) -> bool { match self { - Self::WindowsFastcall | Self::BaldrdashWindows => true, + Self::WindowsFastcall | Self::BaldrdashWindows | Self::WasmtimeFastcall => true, _ => false, } } @@ -75,6 +80,27 @@ impl CallConv { _ => false, } } + + /// Is the calling convention extending the Wasmtime ABI? + pub fn extends_wasmtime(self) -> bool { + match self { + Self::WasmtimeSystemV | Self::WasmtimeFastcall => true, + _ => false, + } + } + + /// Returns the kind of unwind info used for this calling convention + pub fn unwind_info_kind(self) -> UnwindInfoKind { + match self { + #[cfg(feature = "unwind")] + CallConv::Fast | CallConv::Cold | CallConv::SystemV | CallConv::WasmtimeSystemV => { + UnwindInfoKind::SystemV + } + #[cfg(feature = "unwind")] + CallConv::WindowsFastcall | CallConv::WasmtimeFastcall => UnwindInfoKind::Windows, + _ => UnwindInfoKind::None, + } + } } impl fmt::Display for CallConv { @@ -89,6 +115,8 @@ impl fmt::Display for CallConv { Self::BaldrdashWindows => "baldrdash_windows", Self::Baldrdash2020 => "baldrdash_2020", Self::Probestack => "probestack", + Self::WasmtimeSystemV => "wasmtime_system_v", + Self::WasmtimeFastcall => "wasmtime_fastcall", }) } } @@ -106,6 +134,8 @@ impl str::FromStr for CallConv { "baldrdash_windows" => Ok(Self::BaldrdashWindows), "baldrdash_2020" => Ok(Self::Baldrdash2020), "probestack" => Ok(Self::Probestack), + "wasmtime_system_v" => Ok(Self::WasmtimeSystemV), + "wasmtime_fastcall" => Ok(Self::WasmtimeFastcall), _ => Err(()), } } diff --git a/cranelift/codegen/src/isa/x64/abi.rs b/cranelift/codegen/src/isa/x64/abi.rs index f55478a1c8d3..29a0dbd98250 100644 --- a/cranelift/codegen/src/isa/x64/abi.rs +++ b/cranelift/codegen/src/isa/x64/abi.rs @@ -237,10 +237,20 @@ impl ABIMachineSpec for X64ABIMachineSpec { extension: param.extension, }); } else { - // Compute size. Every arg takes a minimum slot of 8 bytes. (16-byte - // stack alignment happens separately after all args.) + // Compute size. For the wasmtime ABI it differs from native + // ABIs in how multiple values are returned, so we take a + // leaf out of arm64's book by not rounding everything up to + // 8 bytes. For all ABI arguments, and other ABI returns, + // though, each slot takes a minimum of 8 bytes. + // + // Note that in all cases 16-byte stack alignment happens + // separately after all args. let size = (reg_ty.bits() / 8) as u64; - let size = std::cmp::max(size, 8); + let size = if args_or_rets == ArgsOrRets::Rets && call_conv.extends_wasmtime() { + size + } else { + std::cmp::max(size, 8) + }; // Align. debug_assert!(size.is_power_of_two()); next_stack = align_to(next_stack, size); @@ -824,15 +834,7 @@ impl From for SyntheticAmode { } fn get_intreg_for_arg(call_conv: &CallConv, idx: usize, arg_idx: usize) -> Option { - let is_fastcall = match call_conv { - CallConv::Fast - | CallConv::Cold - | CallConv::SystemV - | CallConv::BaldrdashSystemV - | CallConv::Baldrdash2020 => false, - CallConv::WindowsFastcall => true, - _ => panic!("int args only supported for SysV or Fastcall calling convention"), - }; + let is_fastcall = call_conv.extends_windows_fastcall(); // Fastcall counts by absolute argument number; SysV counts by argument of // this (integer) class. @@ -853,15 +855,7 @@ fn get_intreg_for_arg(call_conv: &CallConv, idx: usize, arg_idx: usize) -> Optio } fn get_fltreg_for_arg(call_conv: &CallConv, idx: usize, arg_idx: usize) -> Option { - let is_fastcall = match call_conv { - CallConv::Fast - | CallConv::Cold - | CallConv::SystemV - | CallConv::BaldrdashSystemV - | CallConv::Baldrdash2020 => false, - CallConv::WindowsFastcall => true, - _ => panic!("float args only supported for SysV or Fastcall calling convention"), - }; + let is_fastcall = call_conv.extends_windows_fastcall(); // Fastcall counts by absolute argument number; SysV counts by argument of // this (floating-point) class. @@ -894,7 +888,10 @@ fn get_intreg_for_retval( 1 => Some(regs::rdx()), _ => None, }, - CallConv::BaldrdashSystemV | CallConv::Baldrdash2020 => { + CallConv::BaldrdashSystemV + | CallConv::Baldrdash2020 + | CallConv::WasmtimeSystemV + | CallConv::WasmtimeFastcall => { if intreg_idx == 0 && retval_idx == 0 { Some(regs::rax()) } else { @@ -922,7 +919,10 @@ fn get_fltreg_for_retval( 1 => Some(regs::xmm1()), _ => None, }, - CallConv::BaldrdashSystemV | CallConv::Baldrdash2020 => { + CallConv::BaldrdashSystemV + | CallConv::Baldrdash2020 + | CallConv::WasmtimeFastcall + | CallConv::WasmtimeSystemV => { if fltreg_idx == 0 && retval_idx == 0 { Some(regs::xmm0()) } else { @@ -992,12 +992,12 @@ fn get_callee_saves(call_conv: &CallConv, regs: &Set>) -> Vec< CallConv::BaldrdashWindows => { todo!("baldrdash windows"); } - CallConv::Fast | CallConv::Cold | CallConv::SystemV => regs + CallConv::Fast | CallConv::Cold | CallConv::SystemV | CallConv::WasmtimeSystemV => regs .iter() .cloned() .filter(|r| is_callee_save_systemv(r.to_reg())) .collect(), - CallConv::WindowsFastcall => regs + CallConv::WindowsFastcall | CallConv::WasmtimeFastcall => regs .iter() .cloned() .filter(|r| is_callee_save_fastcall(r.to_reg())) diff --git a/cranelift/codegen/src/isa/x86/abi.rs b/cranelift/codegen/src/isa/x86/abi.rs index c6df87ae2203..83b44be1b09a 100644 --- a/cranelift/codegen/src/isa/x86/abi.rs +++ b/cranelift/codegen/src/isa/x86/abi.rs @@ -503,10 +503,12 @@ fn callee_saved_regs_used(isa: &dyn TargetIsa, func: &ir::Function) -> RegisterS pub fn prologue_epilogue(func: &mut ir::Function, isa: &dyn TargetIsa) -> CodegenResult<()> { match func.signature.call_conv { // For now, just translate fast and cold as system_v. - CallConv::Fast | CallConv::Cold | CallConv::SystemV => { + CallConv::Fast | CallConv::Cold | CallConv::SystemV | CallConv::WasmtimeSystemV => { system_v_prologue_epilogue(func, isa) } - CallConv::WindowsFastcall => fastcall_prologue_epilogue(func, isa), + CallConv::WindowsFastcall | CallConv::WasmtimeFastcall => { + fastcall_prologue_epilogue(func, isa) + } CallConv::BaldrdashSystemV | CallConv::BaldrdashWindows => { baldrdash_prologue_epilogue(func, isa) } @@ -1084,16 +1086,17 @@ pub fn create_unwind_info( isa: &dyn TargetIsa, ) -> CodegenResult> { use crate::isa::unwind::UnwindInfo; + use crate::machinst::UnwindInfoKind; // Assumption: RBP is being used as the frame pointer for both calling conventions // In the future, we should be omitting frame pointer as an optimization, so this will change - Ok(match func.signature.call_conv { - CallConv::Fast | CallConv::Cold | CallConv::SystemV => { + Ok(match func.signature.call_conv.unwind_info_kind() { + UnwindInfoKind::SystemV => { super::unwind::systemv::create_unwind_info(func, isa)?.map(|u| UnwindInfo::SystemV(u)) } - CallConv::WindowsFastcall => { + UnwindInfoKind::Windows => { super::unwind::winx64::create_unwind_info(func, isa)?.map(|u| UnwindInfo::WindowsX64(u)) } - _ => None, + UnwindInfoKind::None => None, }) } diff --git a/cranelift/codegen/src/isa/x86/unwind/systemv.rs b/cranelift/codegen/src/isa/x86/unwind/systemv.rs index f3e1cbea84a1..26e1ccbda7d0 100644 --- a/cranelift/codegen/src/isa/x86/unwind/systemv.rs +++ b/cranelift/codegen/src/isa/x86/unwind/systemv.rs @@ -3,7 +3,7 @@ use crate::ir::Function; use crate::isa::{ unwind::systemv::{RegisterMappingError, UnwindInfo}, - CallConv, RegUnit, TargetIsa, + RegUnit, TargetIsa, }; use crate::result::CodegenResult; use gimli::{write::CommonInformationEntry, Encoding, Format, Register, X86_64}; @@ -97,8 +97,8 @@ pub(crate) fn create_unwind_info( isa: &dyn TargetIsa, ) -> CodegenResult> { // Only System V-like calling conventions are supported - match func.signature.call_conv { - CallConv::Fast | CallConv::Cold | CallConv::SystemV => {} + match func.signature.call_conv.unwind_info_kind() { + crate::machinst::UnwindInfoKind::SystemV => {} _ => return Ok(None), } diff --git a/cranelift/codegen/src/machinst/abi_impl.rs b/cranelift/codegen/src/machinst/abi_impl.rs index 17995e18df90..765712c2cf5c 100644 --- a/cranelift/codegen/src/machinst/abi_impl.rs +++ b/cranelift/codegen/src/machinst/abi_impl.rs @@ -647,7 +647,8 @@ impl ABICalleeImpl { || call_conv == isa::CallConv::Cold || call_conv.extends_baldrdash() || call_conv.extends_windows_fastcall() - || call_conv == isa::CallConv::AppleAarch64, + || call_conv == isa::CallConv::AppleAarch64 + || call_conv == isa::CallConv::WasmtimeSystemV, "Unsupported calling convention: {:?}", call_conv ); @@ -1372,15 +1373,7 @@ impl ABICallee for ABICalleeImpl { } fn unwind_info_kind(&self) -> UnwindInfoKind { - match self.sig.call_conv { - #[cfg(feature = "unwind")] - isa::CallConv::Fast | isa::CallConv::Cold | isa::CallConv::SystemV => { - UnwindInfoKind::SystemV - } - #[cfg(feature = "unwind")] - isa::CallConv::WindowsFastcall => UnwindInfoKind::Windows, - _ => UnwindInfoKind::None, - } + self.sig.call_conv.unwind_info_kind() } } diff --git a/cranelift/wasm/src/environ/dummy.rs b/cranelift/wasm/src/environ/dummy.rs index 2bcd004dbe1e..7a56712b7cee 100644 --- a/cranelift/wasm/src/environ/dummy.rs +++ b/cranelift/wasm/src/environ/dummy.rs @@ -660,8 +660,9 @@ impl TargetEnvironment for DummyEnvironment { } impl<'data> ModuleEnvironment<'data> for DummyEnvironment { - fn declare_type_func(&mut self, _wasm: WasmFuncType, sig: ir::Signature) -> WasmResult<()> { - self.info.signatures.push(sig); + fn declare_type_func(&mut self, _wasm: WasmFuncType) -> WasmResult<()> { + // TODO + // self.info.signatures.push(sig); Ok(()) } diff --git a/cranelift/wasm/src/environ/spec.rs b/cranelift/wasm/src/environ/spec.rs index 0bd3c48745f9..3a2389c44d8e 100644 --- a/cranelift/wasm/src/environ/spec.rs +++ b/cranelift/wasm/src/environ/spec.rs @@ -702,11 +702,7 @@ pub trait ModuleEnvironment<'data>: TargetEnvironment { } /// Declares a function signature to the environment. - fn declare_type_func( - &mut self, - wasm_func_type: WasmFuncType, - sig: ir::Signature, - ) -> WasmResult<()>; + fn declare_type_func(&mut self, wasm_func_type: WasmFuncType) -> WasmResult<()>; /// Declares a module type signature to the environment. fn declare_type_module( diff --git a/cranelift/wasm/src/sections_translator.rs b/cranelift/wasm/src/sections_translator.rs index e9f0f784a5b1..c617c9943c89 100644 --- a/cranelift/wasm/src/sections_translator.rs +++ b/cranelift/wasm/src/sections_translator.rs @@ -18,7 +18,6 @@ use crate::wasm_unsupported; use core::convert::TryFrom; use core::convert::TryInto; use cranelift_codegen::ir::immediates::V128Imm; -use cranelift_codegen::ir::{self, AbiParam, Signature}; use cranelift_entity::packed_option::ReservedValue; use cranelift_entity::EntityRef; use std::boxed::Box; @@ -110,18 +109,7 @@ pub fn parse_type_section<'a>( for entry in types { match entry? { TypeDef::Func(wasm_func_ty) => { - let mut sig = Signature::new(environ.target_config().default_call_conv); - sig.params.extend(wasm_func_ty.params.iter().map(|ty| { - let cret_arg: ir::Type = type_to_type(*ty, environ) - .expect("only numeric types are supported in function signatures"); - AbiParam::new(cret_arg) - })); - sig.returns.extend(wasm_func_ty.returns.iter().map(|ty| { - let cret_arg: ir::Type = type_to_type(*ty, environ) - .expect("only numeric types are supported in function signatures"); - AbiParam::new(cret_arg) - })); - environ.declare_type_func(wasm_func_ty.clone().try_into()?, sig)?; + environ.declare_type_func(wasm_func_ty.clone().try_into()?)?; module_translation_state .wasm_types .push((wasm_func_ty.params, wasm_func_ty.returns)); diff --git a/cranelift/wasm/src/translation_utils.rs b/cranelift/wasm/src/translation_utils.rs index c8493d4b51e5..fffb7b375e27 100644 --- a/cranelift/wasm/src/translation_utils.rs +++ b/cranelift/wasm/src/translation_utils.rs @@ -19,6 +19,7 @@ entity_impl!(FuncIndex); /// Index type of a defined function inside the WebAssembly module. #[derive(Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Debug)] +#[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))] pub struct DefinedFuncIndex(u32); entity_impl!(DefinedFuncIndex); diff --git a/crates/cranelift/Cargo.toml b/crates/cranelift/Cargo.toml index f9b763dccc2d..764177223c67 100644 --- a/crates/cranelift/Cargo.toml +++ b/crates/cranelift/Cargo.toml @@ -18,3 +18,4 @@ cranelift-codegen = { path = "../../cranelift/codegen", version = "0.72.0" } cranelift-frontend = { path = "../../cranelift/frontend", version = "0.72.0" } cranelift-entity = { path = "../../cranelift/entity", version = "0.72.0" } wasmparser = "0.77.0" +target-lexicon = "0.12" diff --git a/crates/cranelift/src/func_environ.rs b/crates/cranelift/src/func_environ.rs index a9481aca93ee..29d58dac2045 100644 --- a/crates/cranelift/src/func_environ.rs +++ b/crates/cranelift/src/func_environ.rs @@ -4,20 +4,20 @@ use cranelift_codegen::ir::condcodes::*; use cranelift_codegen::ir::immediates::{Offset32, Uimm64}; use cranelift_codegen::ir::types::*; use cranelift_codegen::ir::{AbiParam, ArgumentPurpose, Function, InstBuilder, Signature}; -use cranelift_codegen::isa::{self, TargetFrontendConfig}; -use cranelift_entity::{EntityRef, PrimaryMap}; +use cranelift_codegen::isa::{self, TargetFrontendConfig, TargetIsa}; +use cranelift_entity::EntityRef; use cranelift_frontend::FunctionBuilder; use cranelift_frontend::Variable; use cranelift_wasm::{ - self, FuncIndex, FuncTranslationState, GlobalIndex, GlobalVariable, MemoryIndex, - SignatureIndex, TableIndex, TargetEnvironment, TypeIndex, WasmError, WasmResult, WasmType, + self, FuncIndex, FuncTranslationState, GlobalIndex, GlobalVariable, MemoryIndex, TableIndex, + TargetEnvironment, TypeIndex, WasmError, WasmResult, WasmType, }; use std::convert::TryFrom; use std::mem; use wasmparser::Operator; use wasmtime_environ::{ - BuiltinFunctionIndex, MemoryPlan, MemoryStyle, Module, TableStyle, Tunables, VMOffsets, - INTERRUPTED, WASM_PAGE_SIZE, + BuiltinFunctionIndex, MemoryPlan, MemoryStyle, Module, TableStyle, Tunables, TypeTables, + VMOffsets, INTERRUPTED, WASM_PAGE_SIZE, }; /// Compute an `ir::ExternalName` for a given wasm function index. @@ -109,14 +109,9 @@ wasmtime_environ::foreach_builtin_function!(declare_function_signatures); /// The `FuncEnvironment` implementation for use by the `ModuleEnvironment`. pub struct FuncEnvironment<'module_environment> { - /// Target-specified configuration. - target_config: TargetFrontendConfig, - - /// The module-level environment which this function-level environment belongs to. + isa: &'module_environment (dyn TargetIsa + 'module_environment), module: &'module_environment Module, - - /// The native signatures for each type signature in this module - native_signatures: &'module_environment PrimaryMap, + types: &'module_environment TypeTables, /// The Cranelift global holding the vmctx address. vmctx: Option, @@ -146,27 +141,27 @@ pub struct FuncEnvironment<'module_environment> { impl<'module_environment> FuncEnvironment<'module_environment> { pub fn new( - target_config: TargetFrontendConfig, + isa: &'module_environment (dyn TargetIsa + 'module_environment), module: &'module_environment Module, - native_signatures: &'module_environment PrimaryMap, + types: &'module_environment TypeTables, tunables: &'module_environment Tunables, ) -> Self { let builtin_function_signatures = BuiltinFunctionSignatures::new( - target_config.pointer_type(), - match target_config.pointer_type() { + isa.pointer_type(), + match isa.pointer_type() { ir::types::I32 => ir::types::R32, ir::types::I64 => ir::types::R64, _ => panic!(), }, - target_config.default_call_conv, + crate::wasmtime_call_conv(isa), ); Self { - target_config, + isa, module, - native_signatures, + types, vmctx: None, builtin_function_signatures, - offsets: VMOffsets::new(target_config.pointer_bytes(), module), + offsets: VMOffsets::new(isa.pointer_bytes(), module), tunables, fuel_var: Variable::new(0), vminterrupts_ptr: Variable::new(0), @@ -178,7 +173,7 @@ impl<'module_environment> FuncEnvironment<'module_environment> { } fn pointer_type(&self) -> ir::Type { - self.target_config.pointer_type() + self.isa.pointer_type() } fn vmctx(&mut self, func: &mut Function) -> ir::GlobalValue { @@ -680,7 +675,7 @@ impl<'module_environment> FuncEnvironment<'module_environment> { impl<'module_environment> TargetEnvironment for FuncEnvironment<'module_environment> { fn target_config(&self) -> TargetFrontendConfig { - self.target_config + self.isa.frontend_config() } fn reference_type(&self, ty: WasmType) -> ir::Type { @@ -1339,7 +1334,8 @@ impl<'module_environment> cranelift_wasm::FuncEnvironment for FuncEnvironment<'m index: TypeIndex, ) -> WasmResult { let index = self.module.types[index].unwrap_function(); - Ok(func.import_signature(self.native_signatures[index].clone())) + let sig = crate::indirect_signature(self.isa, self.types, index); + Ok(func.import_signature(sig)) } fn make_direct_func( @@ -1347,8 +1343,7 @@ impl<'module_environment> cranelift_wasm::FuncEnvironment for FuncEnvironment<'m func: &mut ir::Function, index: FuncIndex, ) -> WasmResult { - let sig_index = self.module.functions[index]; - let sig = self.native_signatures[sig_index].clone(); + let sig = crate::func_signature(self.isa, self.module, self.types, index); let signature = func.import_signature(sig); let name = get_func_name(index); Ok(func.import_function(ir::ExtFuncData { diff --git a/crates/cranelift/src/lib.rs b/crates/cranelift/src/lib.rs index ee330ee6a76c..896294ff39e6 100644 --- a/crates/cranelift/src/lib.rs +++ b/crates/cranelift/src/lib.rs @@ -90,16 +90,18 @@ use crate::func_environ::{get_func_name, FuncEnvironment}; use cranelift_codegen::ir::{self, ExternalName}; +use cranelift_codegen::isa::{CallConv, TargetIsa}; use cranelift_codegen::machinst::buffer::MachSrcLoc; use cranelift_codegen::print_errors::pretty_error; use cranelift_codegen::{binemit, isa, Context}; -use cranelift_wasm::{DefinedFuncIndex, FuncIndex, FuncTranslator}; +use cranelift_wasm::{DefinedFuncIndex, FuncIndex, FuncTranslator, SignatureIndex, WasmType}; use std::convert::TryFrom; use std::sync::Mutex; +use target_lexicon::CallingConvention; use wasmtime_environ::{ CompileError, CompiledFunction, Compiler, FunctionAddressMap, FunctionBodyData, - InstructionAddressMap, ModuleTranslation, Relocation, RelocationTarget, StackMapInformation, - TrapInformation, Tunables, TypeTables, + InstructionAddressMap, Module, ModuleTranslation, Relocation, RelocationTarget, + StackMapInformation, TrapInformation, Tunables, TypeTables, }; mod func_environ; @@ -354,18 +356,12 @@ impl Compiler for Cranelift { let func_index = module.func_index(func_index); let mut context = Context::new(); context.func.name = get_func_name(func_index); - let sig_index = module.functions[func_index]; - context.func.signature = types.native_signatures[sig_index].clone(); + context.func.signature = func_signature(isa, module, types, func_index); if tunables.generate_native_debuginfo { context.func.collect_debug_info(); } - let mut func_env = FuncEnvironment::new( - isa.frontend_config(), - module, - &types.native_signatures, - tunables, - ); + let mut func_env = FuncEnvironment::new(isa, module, types, tunables); // We use these as constant offsets below in // `stack_limit_from_arguments`, so assert their values here. This @@ -457,3 +453,83 @@ impl Compiler for Cranelift { }) } } + +pub fn blank_sig(isa: &dyn TargetIsa, call_conv: CallConv) -> ir::Signature { + let pointer_type = isa.pointer_type(); + let mut sig = ir::Signature::new(call_conv); + // Add the caller/callee `vmctx` parameters. + sig.params.push(ir::AbiParam::special( + pointer_type, + ir::ArgumentPurpose::VMContext, + )); + sig.params.push(ir::AbiParam::new(pointer_type)); + return sig; +} + +pub fn wasmtime_call_conv(isa: &dyn TargetIsa) -> CallConv { + match isa.triple().default_calling_convention() { + Ok(CallingConvention::SystemV) | Ok(CallingConvention::AppleAarch64) | Err(()) => { + CallConv::WasmtimeSystemV + } + Ok(CallingConvention::WindowsFastcall) => CallConv::WasmtimeFastcall, + Ok(unimp) => unimplemented!("calling convention: {:?}", unimp), + } +} + +pub fn push_types( + isa: &dyn TargetIsa, + sig: &mut ir::Signature, + types: &TypeTables, + index: SignatureIndex, +) { + let wasm = &types.wasm_signatures[index]; + + let cvt = |ty: &WasmType| { + ir::AbiParam::new(match ty { + WasmType::I32 => ir::types::I32, + WasmType::I64 => ir::types::I64, + WasmType::F32 => ir::types::F32, + WasmType::F64 => ir::types::F64, + WasmType::V128 => ir::types::I8X16, + WasmType::FuncRef | WasmType::ExternRef => { + wasmtime_environ::reference_type(*ty, isa.pointer_type()) + } + WasmType::ExnRef => unimplemented!(), + }) + }; + sig.params.extend(wasm.params.iter().map(&cvt)); + sig.returns.extend(wasm.returns.iter().map(&cvt)); +} + +pub fn indirect_signature( + isa: &dyn TargetIsa, + types: &TypeTables, + index: SignatureIndex, +) -> ir::Signature { + let mut sig = blank_sig(isa, wasmtime_call_conv(isa)); + push_types(isa, &mut sig, types, index); + return sig; +} + +pub fn func_signature( + isa: &dyn TargetIsa, + module: &Module, + types: &TypeTables, + index: FuncIndex, +) -> ir::Signature { + let call_conv = match module.defined_func_index(index) { + // If this is a defined function in the module and it's never possibly + // exported, then we can optimize this function to use the fastest + // calling convention since it's purely an internal implementation + // detail of the module itself. + Some(idx) if !module.possibly_exported_funcs.contains(&idx) => CallConv::Fast, + + // ... otherwise if it's an imported function or if it's a possibly + // exported function then we use the default ABI wasmtime would + // otherwise select. + _ => wasmtime_call_conv(isa), + }; + let mut sig = blank_sig(isa, call_conv); + push_types(isa, &mut sig, types, module.functions[index]); + return sig; +} diff --git a/crates/environ/src/module.rs b/crates/environ/src/module.rs index ee4ff050dc71..3b520475d6f0 100644 --- a/crates/environ/src/module.rs +++ b/crates/environ/src/module.rs @@ -2,12 +2,11 @@ use crate::tunables::Tunables; use crate::WASM_MAX_PAGES; -use cranelift_codegen::ir; use cranelift_entity::{EntityRef, PrimaryMap}; use cranelift_wasm::*; use indexmap::IndexMap; use serde::{Deserialize, Serialize}; -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use std::sync::Arc; /// Implemenation styles for WebAssembly linear memory. @@ -367,6 +366,10 @@ pub struct Module { /// The type of each nested wasm module this module contains. pub modules: PrimaryMap, + + /// The set of defined functions within this module which are located in + /// element segments. + pub possibly_exported_funcs: HashSet, } /// Initialization routines for creating an instance, encompassing imports, @@ -564,7 +567,6 @@ impl Module { #[allow(missing_docs)] pub struct TypeTables { pub wasm_signatures: PrimaryMap, - pub native_signatures: PrimaryMap, pub module_signatures: PrimaryMap, pub instance_signatures: PrimaryMap, } diff --git a/crates/environ/src/module_environ.rs b/crates/environ/src/module_environ.rs index 57e2f6bb1e7e..8106c1a3fb72 100644 --- a/crates/environ/src/module_environ.rs +++ b/crates/environ/src/module_environ.rs @@ -4,14 +4,14 @@ use crate::module::{ }; use crate::tunables::Tunables; use cranelift_codegen::ir; -use cranelift_codegen::ir::{AbiParam, ArgumentPurpose}; use cranelift_codegen::isa::TargetFrontendConfig; +use cranelift_codegen::packed_option::ReservedValue; use cranelift_entity::PrimaryMap; use cranelift_wasm::{ self, translate_module, Alias, DataIndex, DefinedFuncIndex, ElemIndex, EntityIndex, EntityType, - FuncIndex, Global, GlobalIndex, InstanceIndex, InstanceTypeIndex, Memory, MemoryIndex, - ModuleIndex, ModuleTypeIndex, SignatureIndex, Table, TableIndex, TargetEnvironment, TypeIndex, - WasmError, WasmFuncType, WasmResult, + FuncIndex, Global, GlobalIndex, GlobalInit, InstanceIndex, InstanceTypeIndex, Memory, + MemoryIndex, ModuleIndex, ModuleTypeIndex, SignatureIndex, Table, TableIndex, + TargetEnvironment, TypeIndex, WasmError, WasmFuncType, WasmResult, }; use std::collections::{hash_map::Entry, HashMap}; use std::convert::TryFrom; @@ -357,6 +357,15 @@ impl<'data> ModuleEnvironment<'data> { .module_signatures .push(ModuleSignature { imports, exports }) } + + fn flag_func_possibly_exported(&mut self, func: FuncIndex) { + if func.is_reserved_value() { + return; + } + if let Some(idx) = self.result.module.defined_func_index(func) { + self.result.module.possibly_exported_funcs.insert(idx); + } + } } impl<'data> TargetEnvironment for ModuleEnvironment<'data> { @@ -375,21 +384,17 @@ impl<'data> cranelift_wasm::ModuleEnvironment<'data> for ModuleEnvironment<'data fn reserve_types(&mut self, num: u32) -> WasmResult<()> { let num = usize::try_from(num).unwrap(); self.result.module.types.reserve(num); - self.types.native_signatures.reserve(num); self.types.wasm_signatures.reserve(num); Ok(()) } - fn declare_type_func(&mut self, wasm: WasmFuncType, sig: ir::Signature) -> WasmResult<()> { + fn declare_type_func(&mut self, wasm: WasmFuncType) -> WasmResult<()> { // Deduplicate wasm function signatures through `interned_func_types`, // which also deduplicates across wasm modules with module linking. let sig_index = match self.interned_func_types.get(&wasm) { Some(idx) => *idx, None => { - let sig = translate_signature(sig, self.pointer_type()); - let sig_index = self.types.native_signatures.push(sig); - let sig_index2 = self.types.wasm_signatures.push(wasm.clone()); - debug_assert_eq!(sig_index, sig_index2); + let sig_index = self.types.wasm_signatures.push(wasm.clone()); self.interned_func_types.insert(wasm, sig_index); sig_index } @@ -641,6 +646,9 @@ impl<'data> cranelift_wasm::ModuleEnvironment<'data> for ModuleEnvironment<'data } fn declare_global(&mut self, global: Global) -> WasmResult<()> { + if let GlobalInit::RefFunc(index) = global.initializer { + self.flag_func_possibly_exported(index); + } self.result.module.globals.push(global); Ok(()) } @@ -654,6 +662,7 @@ impl<'data> cranelift_wasm::ModuleEnvironment<'data> for ModuleEnvironment<'data } fn declare_func_export(&mut self, func_index: FuncIndex, name: &str) -> WasmResult<()> { + self.flag_func_possibly_exported(func_index); self.declare_export(EntityIndex::Function(func_index), name) } @@ -678,6 +687,7 @@ impl<'data> cranelift_wasm::ModuleEnvironment<'data> for ModuleEnvironment<'data } fn declare_start_func(&mut self, func_index: FuncIndex) -> WasmResult<()> { + self.flag_func_possibly_exported(func_index); debug_assert!(self.result.module.start_func.is_none()); self.result.module.start_func = Some(func_index); Ok(()) @@ -698,6 +708,9 @@ impl<'data> cranelift_wasm::ModuleEnvironment<'data> for ModuleEnvironment<'data offset: usize, elements: Box<[FuncIndex]>, ) -> WasmResult<()> { + for element in elements.iter() { + self.flag_func_possibly_exported(*element); + } self.result .module .table_initializers @@ -715,6 +728,9 @@ impl<'data> cranelift_wasm::ModuleEnvironment<'data> for ModuleEnvironment<'data elem_index: ElemIndex, segments: Box<[FuncIndex]>, ) -> WasmResult<()> { + for element in segments.iter() { + self.flag_func_possibly_exported(*element); + } let index = self.result.module.passive_elements.len(); self.result.module.passive_elements.push(segments); let old = self @@ -1070,15 +1086,3 @@ and for re-adding support for interface types you can see this issue: Ok(()) } } - -/// Add environment-specific function parameters. -pub fn translate_signature(mut sig: ir::Signature, pointer_type: ir::Type) -> ir::Signature { - // Prepend the vmctx argument. - sig.params.insert( - 0, - AbiParam::special(pointer_type, ArgumentPurpose::VMContext), - ); - // Prepend the caller vmctx argument. - sig.params.insert(1, AbiParam::new(pointer_type)); - sig -} diff --git a/crates/jit/src/lib.rs b/crates/jit/src/lib.rs index 17e6294250c3..12329ea266d7 100644 --- a/crates/jit/src/lib.rs +++ b/crates/jit/src/lib.rs @@ -50,6 +50,7 @@ pub use crate::instantiate::{ CompilationArtifacts, CompiledModule, ModuleCode, SetupError, SymbolizeContext, TypeTables, }; pub use crate::link::link_module; +pub use wasmtime_cranelift::{blank_sig, wasmtime_call_conv}; /// Version number of this crate. pub const VERSION: &str = env!("CARGO_PKG_VERSION"); diff --git a/crates/jit/src/native.rs b/crates/jit/src/native.rs index afcf83d3ccb4..706cbc04b443 100644 --- a/crates/jit/src/native.rs +++ b/crates/jit/src/native.rs @@ -11,9 +11,4 @@ pub fn builder_without_flags() -> cranelift_codegen::isa::Builder { .expect("host machine is not a supported target") } -pub fn call_conv() -> cranelift_codegen::isa::CallConv { - use target_lexicon::HOST; - cranelift_codegen::isa::CallConv::triple_default(&HOST) -} - pub use cranelift_codegen::isa::lookup; diff --git a/crates/jit/src/object.rs b/crates/jit/src/object.rs index 3516be2b133d..e73fa8cf8185 100644 --- a/crates/jit/src/object.rs +++ b/crates/jit/src/object.rs @@ -8,7 +8,7 @@ use std::collections::BTreeSet; use wasmtime_debug::DwarfSection; use wasmtime_environ::isa::{unwind::UnwindInfo, TargetIsa}; use wasmtime_environ::wasm::{FuncIndex, SignatureIndex}; -use wasmtime_environ::{CompiledFunctions, ModuleTranslation, ModuleType, TypeTables}; +use wasmtime_environ::{CompiledFunctions, ModuleTranslation, TypeTables}; use wasmtime_obj::{ObjectBuilder, ObjectBuilderTarget}; pub use wasmtime_obj::utils; @@ -42,18 +42,18 @@ pub(crate) fn build_object( // Build trampolines for every signature that can be used by this module. let signatures = translation .module - .types - .values() - .filter_map(|t| match t { - ModuleType::Function(f) => Some(*f), - _ => None, + .functions + .iter() + .filter_map(|(i, sig)| match translation.module.defined_func_index(i) { + Some(i) if !translation.module.possibly_exported_funcs.contains(&i) => None, + _ => Some(*sig), }) .collect::>(); let mut trampolines = Vec::with_capacity(signatures.len()); let mut cx = FunctionBuilderContext::new(); for i in signatures { - let native_sig = &types.native_signatures[i]; - let func = build_trampoline(isa, &mut cx, native_sig, std::mem::size_of::())?; + let native_sig = wasmtime_cranelift::indirect_signature(isa, &types, i); + let func = build_trampoline(isa, &mut cx, &native_sig, std::mem::size_of::())?; // Preserve trampoline function unwind info. if let Some(info) = &func.unwind_info { unwind_info.push(ObjectUnwindInfo::Trampoline(i, info.clone())) diff --git a/crates/jit/src/trampoline.rs b/crates/jit/src/trampoline.rs index bb470dbcb3fc..b8262fd93fb6 100644 --- a/crates/jit/src/trampoline.rs +++ b/crates/jit/src/trampoline.rs @@ -9,7 +9,7 @@ use wasmtime_runtime::{InstantiationError, VMFunctionBody, VMTrampoline}; pub mod ir { pub(super) use cranelift_codegen::ir::{ - AbiParam, ArgumentPurpose, ConstantOffset, JumpTable, Signature, SourceLoc, + AbiParam, ConstantOffset, JumpTable, Signature, SourceLoc, }; pub use cranelift_codegen::ir::{ ExternalName, Function, InstBuilder, MemFlags, StackSlotData, StackSlotKind, @@ -52,16 +52,8 @@ pub(crate) fn build_trampoline( value_size: usize, ) -> Result { let pointer_type = isa.pointer_type(); - let mut wrapper_sig = ir::Signature::new(isa.frontend_config().default_call_conv); - - // Add the callee `vmctx` parameter. - wrapper_sig.params.push(ir::AbiParam::special( - pointer_type, - ir::ArgumentPurpose::VMContext, - )); - - // Add the caller `vmctx` parameter. - wrapper_sig.params.push(ir::AbiParam::new(pointer_type)); + let mut wrapper_sig = + wasmtime_cranelift::blank_sig(isa, wasmtime_cranelift::wasmtime_call_conv(isa)); // Add the `callee_address` parameter. wrapper_sig.params.push(ir::AbiParam::new(pointer_type)); diff --git a/crates/wasmtime/src/config.rs b/crates/wasmtime/src/config.rs index 43bd809c84aa..55fbcfddca21 100644 --- a/crates/wasmtime/src/config.rs +++ b/crates/wasmtime/src/config.rs @@ -1,6 +1,6 @@ use crate::memory::MemoryCreator; use crate::trampoline::MemoryCreatorProxy; -use crate::{func::HostFunc, Caller, FuncType, IntoFunc, Trap, Val, WasmRet, WasmTy}; +use crate::{func::HostFunc, Caller, FuncType, IntoFunc, Trap, Val, WasmHostResults, WasmTy}; use anyhow::{bail, Result}; use serde::{Deserialize, Serialize}; use std::cmp; @@ -339,7 +339,7 @@ macro_rules! generate_wrap_async_host_func { ) where $($args: WasmTy,)* - R: WasmRet, + R: WasmHostResults, { // Defer the check for async support until engine creation time to not introduce an order dependency self.host_funcs.insert( diff --git a/crates/wasmtime/src/func.rs b/crates/wasmtime/src/func.rs index 2a70df9e3cd2..924a1002d932 100644 --- a/crates/wasmtime/src/func.rs +++ b/crates/wasmtime/src/func.rs @@ -6,6 +6,7 @@ use std::cmp::max; use std::fmt; use std::future::Future; use std::mem; +use std::mem::MaybeUninit; use std::panic::{self, AssertUnwindSafe}; use std::pin::Pin; use std::ptr::{self, NonNull}; @@ -350,7 +351,7 @@ macro_rules! generate_wrap_async_func { where T: 'static, $($args: WasmTy,)* - R: WasmRet, + R: WasmHostResults, { assert!(store.async_support(), concat!("cannot use `wrap", $num, "_async` without enabling async support on the config")); Func::wrap(store, move |caller: Caller<'_>, $($args: $args),*| { @@ -561,15 +562,15 @@ impl Func { /// Any of the Rust types can be returned from the closure as well, in /// addition to some extra types /// - /// | Rust Return Type | WebAssembly Return Type | Meaning | - /// |-------------------|-------------------------|-------------------| - /// | `()` | nothing | no return value | - /// | `Result` | `T` | function may trap | + /// | Rust Return Type | WebAssembly Return Type | Meaning | + /// |-------------------|-------------------------|-----------------------| + /// | `()` | nothing | no return value | + /// | `T` | `T` | a single return value | + /// | `(T1, T2, ...)` | `T1 T2 ...` | multiple returns | /// - /// At this time multi-value returns are not supported, and supporting this - /// is the subject of [#1178]. - /// - /// [#1178]: https://github.com/bytecodealliance/wasmtime/issues/1178 + /// Note that all return types can also be wrapped in `Result<_, Trap>` to + /// indicate that the host function can generate a trap as well as possibly + /// returning a value. /// /// Finally you can also optionally take [`Caller`] as the first argument of /// your closure. If inserted then you're able to inspect the caller's @@ -1260,10 +1261,12 @@ fn enter_wasm_init<'a>(store: &'a Store) -> Result { /// stable over time. /// /// For more information see [`Func::wrap`] -pub unsafe trait WasmRet { +pub unsafe trait WasmHostResults { // Same as `WasmTy::Abi`. #[doc(hidden)] type Abi: Copy; + #[doc(hidden)] + type Retptr: Copy; // Same as `WasmTy::compatible_with_store`. #[doc(hidden)] @@ -1276,100 +1279,47 @@ pub unsafe trait WasmRet { // `invoke_wasm_and_catch_traps` is on the stack, and therefore this method // is unsafe. #[doc(hidden)] - unsafe fn into_abi_for_ret(self, store: &Store) -> Result; + unsafe fn into_abi_for_ret(self, store: &Store, ptr: Self::Retptr) -> Result; + + #[doc(hidden)] + fn func_type(params: impl Iterator) -> FuncType; - // Same as `WasmTy::push`. #[doc(hidden)] - fn valtype() -> Option; + unsafe fn wrap_trampoline(ptr: *mut u128, f: impl FnOnce(Self::Retptr) -> Self::Abi); // Utilities used to convert an instance of this type to a `Result` // explicitly, used when wrapping async functions which always bottom-out // in a function that returns a trap because futures can be cancelled. #[doc(hidden)] - type Fallible: WasmRet; + type Fallible: WasmHostResults; #[doc(hidden)] fn into_fallible(self) -> Self::Fallible; #[doc(hidden)] fn fallible_from_trap(trap: Trap) -> Self::Fallible; } -unsafe impl WasmRet for () { - type Abi = (); - type Fallible = Result<(), Trap>; - - #[inline] - fn compatible_with_store(&self, _store: &Store) -> bool { - true - } - - #[inline] - unsafe fn into_abi_for_ret(self, _store: &Store) -> Result<(), Trap> { - Ok(()) - } - - #[inline] - fn valtype() -> Option { - None - } - - #[inline] - fn into_fallible(self) -> Result<(), Trap> { - Ok(()) - } - - #[inline] - fn fallible_from_trap(trap: Trap) -> Result<(), Trap> { - Err(trap) - } -} - -unsafe impl WasmRet for Result<(), Trap> { - type Abi = (); - type Fallible = Self; - - #[inline] - fn compatible_with_store(&self, _store: &Store) -> bool { - true - } - - #[inline] - unsafe fn into_abi_for_ret(self, _store: &Store) -> Result<(), Trap> { - self - } - - #[inline] - fn valtype() -> Option { - None - } - - #[inline] - fn into_fallible(self) -> Result<(), Trap> { - self - } - - #[inline] - fn fallible_from_trap(trap: Trap) -> Result<(), Trap> { - Err(trap) - } -} - -unsafe impl WasmRet for T +unsafe impl WasmHostResults for T where T: WasmTy, { type Abi = ::Abi; + type Retptr = (); type Fallible = Result; fn compatible_with_store(&self, store: &Store) -> bool { ::compatible_with_store(self, store) } - unsafe fn into_abi_for_ret(self, store: &Store) -> Result { + unsafe fn into_abi_for_ret(self, store: &Store, _retptr: ()) -> Result { Ok(::into_abi(self, store)) } - fn valtype() -> Option { - Some(::valtype()) + fn func_type(params: impl Iterator) -> FuncType { + FuncType::new(params, Some(::valtype())) + } + + unsafe fn wrap_trampoline(ptr: *mut u128, f: impl FnOnce(Self::Retptr) -> Self::Abi) { + *ptr.cast::() = f(()); } fn into_fallible(self) -> Result { @@ -1381,26 +1331,35 @@ where } } -unsafe impl WasmRet for Result +unsafe impl WasmHostResults for Result where - T: WasmTy, + T: WasmHostResults, { - type Abi = ::Abi; + type Abi = ::Abi; + type Retptr = ::Retptr; type Fallible = Self; fn compatible_with_store(&self, store: &Store) -> bool { match self { - Ok(x) => ::compatible_with_store(x, store), + Ok(x) => ::compatible_with_store(x, store), Err(_) => true, } } - unsafe fn into_abi_for_ret(self, store: &Store) -> Result { - self.map(|val| ::into_abi(val, store)) + unsafe fn into_abi_for_ret( + self, + store: &Store, + retptr: Self::Retptr, + ) -> Result { + self.and_then(|val| val.into_abi_for_ret(store, retptr)) + } + + fn func_type(params: impl Iterator) -> FuncType { + T::func_type(params) } - fn valtype() -> Option { - Some(::valtype()) + unsafe fn wrap_trampoline(ptr: *mut u128, f: impl FnOnce(Self::Retptr) -> Self::Abi) { + T::wrap_trampoline(ptr, f) } fn into_fallible(self) -> Result { @@ -1412,6 +1371,163 @@ where } } +macro_rules! impl_wasm_host_results { + ($n:tt $($t:ident)*) => ( + #[allow(non_snake_case)] + unsafe impl<$($t),*> WasmHostResults for ($($t,)*) + where + $($t: WasmTy,)* + ($($t::Abi,)*): HostAbi, + { + type Abi = <($($t::Abi,)*) as HostAbi>::Abi; + type Retptr = <($($t::Abi,)*) as HostAbi>::Retptr; + type Fallible = Result; + + #[inline] + fn compatible_with_store(&self, _store: &Store) -> bool { + let ($($t,)*) = self; + $( $t.compatible_with_store(_store) && )* true + } + + #[inline] + unsafe fn into_abi_for_ret(self, _store: &Store, ptr: Self::Retptr) -> Result { + let ($($t,)*) = self; + let abi = ($($t.into_abi(_store),)*); + Ok(<($($t::Abi,)*) as HostAbi>::into_abi(abi, ptr)) + } + + fn func_type(params: impl Iterator) -> FuncType { + FuncType::new( + params, + std::array::IntoIter::new([$($t::valtype(),)*]), + ) + } + + #[allow(unused_assignments)] + unsafe fn wrap_trampoline(mut _ptr: *mut u128, f: impl FnOnce(Self::Retptr) -> Self::Abi) { + let ($($t,)*) = <($($t::Abi,)*) as HostAbi>::call(f); + $( + *_ptr.cast() = $t; + _ptr = _ptr.add(1); + )* + } + + #[inline] + fn into_fallible(self) -> Result { + Ok(self) + } + + #[inline] + fn fallible_from_trap(trap: Trap) -> Result { + Err(trap) + } + } + ) +} + +for_each_function_signature!(impl_wasm_host_results); + +// Internal trait representing how to communicate tuples of return values across +// an ABI boundary. This internally corresponds to the "wasmtime" ABI inside of +// cranelift itself. Notably the first element of each tuple is returned via the +// typical system ABI (e.g. systemv or fastcall depending on platform) and all +// other values are returned packed via the stack. +// +// This trait helps to encapsulate all the details of that. +#[doc(hidden)] +pub trait HostAbi { + // A value returned from native functions which return `Self` + type Abi: Copy; + // A return pointer, added to the end of the argument list, for native + // functions that return `Self`. Note that a 0-sized type here should get + // elided at the ABI level. + type Retptr: Copy; + + // Converts a value of `self` into its components. Stores necessary values + // into `ptr` and then returns whatever needs to be returned from the + // function. + unsafe fn into_abi(self, ptr: Self::Retptr) -> Self::Abi; + + // Calls `f` with a suitably sized return area and requires `f` to return + // the raw abi value of the first element of our tuple. This will then + // unpack the `Retptr` and assemble it with `Self::Abi` to return an + // instance of the whole tuple. + unsafe fn call(f: impl FnOnce(Self::Retptr) -> Self::Abi) -> Self; +} + +macro_rules! impl_host_abi { + // Base case, everything is `()` + (0) => { + impl HostAbi for () { + type Abi = (); + type Retptr = (); + + unsafe fn into_abi(self, _ptr: Self::Retptr) -> Self::Abi {} + + unsafe fn call(f: impl FnOnce(Self::Retptr) -> Self::Abi) -> Self { + f(()) + } + } + }; + + // In the 1-case the retptr is not present, so it's a 0-sized value. + (1 $a:ident) => { + impl<$a: Copy> HostAbi for ($a,) { + type Abi = $a; + type Retptr = (); + + unsafe fn into_abi(self, _ptr: Self::Retptr) -> Self::Abi { + self.0 + } + + unsafe fn call(f: impl FnOnce(Self::Retptr) -> Self::Abi) -> Self { + (f(()),) + } + } + }; + + // This is where the more interesting case happens. The first element of the + // tuple is returned via `Abi` and all other elements are returned via + // `Retptr`. We create a `TupleRetNN` structure to represent all of the + // return values here. + ($n:tt $t:ident $($u:ident)*) => {paste::paste!{ + #[doc(hidden)] + #[allow(non_snake_case)] + #[repr(C)] + pub struct []<$($u,)*> { + $($u: $u,)* + } + + #[allow(non_snake_case, unused_assignments)] + impl<$t: Copy, $($u: Copy,)*> HostAbi for ($t, $($u,)*) { + type Abi = $t; + type Retptr = *mut []<$($u,)*>; + + unsafe fn into_abi(self, ptr: Self::Retptr) -> Self::Abi { + let ($t, $($u,)*) = self; + // Store the tail of our tuple into the return pointer... + $((*ptr).$u = $u;)* + // ... and return the head raw. + $t + } + + unsafe fn call(f: impl FnOnce(Self::Retptr) -> Self::Abi) -> Self { + // Create space to store all the return values and then invoke + // the function. + let mut space = MaybeUninit::uninit(); + let t = f(space.as_mut_ptr()); + let space = space.assume_init(); + + // Use the return value as the head of the tuple and unpack our + // return area to get the rest of the tuple. + (t, $(space.$u,)*) + } + } + }}; +} + +for_each_function_signature!(impl_host_abi); + /// Internal trait implemented for all arguments that can be passed to /// [`Func::wrap`] and [`Config::wrap_host_func`](crate::Config::wrap_host_func). /// @@ -1533,7 +1649,7 @@ macro_rules! impl_into_func { where F: Fn($($args),*) -> R + 'static, $($args: WasmTy,)* - R: WasmRet, + R: WasmHostResults, { fn into_func(self, registry: Option<&mut SignatureRegistry>) -> (FuncType, InstanceHandle, VMTrampoline) { let f = move |_: Caller<'_>, $($args:$args),*| { @@ -1549,7 +1665,7 @@ macro_rules! impl_into_func { where F: Fn(Caller<'_>, $($args),*) -> R + 'static, $($args: WasmTy,)* - R: WasmRet, + R: WasmHostResults, { fn into_func(self, registry: Option<&mut SignatureRegistry>) -> (FuncType, InstanceHandle, VMTrampoline) { /// This shim is called by Wasm code, constructs a `Caller`, @@ -1563,11 +1679,12 @@ macro_rules! impl_into_func { vmctx: *mut VMContext, caller_vmctx: *mut VMContext, $( $args: $args::Abi, )* + retptr: R::Retptr, ) -> R::Abi where F: Fn(Caller<'_>, $( $args ),*) -> R + 'static, $( $args: WasmTy, )* - R: WasmRet, + R: WasmHostResults, { enum CallResult { Ok(T), @@ -1624,7 +1741,7 @@ macro_rules! impl_into_func { raise_cross_store_trap(); } - match ret.into_abi_for_ret(&store) { + match ret.into_abi_for_ret(&store, retptr) { Ok(val) => CallResult::Ok(val), Err(trap) => CallResult::Trap(trap), } @@ -1654,7 +1771,7 @@ macro_rules! impl_into_func { ) where $($args: WasmTy,)* - R: WasmRet, + R: WasmHostResults, { let ptr = mem::transmute::< *const VMFunctionBody, @@ -1662,6 +1779,7 @@ macro_rules! impl_into_func { *mut VMContext, *mut VMContext, $( $args::Abi, )* + R::Retptr, ) -> R::Abi, >(ptr); @@ -1670,15 +1788,14 @@ macro_rules! impl_into_func { let $args = *args.add(_n).cast::<$args::Abi>(); _n += 1; )* - let ret = ptr(callee_vmctx, caller_vmctx, $( $args ),*); - *args.cast::() = ret; + R::wrap_trampoline(args, |retptr| { + ptr(callee_vmctx, caller_vmctx, $( $args, )* retptr) + }); } - let ty = FuncType::new( + let ty = R::func_type( None::.into_iter() $(.chain(Some($args::valtype())))* - , - R::valtype(), ); let trampoline = host_trampoline::<$($args,)* R>; @@ -1686,7 +1803,7 @@ macro_rules! impl_into_func { // If not given a registry, use a default signature index that is guaranteed to trap // if the function is called indirectly without first being associated with a store (a bug condition). let shared_signature_id = registry - .map(|r| r.register(ty.as_wasm_func_type(), trampoline)) + .map(|r| r.register(ty.as_wasm_func_type(), Some(trampoline))) .unwrap_or(VMSharedSignatureIndex::default()); let instance = unsafe { diff --git a/crates/wasmtime/src/func/typed.rs b/crates/wasmtime/src/func/typed.rs index 5ec884c87840..051c91f446c4 100644 --- a/crates/wasmtime/src/func/typed.rs +++ b/crates/wasmtime/src/func/typed.rs @@ -1,10 +1,10 @@ -use super::invoke_wasm_and_catch_traps; +use super::{invoke_wasm_and_catch_traps, HostAbi}; use crate::{ExternRef, Func, Store, Trap, ValType}; use anyhow::{bail, Result}; use std::marker; use std::mem::{self, MaybeUninit}; use std::ptr; -use wasmtime_runtime::{VMContext, VMFunctionBody, VMTrampoline}; +use wasmtime_runtime::{VMContext, VMFunctionBody}; /// A statically typed WebAssembly function. /// @@ -103,7 +103,6 @@ where let anyfunc = self.func.export.anyfunc.as_ref(); let result = params.invoke::( &self.func.instance.store, - self.func.trampoline, anyfunc.func_ptr.as_ptr(), anyfunc.vmctx, ptr::null_mut(), @@ -181,13 +180,40 @@ macro_rules! primitives { primitives! { i32 => I32 - u32 => I32 i64 => I64 - u64 => I64 f32 => F32 f64 => F64 } +macro_rules! unsigned { + ($($unsigned:ident => $ty:ident/$signed:ident)*) => ($( + unsafe impl WasmTy for $unsigned { + type Abi = $signed; + #[inline] + fn valtype() -> ValType { + ValType::$ty + } + #[inline] + fn compatible_with_store(&self, _: &Store) -> bool { + true + } + #[inline] + fn into_abi(self, _store: &Store) -> Self::Abi { + self as $signed + } + #[inline] + unsafe fn from_abi(abi: Self::Abi, _store: &Store) -> Self { + abi as $unsigned + } + } + )*) +} + +unsigned! { + u32 => I32/i32 + u64 => I64/i64 +} + unsafe impl WasmTy for Option { type Abi = *mut u8; @@ -274,7 +300,6 @@ pub unsafe trait WasmParams { unsafe fn invoke( self, store: &Store, - trampoline: VMTrampoline, func: *const VMFunctionBody, vmctx1: *mut VMContext, vmctx2: *mut VMContext, @@ -296,12 +321,11 @@ where unsafe fn invoke( self, store: &Store, - trampoline: VMTrampoline, func: *const VMFunctionBody, vmctx1: *mut VMContext, vmctx2: *mut VMContext, ) -> R { - <(T,)>::invoke((self,), store, trampoline, func, vmctx1, vmctx2) + <(T,)>::invoke((self,), store, func, vmctx1, vmctx2) } } @@ -333,66 +357,30 @@ macro_rules! impl_wasm_params { unsafe fn invoke( self, store: &Store, - trampoline: VMTrampoline, func: *const VMFunctionBody, vmctx1: *mut VMContext, vmctx2: *mut VMContext, ) -> R { - // Some signatures can go directly into JIT code which uses the - // default platform ABI, but basically only those without - // multiple return values. With multiple return values we can't - // natively in Rust call such a function because there's no way - // to model it (yet). + let fnptr = mem::transmute::< + *const VMFunctionBody, + unsafe extern "C" fn( + *mut VMContext, + *mut VMContext, + $($t::Abi,)* + R::Retptr, + ) -> R::Abi, + >(func); + let ($($t,)*) = self; + // Use the `call` function to acquire a `retptr` which we'll + // forward to the native function. Once we have it we also + // convert all our arguments to abi arguments to go to the raw + // function. // - // To work around that we use the trampoline which passes - // arguments/values via the stack which allows us to match the - // expected ABI. Note that this branch, using the trampoline, - // is slower as a result and has an extra indirect function - // call as well. In the future if this is a problem we should - // consider updating JIT code to use an ABI we can call from - // Rust itself. - if R::uses_trampoline() { - R::with_space(|space1| { - // Figure out whether the parameters or the results - // require more space, and use the bigger one as where - // to store arguments and load return values from. - let mut space2 = [0; $n]; - let space = if space1.len() < space2.len() { - space2.as_mut_ptr() - } else { - space1.as_mut_ptr() - }; - - // ... store the ABI for all values into our storage - // area... - let ($($t,)*) = self; - let mut _n = 0; - $( - *space.add(_n).cast::<$t::Abi>() = $t.into_abi(store); - _n += 1; - )* - - // ... make the indirect call through the trampoline - // which will read from `space` and also write all the - // results to `space`... - trampoline(vmctx1, vmctx2, func, space); - - // ... and then we can decode all the return values - // from `space`. - R::from_storage(space, store) - }) - } else { - let fnptr = mem::transmute::< - *const VMFunctionBody, - unsafe extern "C" fn( - *mut VMContext, - *mut VMContext, - $($t::Abi,)* - ) -> R::Abi, - >(func); - let ($($t,)*) = self; - R::from_abi(fnptr(vmctx1, vmctx2, $($t.into_abi(store),)*), store) - } + // Upon returning `R::call` will convert all the returns back + // into `R`. + R::call(store, |retptr| { + fnptr(vmctx1, vmctx2, $($t.into_abi(store),)* retptr) + }) } } }; @@ -408,80 +396,45 @@ for_each_function_signature!(impl_wasm_params); /// `TypedFunc` is not currently supported. pub unsafe trait WasmResults: WasmParams { #[doc(hidden)] - type Abi; - #[doc(hidden)] - unsafe fn from_abi(abi: Self::Abi, store: &Store) -> Self; - #[doc(hidden)] - fn uses_trampoline() -> bool; - // Provides a stack-allocated array with enough space to store all these - // result values. - // - // It'd be nice if we didn't have to have this API and could do something - // with const-generics (or something like that), but I couldn't figure it - // out. If a future Rust explorer is able to get something like `const LEN: - // usize` working that'd be great! + type Abi: Copy; #[doc(hidden)] - fn with_space(_: impl FnOnce(&mut [u128]) -> R) -> R; + type Retptr: Copy; #[doc(hidden)] - unsafe fn from_storage(ptr: *const u128, store: &Store) -> Self; + unsafe fn call(store: &Store, f: impl FnOnce(Self::Retptr) -> Self::Abi) -> Self; } -unsafe impl WasmResults for T { +// Forwards from a bare type `T` to the 1-tuple type `(T,)` +unsafe impl WasmResults for T +where + (T::Abi,): HostAbi, +{ type Abi = <(T,) as WasmResults>::Abi; - unsafe fn from_abi(abi: Self::Abi, store: &Store) -> Self { - <(T,) as WasmResults>::from_abi(abi, store).0 - } - fn uses_trampoline() -> bool { - <(T,) as WasmResults>::uses_trampoline() - } - fn with_space(f: impl FnOnce(&mut [u128]) -> R) -> R { - <(T,) as WasmResults>::with_space(f) - } - unsafe fn from_storage(ptr: *const u128, store: &Store) -> Self { - <(T,) as WasmResults>::from_storage(ptr, store).0 + type Retptr = <(T,) as WasmResults>::Retptr; + + unsafe fn call(store: &Store, f: impl FnOnce(Self::Retptr) -> Self::Abi) -> Self { + <(T,) as WasmResults>::call(store, f).0 } } -#[doc(hidden)] -pub enum Void {} - macro_rules! impl_wasm_results { ($n:tt $($t:ident)*) => { #[allow(non_snake_case, unused_variables)] - unsafe impl<$($t: WasmTy,)*> WasmResults for ($($t,)*) { - type Abi = impl_wasm_results!(@abi $n $($t)*); - unsafe fn from_abi(abi: Self::Abi, store: &Store) -> Self { - impl_wasm_results!(@from_abi abi store $n $($t)*) - } - fn uses_trampoline() -> bool { - $n > 1 - } - fn with_space(f: impl FnOnce(&mut [u128]) -> R) -> R { - f(&mut [0; $n]) - } - unsafe fn from_storage(ptr: *const u128, store: &Store) -> Self { - let mut _n = 0; - $( - let $t = $t::from_abi(*ptr.add(_n).cast::<$t::Abi>(), store); - _n += 1; - )* - ($($t,)*) + unsafe impl<$($t: WasmTy,)*> WasmResults for ($($t,)*) + where ($($t::Abi,)*): HostAbi + { + type Abi = <($($t::Abi,)*) as HostAbi>::Abi; + type Retptr = <($($t::Abi,)*) as HostAbi>::Retptr; + + unsafe fn call(store: &Store, f: impl FnOnce(Self::Retptr) -> Self::Abi) -> Self { + // Delegate via the host abi to figure out what the actual ABI + // for dealing with this tuple type is, and then we can re-tuple + // everything and create actual values via `from_abi` after the + // call is complete. + let ($($t,)*) = <($($t::Abi,)*) as HostAbi>::call(f); + ($($t::from_abi($t, store),)*) } } }; - - // 0/1 return values we can use natively, everything else isn't expressible - // and won't be used so define the abi type to Void. - (@abi 0) => (()); - (@abi 1 $t:ident) => ($t::Abi); - (@abi $($t:tt)*) => (Void); - - (@from_abi $abi:ident $store:ident 0) => (()); - (@from_abi $abi:ident $store:ident 1 $t:ident) => (($t::from_abi($abi, $store),)); - (@from_abi $abi:ident $store:ident $($t:tt)*) => ({ - debug_assert!(false); - match $abi {} - }); } for_each_function_signature!(impl_wasm_results); diff --git a/crates/wasmtime/src/sig_registry.rs b/crates/wasmtime/src/sig_registry.rs index c9583972c966..99bf127b8d7f 100644 --- a/crates/wasmtime/src/sig_registry.rs +++ b/crates/wasmtime/src/sig_registry.rs @@ -29,7 +29,7 @@ struct Entry { // Note that the code memory for this trampoline is not owned by this // type, but instead it's expected to be owned by the store that this // registry lives within. - trampoline: VMTrampoline, + trampoline: Option, } impl SignatureRegistry { @@ -37,12 +37,19 @@ impl SignatureRegistry { pub fn register( &mut self, wasm: &WasmFuncType, - trampoline: VMTrampoline, + trampoline: Option, ) -> VMSharedSignatureIndex { let len = self.wasm2index.len(); match self.wasm2index.entry(wasm.clone()) { - hash_map::Entry::Occupied(entry) => *entry.get(), + hash_map::Entry::Occupied(entry) => { + let ret = *entry.get(); + let entry = &mut self.index_map[ret.bits() as usize]; + if entry.trampoline.is_none() { + entry.trampoline = trampoline; + } + ret + } hash_map::Entry::Vacant(entry) => { // Keep `signature_hash` len under 2**32 -- VMSharedSignatureIndex::new(std::u32::MAX) // is reserved for VMSharedSignatureIndex::default(). @@ -75,8 +82,10 @@ impl SignatureRegistry { &self, idx: VMSharedSignatureIndex, ) -> Option<(&WasmFuncType, VMTrampoline)> { - self.index_map + let (wasm, trampoline) = self + .index_map .get(idx.bits() as usize) - .map(|e| (&e.wasm, e.trampoline)) + .map(|e| (&e.wasm, e.trampoline))?; + Some((wasm, trampoline?)) } } diff --git a/crates/wasmtime/src/store.rs b/crates/wasmtime/src/store.rs index f01c86662f15..6e315462f8c2 100644 --- a/crates/wasmtime/src/store.rs +++ b/crates/wasmtime/src/store.rs @@ -202,7 +202,7 @@ impl Store { .inner .signatures .borrow_mut() - .register(ty.as_wasm_func_type(), trampoline); + .register(ty.as_wasm_func_type(), Some(trampoline)); Box::new(anyfunc) }); @@ -322,9 +322,15 @@ impl Store { fn register_signatures(&self, module: &Module) { let mut signatures = self.signatures().borrow_mut(); let types = module.types(); + for (_, ty) in module.compiled_module().module().types.iter() { + if let wasmtime_environ::ModuleType::Function(index) = ty { + let wasm = &types.wasm_signatures[*index]; + signatures.register(wasm, None); + } + } for (index, trampoline) in module.compiled_module().trampolines() { let wasm = &types.wasm_signatures[*index]; - signatures.register(wasm, *trampoline); + signatures.register(wasm, Some(*trampoline)); } } diff --git a/crates/wasmtime/src/trampoline/func.rs b/crates/wasmtime/src/trampoline/func.rs index 7b5eb70e6459..d81ed07bb149 100644 --- a/crates/wasmtime/src/trampoline/func.rs +++ b/crates/wasmtime/src/trampoline/func.rs @@ -18,6 +18,7 @@ use wasmtime_jit::trampoline::{ self, binemit, pretty_error, Context, FunctionBuilder, FunctionBuilderContext, }; use wasmtime_jit::CodeMemory; +use wasmtime_jit::{blank_sig, wasmtime_call_conv}; use wasmtime_runtime::{ Imports, InstanceAllocationRequest, InstanceAllocator, InstanceHandle, OnDemandInstanceAllocator, VMContext, VMFunctionBody, VMSharedSignatureIndex, VMTrampoline, @@ -91,16 +92,7 @@ fn make_trampoline( // Mostly reverse copy of the similar method from wasmtime's // wasmtime-jit/src/compiler.rs. let pointer_type = isa.pointer_type(); - let mut stub_sig = ir::Signature::new(isa.frontend_config().default_call_conv); - - // Add the caller/callee `vmctx` parameters. - stub_sig.params.push(ir::AbiParam::special( - pointer_type, - ir::ArgumentPurpose::VMContext, - )); - - // Add the caller `vmctx` parameter. - stub_sig.params.push(ir::AbiParam::new(pointer_type)); + let mut stub_sig = blank_sig(isa, wasmtime_call_conv(isa)); // Add the `values_vec` parameter. stub_sig.params.push(ir::AbiParam::new(pointer_type)); @@ -220,8 +212,15 @@ fn create_function_trampoline( // reference types which requires safepoints. let isa = config.target_isa_with_reference_types(); - let pointer_type = isa.pointer_type(); - let sig = ft.get_wasmtime_signature(pointer_type); + let mut sig = blank_sig(&*isa, wasmtime_call_conv(&*isa)); + sig.params.extend( + ft.params() + .map(|p| ir::AbiParam::new(p.get_wasmtime_type())), + ); + sig.returns.extend( + ft.results() + .map(|p| ir::AbiParam::new(p.get_wasmtime_type())), + ); let mut fn_builder_ctx = FunctionBuilderContext::new(); let mut module = Module::new(); @@ -271,7 +270,7 @@ pub fn create_function( // If there is no signature registry, use the default signature index which is // guaranteed to trap if there is ever an indirect call on the function (should not happen) let shared_signature_id = registry - .map(|r| r.register(ft.as_wasm_func_type(), trampoline)) + .map(|r| r.register(ft.as_wasm_func_type(), Some(trampoline))) .unwrap_or(VMSharedSignatureIndex::default()); unsafe { diff --git a/crates/wasmtime/src/types.rs b/crates/wasmtime/src/types.rs index 5a090d794cce..2a91af473c91 100644 --- a/crates/wasmtime/src/types.rs +++ b/crates/wasmtime/src/types.rs @@ -298,28 +298,6 @@ impl FuncType { &self.sig } - /// Get the Cranelift-compatible function signature. - pub(crate) fn get_wasmtime_signature(&self, pointer_type: ir::Type) -> ir::Signature { - use wasmtime_environ::ir::{AbiParam, ArgumentPurpose, Signature}; - use wasmtime_jit::native; - let call_conv = native::call_conv(); - let mut params = vec![ - AbiParam::special(pointer_type, ArgumentPurpose::VMContext), - AbiParam::new(pointer_type), - ]; - params.extend(self.params().map(|p| AbiParam::new(p.get_wasmtime_type()))); - let returns = self - .results() - .map(|p| AbiParam::new(p.get_wasmtime_type())) - .collect::>(); - - Signature { - params, - returns, - call_conv, - } - } - pub(crate) fn from_wasm_func_type(sig: &wasm::WasmFuncType) -> FuncType { FuncType { sig: sig.clone() } } diff --git a/tests/all/func.rs b/tests/all/func.rs index 0e691993843f..4159cafd9a5e 100644 --- a/tests/all/func.rs +++ b/tests/all/func.rs @@ -619,3 +619,162 @@ fn trap_doesnt_leak() -> anyhow::Result<()> { assert!(dtor2_run.get()); Ok(()) } + +#[test] +fn wrap_multiple_results() -> anyhow::Result<()> { + fn test(store: &Store, t: T) -> anyhow::Result<()> + where + T: WasmHostResults + + WasmResults + + PartialEq + + Copy + + std::fmt::Debug + + EqualToValues + + 'static, + { + let f = Func::wrap(store, move || t); + assert_eq!(f.typed::<(), T>()?.call(())?, t); + assert!(t.eq_values(&f.call(&[])?)); + + let module = Module::new(store.engine(), &T::gen_wasm())?; + let instance = Instance::new(store, &module, &[f.into()])?; + let f = instance.get_func("foo").unwrap(); + + assert_eq!(f.typed::<(), T>()?.call(())?, t); + assert!(t.eq_values(&f.call(&[])?)); + Ok(()) + } + + let store = Store::default(); + // 0 element + test(&store, ())?; + + // 1 element + test(&store, (1i32,))?; + test(&store, (2u32,))?; + test(&store, (3i64,))?; + test(&store, (4u64,))?; + test(&store, (5.0f32,))?; + test(&store, (6.0f64,))?; + + // 2 element ... + test(&store, (7i32, 8i32))?; + test(&store, (7i32, 8i64))?; + test(&store, (7i32, 8f32))?; + test(&store, (7i32, 8f64))?; + + test(&store, (7i64, 8i32))?; + test(&store, (7i64, 8i64))?; + test(&store, (7i64, 8f32))?; + test(&store, (7i64, 8f64))?; + + test(&store, (7f32, 8i32))?; + test(&store, (7f32, 8i64))?; + test(&store, (7f32, 8f32))?; + test(&store, (7f32, 8f64))?; + + test(&store, (7f64, 8i32))?; + test(&store, (7f64, 8i64))?; + test(&store, (7f64, 8f32))?; + test(&store, (7f64, 8f64))?; + + // and beyond... + test(&store, (1i32, 2i32, 3i32))?; + test(&store, (1i32, 2f32, 3i32))?; + test(&store, (1f64, 2f32, 3i32))?; + test(&store, (1f64, 2i64, 3i32))?; + test(&store, (1f32, 2f32, 3i64, 4f64))?; + test(&store, (1f64, 2i64, 3i32, 4i64, 5f32))?; + test(&store, (1i32, 2f64, 3i64, 4f64, 5f64, 6f32))?; + test(&store, (1i64, 2i32, 3i64, 4f32, 5f32, 6i32, 7u64))?; + test(&store, (1u32, 2f32, 3u64, 4f64, 5i32, 6f32, 7u64, 8u32))?; + test( + &store, + (1f32, 2f64, 3f32, 4i32, 5u32, 6i64, 7f32, 8i32, 9u64), + )?; + return Ok(()); + + trait EqualToValues { + fn eq_values(&self, values: &[Val]) -> bool; + fn gen_wasm() -> String; + } + + macro_rules! equal_tuples { + ($($cnt:tt ($($a:ident),*))*) => ($( + #[allow(non_snake_case)] + impl<$($a: EqualToValue,)*> EqualToValues for ($($a,)*) { + fn eq_values(&self, values: &[Val]) -> bool { + let ($($a,)*) = self; + let mut _values = values.iter(); + _values.len() == $cnt && + $($a.eq_value(_values.next().unwrap()) &&)* + true + } + + fn gen_wasm() -> String { + let mut wasm = String::new(); + wasm.push_str("(module "); + wasm.push_str("(type $t (func (result "); + $( + wasm.push_str($a::wasm_ty()); + wasm.push_str(" "); + )* + wasm.push_str(")))"); + + wasm.push_str("(import \"\" \"\" (func $host (type $t)))"); + wasm.push_str("(func (export \"foo\") (type $t)"); + wasm.push_str("call $host"); + wasm.push_str(")"); + wasm.push_str(")"); + + wasm + } + } + )*) + } + + equal_tuples! { + 0 () + 1 (A1) + 2 (A1, A2) + 3 (A1, A2, A3) + 4 (A1, A2, A3, A4) + 5 (A1, A2, A3, A4, A5) + 6 (A1, A2, A3, A4, A5, A6) + 7 (A1, A2, A3, A4, A5, A6, A7) + 8 (A1, A2, A3, A4, A5, A6, A7, A8) + 9 (A1, A2, A3, A4, A5, A6, A7, A8, A9) + } + + trait EqualToValue { + fn eq_value(&self, value: &Val) -> bool; + fn wasm_ty() -> &'static str; + } + + macro_rules! equal_values { + ($a:ident $($ty:ident $wasm:tt $variant:ident $e:expr,)*) => ($( + impl EqualToValue for $ty { + fn eq_value(&self, val: &Val) -> bool { + if let Val::$variant($a) = *val { + return *self == $e; + } + false + } + + fn wasm_ty() -> &'static str { + $wasm + } + } + )*) + } + + equal_values! { + a + i32 "i32" I32 a, + u32 "i32" I32 a as u32, + i64 "i64" I64 a, + u64 "i64" I64 a as u64, + f32 "f32" F32 f32::from_bits(a), + f64 "f64" F64 f64::from_bits(a), + } +}