Cranelift: implement an "unwinder" crate and exception throws in filetests. (#10919)

This commit introduces the next major piece of machinery (after the
previously-landed `try_call` support) that we will eventually use to
implement Wasm exceptions in Wasmtime. In particular, it implements a
generic unwinder as a new crate that supports (i) walking a stack
produced by Cranelift code, (ii) serializing Cranelift exception
metadata to compact tables (in a way very similar to address maps in
Wasmtime, so they will be mappable directly from disk), (iii) using
these serialized tables to find handlers during a stack-walk, and (iv)
jumping to handlers (i.e., actually unwinding). This crate is currently
used in the filetests runner, and will next be used in Wasmtime.

The commit first performs code-motion: it moves stack-walking code from
Wasmtime to `cranelift-unwinder`. This itself has no functional effect,
but isolates the code that understands contiguous sequences of Cranelift
frames ("activations") from that which is specific to Wasmtime's
activation delimiters and metadata.

It then implements a compact exception-table format. This format uses
the `object` crate's mechanisms for directly referencing in-memory
arrays of little-endian `u32`s in a way that will allow us to find
handlers when mapping exception metadata directly from an ELF section in
a `.cwasm` (for example). The format consists of four sorted `u32`
arrays in a way that allows us to look up a callsite first, then search
its sorted array of handler offsets by tags.

It next implements the actual unwind control flow: it contains an
assembly stub for each supported architecture that transfers control to
a PC, SP, and FP value "up the stack", with payload values placed in the
payload registers we have defined per our exception ABI in Cranelift.

Finally, it puts these pieces together in the filetest runner. Note that
the runtest does a lot "by hand": we don't have entry and exit
trampolines as we do in Wasmtime, so the filetest contains three
functions, with the middle one invoking the "throw hostcall" and entry
and exit trampolines around it grabbing the appropriate entry/exit FPs
and exit PC. The dance to call back to host code is also somewhat
delicate, as we haven't done this before. The `JITModule`'s linking +
relocation support does not seem sufficient to properly define a symbol,
so instead we scan for `func_addr` instructions referencing a well-known
name (`__cranelift_throw`) and replace them with `iconst`s with the
function address at runtime, baking it in. This is somewhat ugly, but it
works. All of these filetest-specific details will be handled much more
nicely in the Wasmtime version of this functionality, as we have proper
abstractions for entry/exit trampolines and hostcalls.

Author: cfallin

Cargo.lock

@@ -249,6 +249,7 @@ wasmtime-fuzzing = { path = "crates/fuzzing" }
 wasmtime-jit-icache-coherence = { path = "crates/jit-icache-coherence", version = "=35.0.0" }
 wasmtime-wit-bindgen = { path = "crates/wit-bindgen", version = "=35.0.0" }
 wasmtime-math = { path = "crates/math", version = "=35.0.0" }
+wasmtime-unwinder = { path = "crates/unwinder", version = "=35.0.0" }
 test-programs-artifacts = { path = 'crates/test-programs/artifacts' }
 wasmtime-test-util = { path = "crates/test-util" }
 

Cargo.toml

@@ -113,6 +113,11 @@ impl TestcaseName {
     pub(crate) fn new<T: AsRef<[u8]>>(v: T) -> Self {
         Self(v.as_ref().into())
     }
+
+    /// Get the raw test case name as bytes.
+    pub fn raw(&self) -> &[u8] {
+        &self.0
+    }
 }
 
 /// The name of an external is either a reference to a user-defined symbol

cranelift/codegen/src/ir/extname.rs

@@ -59,8 +59,8 @@ pub mod write;
 
 pub use crate::entity::packed_option;
 pub use crate::machinst::buffer::{
-    FinalizedMachReloc, FinalizedRelocTarget, MachCallSite, MachSrcLoc, MachTextSectionBuilder,
-    MachTrap, OpenPatchRegion, PatchRegion,
+    FinalizedMachCallSite, FinalizedMachReloc, FinalizedRelocTarget, MachCallSite, MachSrcLoc,
+    MachTextSectionBuilder, MachTrap, OpenPatchRegion, PatchRegion,
 };
 pub use crate::machinst::{
     CallInfo, CompiledCode, Final, MachBuffer, MachBufferFinalized, MachInst, MachInstEmit,

cranelift/codegen/src/lib.rs

@@ -19,9 +19,10 @@ cranelift-frontend = { workspace = true }
 cranelift-interpreter = { workspace = true }
 cranelift-native = { workspace = true }
 cranelift-reader = { workspace = true }
-cranelift-jit = { workspace = true, features = ["selinux-fix"] }
+cranelift-jit = { workspace = true, features = ["selinux-fix", "wasmtime-unwinder"] }
 cranelift-module = { workspace = true }
 cranelift-control = { workspace = true }
+wasmtime-unwinder = { workspace = true, features = ["cranelift"] }
 file-per-thread-logger = { workspace = true }
 filecheck = { workspace = true }
 gimli = { workspace = true, features = ['std'] }

cranelift/filetests/Cargo.toml

@@ -0,0 +1,50 @@
+test run
+set preserve_frame_pointers=true
+target x86_64
+target aarch64
+target riscv64
+target s390x
+
+function %entry() -> i64 tail {
+  fn0 = %main(i64) -> i64 tail
+
+block0:
+  v1 = get_frame_pointer.i64
+  v2 = call fn0(v1)
+  return v2
+}
+
+; run: %entry() == 58
+
+function %main(i64) -> i64 tail {
+  sig0 = (i64, i32, i64, i64) tail
+  fn0 = %throw(i64, i32, i64, i64) tail
+
+block0(v0: i64):
+  v1 = iconst.i64 42
+  v2 = iconst.i64 100
+  v3 = iconst.i32 1
+  try_call fn0(v0, v3, v1, v2), sig0, block1(), [ tag1: block2(exn0, exn1) ]
+
+block1:
+  v4 = iconst.i64 1
+  return v4
+
+block2(v5: i64, v6: i64):
+  v7 = isub.i64 v6, v5
+  return v7
+}
+
+
+function %throw(i64, i32, i64, i64) tail {
+  sig0 = (i64, i64, i64, i32, i64, i64)
+  fn0 = %__cranelift_throw(i64, i64, i64, i32, i64, i64)
+
+block0(v0: i64, v1: i32, v2: i64, v3: i64):
+  v4 = get_frame_pointer.i64
+  v5 = get_return_address.i64
+  v6 = load.i64 v5 ; get caller's FP
+  v7 = func_addr.i64 fn0
+  call_indirect sig0, v7(v0, v4, v6, v1, v2, v3)
+  return
+}

cranelift/filetests/filetests/runtests/throw.clif

@@ -1,9 +1,12 @@
 //! Provides functionality for compiling and running CLIF IR for `run` tests.
 use anyhow::{Result, anyhow};
 use core::mem;
+use cranelift::prelude::Imm64;
+use cranelift_codegen::cursor::{Cursor, FuncCursor};
 use cranelift_codegen::data_value::DataValue;
 use cranelift_codegen::ir::{
-    ExternalName, Function, InstBuilder, Signature, UserExternalName, UserFuncName,
+    ExternalName, Function, InstBuilder, InstructionData, LibCall, Opcode, Signature,
+    UserExternalName, UserFuncName,
 };
 use cranelift_codegen::isa::{OwnedTargetIsa, TargetIsa};
 use cranelift_codegen::{CodegenError, Context, ir, settings};
@@ -14,9 +17,10 @@ use cranelift_module::{FuncId, Linkage, Module, ModuleError};
 use cranelift_native::builder_with_options;
 use cranelift_reader::TestFile;
 use pulley_interpreter::interp as pulley;
+use std::cell::Cell;
 use std::cmp::max;
-use std::collections::HashMap;
 use std::collections::hash_map::Entry;
+use std::collections::{HashMap, HashSet};
 use std::ptr::NonNull;
 use target_lexicon::Architecture;
 use thiserror::Error;
@@ -67,7 +71,7 @@ struct DefinedFunction {
 /// let compiled = compiler.compile().unwrap();
 /// let trampoline = compiled.get_trampoline(&func).unwrap();
 ///
-/// let returned = trampoline.call(&vec![DataValue::I32(2), DataValue::I32(40)]);
+/// let returned = trampoline.call(&compiled, &vec![DataValue::I32(2), DataValue::I32(40)]);
 /// assert_eq!(vec![DataValue::I32(42)], returned);
 /// ```
 pub struct TestFileCompiler {
@@ -255,7 +259,13 @@ impl TestFileCompiler {
     }
 
     /// Defines the body of a function
-    pub fn define_function(&mut self, func: Function, ctrl_plane: &mut ControlPlane) -> Result<()> {
+    pub fn define_function(
+        &mut self,
+        mut func: Function,
+        ctrl_plane: &mut ControlPlane,
+    ) -> Result<()> {
+        Self::replace_hostcall_references(&mut func);
+
         let defined_func = self
             .defined_functions
             .get(&func.name)
@@ -271,6 +281,47 @@ impl TestFileCompiler {
         Ok(())
     }
 
+    fn replace_hostcall_references(func: &mut Function) {
+        // For every `func_addr` referring to a hostcall that we
+        // define, replace with an `iconst` with the actual
+        // address. Then modify the external func references to
+        // harmless libcall references (that will be unused so
+        // ignored).
+        let mut funcrefs_to_remove = HashSet::new();
+        let mut cursor = FuncCursor::new(func);
+        while let Some(_block) = cursor.next_block() {
+            while let Some(inst) = cursor.next_inst() {
+                match &cursor.func.dfg.insts[inst] {
+                    InstructionData::FuncAddr {
+                        opcode: Opcode::FuncAddr,
+                        func_ref,
+                    } => {
+                        let ext_func = &cursor.func.dfg.ext_funcs[*func_ref];
+                        let hostcall_addr = match &ext_func.name {
+                            ExternalName::TestCase(tc) if tc.raw() == b"__cranelift_throw" => {
+                                Some(__cranelift_throw as usize)
+                            }
+                            _ => None,
+                        };
+
+                        if let Some(addr) = hostcall_addr {
+                            funcrefs_to_remove.insert(*func_ref);
+                            cursor.func.dfg.insts[inst] = InstructionData::UnaryImm {
+                                opcode: Opcode::Iconst,
+                                imm: Imm64::new(addr as i64),
+                            };
+                        }
+                    }
+                    _ => {}
+                }
+            }
+        }
+
+        for to_remove in funcrefs_to_remove {
+            func.dfg.ext_funcs[to_remove].name = ExternalName::LibCall(LibCall::Probestack);
+        }
+    }
+
     /// Creates and registers a trampoline for a function if none exists.
     pub fn create_trampoline_for_function(
         &mut self,
@@ -356,6 +407,13 @@ impl Drop for CompiledTestFile {
     }
 }
 
+std::thread_local! {
+    /// TLS slot used to store a CompiledTestFile reference so that it
+    /// can be recovered when a hostcall (such as the exception-throw
+    /// handler) is invoked.
+    pub static COMPILED_TEST_FILE: Cell<*const CompiledTestFile> = Cell::new(std::ptr::null());
+}
+
 /// A callable trampoline
 pub struct Trampoline<'a> {
     module: &'a JITModule,
@@ -366,16 +424,18 @@ pub struct Trampoline<'a> {
 
 impl<'a> Trampoline<'a> {
     /// Call the target function of this trampoline, passing in [DataValue]s using a compiled trampoline.
-    pub fn call(&self, arguments: &[DataValue]) -> Vec<DataValue> {
+    pub fn call(&self, compiled: &CompiledTestFile, arguments: &[DataValue]) -> Vec<DataValue> {
         let mut values = UnboxedValues::make_arguments(arguments, &self.func_signature);
         let arguments_address = values.as_mut_ptr();
 
         let function_ptr = self.module.get_finalized_function(self.func_id);
         let trampoline_ptr = self.module.get_finalized_function(self.trampoline_id);
 
+        COMPILED_TEST_FILE.set(compiled as *const _);
         unsafe {
             self.call_raw(trampoline_ptr, function_ptr, arguments_address);
         }
+        COMPILED_TEST_FILE.set(std::ptr::null());
 
         values.collect_returns(&self.func_signature)
     }
@@ -563,6 +623,71 @@ fn make_trampoline(name: UserFuncName, signature: &ir::Signature, isa: &dyn Targ
     func
 }
 
+/// Hostcall invoked directly from a compiled function body to test
+/// exception throws.
+///
+/// This function does not return normally: it either uses the
+/// unwinder to jump directly to a Cranelift frame further up the
+/// stack, if a handler is found; or it panics, if not.
+#[cfg(any(
+    target_arch = "x86_64",
+    target_arch = "aarch64",
+    target_arch = "s390x",
+    target_arch = "riscv64"
+))]
+extern "C-unwind" fn __cranelift_throw(
+    entry_fp: usize,
+    exit_fp: usize,
+    exit_pc: usize,
+    tag: u32,
+    payload1: usize,
+    payload2: usize,
+) -> ! {
+    let compiled_test_file = unsafe { &*COMPILED_TEST_FILE.get() };
+    let unwind_host = wasmtime_unwinder::UnwindHost;
+    let module_lookup = |pc| {
+        compiled_test_file
+            .module
+            .as_ref()
+            .unwrap()
+            .lookup_wasmtime_exception_data(pc)
+    };
+    unsafe {
+        match wasmtime_unwinder::compute_throw_action(
+            &unwind_host,
+            module_lookup,
+            exit_pc,
+            exit_fp,
+            entry_fp,
+            tag,
+        ) {
+            wasmtime_unwinder::ThrowAction::Handler { pc, sp, fp } => {
+                wasmtime_unwinder::resume_to_exception_handler(pc, sp, fp, payload1, payload2);
+            }
+            wasmtime_unwinder::ThrowAction::None => {
+                panic!("Expected a handler to exit for throw of tag {tag} at pc {exit_pc:x}");
+            }
+        }
+    }
+}
+
+#[cfg(not(any(
+    target_arch = "x86_64",
+    target_arch = "aarch64",
+    target_arch = "s390x",
+    target_arch = "riscv64"
+)))]
+extern "C-unwind" fn __cranelift_throw(
+    _entry_fp: usize,
+    _exit_fp: usize,
+    _exit_pc: usize,
+    _tag: u32,
+    _payload1: usize,
+    _payload2: usize,
+) -> ! {
+    panic!("Throw not implemented on platforms without native backends.");
+}
+
 #[cfg(target_arch = "x86_64")]
 use std::arch::x86_64::__m128i;
 #[cfg(target_arch = "x86_64")]
@@ -655,7 +780,7 @@ mod test {
             .unwrap();
         let compiled = compiler.compile().unwrap();
         let trampoline = compiled.get_trampoline(&function).unwrap();
-        let returned = trampoline.call(&[]);
+        let returned = trampoline.call(&compiled, &[]);
         assert_eq!(returned, vec![DataValue::I8(-1)])
     }
 

cranelift/filetests/src/function_runner.rs

@@ -191,7 +191,7 @@ fn run_test(
                     args.extend_from_slice(run_args);
 
                     let trampoline = testfile.get_trampoline(func).unwrap();
-                    Ok(trampoline.call(&args))
+                    Ok(trampoline.call(&testfile, &args))
                 })
                 .map_err(|s| anyhow::anyhow!("{}", s))?;
         }

cranelift/filetests/src/test_run.rs

@@ -19,6 +19,7 @@ cranelift-native = { workspace = true }
 cranelift-codegen = { workspace = true, features = ["std"] }
 cranelift-entity = { workspace = true }
 cranelift-control = { workspace = true }
+wasmtime-unwinder = { workspace = true, optional = true, features = ["cranelift"] }
 anyhow = { workspace = true }
 region = "3.0.2"
 libc = { workspace = true }
@@ -39,6 +40,8 @@ features = [
 selinux-fix = ['memmap2']
 default = []
 
+wasmtime-unwinder = ["dep:wasmtime-unwinder"]
+
 [dev-dependencies]
 cranelift = { path = "../umbrella" }
 cranelift-frontend = { workspace = true }