diff --git a/cranelift/codegen/src/lib.rs b/cranelift/codegen/src/lib.rs index 24446b83699c..787aff2b118c 100644 --- a/cranelift/codegen/src/lib.rs +++ b/cranelift/codegen/src/lib.rs @@ -71,8 +71,8 @@ pub mod write; pub use crate::entity::packed_option; pub use crate::machinst::buffer::{ ExceptionContextLoc, FinalizedMachCallSite, FinalizedMachExceptionHandler, FinalizedMachReloc, - FinalizedRelocTarget, MachCallSite, MachSrcLoc, MachTextSectionBuilder, MachTrap, - OpenPatchRegion, PatchRegion, + FinalizedRelocTarget, MachCallSite, MachExceptionHandler, MachSrcLoc, MachTextSectionBuilder, + MachTrap, OpenPatchRegion, PatchRegion, }; pub use crate::machinst::{ CallInfo, CompiledCode, Final, FrameLayout, MachBuffer, MachBufferDebugTagList, diff --git a/crates/test-util/src/wast.rs b/crates/test-util/src/wast.rs index c480e53b4e1d..3687e65d63c5 100644 --- a/crates/test-util/src/wast.rs +++ b/crates/test-util/src/wast.rs @@ -567,9 +567,6 @@ impl WastTest { "misc_testsuite/externref-table-dropped-segment-issue-8281.wast", "misc_testsuite/many_table_gets_lead_to_gc.wast", "misc_testsuite/no-panic.wast", - // Winch does not implement exception handlers yet. - "misc_testsuite/traps-skip-catch-all.wast", - "spec_testsuite/throw.wast", ]; if unsupported.iter().any(|part| self.path.ends_with(part)) { diff --git a/tests/all/exceptions.rs b/tests/all/exceptions.rs index 7c5d070de5f0..31d28ff5a28a 100644 --- a/tests/all/exceptions.rs +++ b/tests/all/exceptions.rs @@ -3,8 +3,7 @@ use std::sync::atomic::{AtomicBool, Ordering::Relaxed}; use wasmtime::*; use wasmtime_test_macros::wasmtime_test; -// Winch does not implement catches yet. Re-enable after catch is implemented. -#[wasmtime_test(strategies(not(Winch)), wasm_features(exceptions))] +#[wasmtime_test(wasm_features(exceptions))] #[cfg_attr(miri, ignore)] fn basic_throw(config: &mut Config) -> Result<()> { let engine = Engine::new(config)?; @@ -40,8 +39,7 @@ fn basic_throw(config: &mut Config) -> Result<()> { Ok(()) } -// Winch does not implement catches yet. Re-enable after catch is implemented. -#[wasmtime_test(strategies(not(Winch)), wasm_features(exceptions))] +#[wasmtime_test(wasm_features(exceptions))] #[cfg_attr(miri, ignore)] fn dynamic_tags(config: &mut Config) -> Result<()> { let engine = Engine::new(config)?; @@ -101,6 +99,58 @@ fn dynamic_tags(config: &mut Config) -> Result<()> { Ok(()) } +#[wasmtime_test(wasm_features(exceptions))] +#[cfg_attr(miri, ignore)] +fn nested_handler_scopes(config: &mut Config) -> Result<()> { + let engine = Engine::new(config)?; + let mut store = Store::new(&engine, ()); + + let module = Module::new( + &engine, + r#" + (module + (tag $outer) + (tag $inner) + + (func $throw_outer + (throw $outer)) + + (func $throw_inner + (throw $inner)) + + ;; While both handlers are active, the inner tag does not match and + ;; lookup continues to the outer handler. + (func (export "nested") (result i32) + (block $outer_handler + (try_table (catch $outer $outer_handler) + (block $inner_handler + (try_table (catch $inner $inner_handler) + (call $throw_outer))))) + (i32.const 1)) + + ;; After the inner try_table ends, its handler is no longer active. + ;; The outer catch_all handles the throw instead. + (func (export "after") (result i32) + (block $stale_inner_handler + (block $outer_handler + (try_table (catch_all $outer_handler) + (try_table (catch $inner $stale_inner_handler) + (nop)) + (call $throw_inner))) + (return (i32.const 1))) + (i32.const 2))) + "#, + )?; + + let instance = Instance::new(&mut store, &module, &[])?; + let nested = instance.get_typed_func::<(), i32>(&mut store, "nested")?; + let after = instance.get_typed_func::<(), i32>(&mut store, "after")?; + assert_eq!(nested.call(&mut store, ())?, 1); + assert_eq!(after.call(&mut store, ())?, 1); + + Ok(()) +} + #[wasmtime_test(wasm_features(exceptions))] #[cfg_attr(miri, ignore)] fn exception_escape_to_host(config: &mut Config) -> Result<()> { @@ -193,10 +243,57 @@ fn funcref_exception_payload_escape_to_host(config: &mut Config) -> Result<()> { Ok(()) } +#[wasmtime_test(wasm_features(exceptions, reference_types))] +#[cfg_attr(miri, ignore)] +fn caught_funcref_payload(config: &mut Config) -> Result<()> { + let engine = Engine::new(config)?; + let mut store = Store::new(&engine, ()); + + let module = Module::new( + &engine, + r#" + (module + (tag $e (param funcref i32)) + + (func $throw (param funcref) + (throw $e (local.get 0) (i32.const 42))) + + (func (export "catch") (param funcref) (result funcref i32) + (block $handler (result funcref i32) + (try_table (result funcref i32) (catch $e $handler) + (call $throw (local.get 0)) + (ref.null func) + (i32.const 0))))) + "#, + )?; + + let instance = Instance::new(&mut store, &module, &[])?; + let catch = instance.get_func(&mut store, "catch").unwrap(); + let expected = Func::wrap(&mut store, || 126_i32); + let mut results = [Val::null_func_ref(), Val::I32(0)]; + catch.call(&mut store, &[Val::FuncRef(Some(expected))], &mut results)?; + + let actual = results[0].unwrap_funcref().unwrap(); + let actual = actual.typed::<(), i32>(&store)?; + assert_eq!(actual.call(&mut store, ())?, 126); + assert_eq!(results[1].unwrap_i32(), 42); + + Ok(()) +} + #[wasmtime_test(wasm_features(exceptions, reference_types))] #[cfg_attr(miri, ignore)] fn thrown_externref_payload_survives_gc(config: &mut Config) -> Result<()> { - config.collector(Collector::DeferredReferenceCounting); + for collector in [Collector::Copying, Collector::DeferredReferenceCounting] { + println!("Using GC collector: {collector:?}"); + config.collector(collector); + run_thrown_externref_payload_survives_gc(config)?; + } + + Ok(()) +} + +fn run_thrown_externref_payload_survives_gc(config: &Config) -> Result<()> { let engine = Engine::new(config)?; let mut store = Store::new(&engine, ()); @@ -234,6 +331,62 @@ fn thrown_externref_payload_survives_gc(config: &mut Config) -> Result<()> { Ok(()) } +#[wasmtime_test(wasm_features(exceptions, reference_types))] +#[cfg_attr(miri, ignore)] +fn caught_externref_payload_survives_gc(config: &mut Config) -> Result<()> { + for collector in [Collector::Copying, Collector::DeferredReferenceCounting] { + println!("Using GC collector: {collector:?}"); + config.collector(collector); + run_caught_externref_payload_survives_gc(config)?; + } + + Ok(()) +} + +fn run_caught_externref_payload_survives_gc(config: &Config) -> Result<()> { + let engine = Engine::new(config)?; + let mut store = Store::new(&engine, ()); + + let module = Module::new( + &engine, + r#" + (module + (tag $e (param externref i32)) + + (func $throw (param externref) + (throw $e (local.get 0) (i32.const 42))) + + (func (export "catch") (param externref) (result externref i32) + (block $handler (result externref i32) + (try_table (result externref i32) (catch $e $handler) + (call $throw (local.get 0)) + (ref.null extern) + (i32.const 0))))) + "#, + )?; + + let instance = Instance::new(&mut store, &module, &[])?; + let catch = instance + .get_typed_func::>, (Option>, i32)>( + &mut store, "catch", + )?; + let dropped = Arc::new(AtomicBool::new(false)); + + let caught = { + let mut scope = RootScope::new(&mut store); + let payload = ExternRef::new(&mut scope, SetFlagOnDrop(dropped.clone()))?; + let (caught, value) = catch.call(&mut scope, Some(payload))?; + assert_eq!(value, 42); + caught.unwrap().to_owned_rooted(&mut scope)? + }; + + store.gc(None)?; + assert!(!dropped.load(Relaxed)); + assert!(caught.data(&store)?.is_some()); + + Ok(()) +} + #[wasmtime_test(wasm_features(exceptions))] #[cfg_attr(miri, ignore)] fn throw_with_null_collector(config: &mut Config) -> Result<()> { @@ -261,8 +414,7 @@ fn throw_with_null_collector(config: &mut Config) -> Result<()> { Ok(()) } -// Winch does not implement catches yet. Re-enable after catch is implemented. -#[wasmtime_test(strategies(not(Winch)), wasm_features(exceptions))] +#[wasmtime_test(wasm_features(exceptions))] #[cfg_attr(miri, ignore)] fn exception_from_host(config: &mut Config) -> Result<()> { let engine = Engine::new(config)?; @@ -402,8 +554,7 @@ fn thrown_exception_without_throwing(config: &mut Config) -> Result<()> { Ok(()) } -// Winch does not implement catches yet. Re-enable after catch is implemented. -#[wasmtime_test(strategies(not(Winch)), wasm_features(exceptions))] +#[wasmtime_test(wasm_features(exceptions))] #[cfg_attr(miri, ignore)] fn wasm_exceptions_have_backtraces(config: &mut Config) -> Result<()> { let engine = Engine::new(config)?; @@ -429,8 +580,7 @@ fn wasm_exceptions_have_backtraces(config: &mut Config) -> Result<()> { Ok(()) } -// Winch does not implement catches yet. Re-enable after catch is implemented. -#[wasmtime_test(strategies(not(Winch)), wasm_features(exceptions))] +#[wasmtime_test(wasm_features(exceptions))] #[cfg_attr(miri, ignore)] fn store_pending_exnref_is_cloned(config: &mut Config) -> wasmtime::Result<()> { config.collector(Collector::DeferredReferenceCounting); @@ -485,7 +635,7 @@ fn store_pending_exnref_is_cloned(config: &mut Config) -> wasmtime::Result<()> { Ok(()) } -// Winch does not implement catches yet. Re-enable after catch is implemented. +// Winch does not implement `catch_ref` yet. #[wasmtime_test(strategies(not(Winch)), wasm_features(exceptions, reference_types))] #[cfg_attr(miri, ignore)] fn store_pending_exnref_is_exposed(config: &mut Config) -> wasmtime::Result<()> { diff --git a/tests/disas/winch/aarch64/exceptions/throw-drc.wat b/tests/disas/winch/aarch64/exceptions/throw-drc.wat index 275008a6fd29..178ea237cfde 100644 --- a/tests/disas/winch/aarch64/exceptions/throw-drc.wat +++ b/tests/disas/winch/aarch64/exceptions/throw-drc.wat @@ -2,8 +2,8 @@ ;;! test = "winch" ;;! flags = "-W exceptions -C collector=drc" -;; `throw` builds an exception that escapes to the host, while `try_table` -;; still compiles as a plain block. +;; Calls made while the `try_table` handler is active carry exception metadata. +;; Its landing pad loads the exception's payload and branches to `$h`. (module (tag $e (param i32)) (func (result i32) @@ -21,14 +21,16 @@ ;; movk x17, #0x20 ;; add x16, x16, x17 ;; cmp sp, x16 -;; b.lo #0x120 +;; b.lo #0x160 ;; 2c: mov x9, x0 ;; sub x28, x28, #0x10 ;; mov sp, x28 ;; stur x0, [x28, #8] ;; stur x1, [x28] ;; mov x0, x9 -;; bl #0x25c +;; bl #0x2a0 +;; ├─╼ exception frame offset: SP = FP - 0x20 +;; ╰─╼ exception handler: tag=0, context at [SP+0x8], handler=0x108 ;; 48: ldur x9, [x28, #8] ;; ldur x1, [x9, #0x28] ;; ldur w1, [x1, #8] @@ -45,7 +47,9 @@ ;; ldur w2, [x28, #8] ;; mov x3, #0x28 ;; mov x4, #8 -;; bl #0x20c +;; bl #0x250 +;; ├─╼ exception frame offset: SP = FP - 0x30 +;; ╰─╼ exception handler: tag=0, context at [SP+0x18], handler=0x108 ;; 8c: add x28, x28, #8 ;; mov sp, x28 ;; add x28, x28, #4 @@ -71,16 +75,35 @@ ;; mov sp, x28 ;; mov x0, x9 ;; ldur w1, [x28, #0xc] -;; bl #0x28c +;; bl #0x2d0 +;; ├─╼ exception frame offset: SP = FP - 0x30 +;; ╰─╼ exception handler: tag=0, context at [SP+0x18], handler=0x108 ;; f4: add x28, x28, #0xc ;; mov sp, x28 ;; add x28, x28, #4 ;; mov sp, x28 ;; ldur x9, [x28, #8] +;; mov x28, x29 +;; sub x28, x28, #0x10 +;; mov sp, x28 +;; sub x28, x28, #0x10 +;; mov sp, x28 +;; ldur x9, [x28, #8] +;; ldur x1, [x9, #8] +;; ldur x2, [x1, #0x28] +;; ldur x1, [x1, #0x20] +;; mov x16, x0 +;; add x16, x16, #0x28 +;; cmp x16, x2, uxtx +;; b.hi #0x164 +;; 13c: mov x2, x1 +;; add x2, x2, x0, uxtx +;; ldur w0, [x2, #0x20] ;; add x28, x28, #0x10 ;; mov sp, x28 ;; mov sp, x28 ;; ldr x28, [sp], #0x10 ;; ldp x29, x30, [sp], #0x10 ;; ret -;; 120: udf #0xc11f +;; 160: udf #0xc11f +;; 164: udf #0xc11f diff --git a/tests/disas/winch/aarch64/exceptions/throw-null.wat b/tests/disas/winch/aarch64/exceptions/throw-null.wat index 5fefe1833b81..d73446d07bbb 100644 --- a/tests/disas/winch/aarch64/exceptions/throw-null.wat +++ b/tests/disas/winch/aarch64/exceptions/throw-null.wat @@ -2,8 +2,8 @@ ;;! test = "winch" ;;! flags = "-W exceptions -C collector=null" -;; `throw` builds an exception that escapes to the host, while `try_table` -;; still compiles as a plain block. +;; Calls made while the `try_table` handler is active carry exception metadata. +;; Its landing pad loads the exception's payload and branches to `$h`. (module (tag $e (param i32)) (func (result i32) @@ -21,23 +21,25 @@ ;; movk x17, #0x20 ;; add x16, x16, x17 ;; cmp sp, x16 -;; b.lo #0x190 +;; b.lo #0x1d0 ;; 2c: mov x9, x0 ;; sub x28, x28, #0x10 ;; mov sp, x28 ;; stur x0, [x28, #8] ;; stur x1, [x28] ;; mov x0, x9 -;; bl #0x2d8 +;; bl #0x31c +;; ├─╼ exception frame offset: SP = FP - 0x20 +;; ╰─╼ exception handler: tag=0, context at [SP+0x8], handler=0x178 ;; 48: ldur x9, [x28, #8] ;; ldur x16, [x9, #0x20] ;; ldur w1, [x16] ;; adds w1, w1, #7 -;; b.hs #0x194 +;; b.hs #0x1d4 ;; 5c: and w1, w1, #0xfffffff8 ;; mov w2, w1 ;; adds w2, w2, #0x18 -;; b.hs #0x198 +;; b.hs #0x1d8 ;; 6c: sub x28, x28, #4 ;; mov sp, x28 ;; stur w0, [x28] @@ -62,7 +64,9 @@ ;; stur x0, [x28] ;; mov x0, x9 ;; ldur x1, [x28] -;; bl #0x284 +;; bl #0x2c8 +;; ├─╼ exception frame offset: SP = FP - 0x30 +;; ╰─╼ exception handler: tag=0, context at [SP+0x18], handler=0x178 ;; d0: add x28, x28, #8 ;; mov sp, x28 ;; ldur x9, [x28, #0x10] @@ -99,18 +103,37 @@ ;; mov sp, x28 ;; mov x0, x9 ;; ldur w1, [x28, #0xc] -;; bl #0x308 +;; bl #0x34c +;; ├─╼ exception frame offset: SP = FP - 0x30 +;; ╰─╼ exception handler: tag=0, context at [SP+0x18], handler=0x178 ;; 164: add x28, x28, #0xc ;; mov sp, x28 ;; add x28, x28, #4 ;; mov sp, x28 ;; ldur x9, [x28, #8] +;; mov x28, x29 +;; sub x28, x28, #0x10 +;; mov sp, x28 +;; sub x28, x28, #0x10 +;; mov sp, x28 +;; ldur x9, [x28, #8] +;; ldur x1, [x9, #8] +;; ldur x2, [x1, #0x28] +;; ldur x1, [x1, #0x20] +;; mov x16, x0 +;; add x16, x16, #0x18 +;; cmp x16, x2, uxtx +;; b.hi #0x1dc +;; 1ac: mov x2, x1 +;; add x2, x2, x0, uxtx +;; ldur w0, [x2, #0x10] ;; add x28, x28, #0x10 ;; mov sp, x28 ;; mov sp, x28 ;; ldr x28, [sp], #0x10 ;; ldp x29, x30, [sp], #0x10 ;; ret -;; 190: udf #0xc11f -;; 194: udf #0xc11f -;; 198: udf #0xc11f +;; 1d0: udf #0xc11f +;; 1d4: udf #0xc11f +;; 1d8: udf #0xc11f +;; 1dc: udf #0xc11f diff --git a/tests/disas/winch/aarch64/exceptions/throw.wat b/tests/disas/winch/aarch64/exceptions/throw.wat index 7b75c22df46e..c0c4ceb1555c 100644 --- a/tests/disas/winch/aarch64/exceptions/throw.wat +++ b/tests/disas/winch/aarch64/exceptions/throw.wat @@ -2,8 +2,8 @@ ;;! test = "winch" ;;! flags = "-W exceptions -C collector=copying" -;; `throw` builds an exception that escapes to the host, while `try_table` -;; still compiles as a plain block. +;; Calls made while the `try_table` handler is active carry exception metadata. +;; Its landing pad loads the exception's payload and branches to `$h`. (module (tag $e (param i32)) (func (result i32) @@ -21,14 +21,16 @@ ;; movk x17, #0x20 ;; add x16, x16, x17 ;; cmp sp, x16 -;; b.lo #0x124 +;; b.lo #0x164 ;; 2c: mov x9, x0 ;; sub x28, x28, #0x10 ;; mov sp, x28 ;; stur x0, [x28, #8] ;; stur x1, [x28] ;; mov x0, x9 -;; bl #0x260 +;; bl #0x2a4 +;; ├─╼ exception frame offset: SP = FP - 0x20 +;; ╰─╼ exception handler: tag=0, context at [SP+0x8], handler=0x10c ;; 48: ldur x9, [x28, #8] ;; ldur x1, [x9, #0x28] ;; ldur w1, [x1, #8] @@ -46,7 +48,9 @@ ;; ldur w2, [x28, #8] ;; mov x3, #0x20 ;; mov x4, #0x10 -;; bl #0x210 +;; bl #0x254 +;; ├─╼ exception frame offset: SP = FP - 0x30 +;; ╰─╼ exception handler: tag=0, context at [SP+0x18], handler=0x10c ;; 90: add x28, x28, #8 ;; mov sp, x28 ;; add x28, x28, #4 @@ -72,16 +76,35 @@ ;; mov sp, x28 ;; mov x0, x9 ;; ldur w1, [x28, #0xc] -;; bl #0x290 +;; bl #0x2d4 +;; ├─╼ exception frame offset: SP = FP - 0x30 +;; ╰─╼ exception handler: tag=0, context at [SP+0x18], handler=0x10c ;; f8: add x28, x28, #0xc ;; mov sp, x28 ;; add x28, x28, #4 ;; mov sp, x28 ;; ldur x9, [x28, #8] +;; mov x28, x29 +;; sub x28, x28, #0x10 +;; mov sp, x28 +;; sub x28, x28, #0x10 +;; mov sp, x28 +;; ldur x9, [x28, #8] +;; ldur x1, [x9, #8] +;; ldur x2, [x1, #0x28] +;; ldur x1, [x1, #0x20] +;; mov x16, x0 +;; add x16, x16, #0x20 +;; cmp x16, x2, uxtx +;; b.hi #0x168 +;; 140: mov x2, x1 +;; add x2, x2, x0, uxtx +;; ldur w0, [x2, #0x18] ;; add x28, x28, #0x10 ;; mov sp, x28 ;; mov sp, x28 ;; ldr x28, [sp], #0x10 ;; ldp x29, x30, [sp], #0x10 ;; ret -;; 124: udf #0xc11f +;; 164: udf #0xc11f +;; 168: udf #0xc11f diff --git a/tests/disas/winch/x64/exceptions/throw-drc.wat b/tests/disas/winch/x64/exceptions/throw-drc.wat index 7dcaab3d93b0..0aa883656325 100644 --- a/tests/disas/winch/x64/exceptions/throw-drc.wat +++ b/tests/disas/winch/x64/exceptions/throw-drc.wat @@ -2,8 +2,8 @@ ;;! test = "winch" ;;! flags = "-W exceptions -C collector=drc" -;; `throw` builds an exception that escapes to the host, while `try_table` -;; still compiles as a plain block. +;; Calls made while the `try_table` handler is active carry exception metadata. +;; Its landing pad loads the exception's payload and branches to `$h`. (module (tag $e (param i32)) (func (result i32) @@ -17,13 +17,15 @@ ;; movq 0x18(%r11), %r11 ;; addq $0x20, %r11 ;; cmpq %rsp, %r11 -;; ja 0xf3 +;; ja 0x12a ;; 1c: movq %rdi, %r14 ;; subq $0x10, %rsp ;; movq %rdi, 8(%rsp) ;; movq %rsi, (%rsp) ;; movq %r14, %rdi -;; callq 0x1f8 +;; callq 0x231 +;; ├─╼ exception frame offset: SP = FP - 0x10 +;; ╰─╼ exception handler: tag=0, context at [SP+0x8], handler=0xea ;; movq 8(%rsp), %r14 ;; movq 0x28(%r14), %rcx ;; movl 8(%rcx), %ecx @@ -37,7 +39,9 @@ ;; movl 8(%rsp), %edx ;; movl $0x28, %ecx ;; movl $8, %r8d -;; callq 0x1a9 +;; callq 0x1e2 +;; ├─╼ exception frame offset: SP = FP - 0x20 +;; ╰─╼ exception handler: tag=0, context at [SP+0x18], handler=0xea ;; addq $8, %rsp ;; addq $4, %rsp ;; movq 0xc(%rsp), %r14 @@ -56,11 +60,27 @@ ;; subq $0xc, %rsp ;; movq %r14, %rdi ;; movl 0xc(%rsp), %esi -;; callq 0x225 +;; callq 0x25e +;; ├─╼ exception frame offset: SP = FP - 0x20 +;; ╰─╼ exception handler: tag=0, context at [SP+0x18], handler=0xea ;; addq $0xc, %rsp ;; addq $4, %rsp ;; movq 8(%rsp), %r14 +;; movq %rbp, %rsp +;; subq $0x10, %rsp +;; movq 8(%rsp), %r14 +;; movq 8(%r14), %rcx +;; movq 0x28(%rcx), %rdx +;; movq 0x20(%rcx), %rcx +;; movq %rax, %r11 +;; addq $0x28, %r11 +;; cmpq %rdx, %r11 +;; ja 0x12c +;; 118: movq %rcx, %rdx +;; addq %rax, %rdx +;; movl 0x20(%rdx), %eax ;; addq $0x10, %rsp ;; popq %rbp ;; retq -;; f3: ud2 +;; 12a: ud2 +;; 12c: ud2 diff --git a/tests/disas/winch/x64/exceptions/throw-null.wat b/tests/disas/winch/x64/exceptions/throw-null.wat index 2b08339818bc..7498d39859bd 100644 --- a/tests/disas/winch/x64/exceptions/throw-null.wat +++ b/tests/disas/winch/x64/exceptions/throw-null.wat @@ -2,8 +2,8 @@ ;;! test = "winch" ;;! flags = "-W exceptions -C collector=null" -;; `throw` builds an exception that escapes to the host, while `try_table` -;; still compiles as a plain block. +;; Calls made while the `try_table` handler is active carry exception metadata. +;; Its landing pad loads the exception's payload and branches to `$h`. (module (tag $e (param i32)) (func (result i32) @@ -17,22 +17,24 @@ ;; movq 0x18(%r11), %r11 ;; addq $0x20, %r11 ;; cmpq %rsp, %r11 -;; ja 0x14f +;; ja 0x186 ;; 1c: movq %rdi, %r14 ;; subq $0x10, %rsp ;; movq %rdi, 8(%rsp) ;; movq %rsi, (%rsp) ;; movq %r14, %rdi -;; callq 0x250 +;; callq 0x289 +;; ├─╼ exception frame offset: SP = FP - 0x10 +;; ╰─╼ exception handler: tag=0, context at [SP+0x8], handler=0x146 ;; movq 8(%rsp), %r14 ;; movq 0x20(%r14), %r11 ;; movl (%r11), %ecx ;; addl $7, %ecx -;; jb 0x151 +;; jb 0x188 ;; 4f: andl $0xfffffff8, %ecx ;; movl %ecx, %edx ;; addl $0x18, %edx -;; jb 0x153 +;; jb 0x18a ;; 63: subq $4, %rsp ;; movl %eax, (%rsp) ;; subq $4, %rsp @@ -50,7 +52,9 @@ ;; pushq %rax ;; movq %r14, %rdi ;; movq (%rsp), %rsi -;; callq 0x209 +;; callq 0x242 +;; ├─╼ exception frame offset: SP = FP - 0x20 +;; ╰─╼ exception handler: tag=0, context at [SP+0x18], handler=0x146 ;; addq $8, %rsp ;; movq 0x10(%rsp), %r14 ;; movl (%rsp), %eax @@ -78,13 +82,29 @@ ;; subq $0xc, %rsp ;; movq %r14, %rdi ;; movl 0xc(%rsp), %esi -;; callq 0x27d +;; callq 0x2b6 +;; ├─╼ exception frame offset: SP = FP - 0x20 +;; ╰─╼ exception handler: tag=0, context at [SP+0x18], handler=0x146 ;; addq $0xc, %rsp ;; addq $4, %rsp ;; movq 8(%rsp), %r14 +;; movq %rbp, %rsp +;; subq $0x10, %rsp +;; movq 8(%rsp), %r14 +;; movq 8(%r14), %rcx +;; movq 0x28(%rcx), %rdx +;; movq 0x20(%rcx), %rcx +;; movq %rax, %r11 +;; addq $0x18, %r11 +;; cmpq %rdx, %r11 +;; ja 0x18c +;; 174: movq %rcx, %rdx +;; addq %rax, %rdx +;; movl 0x10(%rdx), %eax ;; addq $0x10, %rsp ;; popq %rbp ;; retq -;; 14f: ud2 -;; 151: ud2 -;; 153: ud2 +;; 186: ud2 +;; 188: ud2 +;; 18a: ud2 +;; 18c: ud2 diff --git a/tests/disas/winch/x64/exceptions/throw.wat b/tests/disas/winch/x64/exceptions/throw.wat index 9dff95865d95..38a058208351 100644 --- a/tests/disas/winch/x64/exceptions/throw.wat +++ b/tests/disas/winch/x64/exceptions/throw.wat @@ -2,8 +2,8 @@ ;;! test = "winch" ;;! flags = "-W exceptions -C collector=copying" -;; `throw` builds an exception that escapes to the host, while `try_table` -;; still compiles as a plain block. +;; Calls made while the `try_table` handler is active carry exception metadata. +;; Its landing pad loads the exception's payload and branches to `$h`. (module (tag $e (param i32)) (func (result i32) @@ -17,13 +17,15 @@ ;; movq 0x18(%r11), %r11 ;; addq $0x20, %r11 ;; cmpq %rsp, %r11 -;; ja 0xf3 +;; ja 0x12a ;; 1c: movq %rdi, %r14 ;; subq $0x10, %rsp ;; movq %rdi, 8(%rsp) ;; movq %rsi, (%rsp) ;; movq %r14, %rdi -;; callq 0x1f8 +;; callq 0x231 +;; ├─╼ exception frame offset: SP = FP - 0x10 +;; ╰─╼ exception handler: tag=0, context at [SP+0x8], handler=0xea ;; movq 8(%rsp), %r14 ;; movq 0x28(%r14), %rcx ;; movl 8(%rcx), %ecx @@ -37,7 +39,9 @@ ;; movl 8(%rsp), %edx ;; movl $0x20, %ecx ;; movl $0x10, %r8d -;; callq 0x1a9 +;; callq 0x1e2 +;; ├─╼ exception frame offset: SP = FP - 0x20 +;; ╰─╼ exception handler: tag=0, context at [SP+0x18], handler=0xea ;; addq $8, %rsp ;; addq $4, %rsp ;; movq 0xc(%rsp), %r14 @@ -56,11 +60,27 @@ ;; subq $0xc, %rsp ;; movq %r14, %rdi ;; movl 0xc(%rsp), %esi -;; callq 0x225 +;; callq 0x25e +;; ├─╼ exception frame offset: SP = FP - 0x20 +;; ╰─╼ exception handler: tag=0, context at [SP+0x18], handler=0xea ;; addq $0xc, %rsp ;; addq $4, %rsp ;; movq 8(%rsp), %r14 +;; movq %rbp, %rsp +;; subq $0x10, %rsp +;; movq 8(%rsp), %r14 +;; movq 8(%r14), %rcx +;; movq 0x28(%rcx), %rdx +;; movq 0x20(%rcx), %rcx +;; movq %rax, %r11 +;; addq $0x20, %r11 +;; cmpq %rdx, %r11 +;; ja 0x12c +;; 118: movq %rcx, %rdx +;; addq %rax, %rdx +;; movl 0x18(%rdx), %eax ;; addq $0x10, %rsp ;; popq %rbp ;; retq -;; f3: ud2 +;; 12a: ud2 +;; 12c: ud2 diff --git a/winch/codegen/src/codegen/call.rs b/winch/codegen/src/codegen/call.rs index a45b8d839dbc..5a0d14844682 100644 --- a/winch/codegen/src/codegen/call.rs +++ b/winch/codegen/src/codegen/call.rs @@ -83,7 +83,9 @@ impl FnCall { /// 3. Spills the value stack. /// 4. Creates the stack space needed for the return area. /// 5. Emits the call. - /// 6. Cleans up the stack space. + /// 6. Records any GC stack map and active exception handlers at the call's + /// return address. + /// 7. Cleans up the stack space. pub fn emit( env: &mut FuncEnv, masm: &mut M, @@ -109,6 +111,13 @@ impl FnCall { if !offsets.is_empty() { masm.emit_stack_map(sp, &offsets)?; } + if !context.exception_handlers.is_empty() { + masm.emit_try_call_site( + sp, + context.frame.vmctx_slot().offset, + context.exception_handlers.handlers(), + )?; + } Ok(()) }, )?; diff --git a/winch/codegen/src/codegen/context.rs b/winch/codegen/src/codegen/context.rs index f31c336b6e63..e57540734355 100644 --- a/winch/codegen/src/codegen/context.rs +++ b/winch/codegen/src/codegen/context.rs @@ -3,7 +3,9 @@ use crate::{ Result, abi::{ABIOperand, ABIResults, RetArea, vmctx}, bail, - codegen::{BranchState, CodeGenError, CodeGenPhase, Emission, Prologue}, + codegen::{ + BranchState, CodeGenError, CodeGenPhase, Emission, Prologue, exceptions::HandlerState, + }, ensure, format_err, frame::Frame, isa::reg::RegClass, @@ -45,6 +47,8 @@ pub(crate) struct CodeGenContext<'a, P: CodeGenPhase> { pub reachable: bool, /// A reference to the VMOffsets. pub vmoffsets: &'a VMOffsets, + /// The exception handlers currently in scope. + pub exception_handlers: HandlerState, } impl<'a> CodeGenContext<'a, Emission> { @@ -123,6 +127,7 @@ impl<'a> CodeGenContext<'a, Prologue> { frame, reachable: true, vmoffsets, + exception_handlers: Default::default(), } } @@ -134,6 +139,7 @@ impl<'a> CodeGenContext<'a, Prologue> { reachable: self.reachable, vmoffsets: self.vmoffsets, frame: self.frame.for_emission(), + exception_handlers: self.exception_handlers, } } } diff --git a/winch/codegen/src/codegen/control.rs b/winch/codegen/src/codegen/control.rs index 77d0376ea2f2..3a6d2d31e809 100644 --- a/winch/codegen/src/codegen/control.rs +++ b/winch/codegen/src/codegen/control.rs @@ -6,7 +6,9 @@ //! next instruction is a control instruction, we could avoid emitting //! a [`crate::masm::MacroAssembler::cmp_with_set`] and instead emit //! a conditional jump inline when emitting the control flow instruction. -use super::{CodeGenContext, CodeGenError, Emission, OperandSize, Reg, TypedReg}; +use super::{ + CodeGenContext, CodeGenError, Emission, OperandSize, Reg, TypedReg, exceptions::TryTableInfo, +}; use crate::{ CallingConvention, Result, abi::{ABI, ABIOperand, ABIResults, ABISig, RetArea}, @@ -244,6 +246,8 @@ pub(crate) enum ControlStackFrame { /// target. By default, this is false, and it's updated when /// emitting a `br` or `br_if`. is_branch_target: bool, + /// Exception-handling information when this block is a `try_table`. + try_table_info: Option, }, Loop { /// The start of the Loop. @@ -279,18 +283,54 @@ impl ControlStackFrame { sig: BlockSig, masm: &mut M, context: &mut CodeGenContext, + ) -> Result { + Self::block_impl(sig, None, masm, context) + } + + /// Returns a block control frame with exception-handler information. + pub fn try_table( + sig: BlockSig, + info: TryTableInfo, + masm: &mut M, + context: &mut CodeGenContext, + ) -> Result { + Self::block_impl(sig, Some(info), masm, context) + } + + fn block_impl( + sig: BlockSig, + try_table_info: Option, + masm: &mut M, + context: &mut CodeGenContext, ) -> Result { let mut control = Self::Block { sig, is_branch_target: false, exit: masm.get_label()?, stack_state: Default::default(), + try_table_info, }; control.emit(masm, context)?; Ok(control) } + /// Returns this block's try-table information, if present. + pub fn try_table_info(&self) -> Option<&TryTableInfo> { + match self { + Self::Block { try_table_info, .. } => try_table_info.as_ref(), + _ => None, + } + } + + /// Takes this block's try-table information, if present. + pub fn take_try_table_info(&mut self) -> Option { + match self { + Self::Block { try_table_info, .. } => try_table_info.take(), + _ => None, + } + } + /// Returns [`ControlStackFrame`] for a loop. pub fn r#loop( sig: BlockSig, diff --git a/winch/codegen/src/codegen/drc.rs b/winch/codegen/src/codegen/drc.rs index 94de6ae249e8..abfc88bea630 100644 --- a/winch/codegen/src/codegen/drc.rs +++ b/winch/codegen/src/codegen/drc.rs @@ -48,7 +48,7 @@ where Ok(()) } - /// Emits a DRC read barrier for a value loaded from `addr`. + /// Emits a DRC read barrier for the value in `gc_ref`. /// /// The loaded reference is first pushed and spilled so it is represented /// in a stack map if this barrier calls `force_gc`. Null and i31 references @@ -59,16 +59,8 @@ where /// Finally it forces a collection when the roots list reaches both the /// proportional and absolute thresholds. /// - /// Leaves the loaded reference on the value stack and marks - /// `storage_base` available for register reuse before returning. - pub(crate) fn emit_drc_read_barrier( - &mut self, - ty: WasmValType, - storage_base: Reg, - addr: M::Address, - ) -> Result<()> { - let gc_ref = self.context.reg_for_type(ty, self.masm)?; - self.masm.load(addr, writable!(gc_ref), ty.try_into()?)?; + /// Leaves the loaded reference on the value stack. + pub(crate) fn emit_drc_read_barrier(&mut self, ty: WasmValType, gc_ref: Reg) -> Result<()> { self.context.stack.push(Val::reg(gc_ref, ty)); // Spill the loaded result into a stack-map-visible slot before the @@ -121,7 +113,6 @@ where self.emit_maybe_force_gc(roots_len, heap_data_reg)?; self.masm.bind(skip_barrier)?; - self.context.free_reg(storage_base); Ok(()) } diff --git a/winch/codegen/src/codegen/exceptions.rs b/winch/codegen/src/codegen/exceptions.rs index 9a0aad1297eb..293df086dc96 100644 --- a/winch/codegen/src/codegen/exceptions.rs +++ b/winch/codegen/src/codegen/exceptions.rs @@ -1,20 +1,262 @@ -use super::{Callee, CodeGen, CodeGenError, Emission, FnCall}; +use super::{Callee, CodeGen, CodeGenError, ControlStackFrame, Emission, FnCall}; use crate::{ - Result, format_err, + Result, + codegen::{UnconditionalBranch, control_index}, + ensure, format_err, masm::{IntScratch, MacroAssembler, OperandSize, RegImm}, - reg::Reg, - stack::TypedReg, + reg::{Reg, writable}, + stack::{TypedReg, Val}, }; -use wasmtime_environ::copying::InlineTraceInfo; +use cranelift_codegen::{MachExceptionHandler, MachLabel, ir::ExceptionTag}; +use smallvec::SmallVec; use wasmtime_environ::{ Collector, GcStructLayout, GcTypeLayouts, ModuleInternedTypeIndex, PtrSize, TagIndex, VMGcKind, - WasmExnType, WasmHeapType, WasmStorageType, WasmValType, + WasmExnType, WasmHeapType, WasmStorageType, WasmValType, packed_option::ReservedValue, }; +use wasmtime_environ::{WasmCompositeInnerType, copying::InlineTraceInfo}; + +/// The exception handlers that are currently in scope. +#[derive(Default)] +pub(crate) struct HandlerState { + handlers: Vec<(Option, MachLabel)>, +} + +/// A checkpoint that can restore the exception handlers in scope. +#[derive(Clone, Copy, Debug)] +pub(crate) struct HandlerStateCheckpoint(usize); + +#[derive(Debug)] +pub(crate) struct CatchInfo { + pub(crate) tag: Option, + pub(crate) target_depth: u32, + pub(crate) landing_pad: MachLabel, +} + +#[derive(Debug)] +pub(crate) struct TryTableInfo { + pub(crate) checkpoint: HandlerStateCheckpoint, + pub(crate) catches: Vec, +} + +impl HandlerState { + /// Adds an exception handler. + pub(crate) fn add_handler(&mut self, tag: Option, label: MachLabel) { + self.handlers.push((tag, label)); + } + + /// Takes a checkpoint of the exception handlers currently in scope. + pub(crate) fn take_checkpoint(&self) -> HandlerStateCheckpoint { + HandlerStateCheckpoint(self.handlers.len()) + } + + /// Restores the exception handlers to a previous checkpoint. + pub(crate) fn restore_checkpoint(&mut self, checkpoint: HandlerStateCheckpoint) { + assert!(checkpoint.0 <= self.handlers.len()); + self.handlers.truncate(checkpoint.0); + } + + /// Iterates over exception handlers from the innermost to the outermost. + pub(crate) fn handlers(&self) -> impl Iterator + '_ { + self.handlers + .iter() + .copied() + .rev() + .map(|(tag, label)| match tag { + Some(tag) => MachExceptionHandler::Tag(tag, label), + None => MachExceptionHandler::Default(label), + }) + } + + /// Returns whether there are no exception handlers in scope. + pub(crate) fn is_empty(&self) -> bool { + self.handlers.is_empty() + } +} impl<'a, 'translation, 'data, M> CodeGen<'a, 'translation, 'data, M, Emission> where M: MacroAssembler, { + /// Emits the end of a try-table block and its exception landing pads. + pub(crate) fn emit_try_table_end( + &mut self, + mut control: ControlStackFrame, + info: TryTableInfo, + ) -> Result<()> { + let stack_state = *control.stack_state(); + + let fallthrough_reachable = self.context.reachable; + let end_reachable = fallthrough_reachable || control.is_next_sequence_reachable(); + + if fallthrough_reachable { + ensure!( + control.stack_state().target_len == self.context.stack.len(), + CodeGenError::control_frame_state_mismatch() + ); + + control.pop_abi_results(&mut self.context, self.masm, |results, _, _| { + Ok(results.ret_area().copied()) + })?; + + self.masm.jmp(*control.label())?; + } + + for catch in info.catches { + self.masm.bind(catch.landing_pad)?; + + self.context.reachable = true; + let exception_reg = self + .masm + .prepare_for_exception_handler(stack_state.base_offset)?; + self.context.truncate_stack_to(stack_state.base_len)?; + self.context.load_vmctx(self.masm)?; + + if let Some(tag) = catch.tag { + let exception_reg = self.context.reg(exception_reg, self.masm)?; + self.emit_load_exception_payload_fields(tag, exception_reg)?; + } + self.emit_catch_branch(catch.target_depth)?; + } + + self.context.reachable = end_reachable; + + if end_reachable { + if !fallthrough_reachable { + control.ensure_stack_state(self.masm, &mut self.context)?; + } + control.bind_end(self.masm, &mut self.context) + } else { + Ok(()) + } + } + + fn emit_catch_branch(&mut self, target_depth: u32) -> Result<()> { + let index = control_index(target_depth, self.control_frames.len())?; + let frame = &mut self.control_frames[index]; + + self.context + .br::<_, _, UnconditionalBranch>(frame, self.masm, |masm, context, frame| { + frame.pop_abi_results::(context, masm, |results, _, _| { + Ok(results.ret_area().copied()) + }) + }) + } + + fn emit_load_exception_payload_fields( + &mut self, + tag_index: TagIndex, + exception_reg: Reg, + ) -> Result<()> { + let interned = self.env.translation.module.tags[tag_index] + .exception + .unwrap_module_type_index(); + + let exn_ty = match &self.env.types[interned].composite_type.inner { + WasmCompositeInnerType::Exn(exn_ty) => exn_ty, + _ => return Err(format_err!(CodeGenError::unsupported_wasm_type())), + }; + let gc_codegen_config = self.require_gc_codegen_config(); + let layouts = gc_codegen_config.layouts(); + + let layout = layouts + .exn_layout(exn_ty) + .map_err(|_| format_err!(CodeGenError::unsupported_wasm_type()))?; + + let fields: SmallVec<[(WasmStorageType, u32); 8]> = exn_ty + .fields + .iter() + .zip(layout.fields.iter()) + .map(|(field_ty, field_layout)| (field_ty.element_type, field_layout.offset)) + .collect(); + + let (heap_base, heap_bound) = self.emit_load_gc_heap_base_and_bound()?; + + self.emit_gc_ref_bounds_check(exception_reg, heap_bound, i64::from(layout.size))?; + + self.context.free_reg(heap_bound); + + let mut object_addr = self.emit_gc_ref_addr(exception_reg, heap_base)?; + self.context.free_reg(heap_base); + self.context.free_reg(exception_reg); + for (field_ty, field_offset) in fields { + let ty = match field_ty { + WasmStorageType::Val(ty @ WasmValType::Ref(r)) + if r.heap_type == WasmHeapType::Func => + { + let func_ref_id = self.context.any_gpr(self.masm)?; + let addr = self.masm.address_at_reg(object_addr, field_offset)?; + self.masm + .load(addr, writable!(func_ref_id), OperandSize::S32)?; + + // The builtin call can clobber allocated registers. Preserve + // the object address beneath the call's arguments so later + // payload fields can still be loaded. + self.context.stack.push(TypedReg::i64(object_addr).into()); + self.context.stack.push(TypedReg::i32(func_ref_id).into()); + self.context.stack.push(Val::i32( + ModuleInternedTypeIndex::reserved_value() + .as_bits() + .cast_signed(), + )); + + let get = self.env.builtins.get_interned_func_ref::()?; + FnCall::emit::( + &mut self.env, + self.masm, + &mut self.context, + Callee::Builtin(get), + )?; + + let func_ref = self.context.pop_to_reg(self.masm, None)?; + object_addr = self.context.pop_to_reg(self.masm, None)?.reg; + self.context + .stack + .push(TypedReg::new(ty, func_ref.reg).into()); + continue; + } + WasmStorageType::Val(ty @ WasmValType::Ref(r)) + if r.heap_type == WasmHeapType::Extern => + { + let addr = self.masm.address_at_reg(object_addr, field_offset)?; + if gc_codegen_config.collector() == Collector::DeferredReferenceCounting { + // The DRC read barrier can make an out-of-line call. + // Preserve the object address across the call so later + // payload fields can still be loaded. + let gc_ref = self.context.reg_for_type(ty, self.masm)?; + self.masm.load(addr, writable!(gc_ref), ty.try_into()?)?; + self.context.stack.push(TypedReg::i64(object_addr).into()); + self.emit_drc_read_barrier(ty, gc_ref)?; + let payload = self.context.pop_to_reg(self.masm, None)?; + object_addr = self.context.pop_to_reg(self.masm, None)?.reg; + self.context.stack.push(payload.into()); + } else { + let value = self.context.reg_for_type(ty, self.masm)?; + self.masm.load(addr, writable!(value), ty.try_into()?)?; + self.context.stack.push(TypedReg::new(ty, value).into()); + } + continue; + } + WasmStorageType::Val(WasmValType::Ref(_)) => { + return Err(format_err!(CodeGenError::unsupported_wasm_type())); + } + WasmStorageType::Val(ty) => ty, + WasmStorageType::I8 | WasmStorageType::I16 => { + return Err(format_err!(CodeGenError::unsupported_wasm_type())); + } + }; + + let value = self.context.reg_for_type(ty, self.masm)?; + let addr = self.masm.address_at_reg(object_addr, field_offset)?; + + self.masm.load(addr, writable!(value), ty.try_into()?)?; + + self.context.stack.push(TypedReg::new(ty, value).into()); + } + + self.context.free_reg(object_addr); + Ok(()) + } + /// Allocates an exception and initializes its tag identity. /// /// Exception tags are identified by the defining instance and the tag's @@ -99,7 +341,7 @@ where /// /// This method consumes and frees `object_addr`. It returns `gc_ref`, which /// remains owned by the caller. - pub(crate) fn emit_exception_payload_fields( + pub(crate) fn emit_store_exception_payload_fields( &mut self, exn_ty: &WasmExnType, layout: &GcStructLayout, diff --git a/winch/codegen/src/codegen/mod.rs b/winch/codegen/src/codegen/mod.rs index 1c37f2ff9dcc..0f69ee4ce2c1 100644 --- a/winch/codegen/src/codegen/mod.rs +++ b/winch/codegen/src/codegen/mod.rs @@ -42,6 +42,7 @@ pub use builtin::*; pub(crate) mod bounds; mod drc; mod exceptions; +pub(crate) use exceptions::{CatchInfo, TryTableInfo}; mod gc; use gc::GcCodegenConfig; @@ -318,9 +319,17 @@ where /// Pops a control frame from the control frame stack. pub fn pop_control_frame(&mut self) -> Result { - self.control_frames + let frame = self + .control_frames .pop() - .ok_or_else(|| format_err!(CodeGenError::control_frame_expected())) + .ok_or_else(|| format_err!(CodeGenError::control_frame_expected()))?; + if let Some(info) = frame.try_table_info() { + self.context + .exception_handlers + .restore_checkpoint(info.checkpoint); + } + + Ok(frame) } /// Derives a [RelSourceLoc] from a [SourceLoc]. @@ -358,6 +367,9 @@ where pub fn handle_unreachable_end(&mut self) -> Result<()> { let mut frame = self.pop_control_frame()?; + if let Some(info) = frame.take_try_table_info() { + return self.emit_try_table_end(frame, info); + } // We just popped the outermost block. let is_outermost = self.control_frames.len() == 0; @@ -439,7 +451,7 @@ where fn visit_op_when_unreachable(op: &Operator) -> bool { use Operator::*; match op { - If { .. } | Block { .. } | Loop { .. } | Else | End => true, + If { .. } | Block { .. } | TryTable { .. } | Loop { .. } | Else | End => true, _ => false, } } diff --git a/winch/codegen/src/isa/aarch64/masm.rs b/winch/codegen/src/isa/aarch64/masm.rs index 0c1328736bc3..08b89edd56f8 100644 --- a/winch/codegen/src/isa/aarch64/masm.rs +++ b/winch/codegen/src/isa/aarch64/masm.rs @@ -29,14 +29,16 @@ use crate::{ stack::{TypedReg, Val}, }; use cranelift_codegen::{ - Final, MachBufferFinalized, MachLabel, + ExceptionContextLoc, Final, MachBufferFinalized, MachExceptionHandler, MachLabel, binemit::CodeOffset, ir::{MemFlagsData, RelSourceLoc, SourceLoc, types}, - isa::aarch64, - isa::aarch64::inst::{ - self, Cond, ExtendOp, Imm12, ImmLogic, ImmShift, SImm7Scaled, SImm9, ScalarSize, - VecALUModOp, VecALUOp, VecExtendOp, VecLanesOp, VecMisc2, VecRRLongOp, VecRRNarrowOp, - VecRRPairLongOp, VecRRRLongModOp, VecRRRLongOp, VecShiftImmOp, VectorSize, + isa::aarch64::{ + self, + inst::{ + self, Cond, ExtendOp, Imm12, ImmLogic, ImmShift, SImm7Scaled, SImm9, ScalarSize, + VecALUModOp, VecALUOp, VecExtendOp, VecLanesOp, VecMisc2, VecRRLongOp, VecRRNarrowOp, + VecRRPairLongOp, VecRRRLongModOp, VecRRRLongOp, VecShiftImmOp, VectorSize, + }, }, settings, }; @@ -319,6 +321,28 @@ impl Masm for MacroAssembler { Ok(()) } + fn prepare_for_exception_handler(&mut self, target_offset: SPOffset) -> Result { + let shadow_sp = regs::shadow_sp(); + + self.asm + .mov_rr(regs::fp(), writable!(shadow_sp), OperandSize::S64); + + let initial_offset = + Imm12::maybe_from_u64(u64::from(SHADOW_STACK_POINTER_SLOT_SIZE)).unwrap(); + self.asm.sub_ir( + initial_offset, + shadow_sp, + writable!(shadow_sp), + OperandSize::S64, + ); + + self.move_shadow_sp_to_sp(); + self.sp_offset = 0; + self.reserve_stack(target_offset.as_u32())?; + + Ok(regs::xreg(0)) + } + fn local_address(&mut self, local: &LocalSlot) -> Result
{ let (reg, offset) = local .addressed_from_sp() @@ -1362,6 +1386,30 @@ impl Masm for MacroAssembler { Ok(()) } + fn emit_try_call_site( + &mut self, + sp_offset: SPOffset, + vmctx_slot_offset: u32, + handlers: impl Iterator, + ) -> Result<()> { + let frame_offset = sp_offset.as_u32() + u32::from(SHADOW_STACK_POINTER_SLOT_SIZE); + let vmctx_offset = sp_offset + .as_u32() + .checked_sub(vmctx_slot_offset) + .ok_or_else(CodeGenError::invalid_local_offset)?; + + let handlers = std::iter::once(MachExceptionHandler::Context( + ExceptionContextLoc::SPOffset(vmctx_offset), + )) + .chain(handlers); + + self.asm + .buffer_mut() + .add_try_call_site(Some(frame_offset), handlers); + + Ok(()) + } + fn current_code_offset(&self) -> Result { Ok(self.asm.buffer().cur_offset()) } diff --git a/winch/codegen/src/isa/x64/masm.rs b/winch/codegen/src/isa/x64/masm.rs index e7fe439b243a..a1c9c456dd51 100644 --- a/winch/codegen/src/isa/x64/masm.rs +++ b/winch/codegen/src/isa/x64/masm.rs @@ -34,7 +34,7 @@ use crate::{ masm::CalleeKind, }; use cranelift_codegen::{ - Final, MachBufferFinalized, MachLabel, + ExceptionContextLoc, Final, MachBufferFinalized, MachExceptionHandler, MachLabel, binemit::CodeOffset, ir::{MemFlagsData, RelSourceLoc, SourceLoc}, isa::{ @@ -223,6 +223,13 @@ impl Masm for MacroAssembler { Ok(()) } + fn prepare_for_exception_handler(&mut self, target_offset: SPOffset) -> Result { + self.asm.mov_rr(rbp(), writable!(rsp()), OperandSize::S64); + self.sp_offset = 0; + self.reserve_stack(target_offset.as_u32())?; + Ok(regs::rax()) + } + fn local_address(&mut self, local: &LocalSlot) -> Result
{ let (reg, offset) = if local.addressed_from_sp() { let offset = self @@ -1419,6 +1426,29 @@ impl Masm for MacroAssembler { Ok(()) } + fn emit_try_call_site( + &mut self, + sp_offset: SPOffset, + vmctx_slot_offset: u32, + handlers: impl Iterator, + ) -> Result<()> { + let frame_offset = sp_offset.as_u32(); + let vmctx_offset = frame_offset + .checked_sub(vmctx_slot_offset) + .ok_or_else(CodeGenError::invalid_local_offset)?; + + let handlers = std::iter::once(MachExceptionHandler::Context( + ExceptionContextLoc::SPOffset(vmctx_offset), + )) + .chain(handlers); + + self.asm + .buffer_mut() + .add_try_call_site(Some(frame_offset), handlers); + + Ok(()) + } + fn current_code_offset(&self) -> Result { Ok(self.asm.buffer().cur_offset()) } diff --git a/winch/codegen/src/masm.rs b/winch/codegen/src/masm.rs index b64cd66551fb..f112f54f86f8 100644 --- a/winch/codegen/src/masm.rs +++ b/winch/codegen/src/masm.rs @@ -6,7 +6,7 @@ use crate::isa::{ reg::{Reg, RegClass, WritableReg, writable}, }; use cranelift_codegen::{ - Final, MachBufferFinalized, MachLabel, + Final, MachBufferFinalized, MachExceptionHandler, MachLabel, binemit::CodeOffset, ir::{Endianness, MemFlagsData, RelSourceLoc, SourceLoc, UserExternalNameRef}, }; @@ -1432,6 +1432,10 @@ pub(crate) trait MacroAssembler { /// when dealing with unreachable code. fn reset_stack_pointer(&mut self, offset: SPOffset) -> Result<()>; + /// Prepare to enter an exception handler at the given stack offset and + /// return the register containing the exception reference. + fn prepare_for_exception_handler(&mut self, target_offset: SPOffset) -> Result; + /// Get the address of a local slot. fn local_address(&mut self, local: &LocalSlot) -> Result; @@ -1471,6 +1475,15 @@ pub(crate) trait MacroAssembler { /// a live GC reference. fn emit_stack_map(&mut self, sp_offset: SPOffset, offsets: &[SPOffset]) -> Result<()>; + /// Record the active exception handlers for the call emitted immediately + /// before this point. + fn emit_try_call_site( + &mut self, + sp_offset: SPOffset, + vmctx_slot_offset: u32, + handlers: impl Iterator, + ) -> Result<()>; + /// Acquire a scratch register and execute the given callback. fn with_scratch(&mut self, f: impl FnOnce(&mut Self, Scratch) -> R) -> R; diff --git a/winch/codegen/src/visitor.rs b/winch/codegen/src/visitor.rs index e8ca63754a82..115d6562af54 100644 --- a/winch/codegen/src/visitor.rs +++ b/winch/codegen/src/visitor.rs @@ -6,8 +6,8 @@ use crate::abi::RetArea; use crate::codegen::{ - Callee, CodeGen, CodeGenError, ConditionalBranch, ControlStackFrame, Emission, FnCall, - UnconditionalBranch, control_index, + Callee, CatchInfo, CodeGen, CodeGenError, ConditionalBranch, ControlStackFrame, Emission, + FnCall, TryTableInfo, UnconditionalBranch, control_index, }; use crate::masm::{ AtomicWaitKind, DivKind, Extend, ExtractLaneKind, FloatCmpKind, IntCmpKind, LoadKind, @@ -20,6 +20,7 @@ use crate::masm::{ use crate::reg::{Reg, writable}; use crate::stack::{TypedReg, Val}; use crate::{Result, bail, format_err}; +use cranelift_codegen::ir::ExceptionTag; use regalloc2::RegClass; use smallvec::{SmallVec, smallvec}; use wasmparser::{ @@ -1511,7 +1512,11 @@ where self.handle_unreachable_end() } else { let mut control = self.pop_control_frame()?; - control.emit_end(self.masm, &mut self.context) + if let Some(info) = control.take_try_table_info() { + self.emit_try_table_end(control, info) + } else { + control.emit_end(self.masm, &mut self.context) + } } } @@ -1848,12 +1853,44 @@ where Ok(()) } - // Winch does not implement exception handlers yet, so a `try_table` - // compiles like a `block`. Thrown exceptions escape to the host, and no - // handler metadata is emitted. + // Record the handlers that apply to calls within this `try_table`. Their + // landing pads are emitted when the control frame ends. fn visit_try_table(&mut self, try_table: TryTable) -> Self::Output { - self.control_frames.push(ControlStackFrame::block( + let checkpoint = self.context.exception_handlers.take_checkpoint(); + let mut catches = Vec::with_capacity(try_table.catches.len()); + + for catch in try_table.catches.iter().rev() { + let (tag, target_depth) = match catch { + wasmparser::Catch::One { tag, label } => (Some(TagIndex::from_u32(*tag)), *label), + wasmparser::Catch::All { label } => (None, *label), + wasmparser::Catch::OneRef { .. } | wasmparser::Catch::AllRef { .. } => { + bail!(CodeGenError::unimplemented_wasm_instruction()) + } + }; + + let landing_pad = self.masm.get_label()?; + + let target = control_index(target_depth, self.control_frames.len())?; + self.control_frames[target].set_as_target(); + + let exception_tag = tag.map(|tag| ExceptionTag::from_u32(tag.as_u32())); + self.context + .exception_handlers + .add_handler(exception_tag, landing_pad); + + catches.push(CatchInfo { + tag, + target_depth, + landing_pad, + }); + } + let info = TryTableInfo { + checkpoint, + catches, + }; + self.control_frames.push(ControlStackFrame::try_table( self.env.resolve_block_sig(try_table.ty)?, + info, self.masm, &mut self.context, )?); @@ -1879,7 +1916,8 @@ where let (gc_ref, object_addr) = self.emit_exception_alloc(tag_index, interned, &layout, layouts)?; - let gc_ref = self.emit_exception_payload_fields(exn_ty, &layout, gc_ref, object_addr)?; + let gc_ref = + self.emit_store_exception_payload_fields(exn_ty, &layout, gc_ref, object_addr)?; self.context.stack.push(gc_ref.into()); self.visit_throw_ref() } @@ -2122,14 +2160,14 @@ where let index = GlobalIndex::from_u32(global_index); let (ty, base, offset) = self.emit_get_global_addr(index)?; let addr = self.masm.address_at_reg(base, offset)?; + let gc_ref = self.context.reg_for_type(ty, self.masm)?; + self.masm.load(addr, writable!(gc_ref), ty.try_into()?)?; + self.context.free_reg(base); + if self.gc_barrier_needed(&ty) { - self.emit_drc_read_barrier(ty, base, addr)?; + self.emit_drc_read_barrier(ty, gc_ref)?; } else { - let gc_ref = self.context.reg_for_type(ty, self.masm)?; - self.masm.load(addr, writable!(gc_ref), ty.try_into()?)?; self.context.stack.push(Val::reg(gc_ref, ty)); - - self.context.free_reg(base); } Ok(())