diff --git a/cranelift/codegen/src/isa/pulley_shared/inst/args.rs b/cranelift/codegen/src/isa/pulley_shared/inst/args.rs index e97e3303ef99..79bafbe2fa39 100644 --- a/cranelift/codegen/src/isa/pulley_shared/inst/args.rs +++ b/cranelift/codegen/src/isa/pulley_shared/inst/args.rs @@ -577,6 +577,26 @@ pub struct PulleyCall { pub args: SmallVec<[XReg; 4]>, } +/// Payload of `CallInfo` for indirect-call instructions. +/// +/// Mirror of `PulleyCall` for `Inst::IndirectCall`: the call target is a +/// runtime register (the loaded `wasm_call` pointer at the call_indirect +/// dispatch tail), and the first 0–4 integer ABI args are passed as free +/// registers so the `call_indirect1/2/3/4` opcodes can move them into +/// `x0..x3` as part of the call (saving one `xmov` per arg on the hot +/// dispatch path). Remaining args live in `CallInfo::uses` with fixed +/// pregs, just as for `PulleyCall`. +#[derive(Clone, Debug)] +pub struct PulleyCallIndirect { + /// The register holding the call target (e.g. the `wasm_call` pointer + /// loaded out of a `VMFuncRef`). + pub target: XReg, + /// Up to 4 integer args destined for `x0..x3`. Tracked separately so + /// regalloc doesn't insert moves and the `call_indirectN` opcode moves + /// them itself. + pub args: SmallVec<[XReg; 4]>, +} + pub use super::super::lower::isle::generated_code::AddrO32; impl Copy for AddrO32 {} diff --git a/cranelift/codegen/src/isa/pulley_shared/inst/emit.rs b/cranelift/codegen/src/isa/pulley_shared/inst/emit.rs index 74bff5d97a7d..1f8e8251c0a8 100644 --- a/cranelift/codegen/src/isa/pulley_shared/inst/emit.rs +++ b/cranelift/codegen/src/isa/pulley_shared/inst/emit.rs @@ -233,7 +233,23 @@ fn pulley_emit

( } Inst::IndirectCall { info } => { - enc::call_indirect(sink, info.dest); + // If x0..xN args are already in their correct ABI register + // (because regalloc allocated the producer's vreg there), drop + // them off the end so we can use a narrower `call_indirectN` + // op — mirror of the direct-call shrink loop above. + let target = info.dest.target; + let mut args = &info.dest.args[..]; + while !args.is_empty() && args.last().copied() == XReg::new(x_reg(args.len() - 1)) { + args = &args[..args.len() - 1]; + } + match args { + [] => enc::call_indirect(sink, target), + [x0] => enc::call_indirect1(sink, target, *x0), + [x0, x1] => enc::call_indirect2(sink, target, *x0, *x1), + [x0, x1, x2] => enc::call_indirect3(sink, target, *x0, *x1, *x2), + [x0, x1, x2, x3] => enc::call_indirect4(sink, target, *x0, *x1, *x2, *x3), + _ => unreachable!(), + } if let Some(s) = state.take_stack_map() { let offset = sink.cur_offset(); diff --git a/cranelift/codegen/src/isa/pulley_shared/inst/mod.rs b/cranelift/codegen/src/isa/pulley_shared/inst/mod.rs index 6bbe69795e51..59ac484f069c 100644 --- a/cranelift/codegen/src/isa/pulley_shared/inst/mod.rs +++ b/cranelift/codegen/src/isa/pulley_shared/inst/mod.rs @@ -206,14 +206,25 @@ fn pulley_get_operands(inst: &mut Inst, collector: &mut impl OperandVisitor) { } } Inst::IndirectCall { info } => { - collector.reg_use(&mut info.dest); let CallInfo { uses, defs, + dest, try_call_info, clobbers, .. } = &mut **info; + + // Phase-4: the target and the first up-to-4 integer args live + // in `dest` and are passed as free reg uses; the emitted + // `call_indirect{1,2,3,4}` op moves the args into x0..x3 at + // call time. Remaining args still flow through `uses` with + // fixed pregs as before. + let PulleyCallIndirect { target, args } = dest; + collector.reg_use(target); + for arg in args { + collector.reg_use(arg); + } for CallArgPair { vreg, preg } in uses { collector.reg_fixed_use(vreg, *preg); } @@ -723,7 +734,7 @@ impl Inst { } Inst::IndirectCall { info } => { - let callee = format_reg(*info.dest); + let callee = format_reg(*info.dest.target); let try_call = info .try_call_info .as_ref() diff --git a/cranelift/codegen/src/isa/pulley_shared/lower/isle.rs b/cranelift/codegen/src/isa/pulley_shared/lower/isle.rs index 3068fb1137ff..c065c45a2c6a 100644 --- a/cranelift/codegen/src/isa/pulley_shared/lower/isle.rs +++ b/cranelift/codegen/src/isa/pulley_shared/lower/isle.rs @@ -10,8 +10,8 @@ use crate::ir::{condcodes::*, immediates::*, types::*, *}; use crate::isa::CallConv; use crate::isa::pulley_shared::{ inst::{ - FReg, OperandSize, PulleyCall, ReturnCallInfo, VReg, WritableFReg, WritableVReg, - WritableXReg, XReg, + FReg, OperandSize, PulleyCall, PulleyCallIndirect, ReturnCallInfo, VReg, WritableFReg, + WritableVReg, WritableXReg, XReg, }, lower::{Cond, regs}, *, @@ -30,7 +30,7 @@ type Unit = (); type VecArgPair = Vec; type VecRetPair = Vec; type BoxCallInfo = Box>; -type BoxCallIndInfo = Box>; +type BoxCallIndInfo = Box>; type BoxCallIndirectHostInfo = Box>; type BoxReturnCallInfo = Box>; type BoxReturnCallIndInfo = Box>; @@ -124,7 +124,7 @@ where &mut self, sig: Sig, dest: Reg, - uses: CallArgList, + mut uses: CallArgList, defs: CallRetList, try_call_info: Option, ) -> BoxCallIndInfo { @@ -133,8 +133,30 @@ where self.lower_ctx .abi_mut() .accumulate_outgoing_args_size(stack_ret_space + stack_arg_space); + let call_conv = self.lower_ctx.sigs()[sig].call_conv(); - let dest = XReg::new(dest).unwrap(); + // Mirror of `gen_call_info`: take out the first four integer + // arguments (x0..x3) and pass them through the `args` list so the + // emitted `call_indirect{1,2,3,4}` op can move them at call time. + // Saves one Pulley dispatch per moved arg vs the previous "regalloc + // emits xmov; then `call_indirect`" sequence. + let mut args = SmallVec::new(); + uses.sort_by_key(|arg| arg.preg); + if call_conv != CallConv::PreserveAll { + uses.retain(|arg| { + if arg.preg != regs::x0() + && arg.preg != regs::x1() + && arg.preg != regs::x2() + && arg.preg != regs::x3() + { + return true; + } + args.push(XReg::new(arg.vreg).unwrap()); + false + }); + } + let target = XReg::new(dest).unwrap(); + let dest = PulleyCallIndirect { target, args }; Box::new( self.lower_ctx .gen_call_info(sig, dest, uses, defs, try_call_info, false), diff --git a/cranelift/filetests/filetests/isa/pulley32/call.clif b/cranelift/filetests/filetests/isa/pulley32/call.clif index c2dc9a09f6c9..aece47fc9a19 100644 --- a/cranelift/filetests/filetests/isa/pulley32/call.clif +++ b/cranelift/filetests/filetests/isa/pulley32/call.clif @@ -291,7 +291,7 @@ block0(v0: i32): ; VCode: ; push_frame ; block0: -; indirect_call x0, CallInfo { dest: XReg(p0i), uses: [], defs: [CallRetPair { vreg: Writable { reg: p0i }, location: Reg(p0i, types::I64) }], clobbers: PRegSet { bits: [65534, 4294967295, 4294967295, 0] }, callee_conv: Tail, caller_conv: Fast, callee_pop_size: 0, try_call_info: None, patchable: false } +; indirect_call x0, CallInfo { dest: PulleyCallIndirect { target: XReg(p0i), args: [] }, uses: [], defs: [CallRetPair { vreg: Writable { reg: p0i }, location: Reg(p0i, types::I64) }], clobbers: PRegSet { bits: [65534, 4294967295, 4294967295, 0] }, callee_conv: Tail, caller_conv: Fast, callee_pop_size: 0, try_call_info: None, patchable: false } ; pop_frame ; ret ; diff --git a/cranelift/filetests/filetests/isa/pulley32/exceptions.clif b/cranelift/filetests/filetests/isa/pulley32/exceptions.clif index 2d3dfef3e853..eeed198535d2 100644 --- a/cranelift/filetests/filetests/isa/pulley32/exceptions.clif +++ b/cranelift/filetests/filetests/isa/pulley32/exceptions.clif @@ -77,7 +77,7 @@ function %f2(i32, i32) -> i32, f32, f64 { ; block0: ; fconst64 f1, 4607182418800017408 ; fstore64 Slot(0), f1 // flags = notrap aligned -; indirect_call x1, CallInfo { dest: XReg(p1i), uses: [CallArgPair { vreg: p0i, preg: p0i }], defs: [CallRetPair { vreg: Writable { reg: p0f }, location: Reg(p0f, types::F32) }, CallRetPair { vreg: Writable { reg: p0i }, location: Reg(p0i, types::I32) }, CallRetPair { vreg: Writable { reg: p1i }, location: Reg(p1i, types::I32) }], clobbers: PRegSet { bits: [4294967292, 4294967294, 4294967295, 0] }, callee_conv: Tail, caller_conv: Fast, callee_pop_size: 0, try_call_info: Some(TryCallInfo { continuation: MachLabel(1), exception_handlers: [Default(MachLabel(2))] }), patchable: false }; jump MachLabel(1); catch [default: MachLabel(2)] +; indirect_call x1, CallInfo { dest: PulleyCallIndirect { target: XReg(p1i), args: [XReg(p0i)] }, uses: [], defs: [CallRetPair { vreg: Writable { reg: p0f }, location: Reg(p0f, types::F32) }, CallRetPair { vreg: Writable { reg: p0i }, location: Reg(p0i, types::I32) }, CallRetPair { vreg: Writable { reg: p1i }, location: Reg(p1i, types::I32) }], clobbers: PRegSet { bits: [4294967292, 4294967294, 4294967295, 0] }, callee_conv: Tail, caller_conv: Fast, callee_pop_size: 0, try_call_info: Some(TryCallInfo { continuation: MachLabel(1), exception_handlers: [Default(MachLabel(2))] }), patchable: false }; jump MachLabel(1); catch [default: MachLabel(2)] ; block1: ; xone x0 ; f1 = fload64 Slot(0) // flags = notrap aligned diff --git a/cranelift/filetests/filetests/isa/pulley32/preserve-all.clif b/cranelift/filetests/filetests/isa/pulley32/preserve-all.clif index 67b8297bdd2f..7733ce08fb51 100644 --- a/cranelift/filetests/filetests/isa/pulley32/preserve-all.clif +++ b/cranelift/filetests/filetests/isa/pulley32/preserve-all.clif @@ -15,8 +15,8 @@ block0(v0: i64): ; xmov x3, x0 ; xmov x1, x3 ; xmov x2, x3 -; indirect_call x3, CallInfo { dest: XReg(p3i), uses: [CallArgPair { vreg: p0i, preg: p0i }, CallArgPair { vreg: p1i, preg: p1i }, CallArgPair { vreg: p2i, preg: p2i }, CallArgPair { vreg: p3i, preg: p3i }], defs: [], clobbers: PRegSet { bits: [0, 0, 0, 0] }, callee_conv: PreserveAll, caller_conv: SystemV, callee_pop_size: 0, try_call_info: None, patchable: false } -; indirect_call x3, CallInfo { dest: XReg(p3i), uses: [CallArgPair { vreg: p0i, preg: p0i }, CallArgPair { vreg: p1i, preg: p1i }, CallArgPair { vreg: p2i, preg: p2i }, CallArgPair { vreg: p3i, preg: p3i }], defs: [], clobbers: PRegSet { bits: [0, 0, 0, 0] }, callee_conv: PreserveAll, caller_conv: SystemV, callee_pop_size: 0, try_call_info: None, patchable: false } +; indirect_call x3, CallInfo { dest: PulleyCallIndirect { target: XReg(p3i), args: [] }, uses: [CallArgPair { vreg: p0i, preg: p0i }, CallArgPair { vreg: p1i, preg: p1i }, CallArgPair { vreg: p2i, preg: p2i }, CallArgPair { vreg: p3i, preg: p3i }], defs: [], clobbers: PRegSet { bits: [0, 0, 0, 0] }, callee_conv: PreserveAll, caller_conv: SystemV, callee_pop_size: 0, try_call_info: None, patchable: false } +; indirect_call x3, CallInfo { dest: PulleyCallIndirect { target: XReg(p3i), args: [] }, uses: [CallArgPair { vreg: p0i, preg: p0i }, CallArgPair { vreg: p1i, preg: p1i }, CallArgPair { vreg: p2i, preg: p2i }, CallArgPair { vreg: p3i, preg: p3i }], defs: [], clobbers: PRegSet { bits: [0, 0, 0, 0] }, callee_conv: PreserveAll, caller_conv: SystemV, callee_pop_size: 0, try_call_info: None, patchable: false } ; pop_frame ; ret ; diff --git a/cranelift/filetests/filetests/isa/pulley64/call.clif b/cranelift/filetests/filetests/isa/pulley64/call.clif index 16b271835620..bd6f9bba825f 100644 --- a/cranelift/filetests/filetests/isa/pulley64/call.clif +++ b/cranelift/filetests/filetests/isa/pulley64/call.clif @@ -291,7 +291,7 @@ block0(v0: i64): ; VCode: ; push_frame ; block0: -; indirect_call x0, CallInfo { dest: XReg(p0i), uses: [], defs: [CallRetPair { vreg: Writable { reg: p0i }, location: Reg(p0i, types::I64) }], clobbers: PRegSet { bits: [65534, 4294967295, 4294967295, 0] }, callee_conv: Tail, caller_conv: Fast, callee_pop_size: 0, try_call_info: None, patchable: false } +; indirect_call x0, CallInfo { dest: PulleyCallIndirect { target: XReg(p0i), args: [] }, uses: [], defs: [CallRetPair { vreg: Writable { reg: p0i }, location: Reg(p0i, types::I64) }], clobbers: PRegSet { bits: [65534, 4294967295, 4294967295, 0] }, callee_conv: Tail, caller_conv: Fast, callee_pop_size: 0, try_call_info: None, patchable: false } ; pop_frame ; ret ; diff --git a/cranelift/filetests/filetests/isa/pulley64/exceptions.clif b/cranelift/filetests/filetests/isa/pulley64/exceptions.clif index 88c5528c1935..6a0b7b1577a1 100644 --- a/cranelift/filetests/filetests/isa/pulley64/exceptions.clif +++ b/cranelift/filetests/filetests/isa/pulley64/exceptions.clif @@ -79,7 +79,7 @@ function %f2(i32, i64) -> i32, f32, f64 { ; block0: ; fconst64 f1, 4607182418800017408 ; fstore64 Slot(0), f1 // flags = notrap aligned -; indirect_call x1, CallInfo { dest: XReg(p1i), uses: [CallArgPair { vreg: p0i, preg: p0i }], defs: [CallRetPair { vreg: Writable { reg: p0f }, location: Reg(p0f, types::F32) }, CallRetPair { vreg: Writable { reg: p0i }, location: Reg(p0i, types::I64) }, CallRetPair { vreg: Writable { reg: p1i }, location: Reg(p1i, types::I64) }], clobbers: PRegSet { bits: [4294967292, 4294967294, 4294967295, 0] }, callee_conv: Tail, caller_conv: Fast, callee_pop_size: 0, try_call_info: Some(TryCallInfo { continuation: MachLabel(1), exception_handlers: [Default(MachLabel(2))] }), patchable: false }; jump MachLabel(1); catch [default: MachLabel(2)] +; indirect_call x1, CallInfo { dest: PulleyCallIndirect { target: XReg(p1i), args: [XReg(p0i)] }, uses: [], defs: [CallRetPair { vreg: Writable { reg: p0f }, location: Reg(p0f, types::F32) }, CallRetPair { vreg: Writable { reg: p0i }, location: Reg(p0i, types::I64) }, CallRetPair { vreg: Writable { reg: p1i }, location: Reg(p1i, types::I64) }], clobbers: PRegSet { bits: [4294967292, 4294967294, 4294967295, 0] }, callee_conv: Tail, caller_conv: Fast, callee_pop_size: 0, try_call_info: Some(TryCallInfo { continuation: MachLabel(1), exception_handlers: [Default(MachLabel(2))] }), patchable: false }; jump MachLabel(1); catch [default: MachLabel(2)] ; block1: ; xone x0 ; f1 = fload64 Slot(0) // flags = notrap aligned diff --git a/cranelift/filetests/filetests/isa/pulley64/preserve-all.clif b/cranelift/filetests/filetests/isa/pulley64/preserve-all.clif index 363a6c3538c7..0b13fcc98618 100644 --- a/cranelift/filetests/filetests/isa/pulley64/preserve-all.clif +++ b/cranelift/filetests/filetests/isa/pulley64/preserve-all.clif @@ -15,8 +15,8 @@ block0(v0: i64): ; xmov x3, x0 ; xmov x1, x3 ; xmov x2, x3 -; indirect_call x3, CallInfo { dest: XReg(p3i), uses: [CallArgPair { vreg: p0i, preg: p0i }, CallArgPair { vreg: p1i, preg: p1i }, CallArgPair { vreg: p2i, preg: p2i }, CallArgPair { vreg: p3i, preg: p3i }], defs: [], clobbers: PRegSet { bits: [0, 0, 0, 0] }, callee_conv: PreserveAll, caller_conv: SystemV, callee_pop_size: 0, try_call_info: None, patchable: false } -; indirect_call x3, CallInfo { dest: XReg(p3i), uses: [CallArgPair { vreg: p0i, preg: p0i }, CallArgPair { vreg: p1i, preg: p1i }, CallArgPair { vreg: p2i, preg: p2i }, CallArgPair { vreg: p3i, preg: p3i }], defs: [], clobbers: PRegSet { bits: [0, 0, 0, 0] }, callee_conv: PreserveAll, caller_conv: SystemV, callee_pop_size: 0, try_call_info: None, patchable: false } +; indirect_call x3, CallInfo { dest: PulleyCallIndirect { target: XReg(p3i), args: [] }, uses: [CallArgPair { vreg: p0i, preg: p0i }, CallArgPair { vreg: p1i, preg: p1i }, CallArgPair { vreg: p2i, preg: p2i }, CallArgPair { vreg: p3i, preg: p3i }], defs: [], clobbers: PRegSet { bits: [0, 0, 0, 0] }, callee_conv: PreserveAll, caller_conv: SystemV, callee_pop_size: 0, try_call_info: None, patchable: false } +; indirect_call x3, CallInfo { dest: PulleyCallIndirect { target: XReg(p3i), args: [] }, uses: [CallArgPair { vreg: p0i, preg: p0i }, CallArgPair { vreg: p1i, preg: p1i }, CallArgPair { vreg: p2i, preg: p2i }, CallArgPair { vreg: p3i, preg: p3i }], defs: [], clobbers: PRegSet { bits: [0, 0, 0, 0] }, callee_conv: PreserveAll, caller_conv: SystemV, callee_pop_size: 0, try_call_info: None, patchable: false } ; pop_frame ; ret ; diff --git a/crates/cranelift/src/func_environ.rs b/crates/cranelift/src/func_environ.rs index e791c092c2ab..9ccbdf183cd6 100644 --- a/crates/cranelift/src/func_environ.rs +++ b/crates/cranelift/src/func_environ.rs @@ -1652,8 +1652,11 @@ impl FuncEnvironment<'_> { .unwrap(); // A fixed-size table can't be resized, so its base address won't change - // and its base load is `readonly` and `can_move`. - let is_static = Some(table.limits.min) == table.limits.max; + // and its base load is `readonly` and `can_move`. That holds when the + // table is declared with `min == max`, or when translation proved it + // is never grown or mutated. + let is_static = + Some(table.limits.min) == table.limits.max || !self.translation.tables_mutated[index]; let mut base_flags = ir::MemFlagsData::trusted(); if is_static { base_flags = base_flags.with_readonly().with_can_move(); @@ -2147,6 +2150,14 @@ impl<'a, 'func, 'module_env> Call<'a, 'func, 'module_env> { callee: ir::Value, call_args: &[ir::Value], ) -> WasmResult> { + // Fast path: if we can statically resolve this indirect call to a + // single defined function (immutable funcref table + constant + // callee index + matching signature), emit a direct call instead. + // See `try_static_resolve_indirect_call`. + if let Some(target) = self.try_static_resolve_indirect_call(table_index, ty_index, callee) { + return self.direct_call(target, sig_ref, call_args).map(Some); + } + let (code_ptr, callee_vmctx) = match self.check_and_load_code_and_callee_vmctx( table_index, ty_index, @@ -2161,6 +2172,198 @@ impl<'a, 'func, 'module_env> Call<'a, 'func, 'module_env> { .map(Some) } + /// Try to statically resolve a `call_indirect` site to a single defined + /// function so the call can be lowered as a direct call. + /// + /// All four of these must hold for the resolution to succeed: + /// + /// 1. The target table must be provably immutable for the lifetime of + /// any instance of this module: defined (not imported) and never the + /// target of `table.set` / `table.fill` / `table.copy` (as the dst) + /// / `table.grow` / `table.init`. This is the `tables_mutated` bit + /// populated in `ModuleEnvironment::translate`. + /// + /// 2. The callee index value (the operand to `call_indirect`) must be a + /// compile-time constant — i.e., the wasm did `i32.const N; + /// call_indirect (table $t) (type $sig)`. This is what hand-lowered + /// C++/Rust vtable calls and AOT-compiled JS-to-wasm dispatch tables + /// look like in practice. + /// + /// 3. The slot at index `N` in the table must be precomputable from + /// static `elem` segments: `module.table_initialization + /// .initial_values[defined_index]` must be `TableInitialValue::Null + /// { precomputed }` (i.e., not a fully-dynamic `Expr`-style init), + /// and the index `N` must be in range and resolved to a concrete + /// `FuncIndex` (not the reserved-value sentinel). + /// + /// 4. The function's signature in the module's interned type table + /// must equal the `ty_index` declared by the `call_indirect` site. + /// Otherwise the original semantics are "trap on signature + /// mismatch", which we don't want to replace with a static direct + /// call. + /// + /// Returns the resolved function on success, `None` otherwise (in + /// which case the caller falls back to a normal indirect call). + fn try_static_resolve_indirect_call( + &self, + table_index: TableIndex, + ty_index: TypeIndex, + callee: ir::Value, + ) -> Option { + let translation = self.env.translation; + let module = &translation.module; + + // (1) Table must be provably immutable. Imported tables are + // pre-marked as mutated in `ModuleEnvironment::translate`, so + // this check also rules them out (along with the explicit + // `defined_table_index` check below for clarity). + if translation.tables_mutated[table_index] { + return None; + } + let defined_table = module.defined_table_index(table_index)?; + + // (2) Callee must be a constant `iconst`. Pattern adapted from + // `bounds_checks::statically_known_in_bounds`. + let dfg = &self.builder.func.dfg; + let inst = dfg.value_def(callee).inst()?; + let imm = match dfg.insts[inst] { + ir::InstructionData::UnaryImm { + opcode: ir::Opcode::Iconst, + imm, + } => imm, + _ => return None, + }; + let callee_ty = dfg.value_type(callee); + let callee_idx_u64 = imm + .zero_extend_from_width(callee_ty.bits()) + .bits() + .cast_unsigned(); + + // (3) Slot must be precomputable from the static funcref image. + let precomputed = module.table_initialization.get(defined_table)?; + let slot = usize::try_from(callee_idx_u64).ok()?; + if slot >= precomputed.len() { + return None; + } + let target = precomputed[slot]; + // `FuncIndex::reserved_value()` marks a null (uncovered) slot. + if target.is_reserved_value() { + return None; + } + + // (4) Signature match. The site's declared `ty_index` and the + // target function's declared signature must intern to the same + // module type index. + let expected_ty = module.types[ty_index].unwrap_module_type_index(); + let target_ty = module.functions[target] + .signature + .unwrap_module_type_index(); + if expected_ty != target_ty { + return None; + } + + Some(target) + } + + /// Try to prove that the runtime signature check at a `call_indirect` + /// site through an untyped `funcref` table is redundant. + /// + /// True when: + /// + /// 1. The table is provably immutable (`tables_mutated[table_index] == + /// false`). Defined-not-imported is implied since imported tables + /// are pre-marked as mutated. + /// + /// 2. The table is precomputable from static `elem` segments + /// (`TableInitialValue::Null { precomputed }`). + /// + /// 3. Every non-null entry in `precomputed` has the same module- + /// interned signature as the `ty_index` declared at the call site. + /// Null slots are fine — they trap on the funcref-NULL load that + /// happens after sig-check elision. + /// + /// When this returns true, the caller short-circuits to + /// `CheckIndirectCallTypeSignature::StaticMatch`, which removes the + /// sig load + compare from the hot path. Bounds-check on the table + /// index and the funcref-NULL check are still emitted by the + /// surrounding code, so the call still traps correctly on OOB or + /// null index — only the sig check is elided. + /// + /// This is the static analog of an inline-cache: instead of caching + /// the resolved target per call site, we observe at module-load that + /// the table contents make the sig check uninformative for the + /// lifetime of any instance. + /// True iff every slot in the precomputed `elem`-segment contents for + /// `table_index` is a concrete `FuncIndex` (no + /// `FuncIndex::reserved_value()` "no-entry" sentinel). + /// + /// Caller has already proven the table is immutable, so the contents + /// observed here are stable for the lifetime of any instance — + /// `false` here implies "no slot is ever null at runtime." + /// + /// When this is true, the runtime funcref-NULL check on the loaded + /// funcref pointer is provably redundant: any in-bounds index leads + /// to a non-null funcref. The bounds check still runs (so an + /// out-of-bounds index traps as before with `TRAP_TABLE_OUT_OF_BOUNDS`). + fn precomputed_table_has_no_null_slots(&self, table_index: TableIndex) -> bool { + let module = &self.env.translation.module; + let Some(defined_table) = module.defined_table_index(table_index) else { + return false; + }; + let Some(precomputed) = module.table_initialization.get(defined_table) else { + return false; + }; + if precomputed.is_empty() { + return false; + } + // Slots beyond `precomputed.len()` are null at runtime; coverage + // up to `limits.min` is required (caller proved immutable, so the + // table can't grow beyond min). + let table_min = module.tables[table_index].limits.min; + if (precomputed.len() as u64) < table_min { + return false; + } + precomputed.iter().all(|f| !f.is_reserved_value()) + } + + fn try_elide_sig_check_for_immutable_table( + &self, + table_index: TableIndex, + ty_index: TypeIndex, + ) -> bool { + let translation = self.env.translation; + let module = &translation.module; + + if translation.tables_mutated[table_index] { + return false; + } + let defined_table = match module.defined_table_index(table_index) { + Some(d) => d, + None => return false, + }; + + let precomputed = match module.table_initialization.get(defined_table) { + Some(p) if !p.is_empty() => p, + _ => return false, + }; + + let expected_ty = module.types[ty_index].unwrap_module_type_index(); + for &func_idx in precomputed.iter() { + // Null slots will trap on the funcref-NULL load anyway. + if func_idx.is_reserved_value() { + continue; + } + let actual_ty = module.functions[func_idx] + .signature + .unwrap_module_type_index(); + if actual_ty != expected_ty { + return false; + } + } + + true + } + fn check_and_load_code_and_callee_vmctx( &mut self, table_index: TableIndex, @@ -2218,6 +2421,34 @@ impl<'a, 'func, 'module_env> Call<'a, 'func, 'module_env> { // table of typed functions and that type matches `ty_index`, then // there's no need to perform a typecheck. match table.ref_type.heap_type { + // Untyped `funcref` tables ordinarily need a runtime sig check. + // But if (a) the table is provably immutable (`tables_mutated` + // bit clear) and (b) every non-null entry in the precomputed + // static `elem` segments has the same `VMSharedTypeIndex` as + // the call site, then the runtime check is provably redundant + // and we can elide it the same way we do for typed-funcref + // tables. + // + // This is the AOT-IC-seeding analog: instead of caching the + // resolved target at the call site, we cache the *signature* + // at module-load time and skip the hot-path sig load+compare. + // Helps the megamorphic case (computed `call_indirect` index) + // that the static-monomorphization fast path above can't + // handle. + WasmHeapType::Func + if self.try_elide_sig_check_for_immutable_table(table_index, ty_index) => + { + // If we additionally know every entry in the precomputed + // table is non-null, lower `may_be_null` to false so the + // downstream funcref-NULL check is also elided. This is + // only sound if the table can't be grown or have its + // entries cleared after init (i.e., immutable, which we + // already proved above). + let may_be_null = table.ref_type.nullable + && !self.precomputed_table_has_no_null_slots(table_index); + return CheckIndirectCallTypeSignature::StaticMatch { may_be_null }; + } + // Functions do not have a statically known type in the table, a // typecheck is required. Fall through to below to perform the // actual typecheck. diff --git a/crates/environ/src/compile/module_environ.rs b/crates/environ/src/compile/module_environ.rs index 1c7961323aed..676d974734a7 100644 --- a/crates/environ/src/compile/module_environ.rs +++ b/crates/environ/src/compile/module_environ.rs @@ -110,6 +110,26 @@ pub struct ModuleTranslation<'data> { /// trampolines for each of these signatures are required. pub exported_signatures: Vec, + /// Per-table flag indicating whether the table is ever mutated by any + /// function defined in this module via `table.set` / `table.fill` / + /// `table.copy` (as the destination) / `table.grow` / `table.init`. + /// + /// `false` (the default) means the table's contents are determined + /// entirely by its `elem` segments and any active initializer, and never + /// change at runtime — provably immutable for the lifetime of any + /// instance of this module. + /// + /// `true` means the contents can change at runtime (or the table is + /// imported, in which case we conservatively assume the importer + /// mutates it). + /// + /// This is groundwork for later passes that turn `call_indirect` + /// through provably-immutable function tables into direct calls when + /// the dispatched-to slot is statically known. Set during module + /// translation (see `analyze_table_mutability`); read by Cranelift + /// lowering and by Pulley AOT IC seeding. + pub tables_mutated: SecondaryMap, + /// DWARF debug information, if enabled, parsed from the module. pub debuginfo: DebugInfoData<'data>, @@ -227,6 +247,7 @@ impl<'data> ModuleTranslation<'data> { function_body_inputs: PrimaryMap::default(), known_imported_functions: SecondaryMap::default(), exported_signatures: Vec::default(), + tables_mutated: SecondaryMap::default(), debuginfo: DebugInfoData::default(), has_unparsed_debuginfo: false, data_align: None, @@ -349,6 +370,8 @@ impl<'a, 'data> ModuleEnvironment<'a, 'data> { self.translate_payload(payload?)?; } + analyze_table_mutability(&mut self.result)?; + Ok(self.result) } @@ -1582,3 +1605,83 @@ impl ModuleTranslation<'_> { self.module.startup = ModuleStartup::IfMemoriesNeedInit(ty); } } + +/// Walk every defined function body, recording in +/// `translation.tables_mutated` each table that is the destination of any +/// runtime mutation opcode (`table.set`, `table.fill`, `table.copy` as the +/// destination, `table.grow`, `table.init`). +/// +/// Imported tables are conservatively pre-marked as mutated since the +/// importer can mutate them in ways we can't see. Active `elem` segments +/// applied at instantiation time are NOT counted as mutations — they are +/// part of the table's *initial* state, not a runtime change. +/// +/// `elem.drop` drops a passive element segment but does not write to any +/// table directly, so it is intentionally not counted here. Conservatively, +/// any `table.init` from a passive segment marks the destination table as +/// mutated. +fn analyze_table_mutability<'data>(translation: &mut ModuleTranslation<'data>) -> Result<()> { + // Resize the table-mutability map to cover every table in the module + // (imports + defined). `SecondaryMap` defaults to `false` for all + // unset entries, which is the correct "definitely-not-mutated" default + // for defined tables we haven't observed any mutations on yet. + let num_tables = translation.module.tables.len(); + if num_tables == 0 { + return Ok(()); + } + + // Mark all imported tables as mutated up front. The importer can + // mutate them in ways this module can't see, so the conservative + // assumption is that they are not stable across calls. + let num_imported = translation.module.num_imported_tables; + for i in 0..num_imported { + translation.tables_mutated[TableIndex::from_u32(i as u32)] = true; + } + + // Mark all *exported* tables as mutated as well. A host (or another + // instance importing the export) can call `Table::set` / + // `Table::grow` via the public wasmtime API on any exported table, + // and those mutations are not visible in this module's bytecode. + // The `call_indirect` optimizations that read this bit must + // therefore treat exported tables as conservatively non-stable. + for (_, entity_index) in &translation.module.exports { + if let EntityIndex::Table(table_index) = entity_index { + translation.tables_mutated[*table_index] = true; + } + } + + // Walk every defined function body and look for table-mutation opcodes. + // The cost is O(total opcodes), one extra pass on top of the validator; + // typical large modules (sqlite3 ~50K opcodes) take well under a + // millisecond. + for (_, body_data) in &translation.function_body_inputs { + let mut reader = body_data.body.get_operators_reader()?; + while !reader.eof() { + use wasmparser::Operator; + match reader.read()? { + Operator::TableSet { table } + | Operator::TableFill { table } + | Operator::TableGrow { table } => { + translation.tables_mutated[TableIndex::from_u32(table)] = true; + } + Operator::TableCopy { + dst_table, + src_table: _, + } => { + // `src_table` is read-only in `table.copy`; only the + // destination is mutated. + translation.tables_mutated[TableIndex::from_u32(dst_table)] = true; + } + Operator::TableInit { + table, + elem_index: _, + } => { + translation.tables_mutated[TableIndex::from_u32(table)] = true; + } + _ => {} + } + } + } + + Ok(()) +} diff --git a/crates/environ/tests/table_mutability.rs b/crates/environ/tests/table_mutability.rs new file mode 100644 index 000000000000..562966a708e4 --- /dev/null +++ b/crates/environ/tests/table_mutability.rs @@ -0,0 +1,307 @@ +//! Integration tests for `analyze_table_mutability` and the surrounding +//! precompute ordering invariants. +//! +//! The per-table mutability bit is the foundation of the `call_indirect` +//! optimizations in `crates/cranelift/src/func_environ.rs` +//! (constant-index direct call, sig-check elision, NULL elision, bound- +//! load elision). A false negative here — failing to mark a table as +//! mutated when it actually is — would silently turn correct calls into +//! incorrect direct calls or skip required runtime checks. A false +//! positive — marking an immutable table as mutated — is merely a missed +//! optimization. Pin the analysis behaviour with focused module-level +//! tests so any regression surfaces immediately, not after a downstream +//! optimization fires on a now-invalid premise. +//! +//! Test scenario inspiration drawn from comparable bugs in peer +//! interpreters that have shipped fixes for analogous IC-invalidation +//! mistakes: +//! +//! - **Luau** (`LOP_NAMECALL`): inline cache had to be invalidated on +//! `table.insert` / metatable change. Analogous wasm risk: `table.grow` +//! not invalidating an immutability proof, so see `table_grow_marks…`. +//! - **JavaScriptCore** (`ic_table`): inline-cache corruption from missed +//! shape transitions. Analogous risk: over-marking, e.g. `table.copy` +//! wrongly marking the SOURCE table as mutated would forbid downstream +//! optimizations on a perfectly read-only table. See +//! `table_copy_marks_destination_only_not_source`. +//! - **Hermes** (`HiddenClass` cache): property cache misses with +//! `Object.defineProperty`. Analogous risk: `table.init` (active- +//! segment init at runtime) being treated as a no-op rather than a +//! write. See `table_init_marks_destination`. +//! +//! Lives in `tests/` rather than as a `#[cfg(test)] mod` inside +//! `module_environ.rs` because the latter triggers a pre-existing +//! upstream compile failure in `key.rs` / `module_artifacts.rs` (their +//! `arbitrary::Arbitrary` derives are stale relative to the workspace's +//! pinned `arbitrary 1.4.2`). Integration tests build against the lib +//! as a normal dependency and so do not set `cfg(test)` on +//! `wasmtime-environ` itself. + +use wasmparser::{Parser, Validator, WasmFeatures}; +use wasmtime_environ::{ + ModuleEnvironment, ModuleTypesBuilder, StaticModuleIndex, TableIndex, Tunables, +}; + +/// Translate `wat` and return the resulting `tables_mutated` bits, in +/// table-index order. Helper to keep individual tests short. +fn translate_and_get_mutability(wat: &str) -> Vec { + let bytes = wat::parse_str(wat).expect("WAT parse failed"); + let tunables = Tunables::default_host(); + // WASM2 covers reference-types + bulk-memory, which is what every + // table-mutating opcode below needs (`table.set`, `table.fill`, + // `table.grow`, `table.copy`, `table.init`, `elem.drop`). + let features = WasmFeatures::WASM2; + let mut validator = Validator::new_with_features(features); + let mut types = ModuleTypesBuilder::new(&validator); + let env = ModuleEnvironment::new( + &tunables, + &mut validator, + &mut types, + StaticModuleIndex::from_u32(0), + ); + let parser = Parser::new(0); + let translation = env.translate(parser, &bytes).expect("translate failed"); + let n: u32 = translation.module.tables.len().try_into().unwrap(); + (0..n) + .map(|i| translation.tables_mutated[TableIndex::from_u32(i)]) + .collect() +} + +/// A table only used as the source of `call_indirect` and `table.get` is +/// provably immutable. (Both ops READ the table; neither writes it.) The +/// table is intentionally NOT exported — exported tables are +/// conservatively pre-marked as mutated (see +/// `exported_tables_are_pre_marked` for the export case) since the host +/// can mutate them via the public wasmtime API. +#[test] +fn read_only_table_is_immutable() { + let bits = translate_and_get_mutability( + r#" + (module + (table 4 funcref) + (func $f (result i32) i32.const 42) + (elem (i32.const 0) $f $f $f $f) + (func (export "call_zero") (result i32) + i32.const 0 + call_indirect (param) (result i32)) + (func (export "read_zero") (result funcref) + i32.const 0 + table.get 0)) + "#, + ); + assert_eq!(bits, vec![false], "no opcode mutated this table"); +} + +/// Exported tables are always pre-marked as mutated, regardless of +/// whether any opcode in this module touches them. The host can call +/// `Table::set` / `Table::grow` via the public wasmtime API on any +/// exported table, and another module that imports the export can also +/// mutate it. Without this rule, downstream optimizations would +/// happily elide null traps and sig checks on exported tables on the +/// (false) assumption that the table contents are stable. +#[test] +fn exported_tables_are_pre_marked() { + let bits = translate_and_get_mutability( + r#" + (module + (table (export "t") 4 funcref) + (func $f (result i32) i32.const 42) + (elem (i32.const 0) $f $f $f $f)) + "#, + ); + assert_eq!(bits, vec![true]); +} + +/// `table.set` marks its destination as mutated. +#[test] +fn table_set_marks_destination() { + let bits = translate_and_get_mutability( + r#" + (module + (table 4 funcref) + (func $f (result i32) i32.const 0) + (func (export "do_set") + i32.const 1 + ref.func $f + table.set 0)) + "#, + ); + assert_eq!(bits, vec![true]); +} + +/// `table.fill` marks its destination as mutated. +#[test] +fn table_fill_marks_destination() { + let bits = translate_and_get_mutability( + r#" + (module + (table 4 funcref) + (func $f (result i32) i32.const 0) + (func (export "do_fill") + i32.const 0 + ref.func $f + i32.const 4 + table.fill 0)) + "#, + ); + assert_eq!(bits, vec![true]); +} + +/// `table.grow` is treated as mutating — analogous to Luau's NAMECALL IC +/// needing to invalidate on table-shape change. +#[test] +fn table_grow_marks_destination() { + let bits = translate_and_get_mutability( + r#" + (module + (table 4 funcref) + (func (export "do_grow") (result i32) + ref.null func + i32.const 1 + table.grow 0)) + "#, + ); + assert_eq!(bits, vec![true]); +} + +/// `table.copy` marks the DESTINATION but explicitly NOT the source. The +/// source is read-only (its contents aren't changed by the op); marking +/// it as mutated would forbid downstream optimizations from treating it +/// as immutable, which would be incorrect over-conservatism — the JSC +/// `ic_table` analogue. +#[test] +fn table_copy_marks_destination_only_not_source() { + let bits = translate_and_get_mutability( + r#" + (module + (table $dst (export "dst") 4 funcref) + (table $src 4 funcref) + (func $f (result i32) i32.const 0) + (elem (table $src) (i32.const 0) func $f $f $f $f) + (func (export "do_copy") + i32.const 0 ;; dst offset + i32.const 0 ;; src offset + i32.const 4 ;; len + table.copy $dst $src)) + "#, + ); + assert_eq!( + bits, + vec![true, false], + "dst should be mutated, src should remain immutable", + ); +} + +/// `table.init` writes to the destination table from a passive elem +/// segment, so it is treated as mutation (the destination's contents +/// change at runtime). +#[test] +fn table_init_marks_destination() { + let bits = translate_and_get_mutability( + r#" + (module + (table 4 funcref) + (func $f (result i32) i32.const 0) + (elem $e funcref (ref.func $f) (ref.func $f)) + (func (export "do_init") + i32.const 0 ;; dst + i32.const 0 ;; src offset within elem + i32.const 2 ;; len + table.init 0 $e)) + "#, + ); + assert_eq!(bits, vec![true]); +} + +/// `elem.drop` drops a passive element segment but does NOT write to any +/// table — distinct from `table.init` which DOES write. A pessimistic +/// implementation that marked all tables as mutated on `elem.drop` would +/// hand out false positives and shut off optimizations on perfectly- +/// immutable tables. +#[test] +fn elem_drop_does_not_mark_tables() { + let bits = translate_and_get_mutability( + r#" + (module + (table 4 funcref) + (func $f (result i32) i32.const 0) + (elem $e funcref (ref.func $f)) + (func (export "do_drop") + elem.drop $e)) + "#, + ); + assert_eq!(bits, vec![false]); +} + +/// Imported tables are always pre-marked as mutated, regardless of +/// whether any opcode in this module touches them. The importer can +/// mutate the table in ways this module can't see. +#[test] +fn imported_tables_are_pre_marked() { + let bits = translate_and_get_mutability( + r#" + (module + (import "host" "t" (table 4 funcref))) + "#, + ); + assert_eq!(bits, vec![true]); +} + +/// A mutation in ONE function correctly marks the table — the analysis +/// has to walk every function body, not just the first. +#[test] +fn mutation_in_any_function_counts() { + let bits = translate_and_get_mutability( + r#" + (module + (table 4 funcref) + (func $f (result i32) i32.const 0) + (func (export "innocent") (result i32) + i32.const 0 + call_indirect (param) (result i32)) + (func (export "guilty") + i32.const 0 + ref.func $f + table.set 0)) + "#, + ); + assert_eq!(bits, vec![true]); +} + +/// Two tables, one mutated, one not. The analysis tracks per-table — a +/// mutation on one must not leak to the other. +#[test] +fn mutation_isolated_to_target_table() { + let bits = translate_and_get_mutability( + r#" + (module + (table $a 4 funcref) + (table $b 4 funcref) + (func $f (result i32) i32.const 0) + (func (export "mut_a") + i32.const 0 + ref.func $f + table.set $a)) + "#, + ); + assert_eq!( + bits, + vec![true, false], + "$a should be mutated, $b should remain immutable", + ); +} + +/// Translating without any tables at all must not panic. (Defensive: the +/// analysis indexes a `SecondaryMap` keyed by `TableIndex`, and we want +/// to confirm an empty module produces an empty result rather than e.g. +/// a default-allocated single entry.) +#[test] +fn module_with_no_tables_produces_empty_mutability_vec() { + let bits = translate_and_get_mutability( + r#" + (module + (func (export "noop"))) + "#, + ); + assert!(bits.is_empty(), "no tables ⇒ no mutability bits"); +} diff --git a/pulley/src/interp.rs b/pulley/src/interp.rs index 5b3f79445340..c7533bd5fc09 100644 --- a/pulley/src/interp.rs +++ b/pulley/src/interp.rs @@ -1425,6 +1425,84 @@ impl OpVisitor for Interpreter<'_> { ControlFlow::Continue(()) } + fn call_indirect1(&mut self, dst: XReg, arg1: XReg) -> ControlFlow { + // Phase-4 fusion: combines `xmov x0, arg1` with `call_indirect dst`. + // Read arg1 BEFORE writing x0 so this is safe even when `arg1 == x0`. + let arg1_val = self.state[arg1]; + let target = self.state[dst].get_ptr(); + let return_addr = self.pc.as_ptr(); + self.state.lr = return_addr.as_ptr(); + self.state[XReg::x0] = arg1_val; + // SAFETY: same as `call_indirect`. + unsafe { + self.pc = UnsafeBytecodeStream::new(NonNull::new_unchecked(target)); + } + ControlFlow::Continue(()) + } + + fn call_indirect2(&mut self, dst: XReg, arg1: XReg, arg2: XReg) -> ControlFlow { + let (a1, a2) = (self.state[arg1], self.state[arg2]); + let target = self.state[dst].get_ptr(); + let return_addr = self.pc.as_ptr(); + self.state.lr = return_addr.as_ptr(); + self.state[XReg::x0] = a1; + self.state[XReg::x1] = a2; + // SAFETY: same as `call_indirect`. + unsafe { + self.pc = UnsafeBytecodeStream::new(NonNull::new_unchecked(target)); + } + ControlFlow::Continue(()) + } + + fn call_indirect3( + &mut self, + dst: XReg, + arg1: XReg, + arg2: XReg, + arg3: XReg, + ) -> ControlFlow { + let (a1, a2, a3) = (self.state[arg1], self.state[arg2], self.state[arg3]); + let target = self.state[dst].get_ptr(); + let return_addr = self.pc.as_ptr(); + self.state.lr = return_addr.as_ptr(); + self.state[XReg::x0] = a1; + self.state[XReg::x1] = a2; + self.state[XReg::x2] = a3; + // SAFETY: same as `call_indirect`. + unsafe { + self.pc = UnsafeBytecodeStream::new(NonNull::new_unchecked(target)); + } + ControlFlow::Continue(()) + } + + fn call_indirect4( + &mut self, + dst: XReg, + arg1: XReg, + arg2: XReg, + arg3: XReg, + arg4: XReg, + ) -> ControlFlow { + let (a1, a2, a3, a4) = ( + self.state[arg1], + self.state[arg2], + self.state[arg3], + self.state[arg4], + ); + let target = self.state[dst].get_ptr(); + let return_addr = self.pc.as_ptr(); + self.state.lr = return_addr.as_ptr(); + self.state[XReg::x0] = a1; + self.state[XReg::x1] = a2; + self.state[XReg::x2] = a3; + self.state[XReg::x3] = a4; + // SAFETY: same as `call_indirect`. + unsafe { + self.pc = UnsafeBytecodeStream::new(NonNull::new_unchecked(target)); + } + ControlFlow::Continue(()) + } + fn jump(&mut self, offset: PcRelOffset) -> ControlFlow { self.pc_rel_jump::(offset) } diff --git a/pulley/src/lib.rs b/pulley/src/lib.rs index 36a09cb13a34..8552d21f74b5 100644 --- a/pulley/src/lib.rs +++ b/pulley/src/lib.rs @@ -115,6 +115,17 @@ macro_rules! for_each_op { /// Transfer control to the PC in `reg` and set `lr` to the PC just /// after this instruction. call_indirect = CallIndirect { reg: XReg }; + /// Like `call_indirect`, but also `x0 = arg1`. Saves one Pulley + /// dispatch vs `xmov x0, arg1; call_indirect reg` for the common + /// call_indirect pattern where one ABI register (usually `vmctx`) + /// is set up immediately before the indirect call. + call_indirect1 = CallIndirect1 { reg: XReg, arg1: XReg }; + /// Like `call_indirect`, but also `x0, x1 = arg1, arg2`. + call_indirect2 = CallIndirect2 { reg: XReg, arg1: XReg, arg2: XReg }; + /// Like `call_indirect`, but also `x0, x1, x2 = arg1, arg2, arg3`. + call_indirect3 = CallIndirect3 { reg: XReg, arg1: XReg, arg2: XReg, arg3: XReg }; + /// Like `call_indirect`, but also `x0, x1, x2, x3 = arg1, arg2, arg3, arg4`. + call_indirect4 = CallIndirect4 { reg: XReg, arg1: XReg, arg2: XReg, arg3: XReg, arg4: XReg }; /// Unconditionally transfer control to the PC at the given offset. jump = Jump { offset: PcRelOffset }; diff --git a/tests/disas/call-indirect-immutable-elide-null.wat b/tests/disas/call-indirect-immutable-elide-null.wat new file mode 100644 index 000000000000..e9f43689afeb --- /dev/null +++ b/tests/disas/call-indirect-immutable-elide-null.wat @@ -0,0 +1,121 @@ +;;! target = "x86_64" + +;; Immutable funcref table where every slot is filled by the elem +;; segment (no "no-entry" gaps). With both the sig check AND the +;; funcref-NULL check elided, the dispatch path is reduced to: +;; - bounds check (static) +;; - lazy-init brif + masking +;; - load code+vmctx +;; - call_indirect +;; +;; In particular the cold block that handles the runtime trap-on-null +;; path should not exist after the funcref load: the static-match path +;; with `may_be_null = false` skips both the sig check and any +;; downstream null-handling. + +(module + (table 3 3 funcref) + + (func $f1 (result i32) i32.const 1) + (func $f2 (result i32) i32.const 2) + (func $f3 (result i32) i32.const 3) + + (func (export "call_it") (param i32) (result i32) + local.get 0 + call_indirect (result i32)) + + ;; Fully cover the table — no null slot anywhere. + (elem (i32.const 0) func $f1 $f2 $f3)) +;; function u0:0(i64 vmctx, i64) -> i32 tail { +;; region0 = 8 "VMContext+0x8" +;; region1 = 67108888 "VMStoreContext+0x18" +;; gv0 = vmctx +;; gv1 = load.i64 notrap aligned readonly can_move region0 gv0+8 +;; gv2 = load.i64 notrap aligned region1 gv1+24 +;; stack_limit = gv2 +;; +;; block0(v0: i64, v1: i64): +;; @003f v2 = iconst.i32 1 +;; @0041 jump block1 +;; +;; block1: +;; @0041 return v2 ; v2 = 1 +;; } +;; +;; function u0:1(i64 vmctx, i64) -> i32 tail { +;; region0 = 8 "VMContext+0x8" +;; region1 = 67108888 "VMStoreContext+0x18" +;; gv0 = vmctx +;; gv1 = load.i64 notrap aligned readonly can_move region0 gv0+8 +;; gv2 = load.i64 notrap aligned region1 gv1+24 +;; stack_limit = gv2 +;; +;; block0(v0: i64, v1: i64): +;; @0044 v2 = iconst.i32 2 +;; @0046 jump block1 +;; +;; block1: +;; @0046 return v2 ; v2 = 2 +;; } +;; +;; function u0:2(i64 vmctx, i64) -> i32 tail { +;; region0 = 8 "VMContext+0x8" +;; region1 = 67108888 "VMStoreContext+0x18" +;; gv0 = vmctx +;; gv1 = load.i64 notrap aligned readonly can_move region0 gv0+8 +;; gv2 = load.i64 notrap aligned region1 gv1+24 +;; stack_limit = gv2 +;; +;; block0(v0: i64, v1: i64): +;; @0049 v2 = iconst.i32 3 +;; @004b jump block1 +;; +;; block1: +;; @004b return v2 ; v2 = 3 +;; } +;; +;; function u0:3(i64 vmctx, i64, i32) -> i32 tail { +;; region0 = 8 "VMContext+0x8" +;; region1 = 67108888 "VMStoreContext+0x18" +;; region2 = 671088640 "VMTableDefinition+0x0" +;; region3 = 335544320 "DefinedTable(StaticModuleIndex(0), DefinedTableIndex(0))" +;; region4 = 1610612744 "VMFuncRef+0x8" +;; region5 = 1610612760 "VMFuncRef+0x18" +;; gv0 = vmctx +;; gv1 = load.i64 notrap aligned readonly can_move region0 gv0+8 +;; gv2 = load.i64 notrap aligned region1 gv1+24 +;; sig0 = (i64 vmctx, i64) -> i32 tail +;; sig1 = (i64 vmctx, i32, i64) -> i64 tail +;; fn0 = colocated u805306368:7 sig1 +;; stack_limit = gv2 +;; +;; block0(v0: i64, v1: i64, v2: i32): +;; @0050 v3 = iconst.i32 3 +;; @0050 v4 = icmp uge v2, v3 ; v3 = 3 +;; @0050 v5 = uextend.i64 v2 +;; @0050 v6 = load.i64 notrap aligned readonly can_move region2 v0+48 +;; @0050 v7 = iconst.i64 3 +;; @0050 v8 = ishl v5, v7 ; v7 = 3 +;; @0050 v9 = iadd v6, v8 +;; @0050 v10 = iconst.i64 0 +;; @0050 v11 = select_spectre_guard v4, v10, v9 ; v10 = 0 +;; @0050 v12 = load.i64 user6 aligned region3 v11 +;; @0050 v13 = iconst.i64 -2 +;; @0050 v14 = band v12, v13 ; v13 = -2 +;; @0050 brif v12, block3(v14), block2 +;; +;; block2 cold: +;; @0050 v16 = iconst.i32 0 +;; @0050 v17 = uextend.i64 v2 +;; @0050 v18 = call fn0(v0, v16, v17) ; v16 = 0 +;; @0050 jump block3(v18) +;; +;; block3(v15: i64): +;; @0050 v19 = load.i64 notrap aligned readonly region4 v15+8 +;; @0050 v20 = load.i64 notrap aligned readonly region5 v15+24 +;; @0050 v21 = call_indirect sig0, v19(v20, v0) +;; @0053 jump block1 +;; +;; block1: +;; @0053 return v21 +;; } diff --git a/tests/disas/call-indirect-immutable-elide-sig.wat b/tests/disas/call-indirect-immutable-elide-sig.wat new file mode 100644 index 000000000000..e83dfe0f9d10 --- /dev/null +++ b/tests/disas/call-indirect-immutable-elide-sig.wat @@ -0,0 +1,120 @@ +;;! target = "x86_64" + +;; Immutable funcref table where every elem-segment entry has the same +;; declared type as the call site. This module's `tables_mutated` bit +;; for table 0 is clear (no opcode in any function writes to it), and +;; all three slots resolve to the same module type as the call site. +;; That triggers `try_elide_sig_check_for_immutable_table` → +;; `CheckIndirectCallTypeSignature::StaticMatch`, removing the runtime +;; signature load + compare from the dispatch hot path. +;; +;; Look for the absence of `load.i32 user6 aligned readonly v_+16` (the +;; sig-id load) and the matching `icmp eq / trapz user7` on the call +;; site. Compare with `indirect-call-no-caching.wat` for the +;; non-elided shape. + +(module + (table 10 10 funcref) + + (func $f1 (result i32) i32.const 1) + (func $f2 (result i32) i32.const 2) + (func $f3 (result i32) i32.const 3) + + (func (export "call_it") (param i32) (result i32) + local.get 0 + call_indirect (result i32)) + + (elem (i32.const 0) func $f1 $f2 $f3)) +;; function u0:0(i64 vmctx, i64) -> i32 tail { +;; region0 = 8 "VMContext+0x8" +;; region1 = 67108888 "VMStoreContext+0x18" +;; gv0 = vmctx +;; gv1 = load.i64 notrap aligned readonly can_move region0 gv0+8 +;; gv2 = load.i64 notrap aligned region1 gv1+24 +;; stack_limit = gv2 +;; +;; block0(v0: i64, v1: i64): +;; @003f v2 = iconst.i32 1 +;; @0041 jump block1 +;; +;; block1: +;; @0041 return v2 ; v2 = 1 +;; } +;; +;; function u0:1(i64 vmctx, i64) -> i32 tail { +;; region0 = 8 "VMContext+0x8" +;; region1 = 67108888 "VMStoreContext+0x18" +;; gv0 = vmctx +;; gv1 = load.i64 notrap aligned readonly can_move region0 gv0+8 +;; gv2 = load.i64 notrap aligned region1 gv1+24 +;; stack_limit = gv2 +;; +;; block0(v0: i64, v1: i64): +;; @0044 v2 = iconst.i32 2 +;; @0046 jump block1 +;; +;; block1: +;; @0046 return v2 ; v2 = 2 +;; } +;; +;; function u0:2(i64 vmctx, i64) -> i32 tail { +;; region0 = 8 "VMContext+0x8" +;; region1 = 67108888 "VMStoreContext+0x18" +;; gv0 = vmctx +;; gv1 = load.i64 notrap aligned readonly can_move region0 gv0+8 +;; gv2 = load.i64 notrap aligned region1 gv1+24 +;; stack_limit = gv2 +;; +;; block0(v0: i64, v1: i64): +;; @0049 v2 = iconst.i32 3 +;; @004b jump block1 +;; +;; block1: +;; @004b return v2 ; v2 = 3 +;; } +;; +;; function u0:3(i64 vmctx, i64, i32) -> i32 tail { +;; region0 = 8 "VMContext+0x8" +;; region1 = 67108888 "VMStoreContext+0x18" +;; region2 = 671088640 "VMTableDefinition+0x0" +;; region3 = 335544320 "DefinedTable(StaticModuleIndex(0), DefinedTableIndex(0))" +;; region4 = 1610612744 "VMFuncRef+0x8" +;; region5 = 1610612760 "VMFuncRef+0x18" +;; gv0 = vmctx +;; gv1 = load.i64 notrap aligned readonly can_move region0 gv0+8 +;; gv2 = load.i64 notrap aligned region1 gv1+24 +;; sig0 = (i64 vmctx, i64) -> i32 tail +;; sig1 = (i64 vmctx, i32, i64) -> i64 tail +;; fn0 = colocated u805306368:7 sig1 +;; stack_limit = gv2 +;; +;; block0(v0: i64, v1: i64, v2: i32): +;; @0050 v3 = iconst.i32 10 +;; @0050 v4 = icmp uge v2, v3 ; v3 = 10 +;; @0050 v5 = uextend.i64 v2 +;; @0050 v6 = load.i64 notrap aligned readonly can_move region2 v0+48 +;; @0050 v7 = iconst.i64 3 +;; @0050 v8 = ishl v5, v7 ; v7 = 3 +;; @0050 v9 = iadd v6, v8 +;; @0050 v10 = iconst.i64 0 +;; @0050 v11 = select_spectre_guard v4, v10, v9 ; v10 = 0 +;; @0050 v12 = load.i64 user6 aligned region3 v11 +;; @0050 v13 = iconst.i64 -2 +;; @0050 v14 = band v12, v13 ; v13 = -2 +;; @0050 brif v12, block3(v14), block2 +;; +;; block2 cold: +;; @0050 v16 = iconst.i32 0 +;; @0050 v17 = uextend.i64 v2 +;; @0050 v18 = call fn0(v0, v16, v17) ; v16 = 0 +;; @0050 jump block3(v18) +;; +;; block3(v15: i64): +;; @0050 v19 = load.i64 user7 aligned readonly region4 v15+8 +;; @0050 v20 = load.i64 notrap aligned readonly region5 v15+24 +;; @0050 v21 = call_indirect sig0, v19(v20, v0) +;; @0053 jump block1 +;; +;; block1: +;; @0053 return v21 +;; } diff --git a/tests/disas/call-indirect-immutable-static-bound.wat b/tests/disas/call-indirect-immutable-static-bound.wat new file mode 100644 index 000000000000..4d1f8e4e414f --- /dev/null +++ b/tests/disas/call-indirect-immutable-static-bound.wat @@ -0,0 +1,120 @@ +;;! target = "x86_64" + +;; Table declared with min < max (a "dynamic-declared" table) that is +;; never written to in the module. Without the per-table mutability +;; bit, Cranelift would emit `load.i64 v0+56` per dispatch to fetch +;; the current bound. With it, `make_table` lowers to +;; `TableSize::Static` and the bound becomes an immediate. +;; +;; Look for: bounds-check `iconst.i32 16` (the declared min, used as +;; static bound) and NO `load.i64 ... v0+56` for the current_elements +;; field. (`+48` for the funcref base is still loaded — that's the +;; element-data pointer, separate from the bound.) + +(module + ;; min=16, max=64 — distinct, so without our optimization the + ;; bound would be loaded per dispatch from `current_elements`. + (table 16 64 funcref) + + (func $f1 (result i32) i32.const 1) + (func $f2 (result i32) i32.const 2) + (func $f3 (result i32) i32.const 3) + + (func (export "call_it") (param i32) (result i32) + local.get 0 + call_indirect (result i32)) + + (elem (i32.const 0) func $f1 $f2 $f3)) +;; function u0:0(i64 vmctx, i64) -> i32 tail { +;; region0 = 8 "VMContext+0x8" +;; region1 = 67108888 "VMStoreContext+0x18" +;; gv0 = vmctx +;; gv1 = load.i64 notrap aligned readonly can_move region0 gv0+8 +;; gv2 = load.i64 notrap aligned region1 gv1+24 +;; stack_limit = gv2 +;; +;; block0(v0: i64, v1: i64): +;; @003f v2 = iconst.i32 1 +;; @0041 jump block1 +;; +;; block1: +;; @0041 return v2 ; v2 = 1 +;; } +;; +;; function u0:1(i64 vmctx, i64) -> i32 tail { +;; region0 = 8 "VMContext+0x8" +;; region1 = 67108888 "VMStoreContext+0x18" +;; gv0 = vmctx +;; gv1 = load.i64 notrap aligned readonly can_move region0 gv0+8 +;; gv2 = load.i64 notrap aligned region1 gv1+24 +;; stack_limit = gv2 +;; +;; block0(v0: i64, v1: i64): +;; @0044 v2 = iconst.i32 2 +;; @0046 jump block1 +;; +;; block1: +;; @0046 return v2 ; v2 = 2 +;; } +;; +;; function u0:2(i64 vmctx, i64) -> i32 tail { +;; region0 = 8 "VMContext+0x8" +;; region1 = 67108888 "VMStoreContext+0x18" +;; gv0 = vmctx +;; gv1 = load.i64 notrap aligned readonly can_move region0 gv0+8 +;; gv2 = load.i64 notrap aligned region1 gv1+24 +;; stack_limit = gv2 +;; +;; block0(v0: i64, v1: i64): +;; @0049 v2 = iconst.i32 3 +;; @004b jump block1 +;; +;; block1: +;; @004b return v2 ; v2 = 3 +;; } +;; +;; function u0:3(i64 vmctx, i64, i32) -> i32 tail { +;; region0 = 8 "VMContext+0x8" +;; region1 = 67108888 "VMStoreContext+0x18" +;; region2 = 671088640 "VMTableDefinition+0x0" +;; region3 = 335544320 "DefinedTable(StaticModuleIndex(0), DefinedTableIndex(0))" +;; region4 = 1610612744 "VMFuncRef+0x8" +;; region5 = 1610612760 "VMFuncRef+0x18" +;; gv0 = vmctx +;; gv1 = load.i64 notrap aligned readonly can_move region0 gv0+8 +;; gv2 = load.i64 notrap aligned region1 gv1+24 +;; sig0 = (i64 vmctx, i64) -> i32 tail +;; sig1 = (i64 vmctx, i32, i64) -> i64 tail +;; fn0 = colocated u805306368:7 sig1 +;; stack_limit = gv2 +;; +;; block0(v0: i64, v1: i64, v2: i32): +;; @0050 v3 = iconst.i32 16 +;; @0050 v4 = icmp uge v2, v3 ; v3 = 16 +;; @0050 v5 = uextend.i64 v2 +;; @0050 v6 = load.i64 notrap aligned readonly can_move region2 v0+48 +;; @0050 v7 = iconst.i64 3 +;; @0050 v8 = ishl v5, v7 ; v7 = 3 +;; @0050 v9 = iadd v6, v8 +;; @0050 v10 = iconst.i64 0 +;; @0050 v11 = select_spectre_guard v4, v10, v9 ; v10 = 0 +;; @0050 v12 = load.i64 user6 aligned region3 v11 +;; @0050 v13 = iconst.i64 -2 +;; @0050 v14 = band v12, v13 ; v13 = -2 +;; @0050 brif v12, block3(v14), block2 +;; +;; block2 cold: +;; @0050 v16 = iconst.i32 0 +;; @0050 v17 = uextend.i64 v2 +;; @0050 v18 = call fn0(v0, v16, v17) ; v16 = 0 +;; @0050 jump block3(v18) +;; +;; block3(v15: i64): +;; @0050 v19 = load.i64 user7 aligned readonly region4 v15+8 +;; @0050 v20 = load.i64 notrap aligned readonly region5 v15+24 +;; @0050 v21 = call_indirect sig0, v19(v20, v0) +;; @0053 jump block1 +;; +;; block1: +;; @0053 return v21 +;; } diff --git a/tests/disas/call-indirect-mutable-keeps-sigcheck.wat b/tests/disas/call-indirect-mutable-keeps-sigcheck.wat new file mode 100644 index 000000000000..2b1f8cf7a110 --- /dev/null +++ b/tests/disas/call-indirect-mutable-keeps-sigcheck.wat @@ -0,0 +1,166 @@ +;;! target = "x86_64" + +;; Counterpart to `call-indirect-immutable-elide-sig.wat`. Same module +;; shape — same elem segment, same uniform call-site type — but one +;; function writes to the table via `table.set`. That sets the +;; `tables_mutated` bit and disables sig-check elision. +;; +;; Look for the runtime sig load + compare on the call site: +;; load.i32 user6 aligned readonly v_+16 +;; icmp eq +;; trapz user7 +;; (versus the elided form in the immutable test). + +(module + (table 10 10 funcref) + + (func $f1 (result i32) i32.const 1) + (func $f2 (result i32) i32.const 2) + (func $f3 (result i32) i32.const 3) + + ;; Mutator: this clears the immutability proof for table 0. + (func (export "mutate") (param i32) + local.get 0 + ref.func $f1 + table.set 0) + + (func (export "call_it") (param i32) (result i32) + local.get 0 + call_indirect (result i32)) + + (elem (i32.const 0) func $f1 $f2 $f3)) +;; function u0:0(i64 vmctx, i64) -> i32 tail { +;; region0 = 8 "VMContext+0x8" +;; region1 = 67108888 "VMStoreContext+0x18" +;; gv0 = vmctx +;; gv1 = load.i64 notrap aligned readonly can_move region0 gv0+8 +;; gv2 = load.i64 notrap aligned region1 gv1+24 +;; stack_limit = gv2 +;; +;; block0(v0: i64, v1: i64): +;; @004d v2 = iconst.i32 1 +;; @004f jump block1 +;; +;; block1: +;; @004f return v2 ; v2 = 1 +;; } +;; +;; function u0:1(i64 vmctx, i64) -> i32 tail { +;; region0 = 8 "VMContext+0x8" +;; region1 = 67108888 "VMStoreContext+0x18" +;; gv0 = vmctx +;; gv1 = load.i64 notrap aligned readonly can_move region0 gv0+8 +;; gv2 = load.i64 notrap aligned region1 gv1+24 +;; stack_limit = gv2 +;; +;; block0(v0: i64, v1: i64): +;; @0052 v2 = iconst.i32 2 +;; @0054 jump block1 +;; +;; block1: +;; @0054 return v2 ; v2 = 2 +;; } +;; +;; function u0:2(i64 vmctx, i64) -> i32 tail { +;; region0 = 8 "VMContext+0x8" +;; region1 = 67108888 "VMStoreContext+0x18" +;; gv0 = vmctx +;; gv1 = load.i64 notrap aligned readonly can_move region0 gv0+8 +;; gv2 = load.i64 notrap aligned region1 gv1+24 +;; stack_limit = gv2 +;; +;; block0(v0: i64, v1: i64): +;; @0057 v2 = iconst.i32 3 +;; @0059 jump block1 +;; +;; block1: +;; @0059 return v2 ; v2 = 3 +;; } +;; +;; function u0:3(i64 vmctx, i64, i32) tail { +;; region0 = 8 "VMContext+0x8" +;; region1 = 67108888 "VMStoreContext+0x18" +;; region2 = 671088640 "VMTableDefinition+0x0" +;; region3 = 335544320 "DefinedTable(StaticModuleIndex(0), DefinedTableIndex(0))" +;; gv0 = vmctx +;; gv1 = load.i64 notrap aligned readonly can_move region0 gv0+8 +;; gv2 = load.i64 notrap aligned region1 gv1+24 +;; sig0 = (i64 vmctx, i32) -> i64 tail +;; fn0 = colocated u805306368:6 sig0 +;; stack_limit = gv2 +;; +;; block0(v0: i64, v1: i64, v2: i32): +;; @005e v3 = iconst.i32 0 +;; @005e v4 = call fn0(v0, v3) ; v3 = 0 +;; @0060 v5 = iconst.i32 10 +;; @0060 v6 = icmp uge v2, v5 ; v5 = 10 +;; @0060 v7 = uextend.i64 v2 +;; @0060 v8 = load.i64 notrap aligned readonly can_move region2 v0+48 +;; @0060 v9 = iconst.i64 3 +;; @0060 v10 = ishl v7, v9 ; v9 = 3 +;; @0060 v11 = iadd v8, v10 +;; @0060 v12 = iconst.i64 0 +;; @0060 v13 = select_spectre_guard v6, v12, v11 ; v12 = 0 +;; @0060 v14 = iconst.i64 1 +;; @0060 v15 = bor v4, v14 ; v14 = 1 +;; @0060 store user6 aligned region3 v15, v13 +;; @0062 jump block1 +;; +;; block1: +;; @0062 return +;; } +;; +;; function u0:4(i64 vmctx, i64, i32) -> i32 tail { +;; region0 = 8 "VMContext+0x8" +;; region1 = 67108888 "VMStoreContext+0x18" +;; region2 = 671088640 "VMTableDefinition+0x0" +;; region3 = 335544320 "DefinedTable(StaticModuleIndex(0), DefinedTableIndex(0))" +;; region4 = 40 "VMContext+0x28" +;; region5 = 1677721600 "TypeIdsArray+0x0" +;; region6 = 1610612752 "VMFuncRef+0x10" +;; region7 = 1610612744 "VMFuncRef+0x8" +;; region8 = 1610612760 "VMFuncRef+0x18" +;; gv0 = vmctx +;; gv1 = load.i64 notrap aligned readonly can_move region0 gv0+8 +;; gv2 = load.i64 notrap aligned region1 gv1+24 +;; sig0 = (i64 vmctx, i64) -> i32 tail +;; sig1 = (i64 vmctx, i32, i64) -> i64 tail +;; fn0 = colocated u805306368:7 sig1 +;; stack_limit = gv2 +;; +;; block0(v0: i64, v1: i64, v2: i32): +;; @0067 v3 = iconst.i32 10 +;; @0067 v4 = icmp uge v2, v3 ; v3 = 10 +;; @0067 v5 = uextend.i64 v2 +;; @0067 v6 = load.i64 notrap aligned readonly can_move region2 v0+48 +;; @0067 v7 = iconst.i64 3 +;; @0067 v8 = ishl v5, v7 ; v7 = 3 +;; @0067 v9 = iadd v6, v8 +;; @0067 v10 = iconst.i64 0 +;; @0067 v11 = select_spectre_guard v4, v10, v9 ; v10 = 0 +;; @0067 v12 = load.i64 user6 aligned region3 v11 +;; @0067 v13 = iconst.i64 -2 +;; @0067 v14 = band v12, v13 ; v13 = -2 +;; @0067 brif v12, block3(v14), block2 +;; +;; block2 cold: +;; @0067 v16 = iconst.i32 0 +;; @0067 v17 = uextend.i64 v2 +;; @0067 v18 = call fn0(v0, v16, v17) ; v16 = 0 +;; @0067 jump block3(v18) +;; +;; block3(v15: i64): +;; @0067 v19 = load.i64 notrap aligned readonly can_move region4 v0+40 +;; @0067 v20 = load.i32 notrap aligned readonly can_move region5 v19 +;; @0067 v21 = load.i32 user7 aligned readonly region6 v15+16 +;; @0067 v22 = icmp eq v21, v20 +;; @0067 v23 = uextend.i32 v22 +;; @0067 trapz v23, user8 +;; @0067 v24 = load.i64 notrap aligned readonly region7 v15+8 +;; @0067 v25 = load.i64 notrap aligned readonly region8 v15+24 +;; @0067 v26 = call_indirect sig0, v24(v25, v0) +;; @006a jump block1 +;; +;; block1: +;; @006a return v26 +;; } diff --git a/tests/disas/gc/call-indirect-final-type.wat b/tests/disas/gc/call-indirect-final-type.wat index 55449f05f049..9bc2dbe79046 100644 --- a/tests/disas/gc/call-indirect-final-type.wat +++ b/tests/disas/gc/call-indirect-final-type.wat @@ -19,13 +19,12 @@ ;; region0 = 8 "VMContext+0x8" ;; region1 = 67108888 "VMStoreContext+0x18" ;; region2 = 671088640 "VMTableDefinition+0x0" -;; region3 = 671088648 "VMTableDefinition+0x8" -;; region4 = 335544320 "DefinedTable(StaticModuleIndex(0), DefinedTableIndex(0))" -;; region5 = 40 "VMContext+0x28" -;; region6 = 1677721600 "TypeIdsArray+0x0" -;; region7 = 1610612752 "VMFuncRef+0x10" -;; region8 = 1610612744 "VMFuncRef+0x8" -;; region9 = 1610612760 "VMFuncRef+0x18" +;; region3 = 335544320 "DefinedTable(StaticModuleIndex(0), DefinedTableIndex(0))" +;; region4 = 40 "VMContext+0x28" +;; region5 = 1677721600 "TypeIdsArray+0x0" +;; region6 = 1610612752 "VMFuncRef+0x10" +;; region7 = 1610612744 "VMFuncRef+0x8" +;; region8 = 1610612760 "VMFuncRef+0x18" ;; gv0 = vmctx ;; gv1 = load.i64 notrap aligned readonly can_move region0 gv0+8 ;; gv2 = load.i64 notrap aligned region1 gv1+24 @@ -35,52 +34,43 @@ ;; stack_limit = gv2 ;; ;; block0(v0: i64, v1: i64, v2: i32, v3: i32): -;; @002b v4 = load.i64 notrap aligned region3 v0+56 -;; @002b v8 = load.i64 notrap aligned region2 v0+48 -;; @002b v5 = ireduce.i32 v4 -;; @002b v6 = icmp uge v3, v5 -;; @002b v12 = iconst.i64 0 -;; @002b v7 = uextend.i64 v3 -;; @002b v9 = iconst.i64 3 -;; @002b v10 = ishl v7, v9 ; v9 = 3 -;; @002b v11 = iadd v8, v10 -;; @002b v13 = select_spectre_guard v6, v12, v11 ; v12 = 0 -;; @002b v14 = load.i64 user6 aligned region4 v13 -;; @002b v15 = iconst.i64 -2 -;; @002b v16 = band v14, v15 ; v15 = -2 -;; @002b brif v14, block3(v16), block2 +;; @002b v11 = iconst.i64 0 +;; @002b v13 = load.i64 user6 aligned region3 v11 ; v11 = 0 +;; @002b v14 = iconst.i64 -2 +;; @002b v15 = band v13, v14 ; v14 = -2 +;; @002b brif v13, block3(v15), block2 ;; ;; block2 cold: -;; @002b v18 = iconst.i32 0 -;; @002b v20 = call fn0(v0, v18, v7) ; v18 = 0 -;; @002b jump block3(v20) +;; @002b v4 = iconst.i32 0 +;; @002b v6 = uextend.i64 v3 +;; @002b v19 = call fn0(v0, v4, v6) ; v4 = 0 +;; @002b jump block3(v19) ;; -;; block3(v17: i64): -;; @002b v23 = load.i32 user7 aligned readonly region7 v17+16 -;; @002b v21 = load.i64 notrap aligned readonly can_move region5 v0+40 -;; @002b v22 = load.i32 notrap aligned readonly can_move region6 v21 -;; @002b v24 = icmp eq v23, v22 -;; @002b trapz v24, user8 -;; @002b v26 = load.i64 notrap aligned readonly region8 v17+8 -;; @002b v27 = load.i64 notrap aligned readonly region9 v17+24 -;; @002b v28 = call_indirect sig0, v26(v27, v0, v2) +;; block3(v16: i64): +;; @002b v22 = load.i32 user7 aligned readonly region6 v16+16 +;; @002b v20 = load.i64 notrap aligned readonly can_move region4 v0+40 +;; @002b v21 = load.i32 notrap aligned readonly can_move region5 v20 +;; @002b v23 = icmp eq v22, v21 +;; @002b trapz v23, user8 +;; @002b v25 = load.i64 notrap aligned readonly region7 v16+8 +;; @002b v26 = load.i64 notrap aligned readonly region8 v16+24 +;; @002b v27 = call_indirect sig0, v25(v26, v0, v2) ;; @002e jump block1 ;; ;; block1: -;; @002e return v28 +;; @002e return v27 ;; } ;; ;; function u0:1(i64 vmctx, i64, i32, i32) -> i32 tail { ;; region0 = 8 "VMContext+0x8" ;; region1 = 67108888 "VMStoreContext+0x18" ;; region2 = 671088640 "VMTableDefinition+0x0" -;; region3 = 671088648 "VMTableDefinition+0x8" -;; region4 = 335544320 "DefinedTable(StaticModuleIndex(0), DefinedTableIndex(0))" -;; region5 = 40 "VMContext+0x28" -;; region6 = 1677721600 "TypeIdsArray+0x0" -;; region7 = 1610612752 "VMFuncRef+0x10" -;; region8 = 1610612744 "VMFuncRef+0x8" -;; region9 = 1610612760 "VMFuncRef+0x18" +;; region3 = 335544320 "DefinedTable(StaticModuleIndex(0), DefinedTableIndex(0))" +;; region4 = 40 "VMContext+0x28" +;; region5 = 1677721600 "TypeIdsArray+0x0" +;; region6 = 1610612752 "VMFuncRef+0x10" +;; region7 = 1610612744 "VMFuncRef+0x8" +;; region8 = 1610612760 "VMFuncRef+0x18" ;; gv0 = vmctx ;; gv1 = load.i64 notrap aligned readonly can_move region0 gv0+8 ;; gv2 = load.i64 notrap aligned region1 gv1+24 @@ -90,33 +80,25 @@ ;; stack_limit = gv2 ;; ;; block0(v0: i64, v1: i64, v2: i32, v3: i32): -;; @0035 v4 = load.i64 notrap aligned region3 v0+56 -;; @0035 v8 = load.i64 notrap aligned region2 v0+48 -;; @0035 v5 = ireduce.i32 v4 -;; @0035 v6 = icmp uge v3, v5 -;; @0035 v12 = iconst.i64 0 -;; @0035 v7 = uextend.i64 v3 -;; @0035 v9 = iconst.i64 3 -;; @0035 v10 = ishl v7, v9 ; v9 = 3 -;; @0035 v11 = iadd v8, v10 -;; @0035 v13 = select_spectre_guard v6, v12, v11 ; v12 = 0 -;; @0035 v14 = load.i64 user6 aligned region4 v13 -;; @0035 v15 = iconst.i64 -2 -;; @0035 v16 = band v14, v15 ; v15 = -2 -;; @0035 brif v14, block3(v16), block2 +;; @0035 v11 = iconst.i64 0 +;; @0035 v13 = load.i64 user6 aligned region3 v11 ; v11 = 0 +;; @0035 v14 = iconst.i64 -2 +;; @0035 v15 = band v13, v14 ; v14 = -2 +;; @0035 brif v13, block3(v15), block2 ;; ;; block2 cold: -;; @0035 v18 = iconst.i32 0 -;; @0035 v20 = call fn0(v0, v18, v7) ; v18 = 0 -;; @0035 jump block3(v20) +;; @0035 v4 = iconst.i32 0 +;; @0035 v6 = uextend.i64 v3 +;; @0035 v19 = call fn0(v0, v4, v6) ; v4 = 0 +;; @0035 jump block3(v19) ;; -;; block3(v17: i64): -;; @0035 v23 = load.i32 user7 aligned readonly region7 v17+16 -;; @0035 v21 = load.i64 notrap aligned readonly can_move region5 v0+40 -;; @0035 v22 = load.i32 notrap aligned readonly can_move region6 v21 -;; @0035 v24 = icmp eq v23, v22 -;; @0035 trapz v24, user8 -;; @0035 v26 = load.i64 notrap aligned readonly region8 v17+8 -;; @0035 v27 = load.i64 notrap aligned readonly region9 v17+24 -;; @0035 return_call_indirect sig0, v26(v27, v0, v2) +;; block3(v16: i64): +;; @0035 v22 = load.i32 user7 aligned readonly region6 v16+16 +;; @0035 v20 = load.i64 notrap aligned readonly can_move region4 v0+40 +;; @0035 v21 = load.i32 notrap aligned readonly can_move region5 v20 +;; @0035 v23 = icmp eq v22, v21 +;; @0035 trapz v23, user8 +;; @0035 v25 = load.i64 notrap aligned readonly region7 v16+8 +;; @0035 v26 = load.i64 notrap aligned readonly region8 v16+24 +;; @0035 return_call_indirect sig0, v25(v26, v0, v2) ;; } diff --git a/tests/disas/indirect-call-no-caching.wat b/tests/disas/indirect-call-no-caching.wat index d216ca74abe6..3d4cc4448291 100644 --- a/tests/disas/indirect-call-no-caching.wat +++ b/tests/disas/indirect-call-no-caching.wat @@ -73,11 +73,8 @@ ;; region1 = 67108888 "VMStoreContext+0x18" ;; region2 = 671088640 "VMTableDefinition+0x0" ;; region3 = 335544320 "DefinedTable(StaticModuleIndex(0), DefinedTableIndex(0))" -;; region4 = 40 "VMContext+0x28" -;; region5 = 1677721600 "TypeIdsArray+0x0" -;; region6 = 1610612752 "VMFuncRef+0x10" -;; region7 = 1610612744 "VMFuncRef+0x8" -;; region8 = 1610612760 "VMFuncRef+0x18" +;; region4 = 1610612744 "VMFuncRef+0x8" +;; region5 = 1610612760 "VMFuncRef+0x18" ;; gv0 = vmctx ;; gv1 = load.i64 notrap aligned readonly can_move region0 gv0+8 ;; gv2 = load.i64 notrap aligned region1 gv1+24 @@ -108,17 +105,11 @@ ;; @0050 jump block3(v18) ;; ;; block3(v15: i64): -;; @0050 v19 = load.i64 notrap aligned readonly can_move region4 v0+40 -;; @0050 v20 = load.i32 notrap aligned readonly can_move region5 v19 -;; @0050 v21 = load.i32 user7 aligned readonly region6 v15+16 -;; @0050 v22 = icmp eq v21, v20 -;; @0050 v23 = uextend.i32 v22 -;; @0050 trapz v23, user8 -;; @0050 v24 = load.i64 notrap aligned readonly region7 v15+8 -;; @0050 v25 = load.i64 notrap aligned readonly region8 v15+24 -;; @0050 v26 = call_indirect sig0, v24(v25, v0) +;; @0050 v19 = load.i64 user7 aligned readonly region4 v15+8 +;; @0050 v20 = load.i64 notrap aligned readonly region5 v15+24 +;; @0050 v21 = call_indirect sig0, v19(v20, v0) ;; @0053 jump block1 ;; ;; block1: -;; @0053 return v26 +;; @0053 return v21 ;; } diff --git a/tests/disas/pulley/call.wat b/tests/disas/pulley/call.wat index 233ca7be3c35..d9bc3142fd99 100644 --- a/tests/disas/pulley/call.wat +++ b/tests/disas/pulley/call.wat @@ -8,9 +8,7 @@ ;; wasm[0]::function[1]: ;; push_frame ;; xload32le_o32 x3, x0, 28 -;; xmov x6, x0 -;; xload32le_o32 x0, x6, 36 -;; xmov x1, x6 -;; call_indirect x3 +;; xload32le_o32 x4, x0, 36 +;; call_indirect2 x3, x4, x0 ;; pop_frame ;; ret diff --git a/tests/disas/readonly-funcrefs.wat b/tests/disas/readonly-funcrefs.wat index bb9fc978a844..b21bbcbbae47 100644 --- a/tests/disas/readonly-funcrefs.wat +++ b/tests/disas/readonly-funcrefs.wat @@ -38,11 +38,8 @@ ;; region1 = 67108888 "VMStoreContext+0x18" ;; region2 = 671088640 "VMTableDefinition+0x0" ;; region3 = 335544320 "DefinedTable(StaticModuleIndex(0), DefinedTableIndex(0))" -;; region4 = 40 "VMContext+0x28" -;; region5 = 1677721600 "TypeIdsArray+0x0" -;; region6 = 1610612752 "VMFuncRef+0x10" -;; region7 = 1610612744 "VMFuncRef+0x8" -;; region8 = 1610612760 "VMFuncRef+0x18" +;; region4 = 1610612744 "VMFuncRef+0x8" +;; region5 = 1610612760 "VMFuncRef+0x18" ;; gv0 = vmctx ;; gv1 = load.i64 notrap aligned readonly can_move region0 gv0+8 ;; gv2 = load.i64 notrap aligned region1 gv1+24 @@ -72,14 +69,9 @@ ;; @0031 jump block3(v18) ;; ;; block3(v15: i64): -;; @0031 v21 = load.i32 user7 aligned readonly region6 v15+16 -;; @0031 v19 = load.i64 notrap aligned readonly can_move region4 v0+40 -;; @0031 v20 = load.i32 notrap aligned readonly can_move region5 v19 -;; @0031 v22 = icmp eq v21, v20 -;; @0031 trapz v22, user8 -;; @0031 v24 = load.i64 notrap aligned readonly region7 v15+8 -;; @0031 v25 = load.i64 notrap aligned readonly region8 v15+24 -;; @0031 call_indirect sig0, v24(v25, v0) +;; @0031 v19 = load.i64 user7 aligned readonly region4 v15+8 +;; @0031 v20 = load.i64 notrap aligned readonly region5 v15+24 +;; @0031 call_indirect sig0, v19(v20, v0) ;; @0034 jump block1 ;; ;; block1: diff --git a/tests/disas/startup-elem-active.wat b/tests/disas/startup-elem-active.wat index 4b802eee4480..5fdf9990a768 100644 --- a/tests/disas/startup-elem-active.wat +++ b/tests/disas/startup-elem-active.wat @@ -47,37 +47,21 @@ ;; ;; function u2415919104:0(i64 vmctx, i64) tail { ;; region0 = 671088640 "VMTableDefinition+0x0" -;; region1 = 671088648 "VMTableDefinition+0x8" -;; region2 = 335544320 "DefinedTable(StaticModuleIndex(0), DefinedTableIndex(0))" +;; region1 = 335544320 "DefinedTable(StaticModuleIndex(0), DefinedTableIndex(0))" ;; ;; block0(v0: i64, v1: i64): -;; v4 = load.i64 notrap aligned region1 v0+56 -;; v5 = ireduce.i32 v4 -;; v6 = uextend.i64 v5 -;; v78 = iconst.i64 4 -;; v84 = icmp ult v6, v78 ; v78 = 4 -;; trapnz v84, user6 -;; v13 = load.i64 notrap aligned region0 v0+48 -;; v95 = iconst.i32 21 -;; v2 = iconst.i32 1 -;; v106 = icmp ule v5, v2 ; v2 = 1 -;; v71 = iconst.i64 0 -;; v17 = iadd v13, v78 ; v78 = 4 -;; v34 = select_spectre_guard v106, v71, v17 ; v71 = 0 -;; store user6 aligned region2 v95, v34 ; v95 = 21 -;; v109 = iconst.i32 23 -;; v115 = iconst.i32 2 -;; v121 = icmp ule v5, v115 ; v115 = 2 -;; v123 = iconst.i64 8 -;; v49 = iadd v13, v123 ; v123 = 8 -;; v51 = select_spectre_guard v121, v71, v49 ; v71 = 0 -;; store user6 aligned region2 v109, v51 ; v109 = 23 -;; v125 = iconst.i32 25 -;; v3 = iconst.i32 3 -;; v136 = icmp ule v5, v3 ; v3 = 3 -;; v138 = iconst.i64 12 -;; v66 = iadd v13, v138 ; v138 = 12 -;; v68 = select_spectre_guard v136, v71, v66 ; v71 = 0 -;; store user6 aligned region2 v125, v68 ; v125 = 25 +;; v96 = iconst.i32 21 +;; v12 = load.i64 notrap aligned readonly can_move region0 v0+48 +;; v75 = iconst.i64 4 +;; v16 = iadd v12, v75 ; v75 = 4 +;; store user6 aligned region1 v96, v16 ; v96 = 21 +;; v113 = iconst.i32 23 +;; v130 = iconst.i64 8 +;; v46 = iadd v12, v130 ; v130 = 8 +;; store user6 aligned region1 v113, v46 ; v113 = 23 +;; v132 = iconst.i32 25 +;; v148 = iconst.i64 12 +;; v62 = iadd v12, v148 ; v148 = 12 +;; store user6 aligned region1 v132, v62 ; v132 = 25 ;; return ;; } diff --git a/tests/disas/startup-table-initial-value.wat b/tests/disas/startup-table-initial-value.wat index 3db39fac9976..3bdc39250ed9 100644 --- a/tests/disas/startup-table-initial-value.wat +++ b/tests/disas/startup-table-initial-value.wat @@ -41,31 +41,24 @@ ;; ;; function u2415919104:0(i64 vmctx, i64) tail { ;; region0 = 671088640 "VMTableDefinition+0x0" -;; region1 = 671088648 "VMTableDefinition+0x8" -;; region2 = 335544320 "DefinedTable(StaticModuleIndex(0), DefinedTableIndex(0))" +;; region1 = 335544320 "DefinedTable(StaticModuleIndex(0), DefinedTableIndex(0))" ;; ;; block0(v0: i64, v1: i64): -;; v9 = load.i64 notrap aligned region1 v0+56 -;; v10 = ireduce.i32 v9 -;; v11 = uextend.i64 v10 -;; v39 = iconst.i64 10 -;; v51 = icmp ult v11, v39 ; v39 = 10 -;; trapnz v51, user6 -;; v18 = load.i64 notrap aligned region0 v0+48 +;; v17 = load.i64 notrap aligned readonly can_move region0 v0+48 ;; v3 = iconst.i32 1 -;; v81 = iconst.i64 36 -;; v83 = iadd v18, v81 ; v81 = 36 -;; v20 = iconst.i64 4 -;; jump block1(v18) +;; v83 = iconst.i64 36 +;; v85 = iadd v17, v83 ; v83 = 36 +;; v19 = iconst.i64 4 +;; jump block1(v17) ;; -;; block1(v29: i64): -;; v86 = iconst.i32 1 -;; store notrap aligned region2 v86, v29 ; v86 = 1 -;; v87 = iadd.i64 v18, v81 ; v81 = 36 -;; v88 = icmp eq v29, v87 -;; v89 = iconst.i64 4 -;; v90 = iadd v29, v89 ; v89 = 4 -;; brif v88, block2, block1(v90) +;; block1(v28: i64): +;; v88 = iconst.i32 1 +;; store notrap aligned region1 v88, v28 ; v88 = 1 +;; v89 = iadd.i64 v17, v83 ; v83 = 36 +;; v90 = icmp eq v28, v89 +;; v91 = iconst.i64 4 +;; v92 = iadd v28, v91 ; v91 = 4 +;; brif v90, block2, block1(v92) ;; ;; block2: ;; return diff --git a/tests/misc_testsuite/immutable-table-call-indirect.wast b/tests/misc_testsuite/immutable-table-call-indirect.wast new file mode 100644 index 000000000000..3b40cb9ab534 --- /dev/null +++ b/tests/misc_testsuite/immutable-table-call-indirect.wast @@ -0,0 +1,71 @@ +;;! reference_types = true + +;; call_indirect through tables that are never grown, exported, or mutated. +;; Compilation may use a constant bound and elide null/signature checks on +;; these shapes; runtime behavior must be unchanged: in-bounds calls work, +;; and out-of-bounds, null-slot, and signature-mismatch accesses still trap. + +;; Mixed-signature immutable table with a null hole. +(module + (type $i2i (func (param i32) (result i32))) + (type $v2i (func (result i32))) + (table 5 funcref) + (elem (i32.const 0) $add1 $ten $add1) + + (func $add1 (type $i2i) (i32.add (local.get 0) (i32.const 1))) + (func $ten (type $v2i) (i32.const 10)) + + (func (export "call-i2i") (param i32 i32) (result i32) + (call_indirect (type $i2i) (local.get 1) (local.get 0))) + (func (export "call-v2i") (param i32) (result i32) + (call_indirect (type $v2i) (local.get 0)))) + +(assert_return (invoke "call-i2i" (i32.const 0) (i32.const 41)) (i32.const 42)) +(assert_return (invoke "call-i2i" (i32.const 2) (i32.const 7)) (i32.const 8)) +(assert_return (invoke "call-v2i" (i32.const 1)) (i32.const 10)) + +;; Signature mismatch still traps. +(assert_trap (invoke "call-i2i" (i32.const 1) (i32.const 0)) "indirect call type mismatch") +(assert_trap (invoke "call-v2i" (i32.const 0)) "indirect call type mismatch") + +;; Null slots still trap: slot 3 was never initialized. +(assert_trap (invoke "call-i2i" (i32.const 3) (i32.const 0)) "uninitialized element") +(assert_trap (invoke "call-v2i" (i32.const 4)) "uninitialized element") + +;; Out of bounds still traps against the constant bound. +(assert_trap (invoke "call-i2i" (i32.const 5) (i32.const 0)) "undefined element") +(assert_trap (invoke "call-i2i" (i32.const -1) (i32.const 0)) "undefined element") + +;; Uniform-signature immutable table, fully initialized. +(module + (type $v2i (func (result i32))) + (table 3 funcref) + (elem (i32.const 0) $a $b $c) + + (func $a (type $v2i) (i32.const 1)) + (func $b (type $v2i) (i32.const 2)) + (func $c (type $v2i) (i32.const 3)) + + (func (export "call") (param i32) (result i32) + (call_indirect (type $v2i) (local.get 0))) + (func (export "call-wrong-type") (param i32 i32) (result i32) + (call_indirect (param i32) (result i32) (local.get 1) (local.get 0)))) + +(assert_return (invoke "call" (i32.const 0)) (i32.const 1)) +(assert_return (invoke "call" (i32.const 1)) (i32.const 2)) +(assert_return (invoke "call" (i32.const 2)) (i32.const 3)) +(assert_trap (invoke "call" (i32.const 3)) "undefined element") + +;; A caller whose expected type differs from the table's uniform type must +;; still observe the mismatch. +(assert_trap (invoke "call-wrong-type" (i32.const 0) (i32.const 0)) "indirect call type mismatch") + +;; Same shapes through a declared-growable (no max) table never actually +;; grown: an empty never-grown table has no valid index. +(module + (table 0 100 funcref) + (func (export "call-empty") (param i32) + (call_indirect (local.get 0)))) + +(assert_trap (invoke "call-empty" (i32.const 0)) "undefined element") +(assert_trap (invoke "call-empty" (i32.const 99)) "undefined element") diff --git a/tests/misc_testsuite/pulley-call-indirect-arg-bundling.wast b/tests/misc_testsuite/pulley-call-indirect-arg-bundling.wast new file mode 100644 index 000000000000..8cbfb87995e9 --- /dev/null +++ b/tests/misc_testsuite/pulley-call-indirect-arg-bundling.wast @@ -0,0 +1,52 @@ +;; The Pulley `call_indirect{1,2,3,4}` ops bundle the first four integer ABI +;; arguments into the indirect-call op; a fifth+ integer arg, and any float +;; arg, must still be passed the normal way. Exercise every count across that +;; boundary through a call_indirect and check the results are unchanged. + +(module + (type $t0 (func (result i32))) + (type $t1 (func (param i32) (result i32))) + (type $t4 (func (param i32 i32 i32 i32) (result i32))) + (type $t6 (func (param i32 i32 i32 i32 i32 i32) (result i32))) + (type $tf (func (param i32 f64 i32 f64 i32) (result f64))) + + (table 5 5 funcref) + (elem (i32.const 0) $f0 $f1 $f4 $f6 $ff) + + (func $f0 (type $t0) (i32.const 100)) + (func $f1 (type $t1) (local.get 0)) + (func $f4 (type $t4) + (i32.add (i32.add (local.get 0) (local.get 1)) + (i32.add (local.get 2) (local.get 3)))) + (func $f6 (type $t6) + (i32.add + (i32.add (i32.add (local.get 0) (local.get 1)) (i32.add (local.get 2) (local.get 3))) + (i32.add (local.get 4) (local.get 5)))) + ;; Interleaved int/float args: the floats never occupy x0..x3, so the + ;; bundling must skip them and still sum the integers + floats correctly. + (func $ff (type $tf) + (f64.add + (f64.add (local.get 1) (local.get 3)) + (f64.convert_i32_s (i32.add (i32.add (local.get 0) (local.get 2)) (local.get 4))))) + + (func (export "c0") (result i32) + (call_indirect (type $t0) (i32.const 0))) + (func (export "c1") (param i32) (result i32) + (call_indirect (type $t1) (local.get 0) (i32.const 1))) + (func (export "c4") (param i32 i32 i32 i32) (result i32) + (call_indirect (type $t4) (local.get 0) (local.get 1) (local.get 2) (local.get 3) + (i32.const 2))) + (func (export "c6") (param i32 i32 i32 i32 i32 i32) (result i32) + (call_indirect (type $t6) (local.get 0) (local.get 1) (local.get 2) + (local.get 3) (local.get 4) (local.get 5) (i32.const 3))) + (func (export "cf") (param i32 f64 i32 f64 i32) (result f64) + (call_indirect (type $tf) (local.get 0) (local.get 1) (local.get 2) + (local.get 3) (local.get 4) (i32.const 4)))) + +(assert_return (invoke "c0") (i32.const 100)) +(assert_return (invoke "c1" (i32.const 42)) (i32.const 42)) +(assert_return (invoke "c4" (i32.const 1) (i32.const 2) (i32.const 3) (i32.const 4)) (i32.const 10)) +(assert_return (invoke "c6" (i32.const 1) (i32.const 2) (i32.const 3) + (i32.const 4) (i32.const 5) (i32.const 6)) (i32.const 21)) +(assert_return (invoke "cf" (i32.const 1) (f64.const 2.5) (i32.const 3) (f64.const 4.5) (i32.const 5)) + (f64.const 16))