From 98f27b00f4ee1df241f6e4b77e4dd170f0ea04f0 Mon Sep 17 00:00:00 2001 From: Taylor Holliday Date: Sun, 18 Oct 2026 08:35:08 -0700 Subject: [PATCH 1/2] Return aggregates from extern functions through an output pointer An extern that returns a tuple, struct, array, enum or f32x4 now receives a trailing output pointer and returns void, so a host can hand back compound results without a platform-specific struct-return ABI: extern fn host_pair(x: i32) -> (i32, i32) void host_pair(void* ctx, int32_t x, int32_t* out) The rule lives in one place, TypeID::extern_returns_via_pointer, and every backend consults it, so the same host function works on the Cranelift, LLVM, VM and stack backends. f32x4 goes through the pointer too (four floats) rather than a vector return register, which libffi cannot marshal and which differs across platforms; Cranelift and LLVM reload the vector after the call. The checker rejects extern return types no backend can marshal (function types, i8/u8/u32) with a message listing the supported set. The output slot is 16-byte aligned on Cranelift and LLVM, since the host stores through it as int32_t*/float*/double*. VM codegen checks the libffi argument budget at compile time instead of panicking on the first call. Tests run the same program and host function on every enabled backend for tuple, struct, array, enum and f32x4 results, plus a golden test for the checker error. Co-Authored-By: Claude Fable 5.1 --- Sources/CLyte/include/lyte.h | 10 + src/checker.rs | 16 ++ src/compiler.rs | 270 ++++++++++++++++++++ src/jit.rs | 50 +++- src/llvm_jit.rs | 43 +++- src/stack_codegen.rs | 28 +- src/types.rs | 26 ++ src/vm.rs | 2 +- src/vm_codegen.rs | 45 +++- tests/cases/checker/extern_fn_ret_type.lyte | 12 + tests/cases/extern_tuple_ret.lyte | 21 ++ 11 files changed, 505 insertions(+), 18 deletions(-) create mode 100644 tests/cases/checker/extern_fn_ret_type.lyte create mode 100644 tests/cases/extern_tuple_ret.lyte diff --git a/Sources/CLyte/include/lyte.h b/Sources/CLyte/include/lyte.h index 4f564c45..d04db3bc 100644 --- a/Sources/CLyte/include/lyte.h +++ b/Sources/CLyte/include/lyte.h @@ -125,6 +125,16 @@ void lyte_globals_bind_slice(uint8_t* globals, size_t offset, const void* data, /// The offset should come from lyte_program_get_global_offset for an extern function. /// The host function must use C calling conventions with a prepended void* context parameter. /// For example, `extern fn foo(i32) -> f32` expects: float (*)(void* ctx, int32_t) +/// A slice parameter expands to a data pointer followed by an int32_t length: +/// `extern fn send(buf: [i8]) -> bool` expects: bool (*)(void* ctx, const int8_t* data, int32_t len) +/// Scalar results (bool, i32, f32, f64) are returned by value. Every other +/// result is written through a trailing output pointer and the function +/// returns void. The pointer is 16-byte aligned and the layout is the Lyte +/// layout with no padding: +/// tuple/struct/array: `extern fn pair() -> (i32, i32)` expects: void (*)(void* ctx, int32_t* out) +/// enum (i32 discriminant): `extern fn mode() -> Mode` expects: void (*)(void* ctx, int32_t* out) +/// f32x4 (four floats): `extern fn vec() -> f32x4` expects: void (*)(void* ctx, float* out) +/// This rule is identical on every backend (JIT, LLVM, VM, stack). /// Pass NULL for context if the function doesn't need state. void lyte_globals_bind_extern(uint8_t* globals, size_t offset, const void* func_ptr, void* context); diff --git a/src/checker.rs b/src/checker.rs index 4e1984d8..c2c3a324 100644 --- a/src/checker.rs +++ b/src/checker.rs @@ -1087,6 +1087,22 @@ impl Checker { }); } + // Extern results must be marshallable by every backend. Slices and + // references were already rejected above; this catches function + // types and the narrow integers libffi has no return slot for. + if func_decl.is_extern + && !type_contains_bad_borrow(func_decl.ret) + && !func_decl.ret.is_supported_extern_return() + { + self.errors.push(TypeError { + location: func_decl.loc, + message: format!( + "extern functions cannot return {} (supported: void, bool, i32, f32, f64, f32x4, tuples, structs, arrays, enums)", + func_decl.ret.pretty_print() + ), + }); + } + let n = func_decl.arena.exprs.len(); self.types.resize(n, mk_type(Type::Void)); self.visited.clear(); diff --git a/src/compiler.rs b/src/compiler.rs index 58aae45b..fea5e60e 100644 --- a/src/compiler.rs +++ b/src/compiler.rs @@ -2766,4 +2766,274 @@ mod tests { "send() was not called" ); } + + // ---- By-pointer results from extern functions ---- + // + // An extern that returns a tuple, struct, array, enum or f32x4 receives a + // trailing output pointer and returns void (`TypeID::extern_returns_via_pointer`): + // extern fn host_pair(x: i32) -> (i32, i32) + // void host_pair(void* ctx, int32_t x, int32_t* out) + // The same host function must work on every backend, so each case below + // runs the identical program and host on all of them. + + /// Compile `prelude` + `code` and bind `host` to the single extern. + fn extern_compiler(prelude: &str, code: &str) -> Compiler { + let mut compiler = Compiler::new(); + compiler.parse(prelude, ""); + compiler.parse(code, "test.lyte"); + assert!(compiler.check(), "type check failed"); + compiler.specialize().expect("specialize failed"); + compiler + } + + /// Offset of the single extern function in the globals buffer. + fn extern_offset(compiler: &Compiler) -> usize { + compiler + .globals_info_with_offset(crate::cancel::CANCEL_FLAG_RESERVED as usize) + .iter() + .find(|g| g.4) + .expect("no extern global found") + .1 + } + + fn run_extern_vm(prelude: &str, code: &str, host: *const u8) -> i32 { + let compiler = extern_compiler(prelude, code); + let vm_program = compiler.compile_vm().expect("VM compile failed"); + let globals_size = vm_program.globals_size; + assert_eq!(vm_program.extern_funcs.len(), 1); + // The C signature ends with the output pointer and returns void. + assert_eq!( + vm_program.extern_funcs[0].param_types.last(), + Some(&crate::vm::ExternType::Ptr) + ); + assert_eq!( + vm_program.extern_funcs[0].ret_type, + crate::vm::ExternType::Void + ); + + let linked = crate::vm::LinkedProgram::from_program(&vm_program); + let mut vm = crate::vm::VM::new(); + let mut globals = vec![0u8; globals_size]; + let offset = vm_program.extern_funcs[0].globals_offset as usize; + unsafe { + crate::ffi::lyte_globals_bind_extern( + globals.as_mut_ptr(), + offset, + host, + std::ptr::null_mut(), + ); + } + let func_idx = *vm_program.entry_points.get(&Name::str("main")).unwrap(); + let result = unsafe { + vm.call_with_external_globals( + &linked, + &vm_program, + func_idx, + globals.as_mut_ptr(), + globals_size, + ) + }; + result as i32 + } + + #[cfg(has_stack_interp)] + fn run_extern_stack(prelude: &str, code: &str, host: *const u8) -> i32 { + let compiler = extern_compiler(prelude, code); + let stack_program = compiler.compile_stack().expect("stack VM compile failed"); + let globals_size = stack_program.globals_size; + let offset = extern_offset(&compiler); + let mut backend = crate::stack_interp_bridge::StackBackend::new(&stack_program); + let mut globals = vec![0u8; globals_size]; + unsafe { + crate::ffi::lyte_globals_bind_extern( + globals.as_mut_ptr(), + offset, + host, + std::ptr::null_mut(), + ); + } + let func_idx = *stack_program.entry_points.get(&Name::str("main")).unwrap(); + backend.call_entry(func_idx, globals.as_mut_ptr()) as i32 + } + + /// Run a native entry point with the extern bound; shared by the + /// Cranelift and LLVM runners. + #[cfg(any(feature = "cranelift", feature = "llvm"))] + unsafe fn call_native_entry( + main_ptr: *const u8, + globals_size: usize, + offset: usize, + host: *const u8, + ) -> i32 { + extern "C" { + fn setjmp(env: *mut u8) -> i32; + } + let mut globals = vec![0u8; globals_size]; + crate::ffi::lyte_globals_bind_extern( + globals.as_mut_ptr(), + offset, + host, + std::ptr::null_mut(), + ); + crate::cancel::set_cancel_callback(globals.as_mut_ptr(), None, std::ptr::null_mut()); + let jmp_buf_ptr = globals.as_mut_ptr().add(crate::cancel::JMPBUF_OFFSET); + if setjmp(jmp_buf_ptr) == 0 { + type Entry = fn(*mut u8, *mut u8) -> i32; + let code_fn = mem::transmute::<_, Entry>(main_ptr); + code_fn(globals.as_mut_ptr(), std::ptr::null_mut()) + } else { + panic!("execution was cancelled unexpectedly"); + } + } + + #[cfg(feature = "cranelift")] + fn run_extern_cranelift(prelude: &str, code: &str, host: *const u8) -> i32 { + let compiler = extern_compiler(prelude, code); + let (map, globals_size, jit) = compiler.jit_multi().unwrap(); + let main_ptr = *map.get(&Name::str("main")).unwrap(); + let offset = extern_offset(&compiler); + let result = unsafe { call_native_entry(main_ptr, globals_size, offset, host) }; + jit.free_memory(); + result + } + + #[cfg(feature = "llvm")] + fn run_extern_llvm(prelude: &str, code: &str, host: *const u8) -> i32 { + let compiler = extern_compiler(prelude, code); + let llvm_prog = { + let jit = crate::llvm_jit::LLVMJIT::new(); + let entry_points = vec![Name::str("main")]; + jit.compile_only(&compiler.decls, &entry_points) + .expect("LLVM compile failed") + }; + let main_ptr = *llvm_prog.entry_points.get(&Name::str("main")).unwrap(); + let offset = extern_offset(&compiler); + unsafe { call_native_entry(main_ptr as *const u8, llvm_prog.globals_size, offset, host) } + } + + /// Assert that every enabled backend produces `expected` for the same + /// program bound to the same host function. + fn assert_extern_all_backends(prelude: &str, code: &str, host: *const u8, expected: i32) { + assert_eq!(run_extern_vm(prelude, code, host), expected, "VM backend"); + #[cfg(has_stack_interp)] + assert_eq!( + run_extern_stack(prelude, code, host), + expected, + "stack backend" + ); + #[cfg(feature = "cranelift")] + assert_eq!( + run_extern_cranelift(prelude, code, host), + expected, + "Cranelift backend" + ); + #[cfg(feature = "llvm")] + assert_eq!( + run_extern_llvm(prelude, code, host), + expected, + "LLVM backend" + ); + } + + /// Host for any two-i32 aggregate: writes (x + 1, x * 2). + unsafe extern "C" fn host_pair(_ctx: *mut u8, x: i32, out: *mut i32) { + *out = x + 1; + *out.add(1) = x * 2; + } + + #[test] + fn test_extern_fn_tuple_return() { + assert_extern_all_backends( + "extern fn host_pair(x: i32) -> (i32, i32)", + r#" + fn main() -> i32 { + var p = host_pair(3) + p.0 * 10 + p.1 + } + "#, + host_pair as *const u8, + 46, + ); + } + + #[test] + fn test_extern_fn_struct_return() { + assert_extern_all_backends( + r#" + struct Pt { x: i32, y: i32 } + extern fn host_pt(x: i32) -> Pt + "#, + r#" + fn main() -> i32 { + var p = host_pt(3) + p.x * 10 + p.y + } + "#, + host_pair as *const u8, + 46, + ); + } + + #[test] + fn test_extern_fn_array_return() { + assert_extern_all_backends( + "extern fn host_arr(x: i32) -> [i32; 2]", + r#" + fn main() -> i32 { + var a = host_arr(3) + a[0] * 10 + a[1] + } + "#, + host_pair as *const u8, + 46, + ); + } + + /// Enum host: the i32 discriminant goes through the output pointer too, + /// so a host returning it by value would leave the slot uninitialized. + unsafe extern "C" fn host_mode(_ctx: *mut u8, out: *mut i32) { + *out = 2; // Mode.Blue + } + + #[test] + fn test_extern_fn_enum_return() { + assert_extern_all_backends( + r#" + enum Mode { Red, Green, Blue } + extern fn host_mode() -> Mode + "#, + r#" + fn main() -> i32 { + var m = host_mode() + if m == .Blue { 7 } else { 0 } + } + "#, + host_mode as *const u8, + 7, + ); + } + + /// f32x4 host: four floats through the output pointer on every backend, + /// never a vector return register. + unsafe extern "C" fn host_vec(_ctx: *mut u8, out: *mut f32) { + *out = 1.0; + *out.add(1) = 2.0; + *out.add(2) = 3.0; + *out.add(3) = 4.0; + } + + #[test] + fn test_extern_fn_f32x4_return() { + assert_extern_all_backends( + "extern fn host_vec() -> f32x4", + r#" + fn main() -> i32 { + var v = host_vec() + (v[0] as i32) * 1000 + (v[1] as i32) * 100 + (v[2] as i32) * 10 + (v[3] as i32) + } + "#, + host_vec as *const u8, + 1234, + ); + } } diff --git a/src/jit.rs b/src/jit.rs index 2e27325c..2cdc5550 100644 --- a/src/jit.rs +++ b/src/jit.rs @@ -931,10 +931,13 @@ impl<'a> FunctionTranslator<'a> { // If return type is pointer, allocate stack space and pass as first arg. let output_slot = if returns_via_pointer(to) { let size = to.size(decls) as u32; + // 16-byte aligned: an extern host writes through + // this pointer as int32_t*/float*/double*, and C + // requires natural alignment for those stores. let slot = self.builder.create_sized_stack_slot(StackSlotData { kind: StackSlotKind::ExplicitSlot, size, - align_shift: 0, + align_shift: 4, key: None, }); let addr = self.builder.ins().stack_addr(I64, slot, 0); @@ -979,6 +982,10 @@ impl<'a> FunctionTranslator<'a> { false }; + // An extern `f32x4` result is written through memory (see + // `TypeID::extern_returns_via_pointer`) but lives in a + // vector register here, so the call branch reloads it. + let mut extern_vec_result: Option = None; let call = if let Some(sym) = math_sym { // Math builtin: use a direct call via declared function. let sig = fn_sig(&self.module, from, to); @@ -1037,10 +1044,29 @@ impl<'a> FunctionTranslator<'a> { sig.params.push(AbiParam::new(pty.cranelift_type())); } } - if !matches!(&*callee_decl.ret, crate::Type::Void) { - sig.returns - .push(AbiParam::new(callee_decl.ret.cranelift_type())); - } + // Tuples, structs, arrays, enums and f32x4 come back + // through a trailing output pointer and the C function + // returns void. The sret slot above already covers the + // indirect types; f32x4 needs its own since it is a + // register value everywhere else in this backend. + let extern_out = if to.extern_returns_via_pointer() { + sig.params.push(AbiParam::new(I64)); // output pointer + Some(output_slot.unwrap_or_else(|| { + let slot = self.builder.create_sized_stack_slot(StackSlotData { + kind: StackSlotKind::ExplicitSlot, + size: to.size(decls) as u32, + align_shift: 4, + key: None, + }); + self.builder.ins().stack_addr(I64, slot, 0) + })) + } else { + if !matches!(&*callee_decl.ret, crate::Type::Void) { + sig.returns + .push(AbiParam::new(callee_decl.ret.cranelift_type())); + } + None + }; sig.call_conv = cranelift_codegen::isa::CallConv::SystemV; let mut args = vec![context]; @@ -1087,8 +1113,16 @@ impl<'a> FunctionTranslator<'a> { args.push(arg_val); } } + if let Some(addr) = extern_out { + args.push(addr); + } let sref = self.builder.import_signature(sig); - self.builder.ins().call_indirect(sref, fn_ptr, &args) + let call = self.builder.ins().call_indirect(sref, fn_ptr, &args); + if let (Some(addr), crate::Type::Float32x4) = (extern_out, &*to) { + extern_vec_result = + Some(self.builder.ins().load(F32X4, MemFlags::new(), addr, 0)); + } + call } else if is_builtin { // Other builtins (assert, print, putc): indirect call. // assert needs the globals pointer as the first arg so @@ -1158,7 +1192,9 @@ impl<'a> FunctionTranslator<'a> { self.builder.ins().call_indirect(sref, fn_ptr, &args) }; - if let Some(addr) = output_slot { + if let Some(vec) = extern_vec_result { + vec + } else if let Some(addr) = output_slot { addr } else if let Some(result) = self.builder.inst_results(call).first() { *result diff --git a/src/llvm_jit.rs b/src/llvm_jit.rs index a9ea099c..5c32642f 100644 --- a/src/llvm_jit.rs +++ b/src/llvm_jit.rs @@ -1286,6 +1286,17 @@ impl<'a, 'ctx> FunctionTranslator<'a, 'ctx> { alloca } + /// A 16-byte aligned byte buffer for a by-pointer call result. Extern + /// hosts store through this pointer as `int32_t*`/`float*`/`double*`, and + /// C requires natural alignment for those stores. + fn aligned_ret_slot(&self, size: u64) -> PointerValue<'ctx> { + let slot = self.entry_array_alloca(self.i8_ty(), size, "ret_slot"); + if let Some(inst) = slot.as_instruction() { + inst.set_alignment(16).expect("alloca alignment"); + } + slot + } + /// Create an array alloca in the entry block. fn entry_array_alloca( &self, @@ -3161,8 +3172,7 @@ impl<'a, 'ctx> FunctionTranslator<'a, 'ctx> { // Allocate output slot if returning via pointer. let output_slot: Option> = if returns_via_pointer(to) { let sz = to.size(self.decls) as u64; - let slot = self.entry_array_alloca(self.i8_ty(), sz, "ret_slot"); - Some(slot) + Some(self.aligned_ret_slot(sz)) } else { None }; @@ -3295,7 +3305,20 @@ impl<'a, 'ctx> FunctionTranslator<'a, 'ctx> { param_tys.push(pty.llvm_basic_type(self.ctx()).into()); } } - let extern_fn_ty = if *to == crate::Type::Void || returns_via_pointer(to) { + // Tuples, structs, arrays, enums and f32x4 come back through a + // trailing output pointer and the C function returns void. The + // sret slot above already covers the indirect types; f32x4 is + // a vector value here so it gets its own slot. + let extern_out = if to.extern_returns_via_pointer() { + param_tys.push(self.ptr_ty().into()); // output pointer + Some( + output_slot + .unwrap_or_else(|| self.aligned_ret_slot(to.size(self.decls) as u64)), + ) + } else { + None + }; + let extern_fn_ty = if *to == crate::Type::Void || extern_out.is_some() { self.ctx().void_type().fn_type(¶m_tys, false) } else { to.llvm_basic_type(self.ctx()).fn_type(¶m_tys, false) @@ -3345,12 +3368,22 @@ impl<'a, 'ctx> FunctionTranslator<'a, 'ctx> { args.push(arg_val.into()); } } + if let Some(slot) = extern_out { + args.push(slot.into()); + } let call = self .builder() .build_indirect_call(extern_fn_ty, fn_ptr, &args, "extern_call") .unwrap(); - if let Some(slot) = output_slot { - slot.into() + if let Some(slot) = extern_out { + if is_llvm_value_type(to) { + // f32x4: reload the vector the host wrote. + self.builder() + .build_load(to.llvm_basic_type(self.ctx()), slot, "extern_vec") + .unwrap() + } else { + slot.into() + } } else { call.try_as_basic_value() .basic() diff --git a/src/stack_codegen.rs b/src/stack_codegen.rs index 438d8810..7432c766 100644 --- a/src/stack_codegen.rs +++ b/src/stack_codegen.rs @@ -2217,6 +2217,17 @@ impl<'a> FunctionTranslator<'a> { .get(&callee_name) .expect("extern function not in globals"); + // Aggregate results (tuples, structs, arrays) come back + // through a trailing output pointer and the C function + // returns void. + let ret_ty = self.expr_type(call_expr); + let output_slot = if ret_ty.extern_returns_via_pointer() { + let size = ret_ty.size(self.decls) as u32; + Some(self.alloc_memory(size)) + } else { + None + }; + // For extern calls, push C-level args. Slices expand // to (data_ptr, i32 len) at the host boundary. let mut c_arg_count: u8 = 0; @@ -2258,21 +2269,32 @@ impl<'a> FunctionTranslator<'a> { c_arg_count += 1; } } + if let Some(slot) = output_slot { + func.emit(StackOp::LocalAddr(slot)); + c_arg_count += 1; + } assert!( c_arg_count <= 8, "stack extern call has too many C-level arguments" ); - let ret_ty = self.expr_type(call_expr); func.emit(StackOp::CallExtern { globals_offset, args: c_arg_count, - ret: stack_extern_ret_type(ret_ty), + ret: if output_slot.is_some() { + StackExternRet::Void + } else { + stack_extern_ret_type(ret_ty) + }, }); // Bridge the return value back to the window the // surrounding codegen expects. The extern handler // pushes a zero placeholder for void returns so // translate_call's +1 invariant still holds. - if matches!(&*ret_ty, Type::Float32) { + if let Some(slot) = output_slot { + // Replace the placeholder with the result address. + func.emit(StackOp::Drop); + func.emit(StackOp::LocalAddr(slot)); + } else if matches!(&*ret_ty, Type::Float32) { func.emit(StackOp::BitsToFF); } else if matches!(&*ret_ty, Type::Float64) { func.emit(StackOp::BitsToDD); diff --git a/src/types.rs b/src/types.rs index b8a36694..3e0fc9fb 100644 --- a/src/types.rs +++ b/src/types.rs @@ -106,6 +106,32 @@ impl TypeID { Self(Intern::new(ty)) } + /// Whether an extern function result crosses the C boundary through a + /// trailing output pointer instead of a return register. + /// + /// This is the single definition of the host ABI rule: every backend + /// consults it, so a host function written against `lyte.h` works the + /// same on the JIT, LLVM, VM and stack backends. Tuples, structs, arrays + /// and enums are written through the pointer in Lyte layout. `f32x4` + /// also goes through the pointer (four floats) because vector return + /// registers are not portable across platforms or libffi. + pub fn extern_returns_via_pointer(self) -> bool { + matches!( + &*self, + Type::Tuple(_) | Type::Array(_, _) | Type::Name(_, _) | Type::Float32x4 + ) + } + + /// Whether a type can be the result of an extern function on every + /// backend. The scalar set is what libffi marshals in the VM; anything + /// in `extern_returns_via_pointer` is handed back through memory. + pub fn is_supported_extern_return(self) -> bool { + matches!( + &*self, + Type::Void | Type::Bool | Type::Int32 | Type::Float32 | Type::Float64 + ) || self.extern_returns_via_pointer() + } + /// Returns true if this type contains any unresolved type variable (Anon or Var). pub fn contains_var(self) -> bool { match &*self { diff --git a/src/vm.rs b/src/vm.rs index 807ca9f5..bcf12672 100644 --- a/src/vm.rs +++ b/src/vm.rs @@ -321,7 +321,7 @@ pub(crate) mod tags { } /// Maximum number of C-level arguments for an extern function (including context). -const MAX_EXTERN_ARGS: usize = 16; +pub(crate) const MAX_EXTERN_ARGS: usize = 16; /// Maximum call stack depth before the VM traps with "call stack overflow". /// Must be a hard cap — an unbounded Vec would grow until OOM. diff --git a/src/vm_codegen.rs b/src/vm_codegen.rs index e552778f..7ac7a47b 100644 --- a/src/vm_codegen.rs +++ b/src/vm_codegen.rs @@ -2578,8 +2578,30 @@ impl<'a> FunctionTranslator<'a> { for p in &f.params { c_param_types.extend(type_to_extern_types(p.ty.unwrap())); } - let ret_types = type_to_extern_types(f.ret); - let ret_type = ret_types[0]; + // Aggregate results (tuples, structs, arrays) come back + // through a trailing output pointer and the C function + // returns void. + let output_slot = if f.ret.extern_returns_via_pointer() { + c_param_types.push(crate::vm::ExternType::Ptr); + Some(self.alloc_local(f.ret.size(self.decls) as u32)) + } else { + None + }; + // libffi marshalling has a fixed argument budget that + // includes the context pointer. Catch it here rather + // than panicking on the first call at runtime. + assert!( + c_param_types.len() + 1 <= crate::vm::MAX_EXTERN_ARGS, + "extern function {} has too many C-level arguments ({} including context; max {})", + f.name, + c_param_types.len() + 1, + crate::vm::MAX_EXTERN_ARGS + ); + let ret_type = if output_slot.is_some() { + crate::vm::ExternType::Void + } else { + type_to_extern_types(f.ret)[0] + }; self.extern_funcs.push(crate::vm::ExternFuncInfo { globals_offset, param_types: c_param_types.clone(), @@ -2640,6 +2662,14 @@ impl<'a> FunctionTranslator<'a> { c_arg_count += 1; } } + if let Some(slot) = output_slot { + let addr_reg = self.alloc_reg(); + func.emit(Opcode::LocalAddr { + dst: addr_reg, + slot, + }); + c_arg_count += 1; + } func.emit(Opcode::CallExtern { args_start, @@ -2647,6 +2677,17 @@ impl<'a> FunctionTranslator<'a> { globals_offset, }); + // An aggregate result lives in the output slot; hand + // back its address like a regular sret call does. + if let Some(slot) = output_slot { + let addr_reg = self.alloc_reg(); + func.emit(Opcode::LocalAddr { + dst: addr_reg, + slot, + }); + return addr_reg; + } + let result_reg = self.alloc_reg(); func.emit(Opcode::Move { dst: result_reg, diff --git a/tests/cases/checker/extern_fn_ret_type.lyte b/tests/cases/checker/extern_fn_ret_type.lyte new file mode 100644 index 00000000..e649654f --- /dev/null +++ b/tests/cases/checker/extern_fn_ret_type.lyte @@ -0,0 +1,12 @@ +// Test: extern functions may only return types every backend can marshal. +// args: --check +// expected stdout: +// ❌ ../tests/cases/checker/extern_fn_ret_type.lyte:8:20: extern functions cannot return i8 (supported: void, bool, i32, f32, f64, f32x4, tuples, structs, arrays, enums) +// extern fn host_byte() -> i8 +// ^ + +extern fn host_byte() -> i8 + +fn main() { + var b = host_byte() +} diff --git a/tests/cases/extern_tuple_ret.lyte b/tests/cases/extern_tuple_ret.lyte new file mode 100644 index 00000000..d4ab39d1 --- /dev/null +++ b/tests/cases/extern_tuple_ret.lyte @@ -0,0 +1,21 @@ +// Test: extern functions may return tuples, structs, arrays, enums and +// f32x4; the host writes the result through a trailing output pointer. +// args: --check + +struct Pt { x: i32, y: i32 } +enum Mode { Off, On } + +extern fn host_pair(x: i32) -> (i32, i32) +extern fn host_pt(x: i32) -> Pt +extern fn host_arr(x: i32) -> [i32; 2] +extern fn host_mode() -> Mode +extern fn host_vec() -> f32x4 + +fn main() -> i32 { + var p = host_pair(3) + var q = host_pt(3) + var a = host_arr(3) + var m = host_mode() + var v = host_vec() + p.0 + p.1 + q.x + a[1] + (v[0] as i32) + (if m == .On { 1 } else { 0 }) +} From c2b07fb2cc95508114d9920b42f9744b394b4d91 Mon Sep 17 00:00:00 2001 From: Taylor Holliday Date: Sun, 18 Oct 2026 08:38:40 -0700 Subject: [PATCH 2/2] Reject extern parameter types no backend can marshal The VM's libffi path handles bool, i32, f32, f64, slices and references and panics in codegen on anything else, while the JIT silently passed a struct or f32x4 parameter as a pointer. Check the parameter types in the type checker so the same program is accepted or rejected everywhere, and document the supported set in lyte.h. Co-Authored-By: Claude Fable 5.1 --- Sources/CLyte/include/lyte.h | 1 + src/checker.rs | 25 ++++++++++++++++--- src/types.rs | 16 ++++++++++++ tests/cases/checker/extern_fn_param_type.lyte | 14 +++++++++++ 4 files changed, 53 insertions(+), 3 deletions(-) create mode 100644 tests/cases/checker/extern_fn_param_type.lyte diff --git a/Sources/CLyte/include/lyte.h b/Sources/CLyte/include/lyte.h index d04db3bc..629b87d1 100644 --- a/Sources/CLyte/include/lyte.h +++ b/Sources/CLyte/include/lyte.h @@ -125,6 +125,7 @@ void lyte_globals_bind_slice(uint8_t* globals, size_t offset, const void* data, /// The offset should come from lyte_program_get_global_offset for an extern function. /// The host function must use C calling conventions with a prepended void* context parameter. /// For example, `extern fn foo(i32) -> f32` expects: float (*)(void* ctx, int32_t) +/// Parameters may be bool, i32, f32, f64, a slice or a reference (a thin pointer). /// A slice parameter expands to a data pointer followed by an int32_t length: /// `extern fn send(buf: [i8]) -> bool` expects: bool (*)(void* ctx, const int8_t* data, int32_t len) /// Scalar results (bool, i32, f32, f64) are returned by value. Every other diff --git a/src/checker.rs b/src/checker.rs index c2c3a324..9c946271 100644 --- a/src/checker.rs +++ b/src/checker.rs @@ -1087,9 +1087,28 @@ impl Checker { }); } - // Extern results must be marshallable by every backend. Slices and - // references were already rejected above; this catches function - // types and the narrow integers libffi has no return slot for. + // Extern parameters and results must be marshallable by every + // backend, otherwise the VM panics in codegen on types the JIT + // happily passes as pointers. + if func_decl.is_extern { + for param in &func_decl.params { + if let Some(ty) = param.ty { + if !ty.is_supported_extern_param() { + self.errors.push(TypeError { + location: func_decl.loc, + message: format!( + "extern function parameter '{}' has unsupported type {} (supported: bool, i32, f32, f64, slices, references)", + *param.name, + ty.pretty_print() + ), + }); + } + } + } + } + // Slices and references were already rejected as results above; this + // catches function types and the narrow integers libffi has no + // return slot for. if func_decl.is_extern && !type_contains_bad_borrow(func_decl.ret) && !func_decl.ret.is_supported_extern_return() diff --git a/src/types.rs b/src/types.rs index 3e0fc9fb..c20d88cb 100644 --- a/src/types.rs +++ b/src/types.rs @@ -132,6 +132,22 @@ impl TypeID { ) || self.extern_returns_via_pointer() } + /// Whether a type can be an extern function parameter on every backend. + /// Scalars are passed by value, a slice expands to (data pointer, i32 + /// length) and a reference is a thin pointer. This is the set libffi + /// marshals in the VM; the other backends accept at least this much. + pub fn is_supported_extern_param(self) -> bool { + matches!( + &*self, + Type::Bool + | Type::Int32 + | Type::Float32 + | Type::Float64 + | Type::Slice(_) + | Type::Reference(_) + ) + } + /// Returns true if this type contains any unresolved type variable (Anon or Var). pub fn contains_var(self) -> bool { match &*self { diff --git a/tests/cases/checker/extern_fn_param_type.lyte b/tests/cases/checker/extern_fn_param_type.lyte new file mode 100644 index 00000000..a04274a0 --- /dev/null +++ b/tests/cases/checker/extern_fn_param_type.lyte @@ -0,0 +1,14 @@ +// Test: extern function parameters may only use types every backend can marshal. +// args: --check +// expected stdout: +// ❌ ../tests/cases/checker/extern_fn_param_type.lyte:10:20: extern function parameter 'p' has unsupported type Pt (supported: bool, i32, f32, f64, slices, references) +// extern fn host_draw(p: Pt, scale: f32) +// ^ + +struct Pt { x: f32, y: f32 } + +extern fn host_draw(p: Pt, scale: f32) + +fn main() { + host_draw(Pt(1.0, 2.0), 3.0) +}