perf: errors build only when raised, recursion keeps its caches, and calls skip the name scan

Author: dylan-sutton-chavez

bench/.snapshot

@@ -30,11 +30,11 @@ Each `Instruction` is 4 bytes. That is a 1-byte `OpCode` (`#[repr(u8)]`), a 2-by
 
 ## Dispatch shape
 
-The hot loop reads `cache.fused_ref()[ip]`, a snapshot of the instruction stream with `LoadAttr` + `Call` sites fused into `CallMethod` + `CallMethodArgs`. Fusion happens after first dispatch when every argument is a single-push load, up to 8 arguments, never across a jump target. The arg loads shift down one slot so the pair sits adjacent at the call. Fusion runs once per chunk and is cached.
+The hot loop reads `cache.fused_ref()[ip]`, a snapshot of the instruction stream with `LoadAttr` + `Call` sites fused into `CallMethod` + `CallMethodArgs`. Fusion happens after first dispatch when every argument is a single-push load, up to 8 arguments, never across a jump target. The arg loads shift down one slot so the pair sits adjacent at the call. Fusion runs once per cache, and a recursion holds one cache per depth.
 
 A build with the `coverage` feature marks every instruction the loop dispatches, and `vm.ran(chunk, ip)` says whether `chunk.instructions[ip]` ran, still after a run that raised. Fusion stays off in that build, so each ip is the one the compiler wrote, and a chunk answers under the same pointer the caller built, an imported module under its entry in `chunk.imports`.
 
-For arith and compare opcodes the loop checks `cache.get_fast(ip)`. A present `FastOp` runs inline without a function call.
+For arith and compare opcodes `exec_arith_or_compare` checks `cache.get_fast(ip)` first. A present `FastOp` runs inside that call and skips the generic handler.
 
 `LoadConst` reads a pre-materialised `Vec<Val>` built on first dispatch. Inline-range ints stay inline. Ints from 2⁴⁷ to 2¹²⁷ allocate a `HeapObj::LongInt` slot. Literals beyond ±2¹²⁷ are rejected at parse time.
 

docs/04-implementation/01-design.mdx

@@ -230,6 +230,9 @@ impl SSAChunk {
         i
     }
 
+    /* The slot of `n` in this chunk, None when the chunk never named it. */
+    pub(crate) fn slot_of(&self, n: &str) -> Option<u16> { self.name_index.get(n).copied() }
+
     /* Builds `prev_slots`, coalesces SSA versions to canonical root, rewrites operands, builds `phi_map`. */
     pub fn finalize_prev_slots(&mut self) {
         let n = self.names.len();

src/parser/types.rs

@@ -40,6 +40,37 @@ struct CacheSlot {
     inst: Option<InstanceCache>,
 }
 
+/* One chunk's caches, each running frame holds one and returns it here. */
+#[derive(Default)]
+pub struct CachePool {
+    active: u32,
+    main: Option<OpcodeCache>,
+    /* Only a recursion fills it, so a plain call never allocates here. */
+    spare: Vec<OpcodeCache>,
+}
+
+impl CachePool {
+    pub fn take(&mut self, chunk: &SSAChunk) -> OpcodeCache {
+        self.active += 1;
+        self.main.take().or_else(|| self.spare.pop()).unwrap_or_else(|| OpcodeCache::new(chunk))
+    }
+
+    /* The outermost frame returning keeps one cache and frees the rest. */
+    pub fn put(&mut self, cache: OpcodeCache) {
+        self.active -= 1;
+        if self.active == 0 {
+            self.spare.clear();
+            self.main = Some(cache);
+        } else if self.main.is_none() {
+            self.main = Some(cache);
+        } else {
+            self.spare.push(cache);
+        }
+    }
+
+    pub fn caches(&self) -> impl Iterator<Item = &OpcodeCache> { self.main.iter().chain(&self.spare) }
+}
+
 pub struct OpcodeCache {
     slots: Vec<CacheSlot>,
     fused: Option<Vec<Instruction>>,

src/vm/cache.rs

@@ -155,11 +155,11 @@ impl<'a> VM<'a> {
         let delta_base = self.pending.delta_save.len();
         let key = chunk as *const _;
 
-        let mut cache = self.opcode_caches.remove(&key).unwrap_or_else(|| OpcodeCache::new(chunk));
+        let mut cache = self.opcode_caches.entry(key).or_default().take(chunk);
         cache.ensure_fused(chunk);
         // Pre-materialise the constant pool here (not in OpcodeCache::new) because Str allocates into the live HeapPool.
         if let Err(e) = cache.ensure_const_vals(chunk, &mut self.heap) {
-            self.opcode_caches.insert(key, cache);
+            self.park_cache(key, cache);
             return Err(e);
         }
 
@@ -290,10 +290,15 @@ impl<'a> VM<'a> {
         self.active_const_pools.pop();
         self.active_slots.pop();
         self.frame_safe = outer_safe;
-        self.opcode_caches.insert(key, cache);
+        self.park_cache(key, cache);
         result
     }
 
+    /* Hands a frame's cache back to its chunk's pool. */
+    fn park_cache(&mut self, key: *const SSAChunk, cache: OpcodeCache) {
+        if let Some(pool) = self.opcode_caches.get_mut(&key) { pool.put(cache); }
+    }
+
     pub(crate) fn exec_from(&mut self, chunk: &SSAChunk, slots: &mut [Val], start_ip: usize) -> Result<Val, VmErr> {
         self.resume_ip = start_ip;
         self.exec(chunk, slots)
@@ -386,12 +391,12 @@ impl<'a> VM<'a> {
         match ins.opcode {
             // Short-circuit jumps, instance `__bool__` / `__len__` may run via `truthy_op`.
             OpCode::JumpIfFalseOrPop => {
-                let v = *self.stack.last().ok_or(cold_runtime("stack underflow"))?;
+                let v = *self.stack.last().ok_or_else(|| cold_runtime("stack underflow"))?;
                 if !self.truthy_op(v, chunk, slots)? { *ip = op as usize; }
                 else { self.pop()?; }
             }
             OpCode::JumpIfTrueOrPop => {
-                let v = *self.stack.last().ok_or(cold_runtime("stack underflow"))?;
+                let v = *self.stack.last().ok_or_else(|| cold_runtime("stack underflow"))?;
                 if self.truthy_op(v, chunk, slots)? { *ip = op as usize; }
                 else { self.pop()?; }
             }
@@ -450,23 +455,26 @@ impl<'a> VM<'a> {
                 }
             }
             OpCode::LoadGlobal => {
-                let name = chunk.names.get(op as usize).ok_or(cold_runtime("LoadGlobal: name index out of bounds"))?;
+                let name = chunk.names.get(op as usize).ok_or_else(|| cold_runtime("LoadGlobal: name index out of bounds"))?;
                 let v = self.module_state.get(name.as_str()).copied().or_else(|| self.global(name)).filter(|v| !v.is_undef());
                 self.push(v.ok_or_else(|| VmErr::Name(name.clone()))?);
             }
             OpCode::StoreGlobal => {
                 let v = self.pop()?;
-                let name = chunk.names.get(op as usize).ok_or(cold_runtime("StoreGlobal: name index out of bounds"))?;
+                let name = chunk.names.get(op as usize).ok_or_else(|| cold_runtime("StoreGlobal: name index out of bounds"))?;
                 // A `global` store rebinds a name some cached result may have read.
                 self.templates.clear();
                 self.note_builtin_binding(name);
                 self.globals_written = true;
-                self.module_state.insert(name.clone(), v);
+                match self.module_state.get_mut(name.as_str()) {
+                    Some(s) => *s = v,
+                    None => { self.module_state.insert(name.clone(), v); }
+                }
             }
             OpCode::LoadConst => {
                 // Constants are pre-materialised at exec entry, so this is a single bounds-checked index instead of a Value->Val conversion.
                 let v = *consts.get(op as usize)
-                    .ok_or(cold_runtime("constant index out of bounds"))?;
+                    .ok_or_else(|| cold_runtime("constant index out of bounds"))?;
                 self.push(v);
             }
 
@@ -593,8 +601,8 @@ impl<'a> VM<'a> {
             OpCode::StoreAttr => self.exec_store_attr(op, chunk, slots)?,
 
             OpCode::LoadModule => {
-                let entry = chunk.imports.get(op as usize).ok_or(cold_runtime("LoadModule: import index out of range"))?;
-                let v = *self.module_table.get(&entry.spec).ok_or(cold_runtime("LoadModule: module not initialised"))?;
+                let entry = chunk.imports.get(op as usize).ok_or_else(|| cold_runtime("LoadModule: import index out of range"))?;
+                let v = *self.module_table.get(&entry.spec).ok_or_else(|| cold_runtime("LoadModule: module not initialised"))?;
                 self.push(v);
             }
 
@@ -611,12 +619,12 @@ impl<'a> VM<'a> {
             OpCode::MatMul => self.handle_matmul(operand, chunk, slots)?,
             OpCode::MakeTypeAlias => {
                 let value = self.pop()?;
-                let name = chunk.names.get(operand as usize).ok_or(cold_runtime("MakeTypeAlias: bad name index"))?.clone();
+                let name = chunk.names.get(operand as usize).ok_or_else(|| cold_runtime("MakeTypeAlias: bad name index"))?.clone();
                 let alias = self.heap.alloc(HeapObj::TypeAlias(name, value))?;
                 self.push(alias);
             }
             OpCode::MakeTypeVar => {
-                let name = chunk.names.get(operand as usize).ok_or(cold_runtime("MakeTypeVar: bad name index"))?.clone();
+                let name = chunk.names.get(operand as usize).ok_or_else(|| cold_runtime("MakeTypeVar: bad name index"))?.clone();
                 let v = self.heap.alloc(HeapObj::TypeVar(name))?;
                 self.push(v);
             }
@@ -641,7 +649,7 @@ impl<'a> VM<'a> {
                 self.push(v);
             }
             OpCode::Dup => {
-                let v = *self.stack.last().ok_or(cold_runtime("stack underflow"))?;
+                let v = *self.stack.last().ok_or_else(|| cold_runtime("stack underflow"))?;
                 self.push(v);
             }
             OpCode::MatchSeq => {
@@ -697,7 +705,7 @@ impl<'a> VM<'a> {
             OpCode::BeginFinally => self.unwind_stack.push(Unwind::Normal),
             // Stages `__exit__` for the plain Call behind it, parking-safe.
             OpCode::WithExit => {
-                let cm = self.with_stack.pop().ok_or(cold_runtime("WithExit without matching WithEnter"))?;
+                let cm = self.with_stack.pop().ok_or_else(|| cold_runtime("WithExit without matching WithEnter"))?;
                 let (exit_fn, class) = self.with_dunder(cm, "__exit__")?;
                 // Reraise selects `__exit__(type, exc, None)`, other exits pass three Nones.
                 let (exc_type, exc) = if matches!(self.unwind_stack.last(), Some(Unwind::Reraise(..))) {
@@ -1009,7 +1017,7 @@ impl<'a> VM<'a> {
     fn exec_store_attr(&mut self, op: u16, chunk: &SSAChunk, slots: &mut [Val]) -> Result<(), VmErr> {
         let value = self.pop()?;
         let obj = self.pop()?;
-        let name = chunk.names.get(op as usize).ok_or(cold_runtime("StoreAttr: bad name index"))?;
+        let name = chunk.names.get(op as usize).ok_or_else(|| cold_runtime("StoreAttr: bad name index"))?;
         self.store_attr(obj, name, value, chunk, slots)
     }
 

src/vm/dispatch.rs

@@ -52,7 +52,7 @@ impl<'a> VM<'a> {
         let heap = &mut self.heap; // split borrow, lets closures take &mut heap while iterating other fields
         for frame in &self.iter_stack { frame.for_each_val(&mut |v| heap.mark(v)); }
         for sf in &self.pending_sync_frames { sf.for_each_val(&mut |v| heap.mark(v)); }
-        for cache in self.opcode_caches.values() {
+        for cache in self.opcode_caches.values().flat_map(|p| p.caches()) {
             if let Some(consts) = cache.const_vals_opt() {
                 for &v in consts { self.heap.mark(v); }
             }

src/vm/gc.rs

@@ -470,7 +470,7 @@ impl<'a> VM<'a> {
     /* Suspend until `s` real seconds elapse. */
     pub fn call_sleep(&mut self) -> Result<(), VmErr> {
         let n = self.pop()?;
-        let secs = num_as_f64(n, &self.heap).ok_or(cold_type("sleep() requires a number"))?.max(0.0);
+        let secs = num_as_f64(n, &self.heap).ok_or_else(|| cold_type("sleep() requires a number"))?.max(0.0);
         let until = self.now_ns().saturating_add((secs * 1_000_000_000.0) as u64);
         self.pending.sleep_until_ns = Some(until);
         // Push None as the yield value, the scheduler ignores it.
@@ -500,7 +500,7 @@ impl<'a> VM<'a> {
         if !matches!(self.heap.try_get(coro), Some(HeapObj::Coroutine(..))) {
             return Err(cold_type("with_timeout() requires a coroutine"));
         }
-        let secs = num_as_f64(secs_v, &self.heap).ok_or(cold_type("with_timeout() seconds must be a number"))?;
+        let secs = num_as_f64(secs_v, &self.heap).ok_or_else(|| cold_type("with_timeout() seconds must be a number"))?;
         let deadline_ns = self.now_ns().saturating_add((secs.max(0.0) * 1_000_000_000.0) as u64);
         self.schedule(&[coro]);
         self.park_on(vec![coro], WaitKind::Timeout { deadline_ns, target: coro });

src/vm/globals/async_ops.rs

@@ -56,7 +56,7 @@ impl<'a> VM<'a> {
     /* `del obj.attr` opcode, pop the object, remove the named attribute in place. */
     pub fn exec_del_attr(&mut self, op: u16, chunk: &crate::parser::SSAChunk) -> Result<(), VmErr> {
         let obj = self.pop()?;
-        let name = chunk.names.get(op as usize).ok_or(cold_runtime("DelAttr: bad name index"))?.clone();
+        let name = chunk.names.get(op as usize).ok_or_else(|| cold_runtime("DelAttr: bad name index"))?.clone();
         self.delete_attr_named(obj, &name)
     }
 

src/vm/globals/attr.rs

@@ -215,7 +215,7 @@ impl<'a> VM<'a> {
     pub fn call_chr(&mut self) -> Result<(), VmErr> {
         let o = self.pop()?;
         if !o.is_int() { return Err(cold_type("chr() requires an integer")); }
-        let c = char::from_u32(o.as_int() as u32).ok_or(cold_value("chr() arg out of range"))?;
+        let c = char::from_u32(o.as_int() as u32).ok_or_else(|| cold_value("chr() arg out of range"))?;
         let mut s = String::with_capacity(4);
         s.push(c);
         self.alloc_and_push_str(s)
@@ -262,7 +262,7 @@ impl<'a> VM<'a> {
             }
             // Ints/bools round to themselves, negative ndigits round to tens/hundreds.
             (Some(o), n) if o.is_bool() || o.is_int() || (o.is_heap() && matches!(self.heap.get(o), HeapObj::LongInt(_))) => {
-                let i = if o.is_bool() { o.as_bool() as i128 } else { self.as_i128(o).ok_or(cold_type("round() requires a number"))? };
+                let i = if o.is_bool() { o.as_bool() as i128 } else { self.as_i128(o).ok_or_else(|| cold_type("round() requires a number"))? };
                 let nd = match n { Some(n) if n.is_int() => n.as_int(), _ => 0 };
                 let r = if nd < 0 { round_int_banker(i, (-nd) as u32) } else { i };
                 self.int_to_val(Some(r))?
@@ -396,7 +396,7 @@ impl<'a> VM<'a> {
                 // Modular exponentiation (a ** b) % c on i128.
                 let (Some(base), Some(modulus)) = (self.as_i128(args[0]), self.as_i128(args[2]))
                 else { return Err(cold_type("pow() with 3 args requires integers")); };
-                let exp = self.as_i128(args[1]).ok_or(cold_type("pow() with 3 args requires integer exponent"))?;
+                let exp = self.as_i128(args[1]).ok_or_else(|| cold_type("pow() with 3 args requires integer exponent"))?;
                 if exp < 0 { return Err(cold_value("pow() exponent must be non-negative")); }
                 if modulus == 0 { return Err(VmErr::ZeroDiv); }
 
@@ -438,11 +438,11 @@ impl<'a> VM<'a> {
                 let mut e = exp;
                 while e > 0 {
                     if e & 1 == 1 {
-                        result = result.checked_mul(base).ok_or(cold_overflow())?;
+                        result = result.checked_mul(base).ok_or_else(cold_overflow)?;
                     }
                     e >>= 1;
                     if e > 0 {
-                        base = base.checked_mul(base).ok_or(cold_overflow())?;
+                        base = base.checked_mul(base).ok_or_else(cold_overflow)?;
                     }
                 }
                 return self.int_to_val(Some(result));

src/vm/globals/numeric.rs

@@ -80,13 +80,13 @@ impl<'a> VM<'a> {
     #[inline] pub(crate) fn push(&mut self, v: Val) { self.stack.push(v); }
 
     #[inline] pub(crate) fn pop(&mut self) -> Result<Val, VmErr> {
-        self.stack.pop().ok_or(cold_runtime("stack underflow"))
+        self.stack.pop().ok_or_else(|| cold_runtime("stack underflow"))
     }
     #[inline] pub(crate) fn pop2(&mut self) -> Result<(Val, Val), VmErr> {
         let b = self.pop()?; let a = self.pop()?; Ok((a, b))
     }
     #[inline] pub(crate) fn pop_n(&mut self, n: usize) -> Result<Vec<Val>, VmErr> {
-        let at = self.stack.len().checked_sub(n).ok_or(cold_runtime("stack underflow"))?;
+        let at = self.stack.len().checked_sub(n).ok_or_else(|| cold_runtime("stack underflow"))?;
         Ok(self.stack.split_off(at))
     }