From b86731ae17f1faabb2bc807a8939b517691a5574 Mon Sep 17 00:00:00 2001 From: sanil-23 Date: Wed, 19 Aug 2026 13:16:25 +0530 Subject: [PATCH] feat(adaptive): measure whether the learning is real MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This crate accumulates knowledge in five ways — a ledger, lessons with help rates, workflow scores, repaired variants, a promotion gate — and had no way to answer the only question that matters about any of it: does an episode go better because earlier episodes happened? Derived from the eval design in a sibling PoC, which had already worked out the hard part: **a success rate cannot answer it.** On ten unrelated tasks solved once each, a learning loop and a plain retry loop produce identical numbers. Neither can a single arm — "it solved six of six" is compatible with the ledger being decorative. What answers it is a family of related tasks run twice in the same order, differing only in whether anything survives between episodes, and the quantity to read is the slope: attempts-to-success falling as the family progresses. Two pieces, because that experiment needed two things the crate lacked. **`evals::slope`** — `Episode`, two arms, the fit, and a comparison that refuses to call a bend a win unless the arm also converges (twenty attempts falling to ten loses to a flat two). `Episode::of` reads an episode off the ledger rows the loop already writes, rather than off a parallel bookkeeping nobody uses. **`evals::arms`** — the control. A loop with learning off is *not* a loop with its ledger removed: this ledger is load-bearing within an episode, holding the exclusion list that stops attempt four repeating attempt two. `Forgetful` blanks exactly the cross-episode reads — lessons, evidence, workflow scores, repair lineage — and leaves the rest. Writes still happen and are never read, so the control arm still pays for consolidation; an arm that skipped the writes would look cheaper for a reason unrelated to learning. Three departures from the design as derived, each one a defect the running example found: **An arm that solved nothing must not tie with an arm that solved four.** The example produced exactly that and reported no evidence, because both slopes read `0.0` — one meaning "flat", the other meaning "two points are needed for a line and there was one". Going from never to once is the largest fall in attempts-to-success there is. `trend()` and `mean_attempts()` are now `Option`, the report emits `null` rather than a flattering zero, and `compare` returns a `Verdict` naming which test it won on: `SolvedMore`, `Converged`, or `NoEvidence`. `SolvedMore` is guarded by "no worse per solve" so it cannot decay into the success-rate metric this design exists to avoid. **A wrong answer the judge accepted is not a success.** `Episode` carries an outside check separately from the verdict, and `Outcome` names all four states — solved, refused (right, judge would not accept), wrong (judge accepted, world refuted), failed. `None` for "not checked" is distinct from `Some(true)`, so a report can say which runs were taken at the judge's word. **A failed episode is a gap, not a shift.** Wins are fitted at their original positions, so a wasted episode makes the measured convergence shallower — the family took an extra episode to cover the same ground. The test states the direction explicitly because the intuition runs the other way. No task set ships here. A family has to be chosen for shared technique and checkable answers, and belongs to whoever runs the eval; a list baked into a host-agnostic crate would be an opinion about somebody else's domain. The example carries a simulated one, and says so. --- crates/adaptive/examples/eval.rs | 279 ++++++++++++++++++ crates/adaptive/src/evals/arms.rs | 136 +++++++++ crates/adaptive/src/evals/mod.rs | 40 +++ crates/adaptive/src/evals/slope.rs | 399 ++++++++++++++++++++++++++ crates/adaptive/src/evals/tests.rs | 442 +++++++++++++++++++++++++++++ crates/adaptive/src/lib.rs | 1 + 6 files changed, 1297 insertions(+) create mode 100644 crates/adaptive/examples/eval.rs create mode 100644 crates/adaptive/src/evals/arms.rs create mode 100644 crates/adaptive/src/evals/mod.rs create mode 100644 crates/adaptive/src/evals/slope.rs create mode 100644 crates/adaptive/src/evals/tests.rs diff --git a/crates/adaptive/examples/eval.rs b/crates/adaptive/examples/eval.rs new file mode 100644 index 0000000..8b2b9e5 --- /dev/null +++ b/crates/adaptive/examples/eval.rs @@ -0,0 +1,279 @@ +//! Two arms over one task family, differing only in whether learning is on. +//! +//! Run it: +//! +//! ```text +//! cargo run -p tinyflows-adaptive --example eval +//! ``` +//! +//! The claim is not "it solves things" — a retry loop does that. The claim is +//! that attempts-to-success **declines across a family**, so the eval needs +//! repetition within a distribution or it measures nothing. Same tasks, same +//! order, both arms; one arm keeps what it learns between episodes and the +//! other starts each episode empty. +//! +//! Everything here is the real crate: a real [`Loop`], a real ledger, real +//! intake and closing. Two things are stand-ins, and both have to be — +//! +//! * **the model**, scripted so the arms differ by the variable under test and +//! not by sampling noise. A real eval points this at a provider and accepts +//! that a null result may be the model rather than the loop; +//! * **the runner**, which "solves" a task in fewer attempts when it has been +//! told how. That is the thing being simulated: a harness that benefits from +//! being handed a lesson. Point it at a real one and the shape is unchanged. +//! +//! What it demonstrates, in order: +//! +//! 1. the two arms — the same [`Loop`] wiring, with [`Forgetful`] and a fresh +//! store on the control side; +//! 2. an [`Episode`] read off what the loop already recorded, rather than off +//! a parallel bookkeeping nobody uses; +//! 3. the report, and the comparison that refuses to call a bend a win unless +//! the arm also converges. +//! +//! **`--live` is not implemented here on purpose.** A real family has to be +//! chosen for shared technique and checkable answers, and belongs to whoever +//! runs the eval; this crate is host-agnostic and a task list baked into it +//! would be an opinion about somebody else's domain. + +use std::sync::Arc; + +use async_trait::async_trait; +use serde_json::{Value, json}; +use tinyflows::caps::mock::mock_capabilities; +use tinyflows::caps::{Capabilities, LlmProvider}; +use tinyflows::error::Result as EngineResult; +use tinyflows::store::{FileWorkflowStore, WorkflowStore}; +use tinyflows_adaptive::contracts::{Budget, Goal}; +use tinyflows_adaptive::driver::{Clock, Loop}; +use tinyflows_adaptive::evals::{Episode, Experiment, Forgetful, LEARNING_OFF, LEARNING_ON}; +use tinyflows_adaptive::execute::{Ran, RunReport, Runner, StepOutcome, StepRecord}; +use tinyflows_adaptive::host::HostFacts; +use tinyflows_adaptive::intake::Attempt; +use tinyflows_adaptive::ledger::{Ledger, memory::MemoryLedger}; + +/// The family. One technique, several surfaces — which is what makes a lesson +/// about *approach* rather than about one task, and therefore what makes the +/// second episode able to benefit from the first. +const FAMILY: &str = "cache-and-build-upward"; + +const TASKS: [&str; 5] = [ + "the longest Collatz chain under a million", + "the number of routes through a 20x20 grid", + "the ways to make 200 pence from the usual coins", + "the ways 100 can be written as a sum of at least two positives", + "the first value expressible as a sum of primes in over five thousand ways", +]; + +struct Frozen; +impl Clock for Frozen { + fn now(&self) -> String { + "2026-01-01T00:00:00Z".to_string() + } +} + +/// A scripted model, and the one place the simulation lives. +/// +/// The judge accepts once the runner has done the work, the author writes a +/// one-step plan, and consolidation promotes the family's technique as a +/// lesson **only when it has actually been observed** — a lesson invented +/// before the evidence would make the control arm lose to a fabrication. +struct Scripted; + +#[async_trait] +impl LlmProvider for Scripted { + async fn complete(&self, request: Value, _conn: Option<&str>) -> EngineResult { + let prompt = request["messages"] + .as_array() + .and_then(|m| m.last()) + .and_then(|m| m["content"].as_str()) + .unwrap_or_default(); + // The lesson is only useful because the planner is shown it — this is + // the whole mechanism the eval is measuring, so it is worth seeing. + let knows_the_technique = prompt.contains("cache the sub-results"); + Ok(match request["tier"].as_str().unwrap_or_default() { + "select" => json!({ "workflow_id": null, "why": "nothing stored fits yet" }), + "author" => json!({ + "why": "compute it, then record the answer", + "declared": [], + "inputs": {}, + "steps": [{ + "id": "solve", + "run": if knows_the_technique { + "echo cached-and-built-upward" + } else { + "echo enumerated" + }, + }], + }), + "judge" => json!({ + "satisfied": prompt.contains("cached-and-built-upward"), + "blocker": "goal_not_met", + "gap": "the obvious enumeration does not finish", + "advanced": true, + }), + // `evidence` — the row numbers the prompt showed — and not + // `cites`. `consolidate` refuses a lesson with nothing behind it, + // so the wrong key means the lesson is dropped in silence and the + // treatment arm learns nothing. Both arms then read flat, which + // looks like a null result rather than a typo. + "consolidate" => json!({ + "lessons": [{ + "kind": "strategy", + "trigger": "a CPU-bound scan whose obvious enumeration blows up", + "mechanism": "the sub-results repeat, so recomputing them dominates", + "claim": "cache the sub-results and build upward", + "evidence": [0], + }], + "corroborate": [], + }), + _ => json!({}), + }) + } +} + +/// A runner that succeeds when the plan reflects the technique, and burns an +/// attempt when it does not. +/// +/// This is the simulated half: a real harness benefits from being told how to +/// approach a problem, and an eval with a runner that ignores its brief would +/// measure nothing whatever the loop did. +struct Simulated; + +#[async_trait] +impl Runner for Simulated { + async fn run(&self, attempt: &Attempt) -> Ran { + let script = attempt + .graph + .nodes + .iter() + .find_map(|node| node.config.get("source").and_then(Value::as_str)) + .unwrap_or_default() + .to_string(); + let worked = script.contains("cached-and-built-upward"); + RunReport { + steps: vec![StepRecord { + node_id: "solve".to_string(), + status: StepOutcome::Success, + output: json!({ "json": { "exit_code": 0, "stdout": script } }), + duration_ms: 1, + null_bindings: Vec::new(), + }], + changed: if worked { + "wrote the answer".to_string() + } else { + "the run did not finish in time".to_string() + }, + cost_usd: 0.25, + ..RunReport::default() + } + .into_ran(&attempt.graph) + } +} + +/// A workflow store under a scratch directory of its own. +/// +/// Cleared on the way in rather than on the way out, so a run that panicked +/// half way through does not quietly seed the next one — the treatment arm +/// starting with a workflow it did not earn would fake the entire result. +fn store(tag: &str) -> Arc { + let root = std::env::temp_dir().join(format!("adaptive-eval-{}-{tag}", std::process::id())); + let _ = std::fs::remove_dir_all(&root); + std::fs::create_dir_all(root.join("workflows")).expect("scratch dir"); + Arc::new(FileWorkflowStore::new( + vec![root.join("workflows")], + root.join("runs"), + )) +} + +/// Run one arm over the whole family, in order. +/// +/// The treatment arm keeps one ledger and one store across every episode. The +/// control arm gets [`Forgetful`] over the same ledger — so it still plans +/// within an episode — and a store per episode, because a kept workflow is +/// learning too and would otherwise leak across the boundary the arms are +/// supposed to differ on. +async fn arm(label: &str, experiment: &mut Experiment) { + let caps = Capabilities { + llm: Arc::new(Scripted), + ..mock_capabilities() + }; + let learning = label == LEARNING_ON; + let shared = MemoryLedger::new(); + let shared_store = store(&format!("{label}-kept")); + + for (index, task) in TASKS.iter().enumerate() { + let workflows = if learning { + shared_store.clone() + } else { + // A kept workflow is learning too, so the control arm gets a store + // per episode. Without this the arms would differ in less than + // they appear to, and the result would be worth nothing. + store(&format!("{label}-{index}")) + }; + let ledger: Box = if learning { + Box::new(shared.clone()) + } else { + Box::new(Forgetful::new(shared.clone())) + }; + + let engine = Loop { + ledger: ledger.as_ref(), + store: &workflows, + caps: &caps, + facts: &HostFacts::unknown(), + runner: &Simulated, + clock: &Frozen, + budget: Budget { + attempts: 4, + ..Budget::default() + }, + conn: None, + }; + + let episode = format!("{label}-{index}"); + let finished = engine + .run(&episode, &Goal::new(format!("Compute {task}."))) + .await + .expect("the episode runs"); + + // Read off the ledger rather than off the runner: the ledger is what + // the loop is claimed to learn from, so measuring anything else would + // measure a bookkeeping nobody uses. + let rows = shared.rows(&episode).await.expect("rows"); + let measured = Episode::of(&format!("t{index}"), FAMILY, &finished, &rows); + println!( + " {}. {:<62} {:?} attempts={} ${:.2}", + index + 1, + task, + measured.outcome(), + measured.attempts, + measured.cost_usd + ); + if std::env::var("EVAL_DEBUG").is_ok() { + let known = shared.lessons(None).await.expect("lessons"); + eprintln!( + " [debug] lessons now: {} {:?}", + known.len(), + known.iter().map(|l| l.claim.as_str()).collect::>() + ); + } + experiment.record(label, measured); + } +} + +#[tokio::main] +async fn main() { + let mut experiment = Experiment::new(FAMILY); + for label in [LEARNING_ON, LEARNING_OFF] { + println!("\n{}\n{label}\n{}", "=".repeat(76), "=".repeat(76)); + arm(label, &mut experiment).await; + } + + println!("\n{}\nREPORT\n{}", "=".repeat(76), "=".repeat(76)); + println!( + "{}", + serde_json::to_string_pretty(&experiment.report()).expect("report") + ); + println!("\nlearning_helps: {}", experiment.learning_helps()); +} diff --git a/crates/adaptive/src/evals/arms.rs b/crates/adaptive/src/evals/arms.rs new file mode 100644 index 0000000..c5679a5 --- /dev/null +++ b/crates/adaptive/src/evals/arms.rs @@ -0,0 +1,136 @@ +//! The control arm: a loop that still retries, and still records, but is told +//! nothing about episodes other than its own. +//! +//! An experiment needs the two arms to differ in **one** thing. That thing is +//! not "does it retry" — the episode ledger is not what is under test, and an +//! arm that could not retry would lose for the wrong reason. It is whether +//! anything survives *to the next episode*. +//! +//! Which makes the control arm awkward here in a way it is not elsewhere: this +//! loop's ledger is load-bearing **within** an episode. It holds the exclusion +//! list that stops attempt four repeating attempt two, and the stall counter +//! that decides when to stand down. Dropping it would not turn learning off; +//! it would turn the loop into something that cannot plan. +//! +//! So [`Forgetful`] blanks exactly the cross-episode reads and leaves +//! everything else alone: +//! +//! | Read | On | Off | +//! | --- | --- | --- | +//! | `rows` / `episode` / `steps` — this episode | real | real | +//! | `lessons` / `evidence` — other episodes | real | empty | +//! | `workflow_score` — evidence from prior runs | real | 0/0 | +//! | `lineage` / `parent_of` / `children_of` — repair families | real | empty | +//! +//! **Writes still happen.** The control arm consolidates, promotes and scores +//! exactly as the treatment arm does, and then never reads any of it. That is +//! deliberate: if the off arm skipped the writes it would also skip their +//! model calls, and `cost_per_solve` would favour it for a reason that has +//! nothing to do with learning. Isolating recall is the point. +//! +//! **The workflow store is the caller's half.** Learned graphs live in the +//! store, not the ledger, so an off arm must also be handed a store that is +//! not carried between episodes — otherwise the second episode can select what +//! the first one kept, and the arms differ in less than they appear to. There +//! is no wrapper for that because there is nothing to wrap: give the arm a +//! fresh store per episode. + +use async_trait::async_trait; + +use crate::execute::StepRecord; +use crate::ledger::{Episode, Ledger, LedgerRow, Lesson, LessonKind, Page, Result, Score}; + +/// A [`Ledger`] that answers every cross-episode question with nothing. +/// +/// Wraps a real one so the control arm's episodes are still recorded — the +/// experiment reads those rows to compute cost and workflow use, so an arm +/// that wrote nowhere could not be measured. +pub struct Forgetful { + inner: L, +} + +impl Forgetful { + /// Wrap `inner`, keeping its within-episode behaviour and blanking the + /// rest. + pub const fn new(inner: L) -> Self { + Self { inner } + } + + /// The ledger underneath, for reading what the arm recorded after the fact. + pub const fn inner(&self) -> &L { + &self.inner + } +} + +#[async_trait] +impl Ledger for Forgetful { + async fn append(&self, row: &LedgerRow) -> Result { + self.inner.append(row).await + } + + async fn rows(&self, episode: &str) -> Result> { + // This episode's own attempts. Blanking these would stop the loop + // planning, not stop it learning. + self.inner.rows(episode).await + } + + async fn promote(&self, lesson: &Lesson, cites: &[String]) -> Result { + // Written and never read — see the module doc on why the control arm + // still pays for consolidation. + self.inner.promote(lesson, cites).await + } + + async fn lessons(&self, _kind: Option) -> Result> { + Ok(Vec::new()) + } + + async fn evidence(&self, _lesson_id: &str) -> Result> { + Ok(Vec::new()) + } + + async fn score_lesson(&self, lesson_id: &str, helped: bool) -> Result<()> { + self.inner.score_lesson(lesson_id, helped).await + } + + async fn score_workflow(&self, workflow_id: &str, helped: bool) -> Result<()> { + self.inner.score_workflow(workflow_id, helped).await + } + + async fn workflow_score(&self, _workflow_id: &str) -> Result { + // Not "never run" as a lie — in this arm it genuinely has no record + // the planner is entitled to see. + Ok(Score::default()) + } + + async fn link_variant(&self, parent: &str, variant: &str) -> Result<()> { + self.inner.link_variant(parent, variant).await + } + + async fn parent_of(&self, _id: &str) -> Result> { + Ok(None) + } + + async fn children_of(&self, _id: &str) -> Result> { + Ok(Vec::new()) + } + + async fn save_episode(&self, episode: &Episode) -> Result<()> { + self.inner.save_episode(episode).await + } + + async fn episode(&self, id: &str) -> Result> { + self.inner.episode(id).await + } + + async fn episodes(&self, running_only: bool, page: Page) -> Result> { + self.inner.episodes(running_only, page).await + } + + async fn save_steps(&self, row_id: &str, steps: &[StepRecord]) -> Result<()> { + self.inner.save_steps(row_id, steps).await + } + + async fn steps(&self, row_id: &str) -> Result> { + self.inner.steps(row_id).await + } +} diff --git a/crates/adaptive/src/evals/mod.rs b/crates/adaptive/src/evals/mod.rs new file mode 100644 index 0000000..d9fefc5 --- /dev/null +++ b/crates/adaptive/src/evals/mod.rs @@ -0,0 +1,40 @@ +//! Measuring whether the learning is real. +//! +//! Everything else in this crate is machinery for accumulating knowledge — a +//! ledger, lessons, scores, repaired variants, a promotion gate. None of it +//! answers the only question that matters about any of it: **does an episode +//! go better because earlier episodes happened?** +//! +//! That question has a shape, and getting the shape wrong is easy enough that +//! it is worth stating before any code. A success rate cannot answer it: on +//! ten unrelated tasks solved once each, a learning loop and a plain retry +//! loop produce identical numbers. Neither can a single arm: "it solved six of +//! six" is compatible with the ledger being decorative. What answers it is a +//! **family** of related tasks, run twice in the same order, where the only +//! difference between the runs is whether anything survives between episodes — +//! and the number to read is the [`Series::slope`]: attempts-to-success +//! falling as the family progresses. +//! +//! Two pieces, because that experiment needs two things this crate did not +//! have: +//! +//! * [`slope`] — the measurement. `Episode`, two arms, the slope, and a +//! comparison that refuses to call a bend a win unless the arm also +//! converges. +//! * [`arms`] — the control. A loop with learning off is not a loop with its +//! ledger removed; [`Forgetful`] blanks the cross-episode reads and leaves +//! the within-episode ones, which is the difference under test. +//! +//! What is deliberately *not* here is a task set. A family has to be chosen +//! for shared technique and checkable answers, and belongs to whoever runs the +//! eval — this crate is host-agnostic, and a Project Euler list baked into it +//! would be an opinion about the host's domain. + +mod arms; +mod slope; + +#[cfg(test)] +mod tests; + +pub use arms::Forgetful; +pub use slope::{Episode, Experiment, LEARNING_OFF, LEARNING_ON, Outcome, Series, Verdict}; diff --git a/crates/adaptive/src/evals/slope.rs b/crates/adaptive/src/evals/slope.rs new file mode 100644 index 0000000..4a97feb --- /dev/null +++ b/crates/adaptive/src/evals/slope.rs @@ -0,0 +1,399 @@ +//! The measured quantity: attempts-to-success **declining across a family**. +//! +//! Not the success rate. On disjoint tasks a learning loop and a plain retry +//! loop produce identical numbers — both solve ten unrelated problems once +//! each — so an eval without within-distribution repetition proves nothing +//! about learning, whatever its success rate says. +//! +//! So the eval is two arms over one family of related tasks, in the same +//! order, differing only in whether anything survives between episodes, and +//! the number that decides it is the **slope**. + +use std::collections::BTreeMap; + +use serde_json::{Value, json}; + +use crate::driver::Finished; +use crate::ledger::{EpisodeStatus, LedgerRow}; + +/// How an episode actually ended, once an outside check has its say. +/// +/// Three states, not two, and the difference is not academic: the first +/// version of this collapsed the last two and reported a run that produced the +/// wrong answer as a success because the judge had accepted it. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Outcome { + /// The judge was satisfied and nothing outside contradicted it. + Solved, + /// Right, but the judge would not accept it. A judging defect, not a + /// solving one — and counting it as a failure hides that. + Refused, + /// The judge was satisfied and the answer is wrong. The most expensive + /// state to be blind to, because everything downstream believes it. + Wrong, + /// It did not solve the task and nothing says otherwise. + Failed, +} + +/// One episode of one arm, as the measurement sees it. +#[derive(Debug, Clone, PartialEq)] +pub struct Episode { + /// Which task, so a family can be read back task by task. + pub task: String, + /// The family it belongs to. Two arms are only comparable within one. + pub family: String, + /// Attempts the loop spent. + pub attempts: u32, + /// What the judge concluded. + pub satisfied: bool, + /// An outside check on the work, when the eval has one — the published + /// answer, a test suite, a diff that must exist. + /// + /// `None` means the eval cannot check, which is honest and common. It is + /// *not* the same as `Some(true)`: an unchecked run is taken at the + /// judge's word, and the report says which runs those were. + pub verified: Option, + /// What the arm spent on this episode, in the host's unit. Zero means not + /// measured. + pub cost_usd: f64, + /// Whether a stored workflow served it, rather than one being authored. + pub used_workflow: bool, + /// How many lessons were put in front of a planner during it. + pub lessons_applied: u32, +} + +impl Episode { + /// Read an episode off what the loop already records. + /// + /// `rows` are that episode's ledger rows. Everything here comes from them + /// or from [`Finished`] rather than from the runner, because the ledger is + /// what the loop is claimed to learn from — measuring anything else would + /// measure a parallel bookkeeping nobody uses. + #[must_use] + pub fn of(task: &str, family: &str, finished: &Finished, rows: &[LedgerRow]) -> Self { + Self { + task: task.to_string(), + family: family.to_string(), + attempts: finished.attempts, + satisfied: matches!(finished.status, EpisodeStatus::Satisfied), + verified: None, + cost_usd: rows.iter().map(|row| row.cost_usd).sum(), + // A stored workflow served this episode if any attempt named one. + // The last attempt is the one that succeeded, but an earlier + // workflow attempt is still the store having been used. + used_workflow: rows.iter().any(|row| row.workflow_id.is_some()), + lessons_applied: 0, + } + } + + /// Record an outside check on the work. + #[must_use] + pub fn checked(mut self, verified: bool) -> Self { + self.verified = Some(verified); + self + } + + /// How many lessons the planners were shown across this episode. + #[must_use] + pub fn with_lessons_applied(mut self, applied: u32) -> Self { + self.lessons_applied = applied; + self + } + + /// Whether this counts as a success for the measurement. + /// + /// The judge, unless an outside check says otherwise. A run the judge + /// accepted and the world refuted is not a data point about converging + /// faster; it is a data point about the judge. + #[must_use] + pub fn solved(&self) -> bool { + self.satisfied && self.verified != Some(false) + } + + /// The four-way reading of how it ended. + #[must_use] + pub fn outcome(&self) -> Outcome { + match (self.satisfied, self.verified) { + (true, Some(false)) => Outcome::Wrong, + (true, _) => Outcome::Solved, + (false, Some(true)) => Outcome::Refused, + (false, _) => Outcome::Failed, + } + } +} + +/// One arm of the experiment — learning on, or learning off. +#[derive(Debug, Clone, Default)] +pub struct Series { + /// What to call this arm in the report. + pub label: String, + /// Its episodes, in the order they ran. Order is the independent variable, + /// so this is a sequence and never a set. + pub episodes: Vec, +} + +impl Series { + /// An empty arm under `label`. + #[must_use] + pub fn new(label: impl Into) -> Self { + Self { + label: label.into(), + episodes: Vec::new(), + } + } + + /// How many episodes counted as solved. + #[must_use] + pub fn solved(&self) -> usize { + self.episodes.iter().filter(|e| e.solved()).count() + } + + /// Mean attempts over the episodes that solved, or `None` if none did. + /// + /// Failures are excluded because an attempt count on a task that never + /// succeeded is a budget, not a measurement of how quickly it converged. + /// + /// `None` rather than `0.0`, and the distinction is load-bearing: zero is + /// the *best possible* attempt count, so an arm that solved nothing would + /// beat every arm that solved something. Absence has to be absence. + #[must_use] + pub fn mean_attempts(&self) -> Option { + let wins: Vec = self + .episodes + .iter() + .filter(|e| e.solved()) + .map(|e| f64::from(e.attempts)) + .collect(); + if wins.is_empty() { + return None; + } + Some(wins.iter().sum::() / wins.len() as f64) + } + + /// Total spend divided by episodes solved. + /// + /// The numerator is *every* episode's cost, including the ones that failed: + /// spending on failures is real spending, and an arm that burns three + /// episodes to win one has not won cheaply. + #[must_use] + pub fn cost_per_solve(&self) -> f64 { + let solved = self.solved(); + if solved == 0 { + return 0.0; + } + self.episodes.iter().map(|e| e.cost_usd).sum::() / solved as f64 + } + + /// The fraction of episodes a stored workflow served. + #[must_use] + pub fn workflow_hit_rate(&self) -> f64 { + if self.episodes.is_empty() { + return 0.0; + } + let hits = self.episodes.iter().filter(|e| e.used_workflow).count(); + hits as f64 / self.episodes.len() as f64 + } + + /// How many episodes ended each way. + #[must_use] + pub fn outcomes(&self) -> BTreeMap<&'static str, usize> { + let mut counts: BTreeMap<&'static str, usize> = + [("solved", 0), ("refused", 0), ("wrong", 0), ("failed", 0)] + .into_iter() + .collect(); + for episode in &self.episodes { + let key = match episode.outcome() { + Outcome::Solved => "solved", + Outcome::Refused => "refused", + Outcome::Wrong => "wrong", + Outcome::Failed => "failed", + }; + *counts.entry(key).or_insert(0) += 1; + } + counts + } + + /// Least-squares slope of attempts against position in the sequence, or + /// `None` when fewer than two episodes solved. + /// + /// Negative means converging faster as the family progresses, which is the + /// entire claim. A flat line is a retry loop wearing a learning costume. + /// + /// Fitted over the solved episodes only, at their *original* positions — + /// a failure in the middle is a gap in the sequence, not a shift of + /// everything after it. + /// + /// `None` rather than `0.0` for the same reason `mean_attempts` is + /// optional. Two points make a line and one makes an anecdote, and + /// reporting the anecdote as "flat" reads as *evidence of no learning* + /// when it is the absence of evidence either way — which is exactly how + /// an arm that solved nothing came to tie with an arm that solved four. + #[must_use] + pub fn trend(&self) -> Option { + let wins: Vec<(f64, f64)> = self + .episodes + .iter() + .enumerate() + .filter(|(_, e)| e.solved()) + .map(|(i, e)| (i as f64, f64::from(e.attempts))) + .collect(); + if wins.len() < 2 { + return None; + } + let mean_x = wins.iter().map(|(x, _)| x).sum::() / wins.len() as f64; + let mean_y = wins.iter().map(|(_, y)| y).sum::() / wins.len() as f64; + let denom: f64 = wins.iter().map(|(x, _)| (x - mean_x).powi(2)).sum(); + if denom == 0.0 { + return None; + } + Some( + wins.iter() + .map(|(x, y)| (x - mean_x) * (y - mean_y)) + .sum::() + / denom, + ) + } +} + +/// Two arms over the same family, differing only in whether learning is on. +#[derive(Debug, Clone, Default)] +pub struct Experiment { + /// The family both arms ran. + pub family: String, + /// The arms, by label. Insertion-ordered for a stable report. + pub arms: BTreeMap, +} + +/// What an experiment concluded, and on which evidence. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Verdict { + /// The treatment arm solved episodes the control could not, without + /// costing more attempts on the ones both solved. + SolvedMore, + /// Both arms converged and the treatment arm converged faster, ending up + /// no worse in attempts. + Converged, + /// No difference the evidence can support — including a tie, and including + /// too few solved episodes to fit anything. + NoEvidence, +} + +/// The conventional label for the arm that keeps what it learns. +pub const LEARNING_ON: &str = "learning_on"; +/// The conventional label for the arm that starts each episode empty. +pub const LEARNING_OFF: &str = "learning_off"; + +impl Experiment { + /// An experiment over `family` with no episodes yet. + #[must_use] + pub fn new(family: impl Into) -> Self { + Self { + family: family.into(), + arms: BTreeMap::new(), + } + } + + /// Add an episode to an arm, creating the arm on first use. + pub fn record(&mut self, label: &str, episode: Episode) { + self.arms + .entry(label.to_string()) + .or_insert_with(|| Series::new(label)) + .episodes + .push(episode); + } + + /// The claim, stated as a comparison rather than an absolute. + #[must_use] + pub fn learning_helps(&self) -> bool { + matches!( + self.compare(LEARNING_ON, LEARNING_OFF), + Verdict::Converged | Verdict::SolvedMore + ) + } + + /// Why the comparison came out the way it did. + /// + /// Two ways an arm can show that attempts-to-success fell, and the first + /// version of this only knew one of them. Running the eval produced a + /// treatment arm that solved four of five against a control that solved + /// **none**, and it reported no evidence — because both slopes read as + /// `0.0`, one meaning "flat" and the other meaning "two points are needed + /// for a line and there was one". Going from *never* to *once* is the + /// largest fall in attempts-to-success there is; a measure that calls it a + /// tie is measuring the wrong quantity. + /// + /// So, in order: + /// + /// 1. **Solved more.** Strictly more episodes solved, and no worse per + /// solve where both arms have a figure. Solving what the other arm + /// cannot is convergence in the only sense that matters, and no slope + /// is needed to see it. + /// 2. **Converged.** Both arms have a real trend, the treatment arm's is + /// lower, and it also ends up no worse in attempts — a curve that bends + /// but never converges is not a win: twenty falling to ten loses to a + /// flat two. + /// + /// Anything else is [`Verdict::NoEvidence`], including a tie, which is the + /// answer disjoint tasks must produce. + #[must_use] + pub fn compare(&self, on: &str, off: &str) -> Verdict { + let (Some(on), Some(off)) = (self.arms.get(on), self.arms.get(off)) else { + // One arm compares with nothing, and an experiment that claims a + // win from a single arm is the failure this module exists to stop. + return Verdict::NoEvidence; + }; + // "No worse per solve" only binds when both arms have solved + // something. An arm with no wins has no attempt count, and treating + // its absence as zero would let it out-argue every arm that worked. + let no_worse = match (on.mean_attempts(), off.mean_attempts()) { + (Some(a), Some(b)) => a <= b, + _ => true, + }; + if on.solved() > off.solved() && no_worse { + return Verdict::SolvedMore; + } + match (on.trend(), off.trend()) { + (Some(a), Some(b)) if a < b && no_worse => Verdict::Converged, + _ => Verdict::NoEvidence, + } + } + + /// Every claim the experiment can speak to, as JSON. + #[must_use] + pub fn report(&self) -> Value { + let arms: serde_json::Map = self + .arms + .iter() + .map(|(label, series)| { + ( + label.clone(), + json!({ + "solved": series.solved(), + "episodes": series.episodes.len(), + "mean_attempts": series.mean_attempts().map(|v| round(v, 3)), + "slope": series.trend().map(|v| round(v, 4)), + "cost_per_solve": round(series.cost_per_solve(), 4), + "workflow_hit_rate": round(series.workflow_hit_rate(), 3), + "outcomes": series.outcomes(), + }), + ) + }) + .collect(); + json!({ + "family": self.family, + "learning_helps": self.learning_helps(), + "verdict": match self.compare(LEARNING_ON, LEARNING_OFF) { + Verdict::SolvedMore => "solved_more", + Verdict::Converged => "converged", + Verdict::NoEvidence => "no_evidence", + }, + "arms": arms, + }) + } +} + +/// Round for a report a person reads. Never used in a comparison. +fn round(value: f64, places: u32) -> f64 { + let factor = 10_f64.powi(places as i32); + (value * factor).round() / factor +} diff --git a/crates/adaptive/src/evals/tests.rs b/crates/adaptive/src/evals/tests.rs new file mode 100644 index 0000000..73d0e72 --- /dev/null +++ b/crates/adaptive/src/evals/tests.rs @@ -0,0 +1,442 @@ +//! Testing the measure, because a broken slope would let a null result look +//! like a win — and a null result that looks like a win is worse than no eval, +//! since it ends the work that would have found the truth. + +use super::{Episode, Experiment, Forgetful, LEARNING_OFF, LEARNING_ON, Outcome, Series, Verdict}; + +fn ep(index: usize, attempts: u32) -> Episode { + Episode { + task: format!("t{index}"), + family: "euler".to_string(), + attempts, + satisfied: true, + verified: None, + cost_usd: 0.0, + used_workflow: false, + lessons_applied: 0, + } +} + +fn series(label: &str, attempts: &[u32]) -> Series { + Series { + label: label.to_string(), + episodes: attempts + .iter() + .enumerate() + .map(|(i, a)| ep(i, *a)) + .collect(), + } +} + +fn experiment(on: &[u32], off: &[u32]) -> Experiment { + let mut experiment = Experiment::new("euler"); + for (i, a) in on.iter().enumerate() { + experiment.record(LEARNING_ON, ep(i, *a)); + } + for (i, a) in off.iter().enumerate() { + experiment.record(LEARNING_OFF, ep(i, *a)); + } + experiment +} + +#[test] +fn a_converging_series_has_a_negative_slope() { + assert!( + series("learning_on", &[5, 4, 3, 2, 1]) + .trend() + .expect("a fit") + < 0.0 + ); +} + +#[test] +fn a_flat_series_has_no_slope() { + // A retry loop wearing a learning costume. + assert!( + series("learning_off", &[3, 3, 3, 3]) + .trend() + .expect("a fit") + .abs() + < f64::EPSILON + ); +} + +#[test] +fn a_worsening_series_has_a_positive_slope() { + assert!(series("bad", &[1, 2, 3, 4]).trend().expect("a fit") > 0.0); +} + +#[test] +fn a_single_episode_cannot_show_a_trend() { + // Two points make a line; one makes an anecdote. + assert_eq!(series("x", &[4]).trend(), None, "one point is an anecdote"); + assert_eq!(series("x", &[]).trend(), None); +} + +#[test] +fn unsolved_episodes_are_excluded_from_the_slope() { + // Attempts on a task that never succeeded say nothing about convergence — + // that number is a budget being spent, not a distance being closed. + let mut mixed = series("x", &[5, 9, 1]); + mixed.episodes[1].satisfied = false; + assert!(mixed.trend().expect("a fit") < 0.0); + assert_eq!(mixed.solved(), 2); +} + +#[test] +fn a_failure_is_a_gap_in_the_sequence_not_a_shift_of_what_follows() { + // A win keeps the position it actually happened at, so the fit measures + // convergence *per episode of the family* — including the episodes that + // bought nothing. + // + // Which makes a failure count against the arm, and it is worth being + // explicit about the direction because the intuition runs the other way: + // preserving the gap makes the slope SHALLOWER, not steeper. Six attempts + // falling to two over four episodes is slower learning than the same fall + // over three, and re-indexing the wins 0,1,2 would report the faster + // number for the worse run. + let mut with_gap = series("x", &[6, 9, 4, 2]); + with_gap.episodes[1].satisfied = false; + let without_the_wasted_episode = series("x", &[6, 4, 2]); + + let gapped = with_gap.trend().expect("a fit"); + let dense = without_the_wasted_episode.trend().expect("a fit"); + assert!(gapped < 0.0, "still converging"); + assert!( + gapped > dense, + "the wasted episode must cost the arm: {gapped} should be shallower than {dense}" + ); +} + +#[test] +fn the_claim_is_a_comparison_not_an_absolute() { + assert!(experiment(&[5, 4, 2, 1], &[4, 4, 4, 4]).learning_helps()); +} + +#[test] +fn learning_does_not_win_on_a_steeper_slope_alone() { + // A curve that bends but never converges is not a win: twenty attempts + // falling to ten loses to a flat two. + assert!(!experiment(&[20, 15, 10], &[2, 2, 2]).learning_helps()); +} + +#[test] +fn disjoint_tasks_cannot_distinguish_the_arms() { + // The warning that motivates the whole eval design. With no repetition + // both arms look identical, and a success-rate metric would report the tie + // as evidence of something. + assert!(!experiment(&[3, 3, 3], &[3, 3, 3]).learning_helps()); +} + +#[test] +fn a_missing_arm_never_claims_a_win() { + let mut one_arm = Experiment::new("euler"); + for (i, a) in [5, 1].iter().enumerate() { + one_arm.record(LEARNING_ON, ep(i, *a)); + } + assert!(!one_arm.learning_helps(), "one arm compares with nothing"); + assert!(!Experiment::new("euler").learning_helps()); +} + +#[test] +fn cost_per_solve_counts_failed_episodes_against_the_arm() { + // Spending on failures is real spending, and an arm that burns three + // episodes to win one has not won cheaply. + let mut arm = series("x", &[1, 9]); + arm.episodes[0].cost_usd = 1.0; + arm.episodes[1].cost_usd = 3.0; + arm.episodes[1].satisfied = false; + assert!((arm.cost_per_solve() - 4.0).abs() < 1e-9); +} + +#[test] +fn an_arm_that_never_solved_reports_no_cost_per_solve_rather_than_infinity() { + let mut arm = series("x", &[9, 9]); + for episode in &mut arm.episodes { + episode.satisfied = false; + episode.cost_usd = 2.0; + } + assert_eq!(arm.cost_per_solve(), 0.0); + assert_eq!( + arm.mean_attempts(), + None, + "no wins is not an attempt count of zero — zero would beat every real arm" + ); +} + +#[test] +fn workflow_hit_rate_tracks_whether_the_store_is_being_used() { + let mut arm = series("x", &[2, 1]); + arm.episodes[1].used_workflow = true; + assert!((arm.workflow_hit_rate() - 0.5).abs() < 1e-9); + assert_eq!(Series::new("empty").workflow_hit_rate(), 0.0); +} + +#[test] +fn an_answer_the_judge_accepted_and_the_world_refuted_is_not_a_solve() { + // The state that must never be folded into success: everything downstream + // believes a run the judge passed, so a wrong one is the most expensive + // thing to be blind to. + let wrong = ep(0, 1).checked(false); + assert_eq!(wrong.outcome(), Outcome::Wrong); + assert!(!wrong.solved(), "the judge does not get the last word"); + + let mut arm = Series::new("x"); + arm.episodes.push(wrong); + assert_eq!(arm.solved(), 0); + assert_eq!(arm.trend(), None, "nothing solved, nothing to fit"); +} + +#[test] +fn right_but_refused_is_reported_apart_from_failing() { + // A judging defect, not a solving one. Counting it as a plain failure + // hides which half of the loop to go and fix. + let mut refused = ep(0, 3); + refused.satisfied = false; + let refused = refused.checked(true); + assert_eq!(refused.outcome(), Outcome::Refused); + assert!(!refused.solved()); + + let mut failed = ep(1, 3); + failed.satisfied = false; + assert_eq!(failed.outcome(), Outcome::Failed); + + let arm = Series { + label: "x".to_string(), + episodes: vec![refused, failed], + }; + let counts = arm.outcomes(); + assert_eq!(counts["refused"], 1); + assert_eq!(counts["failed"], 1); +} + +#[test] +fn an_unchecked_episode_is_taken_at_the_judges_word_but_says_so() { + // `None` is not `Some(true)`: it means the eval could not check. Both + // count as solved; only the report can tell them apart, which is the point + // of keeping the field three-valued. + let unchecked = ep(0, 2); + assert_eq!(unchecked.verified, None); + assert!(unchecked.solved()); + assert_eq!(unchecked.outcome(), Outcome::Solved); +} + +#[test] +fn the_report_carries_every_claim() { + let mut experiment = experiment(&[4, 2], &[3, 3]); + experiment.arms.get_mut(LEARNING_ON).expect("arm").episodes[1].used_workflow = true; + let report = experiment.report(); + + assert_eq!(report["family"], "euler"); + assert_eq!(report["learning_helps"], true); + let on = &report["arms"][LEARNING_ON]; + for key in [ + "solved", + "episodes", + "mean_attempts", + "slope", + "cost_per_solve", + "workflow_hit_rate", + "outcomes", + ] { + assert!(!on[key].is_null(), "report is missing {key}: {on}"); + } + assert!(on["slope"].as_f64().expect("a number") < 0.0); + assert!((on["workflow_hit_rate"].as_f64().expect("a number") - 0.5).abs() < 1e-9); +} + +#[test] +fn solving_what_the_other_arm_cannot_is_the_largest_win_available() { + // The case that exposed the defect this verdict exists for. A live run + // produced exactly this — treatment solved four of five, control solved + // NONE — and the first version reported no evidence, because both slopes + // read `0.0`: one meaning "flat", the other meaning "one point is not a + // line". Going from never to once is the biggest fall in + // attempts-to-success there is. + let mut experiment = Experiment::new("euler"); + for (i, attempts) in [4, 1, 1, 1, 1].iter().enumerate() { + let mut episode = ep(i, *attempts); + episode.satisfied = i > 0; + experiment.record(LEARNING_ON, episode); + } + for i in 0..5 { + let mut episode = ep(i, 4); + episode.satisfied = false; + experiment.record(LEARNING_OFF, episode); + } + + assert_eq!( + experiment.compare(LEARNING_ON, LEARNING_OFF), + Verdict::SolvedMore + ); + assert!(experiment.learning_helps()); + assert_eq!(experiment.report()["verdict"], "solved_more"); +} + +#[test] +fn solving_more_does_not_win_if_each_solve_costs_more() { + // The guard that keeps "solved more" from becoming a success-rate metric + // in disguise: an arm that solves twice as many at ten attempts each has + // not learned anything the control's two-attempt solves did not know. + let mut experiment = Experiment::new("euler"); + for (i, attempts) in [10, 10, 10, 10].iter().enumerate() { + experiment.record(LEARNING_ON, ep(i, *attempts)); + } + for i in 0..4 { + let mut episode = ep(i, 2); + episode.satisfied = i < 2; + experiment.record(LEARNING_OFF, episode); + } + + assert_eq!( + experiment.compare(LEARNING_ON, LEARNING_OFF), + Verdict::NoEvidence, + "more solves at a worse attempt cost is not learning" + ); +} + +#[test] +fn a_converging_arm_reports_which_test_it_won_on() { + let experiment = experiment(&[5, 4, 2, 1], &[4, 4, 4, 4]); + assert_eq!( + experiment.compare(LEARNING_ON, LEARNING_OFF), + Verdict::Converged + ); + assert_eq!(experiment.report()["verdict"], "converged"); +} + +#[test] +fn a_tie_reports_no_evidence_by_name() { + let experiment = experiment(&[3, 3, 3], &[3, 3, 3]); + assert_eq!( + experiment.compare(LEARNING_ON, LEARNING_OFF), + Verdict::NoEvidence + ); + assert_eq!(experiment.report()["verdict"], "no_evidence"); +} + +#[test] +fn an_arm_with_no_wins_reports_null_rather_than_a_flattering_zero() { + // What the report says has to survive being read by somebody who did not + // write it: `"slope": 0.0` on an arm that solved nothing reads as "no + // learning observed", which is a stronger claim than the data supports. + let mut experiment = Experiment::new("euler"); + for i in 0..3 { + let mut episode = ep(i, 4); + episode.satisfied = false; + experiment.record(LEARNING_OFF, episode); + } + let arm = &experiment.report()["arms"][LEARNING_OFF]; + assert!(arm["slope"].is_null(), "{arm}"); + assert!(arm["mean_attempts"].is_null(), "{arm}"); + assert_eq!(arm["solved"], 0); +} + +// --------------------------------------------------------------------------- +// The control arm. +// --------------------------------------------------------------------------- + +use crate::ledger::{Ledger, LedgerRow, Lesson, LessonKind, memory::MemoryLedger}; + +fn row(episode: &str, workflow: Option<&str>) -> LedgerRow { + LedgerRow { + id: String::new(), + episode: episode.to_string(), + attempt: 1, + approach_sig: "selected:sweep".to_string(), + approach_desc: "it matched".to_string(), + workflow_id: workflow.map(ToString::to_string), + outcome: "satisfied".to_string(), + cause: String::new(), + cost_usd: 0.5, + at: "2026-01-01T00:00:00Z".to_string(), + satisfied: true, + advanced: true, + } +} + +#[tokio::test] +async fn the_control_arm_still_sees_its_own_episode() { + // Not a loop with its ledger removed. The exclusion list and the stall + // counter live in these rows, and an arm that could not plan would lose + // the experiment for a reason that has nothing to do with learning. + let ledger = Forgetful::new(MemoryLedger::new()); + ledger + .append(&row("ep-1", Some("sweep"))) + .await + .expect("append"); + + let rows = ledger.rows("ep-1").await.expect("rows"); + assert_eq!(rows.len(), 1, "this episode's attempts are still visible"); + assert_eq!(rows[0].workflow_id.as_deref(), Some("sweep")); + assert_eq!( + ledger.tried("ep-1").await.expect("tried"), + vec!["selected:sweep".to_string()], + "so attempt two does not repeat attempt one" + ); +} + +#[tokio::test] +async fn the_control_arm_is_told_nothing_that_outlived_another_episode() { + let ledger = Forgetful::new(MemoryLedger::new()); + let id = ledger + .promote( + &Lesson { + id: String::new(), + kind: LessonKind::Strategy, + trigger: "a CPU-bound scan over ~1M items".to_string(), + mechanism: "the obvious enumeration is quadratic".to_string(), + claim: "cache the sub-results and build upward".to_string(), + applied: 0, + helped: 0, + scope_key: None, + }, + &[], + ) + .await + .expect("promote"); + ledger.score_workflow("sweep", true).await.expect("score"); + + // Written, and then unreadable — which is exactly the variable under test. + assert!( + ledger.lessons(None).await.expect("lessons").is_empty(), + "a planner in this arm recalls nothing from other episodes" + ); + assert!(ledger.evidence(&id).await.expect("evidence").is_empty()); + let score = ledger.workflow_score("sweep").await.expect("score"); + assert_eq!( + (score.applied, score.helped), + (0, 0), + "and weighs no record it did not earn this episode" + ); + + // The writes did land underneath, so the arm paid for consolidation just + // as the treatment arm did — `cost_per_solve` must not favour it for + // skipping work rather than for learning. + assert_eq!(ledger.inner().lessons(None).await.expect("under").len(), 1); +} + +#[tokio::test] +async fn the_control_arm_sees_no_repair_lineage() { + // A variant is knowledge from a previous episode as much as a lesson is, + // and `collapse_families` reads lineage to decide what the planner is + // offered. + let ledger = Forgetful::new(MemoryLedger::new()); + ledger + .link_variant("sweep", "sweep-fix") + .await + .expect("link"); + assert_eq!(ledger.parent_of("sweep-fix").await.expect("parent"), None); + assert!( + ledger + .children_of("sweep") + .await + .expect("children") + .is_empty() + ); + assert_eq!( + ledger.inner().parent_of("sweep-fix").await.expect("under"), + Some("sweep".to_string()), + "recorded underneath, just not offered" + ); +} diff --git a/crates/adaptive/src/lib.rs b/crates/adaptive/src/lib.rs index 9107b51..91e848d 100644 --- a/crates/adaptive/src/lib.rs +++ b/crates/adaptive/src/lib.rs @@ -17,6 +17,7 @@ pub mod closing; pub mod contracts; pub mod driver; +pub mod evals; pub mod execute; pub mod host; pub mod intake;