From f85a9d15d14bfb81e1a0136d68c4290afcc2d730 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Thu, 20 Aug 2026 22:08:38 -0700 Subject: [PATCH 1/3] feat(calibration): reject a contract a trivial baseline satisfies A milestone contract says which progressions count. It does not say that reaching them is hard. deriveContract proves only that every mark fires on the reference run, so a derived contract can pin a memory channel that moves whenever the game runs at all. A constant button press then earns exactly what an evaluated agent earns, and the score measures elapsed frames. calibrateContract replays the reference and a suite of trivial policies through the same attestRun path: one constant policy per input word, a word the game cannot interpret, a round-robin cycle over the vocabulary, and a seeded pseudo-random walk over it. Every policy is deterministic in the seed. A contract separates only when at least one milestone is out of reach of every baseline and the reference verifies strictly more milestones than the strongest baseline. assertContractSeparates fails closed otherwise and names every trivial milestone with the baseline that earned it. The module stays pure, synchronous, and free of adapter and provider imports. --- authoring.ts | 17 +++ calibration.ts | 205 +++++++++++++++++++++++++++++++++++++ index.ts | 1 + scripts/check-boundary.mjs | 1 + 4 files changed, 224 insertions(+) create mode 100644 calibration.ts diff --git a/authoring.ts b/authoring.ts index ccd88ab..689bac9 100644 --- a/authoring.ts +++ b/authoring.ts @@ -33,6 +33,23 @@ function checkHolds(check: MilestoneCheck, evidence: Evidence): boolean { } } +/** + * Derive a milestone contract from one demonstrated trajectory. + * + * The result is a HYPOTHESIS, not a benchmark. This function proves only that + * every mark fires on the reference run and that the contract validates. It + * cannot know whether the progressions it pinned are hard to reach: a mark + * anchored on a memory channel that moves whenever the game runs at all yields + * a contract that a constant button press satisfies exactly as well as an + * evaluated agent does. + * + * Blind-discovery marks are the sharp case, because nothing in the pipeline + * ever asserts that a discovered channel means progress. + * + * Run `calibrateContract` from calibration.ts on the derived contract and gate + * publication with `assertContractSeparates`. A contract that no trivial policy + * can satisfy is a benchmark; an uncalibrated one is a guess. + */ export function deriveContract( game: Game, seed: number, diff --git a/calibration.ts b/calibration.ts new file mode 100644 index 0000000..4f7d603 --- /dev/null +++ b/calibration.ts @@ -0,0 +1,205 @@ +/** + * Contract calibration — does this contract measure skill, or elapsed frames? + * + * A milestone contract says which progressions count. It does not say that + * reaching them is hard. A contract derived from one demonstrated trajectory + * can pin a memory channel that moves whenever the game runs at all, so a + * constant button press earns the same milestones an evaluated agent earns. + * Such a contract is not a benchmark: it separates nothing. + * + * This module replays the reference trajectory and a suite of trivial policies + * through the same `attestRun` path, then reports which milestones survive the + * comparison. `assertContractSeparates` is the fail-closed gate a target author + * calls before publishing a contract. + * + * Everything here is pure and synchronous, and depends only on the runtime, + * schema, and attestation planes. It imports no adapter and no model provider. + */ +import { attestRun } from './attestation' +import { logFrom } from './runtime' +import type { Game } from './runtime' +import type { MilestoneContract } from './schema' + +/** + * A deterministic input policy that needs no observation of the game. + * + * `inputs` must be a pure function of its three arguments: the same + * vocabulary, turn count, and seed must always produce the same script, so a + * calibration report is reproducible by anyone holding the contract. + */ +export interface BaselinePolicy { + id: string + inputs(vocabulary: readonly string[], turns: number, seed: number): readonly string[] +} + +/** + * A word outside every adapter vocabulary. Unknown inputs are no-ops by the + * `Game` contract, so this policy measures what mere elapsed time earns. + */ +export const UNKNOWN_BASELINE_WORD = 'playproof-unknown-word' + +const constant = (word: string): BaselinePolicy => ({ + id: `constant:${word}`, + inputs: (_vocabulary, turns) => Array.from({ length: turns }, () => word), +}) + +/** + * Linear congruential generator (Numerical Recipes constants) over 32 bits. + * The seed is mixed once so seed 0 is not a degenerate starting state. Not a + * statistically strong generator; it only has to be unpredictable to the + * contract and identical on every machine. + */ +function lcg(seed: number): () => number { + let state = ((seed >>> 0) ^ 0x9e3779b9) >>> 0 + return () => { + state = (Math.imul(state, 1664525) + 1013904223) >>> 0 + return state + } +} + +/** + * The standard suite: one constant policy per input word, a word the game + * cannot interpret, a fixed cycle through the vocabulary, and a seeded + * pseudo-random walk over it. + * + * The per-word constants are load-bearing. On Libbet, only `constant:a` and + * `constant:start` reach the milestones the evaluated agent reached; a suite + * that pressed one representative button would have reported the contract + * healthy. + * + * The vocabulary is a required argument because the constant family cannot be + * built without it. Extra policies can be appended and passed to + * `calibrateContract` through `options.baselines`. + */ +export function trivialBaselines(vocabulary: readonly string[]): BaselinePolicy[] { + return [ + ...vocabulary.map(constant), + constant(UNKNOWN_BASELINE_WORD), + { + id: 'round-robin', + inputs: (words, turns) => Array.from({ length: turns }, (_unused, i) => words[i % words.length]!), + }, + { + id: 'pseudo-random', + inputs: (words, turns, seed) => { + const next = lcg(seed) + return Array.from({ length: turns }, () => words[next() % words.length]!) + }, + }, + ] +} + +export interface BaselineOutcome { + id: string + verified: string[] + verdict: 'clean' | 'rejected' +} + +export interface CalibrationReport { + turns: number + seed: number + vocabulary: string[] + reference: BaselineOutcome + baselines: BaselineOutcome[] + /** milestones no baseline earned */ + separating: string[] + /** milestones at least one baseline earned */ + trivial: string[] + /** the strongest baseline's verified count */ + bestBaselineCount: number + separates: boolean +} + +export interface CalibrateOptions { + /** The demonstrated trajectory the contract was derived from or claims to describe. */ + reference: readonly string[] + /** Replay seed for every run, and the seed handed to each policy. Default 0. */ + seed?: number + /** Every input word the adapter accepts. */ + vocabulary: readonly string[] + /** Defaults to `trivialBaselines(vocabulary)`. */ + baselines?: readonly BaselinePolicy[] + /** Inputs per run. Default: the reference length, so the comparison is length-matched. */ + turns?: number +} + +/** + * Replay the reference and every baseline against the same contract and report + * which milestones the baselines cannot reach. + * + * Every run is played at `turns` inputs; the reference is truncated to that + * length. Raising `turns` above the reference length gives the baselines a + * larger budget than the reference had, which can only weaken the separating + * set — useful when the reference is short and the evaluated agent was not. + * + * The seed is used twice on purpose: it is the replay seed handed to + * `attestRun` and the seed each policy derives its script from, so one number + * reproduces the whole report. + */ +export function calibrateContract( + game: Game, + contract: MilestoneContract, + options: CalibrateOptions, +): CalibrationReport { + const seed = options.seed ?? 0 + const turns = options.turns ?? options.reference.length + const vocabulary = [...options.vocabulary] + if (!Number.isInteger(turns) || turns < 0) throw new Error(`turns must be a non-negative integer, got ${turns}`) + if (vocabulary.length === 0) throw new Error('a contract cannot be calibrated against an empty input vocabulary') + const policies = options.baselines ?? trivialBaselines(vocabulary) + + const play = (id: string, inputs: readonly string[]): BaselineOutcome => { + const attestation = attestRun(game, contract, seed, logFrom(seed, [...inputs]), []) + return { id, verified: attestation.verified, verdict: attestation.verdict } + } + + const reference = play('reference', options.reference.slice(0, turns)) + const baselines = policies.map((policy) => { + const inputs = policy.inputs(vocabulary, turns, seed) + if (inputs.length !== turns) { + throw new Error(`baseline ${policy.id} produced ${inputs.length} inputs for ${turns} turns`) + } + return play(policy.id, inputs) + }) + + const earnedByBaseline = new Set(baselines.flatMap((b) => b.verified)) + const order = contract.milestones.map((m) => m.id) + const separating = reference.verified.filter((id) => !earnedByBaseline.has(id)) + const trivial = order.filter((id) => earnedByBaseline.has(id)) + const bestBaselineCount = baselines.reduce((best, b) => Math.max(best, b.verified.length), 0) + + return { + turns, + seed, + vocabulary, + reference, + baselines, + separating, + trivial, + bestBaselineCount, + separates: separating.length > 0 && reference.verified.length > bestBaselineCount, + } +} + +/** + * Fail closed on a contract that a trivial policy satisfies. + * + * Call this wherever a target is published. A contract that does not separate + * still produces scores; those scores report how many frames elapsed, and + * comparing two agents on them compares nothing. + */ +export function assertContractSeparates(report: CalibrationReport): void { + if (report.separates) return + const best = report.baselines.filter((b) => b.verified.length === report.bestBaselineCount).map((b) => b.id) + const earners = (milestone: string): string => + report.baselines.filter((b) => b.verified.includes(milestone)).map((b) => b.id).join(', ') + const lines = [ + `contract does not separate: the reference verified ${report.reference.verified.length} milestone(s) ` + + `and the best trivial baseline verified ${report.bestBaselineCount} over ${report.turns} turns ` + + `(seed ${report.seed}, strongest: ${best.join(', ') || 'none'})`, + ...report.trivial.map((id) => ` trivial: ${id} — earned by ${earners(id)}`), + ` out of reach of every baseline: ${report.separating.join(', ') || 'nothing'}`, + 'A derived contract is a hypothesis until it separates. Pin a progression a trivial policy cannot reach.', + ] + throw new Error(lines.join('\n')) +} diff --git a/index.ts b/index.ts index 3d968be..32291c5 100644 --- a/index.ts +++ b/index.ts @@ -1,6 +1,7 @@ export * from './runtime' export * from './schema' export * from './authoring' +export * from './calibration' export * from './attestation' export * from './artifact' export * from './platform' diff --git a/scripts/check-boundary.mjs b/scripts/check-boundary.mjs index 2fb5f93..eda5e89 100644 --- a/scripts/check-boundary.mjs +++ b/scripts/check-boundary.mjs @@ -33,6 +33,7 @@ const productionFiles = new Set([ 'artifact.ts', 'attestation.ts', 'authoring.ts', + 'calibration.ts', 'campaign.ts', 'episode.ts', 'episode-loop.ts', From 84439284f1ff5f651e08b872a99476dd600d7074 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Thu, 20 Aug 2026 22:08:53 -0700 Subject: [PATCH 2/3] test(calibration): pin the Libbet contract a constant button press satisfies calibration.test.mts runs offline in the repo test chain. A combination lock that opens on one six-word sequence must separate; a game whose channel moves whenever "a" is pressed must not, and the gate must throw naming constant:a. It also covers policy determinism across calls and seeds, report consistency, a one-word vocabulary, a zero-turn calibration, and the empty-vocabulary and negative-turn errors. The same gate runs on the native-2048 adapter and reds it: NATIVE_2048_REFERENCE is a fixed cycle of four directions, and 2048 merges tiles under almost any input, so a seeded pseudo-random walk of the same length reaches all seven milestones. That target exercises the execution and evidence paths; it does not measure skill, and the test now says so. pyboy-libbet.test.mts pins the finding that made this work exist. A live agent campaign of 70 turns on the packaged blind-discovery contract earned three milestones with a clean, replay-verified run. Over the same 70 turns: reference (packaged trajectory at the same progress level) 3 constant:a 3, the same set constant:start 3, the same set round-robin 3, the same set pseudo-random 3, the same set constant:select 2 constant:up, down, left, right, b 0 an unknown word 0 The regression keeps its existing ROM gate: it skips without the ROM and fails under PLAYPROOF_REQUIRE_ROM=1. --- calibration.test.mts | 262 ++++++++++++++++++++++++++++++++++++++++++ package.json | 2 +- pyboy-libbet.test.mts | 48 +++++++- 3 files changed, 310 insertions(+), 2 deletions(-) create mode 100644 calibration.test.mts diff --git a/calibration.test.mts b/calibration.test.mts new file mode 100644 index 0000000..97ae1d0 --- /dev/null +++ b/calibration.test.mts @@ -0,0 +1,262 @@ +/** + * Contract calibration test — the gate that rejects a contract a trivial + * baseline satisfies. + * + * Two hand-written games make the two outcomes exact. The combination lock can + * only be opened by one long word sequence, so it must separate. The mash game + * pins a channel that moves whenever one button is pressed — the Libbet shape — + * so it must not. The native-2048 adapter then runs the same gate on a real + * out-of-process game. + */ +import { strict as assert } from 'node:assert' +import { deriveContract } from './authoring' +import { + assertContractSeparates, + calibrateContract, + trivialBaselines, + UNKNOWN_BASELINE_WORD, +} from './calibration' +import type { Game } from './runtime' +import { makeNative2048, NATIVE_2048_INPUTS, NATIVE_2048_REFERENCE } from './adapters/native-2048' + +// --- a game that needs an exact sequence ------------------------------------ + +const LOCK_VOCABULARY = ['a', 'b', 'c', 'd', 'e', 'f'] +const LOCK_CODE = ['c', 'a', 'f', 'b', 'e', 'd'] +const LOCK_REFERENCE = ['b', 'a', ...LOCK_CODE] + +interface LockState { + progress: number + opened: number + steps: number +} + +const comboLock: Game = { + id: 'combo-lock', + init: () => ({ progress: 0, opened: 0, steps: 0 }), + step: (s, input) => { + const steps = s.steps + 1 + if (s.opened === 1) return { ...s, steps } + const advances = input === LOCK_CODE[s.progress] + const restarts = !advances && input === LOCK_CODE[0] + const progress = advances ? s.progress + 1 : restarts ? 1 : 0 + return { progress, opened: progress === LOCK_CODE.length ? 1 : 0, steps } + }, + frame: (s) => `steps ${s.steps} · the lock is ${s.opened === 1 ? 'open' : 'shut'}`, + evidence: (s) => ({ engineState: { progress: s.progress, opened: s.opened, steps: s.steps } }), +} + +const lockContract = deriveContract(comboLock, 0, [...LOCK_REFERENCE], [ + { + // Free: any input at all moves the step counter. A contract may carry such + // a milestone and still separate, as long as something is out of reach. + id: 'moved', + tier: 'engine-state', + glitchClass: 'legal', + when: (e) => (e.engineState?.steps ?? 0) >= 1, + sample: (e) => ({ kind: 'state-path', path: 'steps', op: '>=', value: e.engineState?.steps ?? 1 }), + }, + { + id: 'lock-opened', + tier: 'engine-state', + glitchClass: 'legal', + requires: ['moved'], + when: (e) => (e.engineState?.opened ?? 0) >= 1, + sample: (e) => ({ kind: 'state-path', path: 'opened', op: '>=', value: e.engineState?.opened ?? 1 }), + }, +]) + +// --- a game whose channel moves whenever one button is pressed --------------- + +const MASH_VOCABULARY = ['a', 'b', 'up', 'down'] +const MASH_REFERENCE = ['up', 'a', 'down', 'a', 'b', 'up', 'a', 'down'] + +interface MashState { + channel: number + steps: number +} + +const mashGame: Game = { + id: 'mash-channel', + init: () => ({ channel: 0, steps: 0 }), + step: (s, input) => ({ channel: input === 'a' ? s.channel + 1 : s.channel, steps: s.steps + 1 }), + frame: (s) => `steps ${s.steps}`, + evidence: (s) => ({ engineState: { channel: s.channel, steps: s.steps } }), +} + +const mashContract = deriveContract(mashGame, 0, [...MASH_REFERENCE], [ + { + id: 'channel-progressed', + tier: 'engine-state', + glitchClass: 'legal', + when: (e) => (e.engineState?.channel ?? 0) > 0, + sample: (e) => ({ kind: 'state-path', path: 'channel', op: '>=', value: e.engineState?.channel ?? 1 }), + }, +]) + +// (a) a contract that genuinely requires skill separates. +{ + const report = calibrateContract(comboLock, lockContract, { + reference: LOCK_REFERENCE, + vocabulary: LOCK_VOCABULARY, + }) + assert.equal(report.separates, true, `lock contract must separate: ${JSON.stringify(report.separating)}`) + assert.deepEqual(report.separating, ['lock-opened']) + assert.deepEqual(report.trivial, ['moved']) + assert.equal(report.bestBaselineCount, 1) + assert.deepEqual(report.reference.verified, ['moved', 'lock-opened']) + assertContractSeparates(report) + + // (d) the counts are internally consistent. + assert.equal(report.turns, LOCK_REFERENCE.length) + assert.equal(report.seed, 0) + assert.deepEqual(report.vocabulary, LOCK_VOCABULARY) + assert.equal(report.baselines.length, LOCK_VOCABULARY.length + 3) + assert.equal(report.bestBaselineCount, Math.max(...report.baselines.map((b) => b.verified.length))) + assert.ok(report.baselines.every((b) => b.verdict === 'clean')) + assert.ok(report.separating.every((id) => !report.trivial.includes(id))) + for (const id of report.reference.verified) { + assert.ok(report.separating.includes(id) !== report.trivial.includes(id), + `${id} must be either separating or trivial, never both or neither`) + } + // No baseline may earn a milestone that is reported as separating. + for (const baseline of report.baselines) { + for (const id of baseline.verified) assert.ok(!report.separating.includes(id)) + } + // The unknown word is a no-op, so it earns exactly what elapsed time earns. + const unknown = report.baselines.find((b) => b.id === `constant:${UNKNOWN_BASELINE_WORD}`) + assert.deepEqual(unknown?.verified, ['moved']) +} + +// (b) a contract a constant policy satisfies does not separate, and the gate +// throws with the offending baseline named. +{ + const report = calibrateContract(mashGame, mashContract, { + reference: MASH_REFERENCE, + vocabulary: MASH_VOCABULARY, + }) + assert.equal(report.separates, false) + assert.deepEqual(report.separating, []) + assert.deepEqual(report.trivial, ['channel-progressed']) + assert.deepEqual(report.reference.verified, ['channel-progressed']) + assert.equal(report.bestBaselineCount, 1) + const earners = report.baselines.filter((b) => b.verified.includes('channel-progressed')).map((b) => b.id) + assert.deepEqual(earners, ['constant:a', 'round-robin', 'pseudo-random']) + + assert.throws( + () => assertContractSeparates(report), + (error: unknown) => { + const message = (error as Error).message + assert.match(message, /does not separate/u) + assert.match(message, /channel-progressed/u) + assert.match(message, /constant:a/u) + assert.match(message, /reference verified 1 milestone\(s\)/u) + assert.match(message, /best trivial baseline verified 1 over 8 turns/u) + return true + }, + ) +} + +// (c) policies are deterministic across calls, and only pseudo-random moves +// with the seed. +{ + const script = (seed: number) => + Object.fromEntries(trivialBaselines(LOCK_VOCABULARY).map((p) => [p.id, [...p.inputs(LOCK_VOCABULARY, 12, seed)]])) + assert.deepEqual(script(0), script(0)) + assert.deepEqual(script(7), script(7)) + const zero = script(0) + const seven = script(7) + for (const id of Object.keys(zero)) { + if (id === 'pseudo-random') continue + assert.deepEqual(zero[id], seven[id], `${id} must not depend on the seed`) + } + assert.notDeepEqual(zero['pseudo-random'], seven['pseudo-random'], 'pseudo-random must depend on the seed') + assert.deepEqual(zero['constant:c'], Array.from({ length: 12 }, () => 'c')) + assert.deepEqual(zero['round-robin']?.slice(0, 7), ['a', 'b', 'c', 'd', 'e', 'f', 'a']) + assert.ok(zero['pseudo-random']?.every((word) => LOCK_VOCABULARY.includes(word))) + + // A seeded report reproduces exactly. + const options = { reference: LOCK_REFERENCE, vocabulary: LOCK_VOCABULARY, seed: 3 } + assert.deepEqual(calibrateContract(comboLock, lockContract, options), calibrateContract(comboLock, lockContract, options)) +} + +// (e) a one-word vocabulary and a zero-turn calibration are handled. +{ + const single = calibrateContract(mashGame, mashContract, { + reference: MASH_REFERENCE, + vocabulary: ['a'], + }) + assert.deepEqual(single.vocabulary, ['a']) + assert.equal(single.baselines.length, 4) + assert.equal(single.separates, false) + assert.deepEqual(single.trivial, ['channel-progressed']) + + const empty = calibrateContract(comboLock, lockContract, { + reference: [], + vocabulary: LOCK_VOCABULARY, + }) + assert.equal(empty.turns, 0) + assert.deepEqual(empty.reference.verified, []) + assert.deepEqual(empty.separating, []) + assert.deepEqual(empty.trivial, []) + assert.equal(empty.bestBaselineCount, 0) + assert.equal(empty.separates, false) + assert.ok(empty.baselines.every((b) => b.verified.length === 0)) + + const truncated = calibrateContract(comboLock, lockContract, { + reference: LOCK_REFERENCE, + vocabulary: LOCK_VOCABULARY, + turns: 2, + }) + assert.equal(truncated.turns, 2) + assert.deepEqual(truncated.reference.verified, ['moved']) + assert.equal(truncated.separates, false) + + assert.throws( + () => calibrateContract(comboLock, lockContract, { reference: LOCK_REFERENCE, vocabulary: [] }), + /empty input vocabulary/u, + ) + assert.throws( + () => calibrateContract(comboLock, lockContract, { reference: LOCK_REFERENCE, vocabulary: LOCK_VOCABULARY, turns: -1 }), + /non-negative integer/u, + ) +} + +/** + * The same gate on a real out-of-process game, and a second measured finding. + * + * `NATIVE_2048_REFERENCE` is itself a fixed cycle of four directions, and 2048 + * merges tiles under almost any input, so the contract derived from it does not + * separate: a seeded pseudo-random walk of the same length reaches every + * milestone, `tile-32` included. The packaged 2048 target demonstrates the + * execution and evidence paths; it is not a benchmark of skill, and the gate + * says so instead of leaving a reader to assume otherwise. + */ +const adapter = makeNative2048() +try { + const report = calibrateContract(adapter.game, adapter.contract, { + reference: NATIVE_2048_REFERENCE, + vocabulary: NATIVE_2048_INPUTS, + seed: adapter.seed, + }) + assert.equal(report.baselines.length, NATIVE_2048_INPUTS.length + 3) + assert.equal(report.turns, NATIVE_2048_REFERENCE.length) + assert.equal(report.reference.verified.length, adapter.contract.milestones.length) + assert.equal(report.separates, false, `2048 must not separate from a cyclic reference: ${JSON.stringify(report)}`) + assert.deepEqual(report.separating, []) + assert.deepEqual(report.trivial, adapter.contract.milestones.map((m) => m.id)) + assert.equal(report.bestBaselineCount, report.reference.verified.length) + // A constant direction already merges tiles; the unknown word never does. + assert.ok(report.baselines.find((b) => b.id === 'constant:left')?.verified.includes('tile-8-engine')) + assert.deepEqual(report.baselines.find((b) => b.id === `constant:${UNKNOWN_BASELINE_WORD}`)?.verified, []) + assert.throws(() => assertContractSeparates(report), /does not separate/u) + console.log( + `playproof calibration: native-2048 reference ${report.reference.verified.length} milestones vs best baseline ` + + `${report.bestBaselineCount} (${report.baselines.filter((b) => b.verified.length === report.bestBaselineCount).map((b) => b.id).join(', ')}) ` + + `over ${report.turns} turns — packaged target does not separate`, + ) +} finally { + adapter.dispose() +} + +console.log('playproof calibration: separating and non-separating contracts, policy determinism, edge cases OK') diff --git a/package.json b/package.json index 39a429c..81de3d6 100644 --- a/package.json +++ b/package.json @@ -122,7 +122,7 @@ "build": "pnpm clean && tsup && node scripts/copy-assets.mjs", "check:boundary": "node scripts/check-boundary.mjs", "typecheck": "tsc -p tsconfig.json --noEmit", - "test": "tsx playproof.test.mts && tsx episode.test.mts && tsx platform.test.mts && tsx desktop-platforms.test.mts && tsx drivers.test.mts && tsx campaign.test.mts", + "test": "tsx playproof.test.mts && tsx calibration.test.mts && tsx episode.test.mts && tsx platform.test.mts && tsx desktop-platforms.test.mts && tsx drivers.test.mts && tsx campaign.test.mts", "test:pyboy": "tsx pyboy-tetris.test.mts", "test:retro": "tsx stable-retro.test.mts", "test:ale": "tsx ale.test.mts", diff --git a/pyboy-libbet.test.mts b/pyboy-libbet.test.mts index b36ebbc..d764c1e 100644 --- a/pyboy-libbet.test.mts +++ b/pyboy-libbet.test.mts @@ -20,9 +20,15 @@ import { existsSync, readFileSync } from 'node:fs' import { fileURLToPath } from 'node:url' import { autoMarks, loadDiscovery, makePyBoyGeneric } from './adapters/pyboy-generic' import { attestRun } from './attestation' +import { assertContractSeparates, calibrateContract, UNKNOWN_BASELINE_WORD } from './calibration' import { logFrom } from './runtime' import { validateContract } from './schema' +/** The Game Boy pad, matching `BUTTONS` in pyboy/tetris.py. */ +const GAME_BOY_BUTTONS = ['up', 'down', 'left', 'right', 'a', 'b', 'start', 'select'] +/** Turn count of the agent campaign this regression pins. */ +const CAMPAIGN_TURNS = 70 + const DISCOVERY = fileURLToPath(new URL('./pyboy/discovery-libbet.json', import.meta.url)) const WORKER_MATCH = 'pyboy/worker.py' const required = process.env.PLAYPROOF_REQUIRE_ROM === '1' @@ -126,11 +132,51 @@ try { } console.log(`pyboy-libbet: derivation, 3-tier contract, ${all.length} milestones on ${adapter.reference.length} reference inputs, determinism, garbage-input rejection OK`) + + // (e) Calibration regression — this contract is NOT a benchmark, and the + // gate must keep saying so. + // + // A live agent campaign ran 70 turns on this ROM through this adapter and + // this discovery document and earned three milestones: ch_c321-progressed, + // ch_c32d-progressed, ch_ff96-progressed. Its verdict was clean and its run + // replay-verified. Pressing "a" 70 times earns the same three milestones. + // + // The agent's transcript is not packaged, so the reference here is the + // packaged exploration trajectory truncated to the same progress level: the + // inputs before the rank-0 channel first moves. Every baseline still gets the + // campaign's full 70 turns, which can only favour the baselines. + // + // Nobody may read a Libbet milestone count as evidence of competence. + const rank0 = [...doc.channels].sort((a, b) => a.rank - b.rank)[0]! + const report = calibrateContract(adapter.game, adapter.contract, { + reference: adapter.reference.slice(0, rank0.firstChangeStep - 1), + vocabulary: GAME_BOY_BUTTONS, + turns: CAMPAIGN_TURNS, + seed: adapter.seed, + }) + for (const outcome of [report.reference, ...report.baselines]) { + console.log(` ${outcome.id.padEnd(32)} ${String(outcome.verified.length).padStart(2)} ${outcome.verdict} ${outcome.verified.join(',') || '-'}`) + } + assert.deepEqual(report.reference.verified, ['ch_c321-progressed', 'ch_c32d-progressed', 'ch_ff96-progressed'], + 'the reference no longer reproduces the milestone set the live agent earned') + const constantA = report.baselines.find((b) => b.id === 'constant:a') + assert.deepEqual(constantA?.verified, report.reference.verified, + 'pressing "a" 70 times must still earn exactly what the evaluated agent earned') + assert.ok(report.bestBaselineCount >= report.reference.verified.length, + `best baseline ${report.bestBaselineCount} vs reference ${report.reference.verified.length}`) + assert.deepEqual(report.separating, [], 'no Libbet milestone is out of reach of a trivial policy') + assert.equal(report.separates, false, 'the Libbet discovery contract must not claim to separate') + assert.throws(() => assertContractSeparates(report), /constant:a/u) + // The milestones need a button, just not the right one: an uninterpretable + // word earns nothing, so this is not a pure function of elapsed frames. + assert.deepEqual(report.baselines.find((b) => b.id === `constant:${UNKNOWN_BASELINE_WORD}`)?.verified, []) + + console.log(`pyboy-libbet: calibration regression — reference ${report.reference.verified.length} milestones, best trivial baseline ${report.bestBaselineCount} over ${report.turns} turns, separates=${report.separates} OK`) } finally { adapter.dispose() } -// (e) Dispose ends the worker process and closes the transport. +// (f) Dispose ends the worker process and closes the transport. assert.ok(workerPid !== undefined, 'no PyBoy worker process was observed while the adapter was live') const deadline = Date.now() + 5_000 while (workerPids().has(workerPid) && Date.now() < deadline) sleepSync(50) From daeb2ac36bd4732bec2f9416e430614783b9748d Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Thu, 20 Aug 2026 22:09:01 -0700 Subject: [PATCH 3/3] docs(calibration): an uncalibrated derived contract is not a benchmark README gains a calibration section with the Libbet measurement as the worked example and the full baseline table. docs/adapters.md gains a matching section beside the replay-proof discussion, because replay proof answers "did this run happen" and never answers "was that hard". Blind-discovery contracts need the gate most: nothing in that pipeline asserts that a discovered memory channel means progress. CHANGELOG records the API and both measured findings under 0.4.0. --- CHANGELOG.md | 12 +++++++++++ README.md | 54 ++++++++++++++++++++++++++++++++++++++++++++++++ docs/adapters.md | 31 +++++++++++++++++++++++++++ 3 files changed, 97 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3c9c8d7..f1ecf35 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,18 @@ All notable changes to Playproof are documented here. +## 0.4.0 + +### Verification + +- `calibrateContract` replays a reference trajectory and a suite of trivial policies against the same contract, then reports which milestones the trivial policies cannot reach. +- The suite is one constant policy per input word, a word the game cannot interpret, a round-robin cycle over the vocabulary, and a seeded pseudo-random walk over it. Every policy is deterministic in the seed, so a report reproduces from one number. +- `assertContractSeparates` fails closed on a contract a trivial policy satisfies, and names every trivial milestone with the baseline that earned it. +- A contract separates only when at least one milestone is out of reach of every baseline and the reference verifies strictly more milestones than the strongest baseline. +- `deriveContract` is documented as producing a hypothesis, not a benchmark. It proves that a mark fires on the reference run; it cannot prove the mark is hard to reach. +- Measured, and pinned by the Libbet regression in CI: a 70-turn agent campaign on the packaged blind-discovery contract earned three milestones, and pressing `a` seventy times earns the same three. `constant:start`, `round-robin`, and a seeded pseudo-random walk also earn them, while five of the eight buttons and an unknown word earn none. +- Measured on the packaged 2048 target: its reference is a fixed cycle of four directions, so a pseudo-random walk of the same length reaches every milestone. That target exercises the execution and evidence paths and does not measure skill. + ## 0.3.0 ### Game and platform adapters diff --git a/README.md b/README.md index ef0e5f0..0ff6a8b 100644 --- a/README.md +++ b/README.md @@ -185,6 +185,60 @@ Use semantic checks such as `score >= 10` for progression. Exact hashes identify Dependencies between milestones form a declared partial order. A later achievement cannot verify before its prerequisites, even when its raw condition already holds. +## Calibration: does the contract separate? + +A milestone contract says which progressions count. +It does not say that reaching them is hard. +`deriveContract` proves only that every mark fires on the reference run, so a contract can pin a memory channel that moves whenever the game runs at all. +A constant button press then earns exactly what an evaluated agent earns. + +**An uncalibrated derived contract is not a benchmark.** Calibrate it, or do not publish a score from it. + +```ts +import { assertContractSeparates, calibrateContract } from '@tangle-network/playproof' + +const report = calibrateContract(game, contract, { + reference: referenceInputs, + vocabulary: ['up', 'down', 'left', 'right', 'a', 'b', 'start', 'select'], +}) +assertContractSeparates(report) +``` + +`calibrateContract` replays the reference and a suite of trivial policies through the same attestation path: one constant policy per input word, a word the game cannot interpret, a round-robin cycle over the vocabulary, and a seeded pseudo-random walk over it. +Every policy is deterministic in the seed, so a report reproduces from one number. + +The report names `separating` (milestones no baseline earned), `trivial` (milestones at least one baseline earned), and `bestBaselineCount`. +`separates` is true only when something is out of reach of every baseline **and** the reference verifies strictly more milestones than the strongest baseline. +`assertContractSeparates` throws otherwise, and the message names every trivial milestone with the baseline that earned it. + +### The measurement that made this exist + +A live agent campaign ran 70 turns on Libbet and the Magic Floor through `adapters/pyboy-generic` and the packaged `pyboy/discovery-libbet.json` blind-discovery document. +It earned three milestones. Its verdict was clean and its run replay-verified. + +Trivial policies of the same length, on the same ROM and the same derived contract, earn this: + +| Policy | Milestones verified | +|---|---| +| live agent, 70 turns | 3 — `ch_c321-progressed`, `ch_c32d-progressed`, `ch_ff96-progressed` | +| `constant:a` | 3 — the same set | +| `constant:start` | 3 — the same set | +| `round-robin` | 3 — the same set | +| `pseudo-random` | 3 — the same set | +| `constant:select` | 2 | +| `constant:up`, `constant:down`, `constant:left`, `constant:right`, `constant:b` | 0 | +| an unknown word | 0 | + +Pressing `a` seventy times scores what the agent scored. +That contract measures that frames elapsed, not that a game was played well. +`pyboy-libbet.test.mts` pins the result on the free ROM in CI, so no later reader can quote a Libbet milestone count as evidence of competence. + +The gate reds a second packaged target for the same reason. +`NATIVE_2048_REFERENCE` is a fixed cycle of four directions, and 2048 merges tiles under almost any input, so a seeded pseudo-random walk of the same length reaches every milestone the reference reaches, `tile-32` included. +That target exercises the execution, evidence, checkpoint, and signing paths. It does not measure skill. + +Blind discovery needs this gate most, because nothing in that pipeline ever asserts that a discovered memory channel means progress. + ## Execution adapters ### Deterministic native process diff --git a/docs/adapters.md b/docs/adapters.md index 8bc5948..065bbc2 100644 --- a/docs/adapters.md +++ b/docs/adapters.md @@ -45,6 +45,37 @@ ALE was measured the same way and gives the opposite answer: The PyBoy adapter reaches the same conclusion as ALE on its own substrate and does publish a save-state hash. No answer generalizes. A new replay adapter measures its own substrate before it declares a tier. +## Calibration: a derived contract is a hypothesis + +An adapter can prove determinism perfectly and still ship a contract that measures nothing. +Replay proof answers "did this run really happen". +It does not answer "was reaching that milestone hard", and those are different questions. + +`deriveContract` proves that every mark fires on the reference run and that the contract validates. +It cannot know whether the progression it pinned is out of reach of button mashing. +So every adapter that derives a contract must run `calibrateContract` and gate publication with `assertContractSeparates`. +An uncalibrated derived contract is not a benchmark. + +Blind-discovery adapters need this most. +`adapters/pyboy-generic` builds its contract from discovered memory channels alone, and nothing in that pipeline ever asserts that a discovered channel means progress. +A channel that counts frames, animation ticks, or a menu cursor is indistinguishable, at derivation time, from a channel that counts rooms cleared. + +The packaged Libbet contract is the worked example, and it fails the gate. +A live agent campaign of 70 turns earned three milestones on it, with a clean verdict and a replay-verified run. +Over the same 70 turns, on the same ROM and the same contract: + +| Policy | Milestones verified | +|---|---| +| live agent | 3 — `ch_c321-progressed`, `ch_c32d-progressed`, `ch_ff96-progressed` | +| `constant:a`, `constant:start`, `round-robin`, `pseudo-random` | 3 — the same set | +| `constant:select` | 2 | +| `constant:up`, `constant:down`, `constant:left`, `constant:right`, `constant:b` | 0 | +| an unknown word | 0 | + +`pyboy-libbet.test.mts` pins this in CI on the free ROM. +The finding is not a Libbet defect and not a PyBoy defect. +It is what a derived contract is worth before somebody measures it. + ## Libretro consoles through stable-retro ```ts