From c97228c7ad280fd29b890c65f8c53073c1d78218 Mon Sep 17 00:00:00 2001 From: Lann Martin Date: Wed, 12 Aug 2026 09:16:46 -0400 Subject: [PATCH] JS harness: `only` reports the unselected census as deselected (#89) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Runner parity with the reference runner's #90 behavior, closing the one gap the policy doc had to footnote: the shared case loop (js/viewer/harness.mjs, used by the deltic shard workers, the viewer's browser workers, and consumers' node legs) previously skipped emit entirely for cases outside `only`, so a subset run failed lockfile coverage instead of aggregating as a visible subset. Now the whole census is reported: capability first (a tags-excluded case stays not-applicable even outside the selection — aggregate's applicability policing on scheduled streams requires the precedence), then deselected for applicable-but-unselected cases (never executed, no provenance, detail naming the selection). The inventory drift check now covers the whole census too, filtered or not — same census-truth rule as the wasmtime runner's startup cross-check. Counts gain `deselected` and `selected` (census cases matching the filter, applicability aside); mergeCounts sums both and tolerates old-shape counts from downstream-bundled workers. The empty-selection rule lands at whole-census scope: runCases throws when unsharded and nothing matched; sharded stripes may legitimately match nothing, so the pooled coordinator (page-runner) applies the same guard over the merged counts. Downstream note: counts deepEquals will need the two new fields on the next pin bump. The policy doc's "tracked in #89" parenthetical is dropped — the rule is now true on both legs. Verified: just verify-imports (new stub legs: deselected event shape, precedence, instance economy, empty-selection rejection), just verify-deltic (new selftest leg drives the real deltic engine over the fixture suite: 8-event census with 5 deselected, precedence, empty selection), just all (all goldens byte-identical — nothing golden'd uses `only`). --- docs/runner-policy.md | 4 +-- js/node-runner.test.mjs | 64 ++++++++++++++++++++++++++++++++++- js/runner-deltic/selftest.mjs | 54 +++++++++++++++++++++++++++-- js/viewer/harness.mjs | 38 +++++++++++++++++---- js/viewer/page-runner.mjs | 13 ++++++- 5 files changed, 159 insertions(+), 14 deletions(-) diff --git a/docs/runner-policy.md b/docs/runner-policy.md index 9e11b0b..348a60d 100644 --- a/docs/runner-policy.md +++ b/docs/runner-policy.md @@ -198,6 +198,4 @@ Capability takes precedence: a tags-excluded case stays aggregation's applicability policing rejects `deselected` where the manifest says the case could never have run (selection must not hide capability). An empty selection (`--only` matching nothing) is a run -error, not a vacuous green. (The JS leg's `only` option predates this -rule and still omits filtered cases instead of reporting them — -tracked in #89.) +error, not a vacuous green. diff --git a/js/node-runner.test.mjs b/js/node-runner.test.mjs index e077a11..7f553f8 100644 --- a/js/node-runner.test.mjs +++ b/js/node-runner.test.mjs @@ -68,7 +68,15 @@ const counts = await runSuiteJsonl({ missing: ["hsm"], emit: (line) => lines.push(line), }); -assert.deepEqual(counts, { passed: 1, failed: 1, skipped: 0, na: 1, total: 3 }); +assert.deepEqual(counts, { + passed: 1, + failed: 1, + skipped: 0, + na: 1, + deselected: 0, + selected: 3, + total: 3, +}); assert.equal(lines.length, 5, "envelope + three events + terminator"); const head = JSON.parse(lines[0]); assert.equal(head.suite.name, "sample_suite", "envelope normalizes the transpile name"); @@ -83,6 +91,60 @@ assert.equal(events[1].detail, "boom"); // census + one fresh instance per executed case (the N/A case never runs) assert.equal(instances, 3); +// `only` reports the unselected census as deselected (#89): full +// coverage, subsetting visible, capability (not-applicable) winning +// over selection, deselected cases never executed. +{ + const lines = []; + const before = instances; + const counts = await runSuiteJsonl({ + newTests, + tagsOf: (name) => tags[name], + target: "stub-target", + suiteName: "sample-suite", + missing: ["hsm"], + only: "pass", + emit: (line) => lines.push(line), + }); + assert.deepEqual(counts, { + passed: 1, + failed: 0, + skipped: 0, + na: 1, + deselected: 1, + selected: 1, + total: 3, + }); + const events = lines.slice(1, -1).map((l) => JSON.parse(l)); + assert.deepEqual( + events.map((e) => [e.case, e.status]), + [ + ["basic/pass", "pass"], + ["basic/fail", "deselected"], + ["gated/probe", "not-applicable"], + ], + ); + assert.equal(events[1].detail, "only pass"); + assert.equal(events[1].provenance, undefined, "deselected cases never executed"); + // census + one fresh instance for the one executed case + assert.equal(instances - before, 2); +} + +// A selection matching nothing is a run error, like the reference runner. +await assert.rejects( + () => + runSuiteJsonl({ + newTests, + tagsOf: (name) => tags[name], + target: "t", + suiteName: "s", + missing: ["hsm"], + only: "zzz", + emit: () => {}, + }), + /only `zzz` matches no cases \(empty selection is a run error\)/, +); + await assert.rejects( () => runSuiteJsonl({ diff --git a/js/runner-deltic/selftest.mjs b/js/runner-deltic/selftest.mjs index 5ed37ec..e3a345b 100644 --- a/js/runner-deltic/selftest.mjs +++ b/js/runner-deltic/selftest.mjs @@ -35,13 +35,14 @@ async function suiteOf(path, env) { }); } -async function run(engine, { missing = [], shard } = {}) { +async function run(engine, { missing = [], only, shard } = {}) { const events = []; const counts = await runCases({ cases: await (await engine.newTests()).all(), Context: engine.Context, tagsOf: engine.tagsOf, missing, + only, shard, emit: (event, index) => events.push({ index, event }), freshCases: async () => (await engine.newTests()).all(), @@ -54,7 +55,15 @@ async function run(engine, { missing = [], shard } = {}) { { const engine = await suiteOf(samplePath); const { counts, events } = await run(engine); - assert.deepEqual(counts, { passed: 1, failed: 1, skipped: 1, na: 0, total: 3 }); + assert.deepEqual(counts, { + passed: 1, + failed: 1, + skipped: 1, + na: 0, + deselected: 0, + selected: 3, + total: 3, + }); const byCase = Object.fromEntries(events.map((e) => [e.case, e])); assert.equal(byCase["sample/math/add"].status, "pass"); assert.equal(byCase["sample/math/mul"].status, "fail"); @@ -66,7 +75,15 @@ async function run(engine, { missing = [], shard } = {}) { { const engine = await suiteOf(fixturePath); const { counts, events } = await run(engine, { missing: ["hsm"] }); - assert.deepEqual(counts, { passed: 6, failed: 1, skipped: 0, na: 1, total: 8 }); + assert.deepEqual(counts, { + passed: 6, + failed: 1, + skipped: 0, + na: 1, + deselected: 0, + selected: 8, + total: 8, + }); const byCase = Object.fromEntries(events.map((e) => [e.case, e])); assert.equal(byCase["fixture/trap/boom"].status, "fail"); assert.equal(byCase["fixture/trap/boom"].provenance, "trap"); @@ -81,6 +98,37 @@ async function run(engine, { missing = [], shard } = {}) { assert.equal(byCase["fixture/hsm/declined"].status, "pass"); console.log("selftest: fixture trap + tag scheduling ok"); + // Selection (#89): the unselected census is reported deselected — + // full coverage, never executed, capability winning over selection + // (hsm/attest stays not-applicable outside the filter, exactly the + // reference runner's precedence). The trap case is outside the + // selection: nothing fails. + const sub = await run(engine, { missing: ["hsm"], only: "gen" }); + assert.deepEqual(sub.counts, { + passed: 2, + failed: 0, + skipped: 0, + na: 1, + deselected: 5, + selected: 2, + total: 8, + }); + const subByCase = Object.fromEntries(sub.events.map((e) => [e.case, e])); + assert.deepEqual(subByCase["fixture/trap/boom"], { + case: "fixture/trap/boom", + status: "deselected", + detail: "only gen", + }); + assert.equal(subByCase["fixture/hsm/attest"].status, "not-applicable"); + assert.equal(subByCase["fixture/gen/tc1"].status, "pass"); + assert.equal(sub.events.length, 8, "full census reported"); + // A selection matching nothing is a run error, not a vacuous green. + await assert.rejects( + () => run(engine, { only: "zzz" }), + /empty selection is a run error/, + ); + console.log("selftest: only -> deselected census ok"); + // Striping partition equality (harness semantics over the deltic engine): // two shards merge to the full counts, disjoint cases, full union. const s0 = await run(engine, { missing: ["hsm"], shard: { index: 0, count: 2 } }); diff --git a/js/viewer/harness.mjs b/js/viewer/harness.mjs index 1989d79..5362172 100644 --- a/js/viewer/harness.mjs +++ b/js/viewer/harness.mjs @@ -143,12 +143,23 @@ export function envelope(target, suite) { * @param {new (onDiagnostic: (msg: string) => void) => object} options.Context * @param {(name: string) => string[] | undefined} options.tagsOf * @param {string[]} options.missing - * @param {string} [options.only] Substring filter (skips emit entirely). + * @param {string} [options.only] Substring selection: census cases + * outside it are reported `deselected` (never executed) rather than + * omitted, so subset runs keep full coverage with the subsetting + * visible as selection policy (docs/runner-policy.md "Selection is + * not capability"). Capability wins: a tags-excluded case stays + * `not-applicable` even outside the selection. A filter matching no + * census case throws (empty selection is a run error) when the loop + * sees the whole census — unsharded; pooled coordinators apply the + * same guard over merged counts. * @param {(event: object, index: number) => void} options.emit * @param {{ index: number, count: number }} [options.shard] * @param {() => Promise} [options.freshCases] * @param {number} [options.caseTimeoutMs] - * @returns {Promise<{passed, failed, skipped, na, total}>} + * @returns {Promise<{passed, failed, skipped, na, deselected, selected, total}>} + * `selected` counts census cases matching the selection (all of + * them without `only`) regardless of applicability; `total` = + * executed + na + deselected. */ export async function runCases({ cases, @@ -162,16 +173,17 @@ export async function runCases({ caseTimeoutMs, }) { const { index: shardIndex, count: shardCount } = shard ?? { index: 0, count: 1 }; - let passed = 0, failed = 0, skipped = 0, na = 0, total = 0; + let passed = 0, failed = 0, skipped = 0, na = 0, deselected = 0, selected = 0, total = 0; for (const [caseIndex, testCase] of cases.entries()) { if (caseIndex % shardCount !== shardIndex) continue; total++; const name = String(await testCase.name()); - if (only && !name.includes(only)) continue; const tags = tagsOf(name); if (tags === undefined) { throw new Error(`inventory drift: no tags record covers ${name}`); } + const isSelected = !only || name.includes(only); + if (isSelected) selected++; if (!applies(tags, missing)) { na++; const excluding = tags.find((t) => @@ -180,6 +192,11 @@ export async function runCases({ emit({ case: name, status: "not-applicable", detail: excluding ?? "" }, caseIndex); continue; } + if (!isSelected) { + deselected++; + emit({ case: name, status: "deselected", detail: `only ${only}` }, caseIndex); + continue; + } let executed = testCase; if (freshCases) { // Positional relocation: enumeration order is deterministic @@ -252,17 +269,26 @@ export async function runCases({ if (diags.length > 0) event.diagnostics = diags; emit(event, caseIndex); } - return { passed, failed, skipped, na, total }; + // The reference runner's empty-selection rule (a typo'd filter must + // not exit green with the whole census deselected), applied where + // the whole census is visible. Sharded stripes may legitimately + // match nothing; their coordinator guards over merged counts. + if (only && shardCount === 1 && selected === 0) { + throw new Error(`only \`${only}\` matches no cases (empty selection is a run error)`); + } + return { passed, failed, skipped, na, deselected, selected, total }; } /** Merge per-shard `runCases` counts (shards partition the suite). */ export function mergeCounts(parts) { - const out = { passed: 0, failed: 0, skipped: 0, na: 0, total: 0 }; + const out = { passed: 0, failed: 0, skipped: 0, na: 0, deselected: 0, selected: 0, total: 0 }; for (const c of parts) { out.passed += c.passed; out.failed += c.failed; out.skipped += c.skipped; out.na += c.na; + out.deselected += c.deselected ?? 0; + out.selected += c.selected ?? 0; out.total += c.total; } return out; diff --git a/js/viewer/page-runner.mjs b/js/viewer/page-runner.mjs index f525a2f..38af6a2 100644 --- a/js/viewer/page-runner.mjs +++ b/js/viewer/page-runner.mjs @@ -73,13 +73,24 @@ export async function runSuitesInPage({ workerUrl, suites, jobs }) { ); const events = shards.flatMap((s) => s.events); events.sort((a, b) => a.index - b.index); + const counts = mergeCounts(shards.map((s) => s.counts)); + // The empty-selection rule over the whole pool: single stripes + // may legitimately match nothing (harness.mjs suppresses its + // per-census guard when sharded), so the merged census is where + // a dead filter surfaces. + if (config.only && counts.selected === 0) { + throw new Error( + `suite ${suite}: only \`${config.only}\` matches no cases ` + + "(empty selection is a run error)", + ); + } out[key] = { lines: [ JSON.stringify(envelope(target, suite)), ...events.map((e) => JSON.stringify(e.event)), '{"segment-end":true}', ], - counts: mergeCounts(shards.map((s) => s.counts)), + counts, }; } window.__report(out);