From 2f9c56f1a5487ff3e5594dd0dd2b2dc2c87630d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Fri, 16 Oct 2026 15:19:21 +0200 Subject: [PATCH 1/2] test(coverage): split gap rows into owed work and accepted limitations Every coverage-gap row is now one of two kinds: a known-gap row is owed work and must carry a tracking issue that owns it; a new accepted-gap row states a non-empty reason why no evidence lane will cover it and has no tracking issue. Enforce both in a shared manifest gate. Linux (9 rows) and web (1 row) carried rows pointing at closed issues #1915 and #1900; each states its permanent limitation instead, so the closed constants are removed. macOS moves doctor, shutdown, events, test, trace, and react-native to accepted limitations per the #1916 decision, keeping the scheduled interaction rows on the open umbrella. tvOS keeps all 38 rows as owed work on #1914. --- .../command-coverage/declarations.ts | 84 +++++++++++++++---- test/integration/command-coverage/entries.ts | 32 ++++--- test/integration/command-coverage/evidence.ts | 5 +- test/integration/linux-e2e/coverage.ts | 12 +-- test/integration/smoke-linux-coverage.test.ts | 37 ++++---- test/integration/smoke-macos-coverage.test.ts | 13 ++- .../smoke-tvos-platform-coverage.test.ts | 13 ++- .../smoke-web-platform-coverage.test.ts | 34 ++++---- .../support/coverage-classification.ts | 50 ++++++++++- test/integration/web-e2e/coverage.ts | 10 +-- 10 files changed, 190 insertions(+), 100 deletions(-) diff --git a/test/integration/command-coverage/declarations.ts b/test/integration/command-coverage/declarations.ts index 888fff4428..c53aaa2ae6 100644 --- a/test/integration/command-coverage/declarations.ts +++ b/test/integration/command-coverage/declarations.ts @@ -42,6 +42,10 @@ const C = PUBLIC_COMMANDS; * load time (`projectCoverage`), so adding a public command is one edit here rather than six * parallel edits in six manifests. * + * A gap row is exactly one of two kinds: a `known-gap` row is owed work and its builder pins the + * platform's open tracking issue; an `accepted-gap` row states a `reason` for why no evidence lane + * will cover it and has no tracking issue. `assertCoverageGapRowsAreOwned` enforces both. + * * `satisfies Record` is the exhaustiveness check: * a command added to the catalog without a row here fails type-checking, and each platform's * coverage smoke test fails the same omission at runtime. @@ -68,8 +72,9 @@ const COMMAND_COVERAGE_DECLARATIONS = { 'artifacts lists a daemon-tracked artifact produced during a web session', 'daemon artifact listing round-trips a tracked artifact for a web-backed session', ), - linux: linux.gap( - 'No Linux live command creates a downloadable daemon artifact for inventory yet', + linux: linux.accepted( + 'No Linux live command creates a downloadable daemon artifact for inventory', + 'The Linux lanes run the local CLI, where every command output is a caller-owned local path; the daemon only inventories artifacts carried over remote artifact transport, so a Linux lane cannot create a downloadable inventory entry without a local-transport product decision.', ), }, [C.devices]: { @@ -135,7 +140,10 @@ const COMMAND_COVERAGE_DECLARATIONS = { 'smoke:inventory-install', 'doctor discovers the installed fixture app', ), - macos: macos.gap('No command-specific macOS doctor evidence exists yet'), + macos: macos.accepted( + 'No command-specific macOS doctor evidence exists', + "The doctor checks are daemon-owned host diagnostics (version, state dir, inventory probes, Metro) with no macOS device branch, already pinned by the daemon's own doctor tests; a macOS lane would re-run that identical code, so macOS carries no command-specific doctor evidence.", + ), tvos: tvos.gap('No tvOS-specific doctor command evidence exists yet'), web: web.contract( 'packages/platform-web/src/__tests__/doctor.test.ts', @@ -204,14 +212,20 @@ const COMMAND_COVERAGE_DECLARATIONS = { 'full:device-lifecycle', 'shutdown succeeds and inventory reports the selected simulator stopped', ), - macos: macos.gap('No command-specific macOS shutdown evidence exists yet'), + macos: macos.accepted( + 'No command-specific macOS shutdown evidence exists', + 'The Apple owner declares shutdownTarget unavailable on the macOS leaf (unsupported-platform-leaf): agent-device never powers off the host Mac it runs on, so the command refuses at admission and no lane will observe a successful macOS shutdown.', + ), tvos: tvos.gap('No tvOS-specific shutdown command evidence exists yet'), web: web.contract( 'packages/platform-web/src/runtime.test.ts', 'boot and shutdown report the runtime-owned unavailable readiness fact', 'the web runtime fact rejects device shutdown', ), - linux: linux.gap('No Linux-specific shutdown command evidence exists yet'), + linux: linux.accepted( + 'No Linux-specific shutdown command evidence exists', + 'The Linux owner declares shutdownTarget unavailable (unsupported-platform-leaf): the Linux desktop is the host machine itself and agent-device never powers off the machine it runs on, so the command refuses at admission and no lane will observe a successful shutdown.', + ), }, [C.appState]: { androidEmulator: androidEmulator.live( @@ -280,7 +294,10 @@ const COMMAND_COVERAGE_DECLARATIONS = { 'logs reports the runtime-owned unavailable app-log facts', 'the web runtime fact rejects native app-log commands', ), - linux: linux.gap('No Linux-specific app-log command evidence exists yet'), + linux: linux.accepted( + 'No Linux-specific app-log command evidence exists', + 'The Linux runtime declares the whole app-log family unavailable (unsupported-platform-leaf) by owner decision: a Linux desktop session has no runtime-owned app log stream, so the command refuses at admission and no lane will observe a Linux app-log stream.', + ), }, [C.events]: { androidEmulator: androidEmulator.live( @@ -291,7 +308,10 @@ const COMMAND_COVERAGE_DECLARATIONS = { 'full:observability-artifacts', 'timeline contains commands from this session', ), - macos: macos.gap('No command-specific macOS event timeline evidence exists yet'), + macos: macos.accepted( + 'No command-specific macOS event timeline evidence exists', + 'The events timeline is daemon-owned session bookkeeping with no platform branch, already pinned for a daemon-backed session by request-router-events.test.ts; a macOS lane would re-run that identical code, so macOS carries no command-specific event timeline evidence.', + ), tvos: tvos.gap('No tvOS-specific session-event command evidence exists yet'), web: web.contract( 'packages/daemon/src/server/__tests__/request-router-events.test.ts', @@ -386,10 +406,14 @@ const COMMAND_COVERAGE_DECLARATIONS = { 'full:fixture-replays', 'fixture scripts run as a suite with JUnit artifacts', ), - macos: macos.gap('No command-specific macOS test-suite evidence exists yet'), + macos: macos.accepted( + 'No command-specific macOS test-suite evidence exists', + "The CI macOS replay lane dogfoods the test command to run the replay suite, whose per-command macOS claims are already carried by the replay:system-settings live rows; no lane asserts the test command's own suite result and JUnit output on macOS, and none will.", + ), tvos: tvos.gap('No tvOS-specific test-suite command evidence exists yet'), - web: web.gap( - "Web test-suite execution has no executable web evidence: ReplayTestPlatform = Exclude structurally excludes web from the declared-platform filter, so `test --platform web` can never select a script (proven by a regression test in session-command-replay.test.ts) — that is evidence of what the command cannot do, not that it works on web", + web: web.accepted( + 'No command-specific web test-suite evidence exists', + "ReplayTestPlatform = Exclude structurally excludes web from the declared-platform filter, so `test --platform web` can never select a script (proven by a regression test in session-command-replay.test.ts); there is no successful web test-suite execution for any lane to observe, and adding web replay-test support is a product decision, not scheduled work.", ), linux: linux.commandEvidenceLive( 'the command-evidence lane runs a dedicated Linux script as a test suite', @@ -474,7 +498,10 @@ const COMMAND_COVERAGE_DECLARATIONS = { 'install and reinstall share the runtime-owned unavailable deploy fact', 'the web runtime fact rejects native app installation', ), - linux: linux.gap('No Linux-specific application installation command evidence exists yet'), + linux: linux.accepted( + 'No Linux-specific application installation command evidence exists', + 'The Linux owner declares the app-deployment family unavailable (unsupported-platform-leaf): a Linux desktop app is launched by name through the desktop interactor and there is no installable package model, so the command refuses at admission and no lane will observe a Linux install.', + ), }, [C.reinstall]: { androidEmulator: androidEmulator.contract( @@ -500,7 +527,10 @@ const COMMAND_COVERAGE_DECLARATIONS = { 'install and reinstall share the runtime-owned unavailable deploy fact', 'the web runtime fact rejects native app reinstallation', ), - linux: linux.gap('No Linux-specific application reinstallation command evidence exists yet'), + linux: linux.accepted( + 'No Linux-specific application reinstallation command evidence exists', + 'Reinstall reuses the app-deployment family the Linux runtime declares unavailable (unsupported-platform-leaf): there is no installable package model for the desktop interactor to reinstall, so the command refuses at admission and no lane will observe a Linux reinstall.', + ), }, [C.push]: { androidEmulator: androidEmulator.live( @@ -527,7 +557,10 @@ const COMMAND_COVERAGE_DECLARATIONS = { 'push reports the runtime-owned unavailable readiness and push facts', 'the web runtime fact rejects native push notification delivery', ), - linux: linux.gap('No Linux-specific push delivery command evidence exists yet'), + linux: linux.accepted( + 'No Linux-specific push delivery command evidence exists', + 'The Linux runtime declares push delivery unavailable (unsupported-platform-leaf): the desktop interactor has no OS push channel to deliver through, so the command refuses at admission and no lane will observe a Linux push delivery.', + ), }, [C.triggerAppEvent]: { androidEmulator: androidEmulator.contract( @@ -763,7 +796,10 @@ const COMMAND_COVERAGE_DECLARATIONS = { 'full:observability-artifacts', 'Release fixture returns typed detected=false and dismissed=false overlay state', ), - macos: macos.gap('No command-specific macOS React Native inspection evidence exists yet'), + macos: macos.accepted( + 'No command-specific macOS React Native inspection evidence exists', + "react-native dismiss-overlay only answers meaningfully on a surface that carries a React Native overlay, and the repository's React Native fixtures run on iOS simulators and Android emulators; the macOS lanes drive desktop apps, so no macOS lane will cover this row.", + ), tvos: tvos.gap('No tvOS-specific React Native inspection command evidence exists yet'), // R61: no owner fact refuses this command on a browser — its whole device work is one bound // `tapPoint` the web target admits — so it now runs and reports that no overlay is present. @@ -800,7 +836,10 @@ const COMMAND_COVERAGE_DECLARATIONS = { 'start web recording', 'web recording starts and stops through the scoped provider', ), - linux: linux.gap('No Linux-specific recording command evidence exists yet'), + linux: linux.accepted( + 'No Linux-specific recording command evidence exists', + 'The Linux runtime declares the screen-recording family unavailable (unsupported-platform-leaf): there is no Linux capture pipeline, so the command refuses at admission and no lane will produce a playable Linux recording.', + ), }, [C.trace]: { androidEmulator: androidEmulator.live( @@ -811,14 +850,20 @@ const COMMAND_COVERAGE_DECLARATIONS = { 'full:observability-artifacts', 'typed start/stop lifecycle retains the requested path and captures non-empty diagnostics', ), - macos: macos.gap('No command-specific macOS trace evidence exists yet'), + macos: macos.accepted( + 'No command-specific macOS trace evidence exists', + "trace never binds a device runtime: its session-scoped start/stop bookkeeping is platform-neutral and already pinned by the daemon's trace-runtime tests, so a macOS lane would re-run identical code and macOS carries no command-specific trace evidence.", + ), tvos: tvos.gap('No tvOS-specific trace command evidence exists yet'), web: web.contract( 'packages/daemon/src/sessions/handlers/__tests__/trace-runtime.test.ts', 'starts and stops one trace through the session-owned trace slot on a web session', 'session-scoped trace start/stop bookkeeping works the same for a web session as any other platform', ), - linux: linux.gap('No Linux-specific trace command evidence exists yet'), + linux: linux.accepted( + 'No Linux-specific trace command evidence exists', + "trace never binds a device runtime: its session-scoped start/stop bookkeeping is platform-neutral and already pinned by the daemon's trace-runtime tests, so a Linux lane would re-run identical code and Linux carries no command-specific trace evidence.", + ), }, [C.find]: { androidEmulator: androidEmulator.live( @@ -1386,7 +1431,10 @@ const COMMAND_COVERAGE_DECLARATIONS = { 'install-from-source reports the runtime-owned unavailable materialize and deploy facts', 'the web runtime fact rejects source-based app installation', ), - linux: linux.gap('No Linux-specific source-install command evidence exists yet'), + linux: linux.accepted( + 'No Linux-specific source-install command evidence exists', + 'Source installation runs through the app-deployment family the Linux runtime declares unavailable (unsupported-platform-leaf): the desktop has no materialized-app install route to source from, so the command refuses at admission and no lane will observe a Linux source install.', + ), }, } satisfies Record; diff --git a/test/integration/command-coverage/entries.ts b/test/integration/command-coverage/entries.ts index 74ac73fc6a..9d330d699d 100644 --- a/test/integration/command-coverage/entries.ts +++ b/test/integration/command-coverage/entries.ts @@ -7,11 +7,9 @@ import { } from '../macos-e2e/live-scenarios.ts'; import { LINUX_COMMAND_EVIDENCE, - LINUX_COVERAGE_GAP_ISSUE, LINUX_REPLAY_EVIDENCE, MACOS_COVERAGE_GAP_ISSUE, TVOS_COVERAGE_GAP_ISSUE, - WEB_COVERAGE_GAP_ISSUE, WEB_SMOKE_EVIDENCE, type RepositoryEvidence, } from './evidence.ts'; @@ -58,6 +56,11 @@ export type MacOsPlatformCoverageEntry = assertion: string; level: 'known-gap'; trackingIssue: number; + } + | { + assertion: string; + level: 'accepted-gap'; + reason: string; }; export type TvOsPlatformCoverageEntry = @@ -91,8 +94,8 @@ export type WebPlatformCoverageEntry = } | { assertion: string; - level: 'known-gap'; - trackingIssue: number; + level: 'accepted-gap'; + reason: string; }; export type LinuxPlatformCoverageEntry = @@ -103,8 +106,8 @@ export type LinuxPlatformCoverageEntry = } | { assertion: string; - level: 'known-gap'; - trackingIssue: number; + level: 'accepted-gap'; + reason: string; }; /** One command's coverage judgment on every platform that classifies the public catalog. */ @@ -170,6 +173,11 @@ export const macos = { level: 'known-gap', trackingIssue: MACOS_COVERAGE_GAP_ISSUE, }), + accepted: (assertion: string, reason: string): MacOsPlatformCoverageEntry => ({ + assertion, + level: 'accepted-gap', + reason, + }), }; export const tvos = { @@ -207,10 +215,10 @@ export const web = { level: 'command-contract', owner: { path, test }, }), - gap: (assertion: string): WebPlatformCoverageEntry => ({ + accepted: (assertion: string, reason: string): WebPlatformCoverageEntry => ({ assertion, - level: 'known-gap', - trackingIssue: WEB_COVERAGE_GAP_ISSUE, + level: 'accepted-gap', + reason, }), }; @@ -230,10 +238,10 @@ export const linux = { level: 'command-contract', owner: { path, test }, }), - gap: (assertion: string): LinuxPlatformCoverageEntry => ({ + accepted: (assertion: string, reason: string): LinuxPlatformCoverageEntry => ({ assertion, - level: 'known-gap', - trackingIssue: LINUX_COVERAGE_GAP_ISSUE, + level: 'accepted-gap', + reason, }), }; diff --git a/test/integration/command-coverage/evidence.ts b/test/integration/command-coverage/evidence.ts index f43f1720ee..5f16a2a5f7 100644 --- a/test/integration/command-coverage/evidence.ts +++ b/test/integration/command-coverage/evidence.ts @@ -12,11 +12,10 @@ export type RepositoryEvidence = { const C = PUBLIC_COMMANDS; -// Tracking issues for the known-gap rows each platform still carries. +// The tracking issues that own the `known-gap` rows each platform still carries. An +// accepted-gap row states a `reason` instead of citing one of these. export const MACOS_COVERAGE_GAP_ISSUE = 1916; export const TVOS_COVERAGE_GAP_ISSUE = 1914; -export const WEB_COVERAGE_GAP_ISSUE = 1900; -export const LINUX_COVERAGE_GAP_ISSUE = 1915; // Android contract evidence that only the coverage declarations cite. Evidence shared with a // production test module keeps living next to that module and is imported by declarations.ts. diff --git a/test/integration/linux-e2e/coverage.ts b/test/integration/linux-e2e/coverage.ts index 22e96e79d8..5c4e968d6c 100644 --- a/test/integration/linux-e2e/coverage.ts +++ b/test/integration/linux-e2e/coverage.ts @@ -3,21 +3,17 @@ import type { PublicCommand } from '../command-coverage/entries.ts'; import { LINUX_COMMAND_EVIDENCE, LINUX_REPLAY_EVIDENCE } from '../command-coverage/evidence.ts'; import { buildCoverageClassificationSummary } from '../support/coverage-classification.ts'; -export { - LINUX_COMMAND_EVIDENCE, - LINUX_COVERAGE_GAP_ISSUE, - LINUX_REPLAY_EVIDENCE, -} from '../command-coverage/evidence.ts'; +export { LINUX_COMMAND_EVIDENCE, LINUX_REPLAY_EVIDENCE } from '../command-coverage/evidence.ts'; /** * One primary, observable owner for every public command on the Linux desktop. * * Live rows cite either the existing Linux replay or the separate command-evidence * lane. The existing replay scope stays unchanged. Contract rows cite the existing - * provider scenario or dedicated Linux unit/runtime evidence; they do not turn + * provider scenario or dedicated Linux runtime-fact evidence; they do not turn * mocked provider calls into live desktop claims. Capability denials are derived - * from the owning command-descriptor matrix. Known gaps are explicit follow-up - * work, not an implicit claim that a generic command works. + * from the owning command-descriptor matrix. Every remaining gap is an accepted + * limitation naming why no lane will cover it; Linux carries no open tracking issue. * * Projected from the command-owned declaration table; the rows are authored there. */ diff --git a/test/integration/smoke-linux-coverage.test.ts b/test/integration/smoke-linux-coverage.test.ts index 137bf05a29..30dfb7ee66 100644 --- a/test/integration/smoke-linux-coverage.test.ts +++ b/test/integration/smoke-linux-coverage.test.ts @@ -5,9 +5,11 @@ import test from 'node:test'; import { parseReplayScriptDetailed } from '@agent-device/ad-script'; import { PUBLIC_COMMANDS } from '@agent-device/command-registry/catalog'; -import { assertCoverageClassificationSummaryWiredToManifest } from './support/coverage-classification.ts'; import { - LINUX_COVERAGE_GAP_ISSUE, + assertCoverageClassificationSummaryWiredToManifest, + assertCoverageGapRowsAreOwned, +} from './support/coverage-classification.ts'; +import { LINUX_COMMAND_EVIDENCE, LINUX_PLATFORM_COVERAGE, LINUX_PLATFORM_COVERAGE_CLASSIFICATION_SUMMARY, @@ -24,6 +26,7 @@ const publicCommands = Object.values(PUBLIC_COMMANDS).sort(); test('Linux coverage exhaustively classifies the public catalog', () => { assert.deepEqual(Object.keys(LINUX_PLATFORM_COVERAGE).sort(), publicCommands); + assertCoverageGapRowsAreOwned('Linux', LINUX_PLATFORM_COVERAGE); for (const command of publicCommands) { const entry = LINUX_PLATFORM_COVERAGE[command]; @@ -32,13 +35,6 @@ test('Linux coverage exhaustively classifies the public catalog', () => { assert.ok(entry.owner.path.trim().length > 0, `${command} needs an evidence path`); assert.ok(entry.owner.test.trim().length > 0, `${command} needs named evidence`); } - if (entry.level === 'known-gap') { - assert.equal( - entry.trackingIssue, - LINUX_COVERAGE_GAP_ISSUE, - `${command} has the wrong Linux gap issue`, - ); - } } }); @@ -117,11 +113,20 @@ test('Linux contract claims name existing executable evidence', () => { } }); -test('Linux known gaps use one grouped tracking issue', () => { - const gapIssues = new Set( - Object.values(LINUX_PLATFORM_COVERAGE) - .filter((entry) => entry.level === 'known-gap') - .map((entry) => entry.trackingIssue), - ); - assert.deepEqual([...gapIssues], [LINUX_COVERAGE_GAP_ISSUE]); +test('Linux accepted gaps name the permanent Linux limitation behind each row', () => { + const acceptedCommands = Object.entries(LINUX_PLATFORM_COVERAGE) + .filter(([, entry]) => entry.level === 'accepted-gap') + .map(([command]) => command) + .sort(); + assert.deepEqual(acceptedCommands, [ + 'artifacts', + 'install', + 'install-from-source', + 'logs', + 'push', + 'record', + 'reinstall', + 'shutdown', + 'trace', + ]); }); diff --git a/test/integration/smoke-macos-coverage.test.ts b/test/integration/smoke-macos-coverage.test.ts index cd5bbb6215..3d9832089b 100644 --- a/test/integration/smoke-macos-coverage.test.ts +++ b/test/integration/smoke-macos-coverage.test.ts @@ -8,6 +8,7 @@ import { mkdtempForTest } from '@agent-device/test-kit'; import { PUBLIC_COMMANDS } from '@agent-device/command-registry/catalog'; import { assertCoverageClassificationSummaryWiredToManifest, + assertCoverageGapRowsAreOwned, assertLiveCoverageMatchesEvidence, } from './support/coverage-classification.ts'; import type { MacOsLiveScenario } from './macos-e2e/live-scenarios.ts'; @@ -24,16 +25,12 @@ const publicCommands = Object.values(PUBLIC_COMMANDS).sort(); test('macOS coverage exhaustively classifies the public catalog', () => { assert.deepEqual(Object.keys(MACOS_PLATFORM_COVERAGE).sort(), publicCommands); + assertCoverageGapRowsAreOwned('macOS', MACOS_PLATFORM_COVERAGE); for (const command of publicCommands) { const entry = MACOS_PLATFORM_COVERAGE[command]; assert.ok(entry.assertion.trim().length > 0, `${command} needs an observable assertion`); - if (entry.level === 'known-gap') { - assert.equal( - entry.trackingIssue, - MACOS_COVERAGE_GAP_ISSUE, - `${command} has the wrong gap issue`, - ); + if (entry.level === 'known-gap' || entry.level === 'accepted-gap') { continue; } assert.ok(entry.owner.path.trim().length > 0, `${command} needs an evidence path`); @@ -93,7 +90,9 @@ test('macOS live claims reference commands in existing executable scenarios', () test('macOS non-live evidence owners name existing executable repository evidence', () => { for (const [command, entry] of Object.entries(MACOS_PLATFORM_COVERAGE)) { - if (entry.level === 'live' || entry.level === 'known-gap') continue; + if (entry.level === 'live' || entry.level === 'known-gap' || entry.level === 'accepted-gap') { + continue; + } const evidencePath = path.resolve(entry.owner.path); assert.equal(fs.existsSync(evidencePath), true, `${command} owner does not exist`); assert.equal( diff --git a/test/integration/smoke-tvos-platform-coverage.test.ts b/test/integration/smoke-tvos-platform-coverage.test.ts index 63d19f261e..f3dfc59592 100644 --- a/test/integration/smoke-tvos-platform-coverage.test.ts +++ b/test/integration/smoke-tvos-platform-coverage.test.ts @@ -5,7 +5,10 @@ import test from 'node:test'; import { TVOS_SIMULATOR } from '@agent-device/test-kit'; import { PUBLIC_COMMANDS } from '@agent-device/command-registry/catalog'; -import { assertCoverageClassificationSummaryWiredToManifest } from './support/coverage-classification.ts'; +import { + assertCoverageClassificationSummaryWiredToManifest, + assertCoverageGapRowsAreOwned, +} from './support/coverage-classification.ts'; import { createPlatformRuntimeGateway } from '../../src/platform-runtime.ts'; import { gestureRefusalMessage } from '@agent-device/contracts/gesture-admission'; import { @@ -21,6 +24,7 @@ const publicCommands = Object.values(PUBLIC_COMMANDS).sort(); test('tvOS coverage exhaustively classifies the public catalog', () => { assert.deepEqual(Object.keys(TVOS_PLATFORM_COVERAGE).sort(), publicCommands); + assertCoverageGapRowsAreOwned('tvOS', TVOS_PLATFORM_COVERAGE); for (const command of publicCommands) { const entry = TVOS_PLATFORM_COVERAGE[command]; @@ -29,13 +33,6 @@ test('tvOS coverage exhaustively classifies the public catalog', () => { assert.ok(entry.owner.path.trim().length > 0, `${command} needs an evidence path`); assert.ok(entry.owner.test.trim().length > 0, `${command} needs named evidence`); } - if (entry.level === 'known-gap') { - assert.equal( - entry.trackingIssue, - TVOS_COVERAGE_GAP_ISSUE, - `${command} has the wrong gap issue`, - ); - } } }); diff --git a/test/integration/smoke-web-platform-coverage.test.ts b/test/integration/smoke-web-platform-coverage.test.ts index 15c41ba2fe..5efe8eeee0 100644 --- a/test/integration/smoke-web-platform-coverage.test.ts +++ b/test/integration/smoke-web-platform-coverage.test.ts @@ -7,10 +7,10 @@ import { mkdtempForTest } from '@agent-device/test-kit'; import { PUBLIC_COMMANDS } from '@agent-device/command-registry/catalog'; import { assertCoverageClassificationSummaryWiredToManifest, + assertCoverageGapRowsAreOwned, assertLiveCoverageMatchesEvidence, } from './support/coverage-classification.ts'; import { - WEB_COVERAGE_GAP_ISSUE, WEB_PLATFORM_COVERAGE, WEB_PLATFORM_COVERAGE_CLASSIFICATION_SUMMARY, WEB_SMOKE_EVIDENCE, @@ -23,6 +23,7 @@ const publicCommands = Object.values(PUBLIC_COMMANDS).sort(); test('web coverage exhaustively classifies the public catalog', () => { assert.deepEqual(Object.keys(WEB_PLATFORM_COVERAGE).sort(), publicCommands); + assertCoverageGapRowsAreOwned('web', WEB_PLATFORM_COVERAGE); for (const command of publicCommands) { const entry = WEB_PLATFORM_COVERAGE[command]; @@ -31,13 +32,6 @@ test('web coverage exhaustively classifies the public catalog', () => { assert.ok(entry.owner.path.trim().length > 0, `${command} needs an evidence path`); assert.ok(entry.owner.test.trim().length > 0, `${command} needs named evidence`); } - if (entry.level === 'known-gap') { - assert.equal( - entry.trackingIssue, - WEB_COVERAGE_GAP_ISSUE, - `${command} has the wrong gap issue`, - ); - } } }); @@ -78,21 +72,21 @@ test('web contract claims name existing executable evidence', () => { }); // #1900 closed 14 of the 15 known-gap rows this manifest originally carried (see the comment -// above `WEB_PLATFORM_COVERAGE`); `test` stays a known gap because its only candidate evidence -// proves the command CANNOT be targeted at web, not that it works. This is the planted-red proof -// that no other row can silently regress back to 'known-gap', and that `test`'s stays pinned to -// the one tracking issue. -test('web known gaps use one grouped tracking issue', () => { - const gapIssues = new Set( - Object.values(WEB_PLATFORM_COVERAGE) - .filter((entry) => entry.level === 'known-gap') - .map((entry) => entry.trackingIssue), - ); +// above `WEB_PLATFORM_COVERAGE`), and the rule keeps `test` the only gap row left: the web entry +// type has no `known-gap` variant, so a row that regresses to owed work without an owning issue +// fails to compile. This pins that `test` is the one accepted limitation and its reason survives. +test('web accepted gap is exactly the structurally excluded test command', () => { const gapCommands = Object.entries(WEB_PLATFORM_COVERAGE) - .filter(([, entry]) => entry.level === 'known-gap') + .filter(([, entry]) => entry.level === 'accepted-gap') .map(([command]) => command); assert.deepEqual(gapCommands, ['test']); - assert.deepEqual([...gapIssues], [WEB_COVERAGE_GAP_ISSUE]); + const testEntry = WEB_PLATFORM_COVERAGE.test; + assert.equal(testEntry.level, 'accepted-gap'); + assert.match( + testEntry.level === 'accepted-gap' ? testEntry.reason : '', + /ReplayTestPlatform/, + 'the accepted web test row must name the structural exclusion as its reason', + ); }); test('web coverage report persists the manifest rollup and live command list', async () => { diff --git a/test/integration/support/coverage-classification.ts b/test/integration/support/coverage-classification.ts index 7da1269407..fc296e378a 100644 --- a/test/integration/support/coverage-classification.ts +++ b/test/integration/support/coverage-classification.ts @@ -1,6 +1,10 @@ import assert from 'node:assert/strict'; -export type CoverageClassificationLevel = 'live' | 'command-contract' | 'known-gap'; +export type CoverageClassificationLevel = + | 'live' + | 'command-contract' + | 'known-gap' + | 'accepted-gap'; export type CoverageClassificationSummary = { contract: number; @@ -14,9 +18,52 @@ type CoverageBucket = 'live' | 'contract' | 'gap'; const BUCKET_BY_LEVEL: Readonly> = { live: 'live', 'command-contract': 'contract', + // Both gap kinds roll up into the one `gap` bucket: the report counts how many + // commands a platform does not prove, not which kind of gap each one is. 'known-gap': 'gap', + 'accepted-gap': 'gap', }; +type CoverageGapRow = { + level: string; + trackingIssue?: number; + reason?: string; +}; + +/** + * Every gap row is exactly one of two kinds, so a gap is owned rather than merely visible: + * + * - a `known-gap` row is owed work and carries a positive `trackingIssue` naming an open + * issue that owns it, an umbrella or a child of one; + * - an `accepted-gap` row is a permanent limitation: it carries a non-empty `reason` saying + * why no evidence lane will cover it, and no tracking issue. + * + * This mirrors the `gap:` waiver rule ADR 0011 enforces for interaction guarantees. + */ +export function assertCoverageGapRowsAreOwned( + platform: string, + manifest: Readonly>, +): void { + for (const [command, entry] of Object.entries(manifest)) { + if (entry.level === 'known-gap') { + assert.ok( + Number.isInteger(entry.trackingIssue) && (entry.trackingIssue ?? 0) > 0, + `${platform} ${command}: a known-gap row must carry a tracking issue that owns the work`, + ); + } + if (entry.level === 'accepted-gap') { + assert.ok( + typeof entry.reason === 'string' && entry.reason.trim().length > 0, + `${platform} ${command}: an accepted-gap row must carry a non-empty reason`, + ); + assert.ok( + entry.trackingIssue === undefined, + `${platform} ${command}: an accepted-gap row carries a reason, not a tracking issue`, + ); + } + } +} + /** * Proves a published summary is still the rollup of the manifest it is exported beside, and that the * manifest still covers the public catalog. It catches a summary wired to the wrong array, a @@ -98,6 +145,7 @@ export function buildCoverageClassificationSummary( summary.contract += 1; break; case 'known-gap': + case 'accepted-gap': summary.gap += 1; break; } diff --git a/test/integration/web-e2e/coverage.ts b/test/integration/web-e2e/coverage.ts index f141b79e51..0c0977a73c 100644 --- a/test/integration/web-e2e/coverage.ts +++ b/test/integration/web-e2e/coverage.ts @@ -2,11 +2,7 @@ import { projectCoverage } from '../command-coverage/declarations.ts'; import type { PublicCommand } from '../command-coverage/entries.ts'; import { buildCoverageClassificationSummary } from '../support/coverage-classification.ts'; -export { - WEB_COVERAGE_GAP_ISSUE, - WEB_SMOKE_EVIDENCE, - WEB_SMOKE_TEST_NAME, -} from '../command-coverage/evidence.ts'; +export { WEB_SMOKE_EVIDENCE, WEB_SMOKE_TEST_NAME } from '../command-coverage/evidence.ts'; /** * One primary, observable owner for every public command on the managed web target. @@ -21,12 +17,12 @@ export { * permanent denial (e.g. `boot`, `push`). * * #1900 closed 14 of the 15 known-gap rows this manifest originally carried using the shapes - * above. `test` stays `known-gap`: its declared-platform filter structurally excludes web + * above. `test` is an accepted-gap row: its declared-platform filter structurally excludes web * (`ReplayTestPlatform = Exclude`), so `test --platform web` can never * select a script — real, tested, command-specific behavior (see * `session-command-replay.test.ts`), but evidence of what the command cannot do, not executable * evidence that it works on web. Closing this row needs a separate product decision: either web - * replay-test support, or an explicit denial. + * replay-test support, or an explicit denial. Until one is taken, web carries no open gap row. * * Projected from the command-owned declaration table; the rows are authored there. */ From 8f4a7b7aa99d1a88f2a609062b805e8e26e6164f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Fri, 16 Oct 2026 16:47:26 +0200 Subject: [PATCH 2/2] test(coverage): keep macOS test-suite evidence as owed work and audit accepted reasons The CI macOS replay lane runs agent-device test with --report-junit and gates on its exit status, so the test row is closeable with real lane evidence and stays known-gap on #1916. Each accepted-gap reason now states the absence it claims against the repository's lanes, and the gate's docstring encodes that rule. --- .../command-coverage/declarations.ts | 17 +++++++---------- .../support/coverage-classification.ts | 4 +++- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/test/integration/command-coverage/declarations.ts b/test/integration/command-coverage/declarations.ts index c53aaa2ae6..a3b9cc40fb 100644 --- a/test/integration/command-coverage/declarations.ts +++ b/test/integration/command-coverage/declarations.ts @@ -74,7 +74,7 @@ const COMMAND_COVERAGE_DECLARATIONS = { ), linux: linux.accepted( 'No Linux live command creates a downloadable daemon artifact for inventory', - 'The Linux lanes run the local CLI, where every command output is a caller-owned local path; the daemon only inventories artifacts carried over remote artifact transport, so a Linux lane cannot create a downloadable inventory entry without a local-transport product decision.', + 'The Linux lanes run the local CLI, where command outputs are caller-owned local paths; request finalization inventories a response artifact only when the request carries a client artifact path (remote transport) or it is the sparse-snapshot fallback screenshot, which no Linux lane observes, so a Linux lane cannot produce a downloadable inventory entry without a local-transport product decision.', ), }, [C.devices]: { @@ -142,7 +142,7 @@ const COMMAND_COVERAGE_DECLARATIONS = { ), macos: macos.accepted( 'No command-specific macOS doctor evidence exists', - "The doctor checks are daemon-owned host diagnostics (version, state dir, inventory probes, Metro) with no macOS device branch, already pinned by the daemon's own doctor tests; a macOS lane would re-run that identical code, so macOS carries no command-specific doctor evidence.", + 'The doctor checks are daemon-owned host diagnostics — version, state dir, inventory probes, Metro — pinned platform-neutrally by the daemon doctor tests, and no macOS lane invokes the command: neither the System Settings replay nor the desktop and recording provider scenarios carry a doctor step.', ), tvos: tvos.gap('No tvOS-specific doctor command evidence exists yet'), web: web.contract( @@ -310,7 +310,7 @@ const COMMAND_COVERAGE_DECLARATIONS = { ), macos: macos.accepted( 'No command-specific macOS event timeline evidence exists', - 'The events timeline is daemon-owned session bookkeeping with no platform branch, already pinned for a daemon-backed session by request-router-events.test.ts; a macOS lane would re-run that identical code, so macOS carries no command-specific event timeline evidence.', + 'The events timeline is daemon-owned session bookkeeping, pinned by request-router-events.test.ts for a daemon-backed web session, and no macOS lane invokes the command: neither the System Settings replay nor the desktop and recording provider scenarios carry an events step.', ), tvos: tvos.gap('No tvOS-specific session-event command evidence exists yet'), web: web.contract( @@ -406,10 +406,7 @@ const COMMAND_COVERAGE_DECLARATIONS = { 'full:fixture-replays', 'fixture scripts run as a suite with JUnit artifacts', ), - macos: macos.accepted( - 'No command-specific macOS test-suite evidence exists', - "The CI macOS replay lane dogfoods the test command to run the replay suite, whose per-command macOS claims are already carried by the replay:system-settings live rows; no lane asserts the test command's own suite result and JUnit output on macOS, and none will.", - ), + macos: macos.gap('No command-specific macOS test-suite evidence exists yet'), tvos: tvos.gap('No tvOS-specific test-suite command evidence exists yet'), web: web.accepted( 'No command-specific web test-suite evidence exists', @@ -798,7 +795,7 @@ const COMMAND_COVERAGE_DECLARATIONS = { ), macos: macos.accepted( 'No command-specific macOS React Native inspection evidence exists', - "react-native dismiss-overlay only answers meaningfully on a surface that carries a React Native overlay, and the repository's React Native fixtures run on iOS simulators and Android emulators; the macOS lanes drive desktop apps, so no macOS lane will cover this row.", + "react-native dismiss-overlay only answers meaningfully on a surface that carries a React Native overlay, the repository's React Native overlay evidence runs on iOS simulators and Android emulators, and no macOS lane invokes the command: neither the System Settings replay nor the desktop and recording provider scenarios carry a react-native step.", ), tvos: tvos.gap('No tvOS-specific React Native inspection command evidence exists yet'), // R61: no owner fact refuses this command on a browser — its whole device work is one bound @@ -852,7 +849,7 @@ const COMMAND_COVERAGE_DECLARATIONS = { ), macos: macos.accepted( 'No command-specific macOS trace evidence exists', - "trace never binds a device runtime: its session-scoped start/stop bookkeeping is platform-neutral and already pinned by the daemon's trace-runtime tests, so a macOS lane would re-run identical code and macOS carries no command-specific trace evidence.", + 'trace never binds a device runtime: handleTraceCommand carries no bindDevice or inspectFacts call, its start/stop bookkeeping is pinned by the daemon trace-runtime tests, and no macOS lane invokes the command: neither the System Settings replay nor the desktop and recording provider scenarios carry a trace step.', ), tvos: tvos.gap('No tvOS-specific trace command evidence exists yet'), web: web.contract( @@ -862,7 +859,7 @@ const COMMAND_COVERAGE_DECLARATIONS = { ), linux: linux.accepted( 'No Linux-specific trace command evidence exists', - "trace never binds a device runtime: its session-scoped start/stop bookkeeping is platform-neutral and already pinned by the daemon's trace-runtime tests, so a Linux lane would re-run identical code and Linux carries no command-specific trace evidence.", + 'trace never binds a device runtime: handleTraceCommand carries no bindDevice or inspectFacts call and its start/stop bookkeeping is pinned by the daemon trace-runtime tests, and no Linux lane invokes the command: neither the replay script nor the command-evidence lane carries a trace step.', ), }, [C.find]: { diff --git a/test/integration/support/coverage-classification.ts b/test/integration/support/coverage-classification.ts index fc296e378a..31e502dd92 100644 --- a/test/integration/support/coverage-classification.ts +++ b/test/integration/support/coverage-classification.ts @@ -36,7 +36,9 @@ type CoverageGapRow = { * - a `known-gap` row is owed work and carries a positive `trackingIssue` naming an open * issue that owns it, an umbrella or a child of one; * - an `accepted-gap` row is a permanent limitation: it carries a non-empty `reason` saying - * why no evidence lane will cover it, and no tracking issue. + * why no evidence lane will cover it, and no tracking issue. An accepted-gap reason may + * only claim an absence that holds against the repository's lanes — if a lane runs or + * asserts the command, the row is evidence-citable or owed, never accepted. * * This mirrors the `gap:` waiver rule ADR 0011 enforces for interaction guarantees. */