From d25264c3d812e5e88eb81ff255c6a98b83a19375 Mon Sep 17 00:00:00 2001 From: Chris Huber Date: Sat, 29 Aug 2026 10:04:15 -0400 Subject: [PATCH 1/4] refactor(smoke): discover test files instead of registering them The smoke manifest enumerated 128 commands by hand, and a test only ran if someone remembered to add it. #2402 measured the result: 153 test scripts that no gate invoked, and a package.json that absorbed 15 percent of all commits acting as the de facto test registry. Discover tests/*.test.{ts,mjs} and scripts/*-smoke.{ts,php} by convention. Adding a test file is now enough to make it run. Anything that must not run is listed in DISCOVERY_EXCLUSIONS with a reason, so exclusions stay short and reviewable rather than being the silent default. The manifest keeps only work that is not a single test file: 'build' for tsc -b plus the CLI bin and provenance steps, and 'test:cloudflare-runtime' for tsc -p packages/runtime-cloudflare --noEmit. It drops from 198 lines and 128 entries to 50 lines and 2. Coverage is verified, not assumed. Of the 147 files the old manifest executed, 145 are in the discovered set; the other two are build helpers that 'build' still runs. Discovery adds 212 files that were never gated. Discovered files are independent processes, so they run at concurrency 8 and report every failure instead of stopping at the first. Five files contend on the shared Playground archive cache and run in a serial phase afterwards; they are correct in isolation and are listed in DISCOVERY_SERIAL. Measured: 359 commands over 357 files in 7m59s, against 128 commands over 147 files before. 143 percent more files for about three more minutes. --- docs/browser-runtime-dependency-audit.md | 2 +- docs/transfer-namespace-plan.md | 2 +- package.json | 2 +- scripts/run-smoke.ts | 121 +++++++++++++- scripts/smoke-discovery.ts | 95 +++++++++++ scripts/smoke-manifest.ts | 198 ++--------------------- 6 files changed, 227 insertions(+), 193 deletions(-) create mode 100644 scripts/smoke-discovery.ts diff --git a/docs/browser-runtime-dependency-audit.md b/docs/browser-runtime-dependency-audit.md index 496cd5b84..da49ebc24 100644 --- a/docs/browser-runtime-dependency-audit.md +++ b/docs/browser-runtime-dependency-audit.md @@ -69,7 +69,7 @@ Packaging coverage is intentional: 1. `npm run build` compiles the runtime packages and CLI package. 2. `npm pack --workspace @automattic/wp-codebox-cli --dry-run --json` proves the CLI package includes compiled `dist` files and omits TypeScript source. 3. `npm run package:wordpress-plugin` builds the plugin zip from source files, checked-in plugin assets, and the staged CLI release bundle. -4. `npm run smoke -- --group package` asserts the CLI pack shape and plugin zip shape, including the browser runtime asset and vendored CLI/Node runtime path. +4. `npm run check` asserts the CLI pack shape and plugin zip shape, including the browser runtime asset and vendored CLI/Node runtime path. 6. `npm run smoke -- --command wordpress-plugin-smoke` verifies the browser Playground session ability and runtime dependency metadata shape. ## Provenance And Transfer Review diff --git a/docs/transfer-namespace-plan.md b/docs/transfer-namespace-plan.md index c9a43613b..fc0930420 100644 --- a/docs/transfer-namespace-plan.md +++ b/docs/transfer-namespace-plan.md @@ -158,7 +158,7 @@ After the target npm scope and repository URL are confirmed: - Rebuild generated `dist` output with `npm run build` so compiled imports match the source package names. - Rebuild the WordPress plugin zip with `npm run package:wordpress-plugin`. -- Run `npm run smoke -- --group package` to verify npm packing and plugin zip +- Run `npm run check` to verify npm packing and plugin zip contents after the rename. - Run `npm run check` before publishing or tagging a release candidate. diff --git a/package.json b/package.json index 485e93d44..0c015f67e 100644 --- a/package.json +++ b/package.json @@ -332,7 +332,7 @@ "test:wp-cli-recipe-result": "tsx tests/wp-cli-recipe-result.test.ts", "test:browser-sdk-facade": "tsx tests/browser-sdk-facade.test.ts", "test:browser-viewport-replay": "node ./node_modules/typescript/bin/tsc -b packages/runtime-core --force && node ./node_modules/typescript/bin/tsc -b packages/runtime-playground packages/cli --force && node scripts/ensure-cli-bin-executable.mjs && tsx tests/browser-viewport-replay.test.ts", - "check": "npm run test:production-boundary-enforcement && npm run smoke -- --group=check" + "check": "npm run smoke -- --group=check" }, "workspaces": [ "packages/*" diff --git a/scripts/run-smoke.ts b/scripts/run-smoke.ts index f5df7d68e..90179ac69 100644 --- a/scripts/run-smoke.ts +++ b/scripts/run-smoke.ts @@ -1,5 +1,10 @@ import { spawn } from "node:child_process" +import { fileURLToPath } from "node:url" import { smokeGroups, smokeManifest, type SmokeCommand } from "./smoke-manifest.ts" +import { discoveredParallelCommands, discoveredSerialCommands } from "./smoke-discovery.ts" + +const repositoryRoot = fileURLToPath(new URL("..", import.meta.url)) +const DEFAULT_CONCURRENCY = 8 type ResolvedGroup = { name: string @@ -20,15 +25,21 @@ function usage(): string { ].join("\n") } -function parseArgs(args: string[]): { group?: string; command?: string; all: boolean; list: boolean } { +function parseArgs(args: string[]): { group?: string; command?: string; all: boolean; list: boolean; concurrency: number } { let group: string | undefined let command: string | undefined let all = false let list = false + let concurrency = DEFAULT_CONCURRENCY for (let index = 0; index < args.length; index += 1) { const arg = args[index] - if (arg === "--all") { + if (arg.startsWith("--concurrency=")) { + concurrency = Number(arg.slice("--concurrency=".length)) + if (!Number.isInteger(concurrency) || concurrency < 1) { + throw new Error("--concurrency requires a positive integer") + } + } else if (arg === "--all") { all = true } else if (arg === "--list") { list = true @@ -55,7 +66,7 @@ function parseArgs(args: string[]): { group?: string; command?: string; all: boo } } - return { group, command, all, list } + return { group, command, all, list, concurrency } } function resolveGroup(groupName: string): ResolvedGroup { @@ -108,6 +119,63 @@ function runCommand(command: SmokeCommand): Promise { }) } +type CommandOutcome = { command: SmokeCommand; error?: Error; output: string } + +/* + * Discovered files run as independent processes, so the parallel phase captures + * output per command and reports every failure instead of stopping at the first. + */ +function runCapturedCommand(command: SmokeCommand): Promise { + return new Promise((resolve) => { + let output = "" + const child = spawn(command.command, command.args, { + cwd: new URL("..", import.meta.url), + stdio: ["ignore", "pipe", "pipe"], + shell: process.platform === "win32", + }) + + child.stdout?.on("data", (chunk) => { output += String(chunk) }) + child.stderr?.on("data", (chunk) => { output += String(chunk) }) + child.on("error", (error) => resolve({ command, error, output })) + child.on("exit", (code, signal) => { + if (code === 0) { + resolve({ command, output }) + return + } + resolve({ + command, + error: new Error(`${command.name} failed${signal ? ` with signal ${signal}` : ` with exit code ${code ?? "unknown"}`}`), + output, + }) + }) + }) +} + +async function runInParallel(commands: SmokeCommand[], concurrency: number): Promise { + const failures: Error[] = [] + let cursor = 0 + let finished = 0 + + async function worker(): Promise { + while (cursor < commands.length) { + const command = commands[cursor] + cursor += 1 + const outcome = await runCapturedCommand(command) + finished += 1 + if (outcome.error) { + failures.push(outcome.error) + console.log(`\n[smoke] FAIL (${finished}/${commands.length}) ${command.name}`) + console.log(outcome.output.trimEnd()) + } else if (finished % 25 === 0) { + console.log(`[smoke] ${finished}/${commands.length} complete`) + } + } + } + + await Promise.all(Array.from({ length: Math.max(1, concurrency) }, worker)) + return failures +} + async function main(): Promise { const options = parseArgs(process.argv.slice(2)) @@ -121,12 +189,51 @@ async function main(): Promise { throw new Error("Use only one of --all, --group, or --command.") } - const group = options.command ? resolveCommand(options.command) : resolveGroup(options.all ? "check" : options.group ?? "check") - console.log(`[smoke] Running ${group.commands.length} command(s) from ${group.name}`) + if (options.command) { + const group = resolveCommand(options.command) + for (const command of group.commands) await runCommand(command) + return + } + + const name = options.all ? "check" : options.group ?? "check" + + if (name !== "check") { + const group = resolveGroup(name) + console.log(`[smoke] Running ${group.commands.length} command(s) from ${group.name}`) + for (const command of group.commands) await runCommand(command) + return + } + + // The aggregate: declared commands first (they build artifacts the discovered + // files rely on), then discovered files in parallel, then the serial tail. + const declared = resolveGroup("check") + const parallel = discoveredParallelCommands(repositoryRoot) + const serial = discoveredSerialCommands(repositoryRoot) + + console.log( + `[smoke] check: ${declared.commands.length} declared, ${parallel.length} discovered (concurrency ${options.concurrency}), ${serial.length} serial`, + ) + + for (const command of declared.commands) await runCommand(command) + + console.log(`\n[smoke] discovered phase: ${parallel.length} files at concurrency ${options.concurrency}`) + const failures = await runInParallel(parallel, options.concurrency) - for (const command of group.commands) { - await runCommand(command) + console.log(`\n[smoke] serial phase: ${serial.length} files`) + for (const command of serial) { + const outcome = await runCapturedCommand(command) + if (outcome.error) { + failures.push(outcome.error) + console.log(`\n[smoke] FAIL ${command.name}`) + console.log(outcome.output.trimEnd()) + } + } + + if (failures.length > 0) { + throw new Error(`${failures.length} smoke command(s) failed:\n` + failures.map((f) => ` - ${f.message}`).join("\n")) } + + console.log(`\n[smoke] check passed: ${declared.commands.length + parallel.length + serial.length} command(s)`) } main().catch((error: unknown) => { diff --git a/scripts/smoke-discovery.ts b/scripts/smoke-discovery.ts new file mode 100644 index 000000000..0324950f4 --- /dev/null +++ b/scripts/smoke-discovery.ts @@ -0,0 +1,95 @@ +import { readdirSync, statSync } from "node:fs" +import { join } from "node:path" + +import type { SmokeCommand } from "./smoke-manifest.js" + +/* + * Test files are discovered by convention rather than registered by hand. + * Adding tests/.test.ts or scripts/-smoke.ts is enough to make it + * run. Anything that must not run has to be listed below with a reason, so the + * exclusions stay short, visible, and reviewable. + */ + +export const DISCOVERY_PATTERNS = { + tests: /^[^/]+\.test\.(ts|mjs)$/, + scripts: /^[^/]+-smoke\.(ts|php)$/, +} as const + +type Exclusion = { file: string; reason: string } + +export const DISCOVERY_EXCLUSIONS: readonly Exclusion[] = [ + { file: "scripts/run-smoke.ts", reason: "the runner itself; discovering it would recurse" }, + + // Require an environment the aggregate does not provision. + { file: "tests/mysqli-poll.integration.test.ts", reason: "requires Docker; runs in the agent-task-contracts workflow" }, + { file: "tests/runtime-sources-playground-integration.test.ts", reason: "exceeds the per-file budget; runs in the agent-task-contracts workflow" }, + { file: "tests/release-package-coverage.test.ts", reason: "needs the 427 MB plugin zip from package:wordpress-plugin; runs in the Homeboy gate" }, + + // Known-failing and unmaintained. Tracked for triage; see the discovery audit + // in issue #2402. These were added in June 2026, never wired to a gate, and + // have not been touched since. + { file: "tests/artifact-reference-dtos.test.ts", reason: "failing and unmaintained; pending triage" }, + { file: "tests/browser-blueprint-ref-permission.test.ts", reason: "failing and unmaintained; pending triage" }, + { file: "tests/command-diagnostics.test.ts", reason: "failing and unmaintained; pending triage" }, + { file: "tests/docs-boundary-language.test.ts", reason: "failing and unmaintained; pending triage" }, + { file: "tests/performance-observation-contracts.test.ts", reason: "blocked on raw NUL in generated PHP; pending triage" }, + { file: "tests/rest-request-query-params.test.ts", reason: "blocked on raw NUL in generated PHP; pending triage" }, + { file: "tests/wordpress-crud-contracts.test.ts", reason: "blocked on raw NUL in generated PHP; pending triage" }, + { file: "tests/temp-runtime-cleanup.test.ts", reason: "failing and unmaintained; pending triage" }, + { file: "tests/wordpress-runtime-discovery-coverage-plan.test.ts", reason: "failing and unmaintained; pending triage" }, + { file: "scripts/agent-runtime-task-ability-smoke.ts", reason: "failing and unmaintained; pending triage" }, +] + +/* + * These contend on the shared Playground WordPress archive cache and fail when + * run alongside each other. They are correct in isolation, so they run in a + * serial phase after the parallel one rather than being excluded. + */ +export const DISCOVERY_SERIAL: readonly string[] = [ + "scripts/doctor-command-smoke.ts", + "tests/bounded-recipe-plan.integration.test.ts", + "tests/phpunit-runtime-rejection.test.ts", + "tests/playground-readonly-mounts.test.ts", + "tests/playground-phpunit-bootstrap-failure.integration.test.ts", +] + +const excluded = new Set(DISCOVERY_EXCLUSIONS.map((entry) => entry.file)) +const serial = new Set(DISCOVERY_SERIAL) + +function listFiles(root: string, directory: string, pattern: RegExp): string[] { + return readdirSync(join(root, directory)) + .filter((entry) => pattern.test(entry)) + .filter((entry) => statSync(join(root, directory, entry)).isFile()) + .map((entry) => `${directory}/${entry}`) +} + +export function discoverSmokeFiles(root = process.cwd()): string[] { + const files = [ + ...listFiles(root, "tests", DISCOVERY_PATTERNS.tests), + ...listFiles(root, "scripts", DISCOVERY_PATTERNS.scripts), + ] + return files.filter((file) => !excluded.has(file)).sort() +} + +function toCommand(file: string): SmokeCommand { + return { + name: file, + command: file.endsWith(".php") ? "php" : file.endsWith(".mjs") ? "node" : "tsx", + args: [file], + } +} + +export function discoveredCommands(root = process.cwd()): SmokeCommand[] { + return discoverSmokeFiles(root).map(toCommand) +} + +/** Files safe to run concurrently. */ +export function discoveredParallelCommands(root = process.cwd()): SmokeCommand[] { + return discoverSmokeFiles(root).filter((file) => !serial.has(file)).map(toCommand) +} + +/** Files that must run one at a time, after the parallel phase. */ +export function discoveredSerialCommands(root = process.cwd()): SmokeCommand[] { + const found = new Set(discoverSmokeFiles(root)) + return DISCOVERY_SERIAL.filter((file) => found.has(file)).map(toCommand) +} diff --git a/scripts/smoke-manifest.ts b/scripts/smoke-manifest.ts index 832771425..bc2602d8b 100644 --- a/scripts/smoke-manifest.ts +++ b/scripts/smoke-manifest.ts @@ -17,192 +17,24 @@ function npmScript(name: string): SmokeCommand { } } -function tsxSmoke(name: string, script = name): SmokeCommand { - return { - name, - command: "tsx", - args: [`scripts/${script}.ts`], - } -} - -function phpSmoke(name: string, script = name): SmokeCommand { - return { - name, - command: "php", - args: [`scripts/${script}.php`], - } -} - +/* + * Test files are not registered here. `scripts/smoke-discovery.ts` finds + * tests/*.test.{ts,mjs} and scripts/*-smoke.{ts,php} by convention, and + * `npm run check` runs them after the commands below. + * + * This group is only for work that is not a single test file: compilation and + * typechecking. Everything else belongs in a discovered file. + */ export const smokeGroups = { - core: { - description: "Build and core command contract smoke checks.", - commands: [ - npmScript("build"), - npmScript("test:generic-primitives"), - npmScript("test:php-json-codec"), - npmScript("test:primitive-contract-parity"), - npmScript("test:php-primitive-contract-parity"), - npmScript("test:browser-task-builder"), - npmScript("test:host-recipe-builder"), - npmScript("test:browser-runner-template"), - npmScript("test:browser-runtime-file-ops"), - npmScript("test:browser-prepared-runtime-filesystem-overlays"), - npmScript("test:browser-provider-bridge-inheritance"), - npmScript("test:host-http-transport"), - npmScript("test:browser-preview-routing"), - npmScript("test:browser-routed-command-security"), - tsxSmoke("runtime-backend-registry-smoke"), - tsxSmoke("backend-package-adapter-registry-smoke"), - tsxSmoke("command-registry-smoke"), - tsxSmoke("browser-probe-contract-smoke"), - tsxSmoke("command-codecs-smoke"), - tsxSmoke("command-args-smoke"), - tsxSmoke("host-tool-registry-smoke"), - tsxSmoke("host-command-tool-smoke"), - tsxSmoke("runtime-env-smoke"), - tsxSmoke("task-input-contract-smoke"), - tsxSmoke("status-taxonomy-smoke"), - npmScript("test:schema-parity"), - npmScript("test:recipe-validation-descriptors"), - npmScript("test:recipe-runtime-backend-normalization"), - npmScript("test:runtime-preset-registry"), - npmScript("test:provider-runtime-contracts"), - tsxSmoke("discovery-command-smoke"), - npmScript("test:doctor-archive-inspection"), - tsxSmoke("doctor-command-smoke"), - tsxSmoke("cli-json-failure-smoke"), - tsxSmoke("source-checkout-entrypoint-smoke"), - tsxSmoke("cli-unsettled-command-smoke"), - tsxSmoke("php-snippets-smoke"), - tsxSmoke("agent-runtime-failure-smoke"), - tsxSmoke("recipe-run-terminal-phase-failure-smoke"), - ], - }, - policy: { - description: "Workspace and runtime policy smoke checks.", - commands: [ - tsxSmoke("file-tree-policy-smoke"), - tsxSmoke("policy-validation-smoke"), - tsxSmoke("workspace-policy-smoke"), - phpSmoke("php-runner-workspace-tools-smoke"), - phpSmoke("php-runner-workspace-executor-dispatch-smoke"), - tsxSmoke("source-policy-smoke"), - tsxSmoke("overlay-preparer-registry-smoke"), - ], - }, - artifact: { - description: "Artifact contract and normalization smoke checks.", - commands: [ - tsxSmoke("artifact-bundle-verifier-smoke"), - tsxSmoke("artifact-layout-writer-smoke"), - tsxSmoke("artifact-apply-adapter-smoke"), - tsxSmoke("transfer-proof-smoke"), - tsxSmoke("artifact-redaction-smoke"), - tsxSmoke("artifact-patch-git-apply-smoke"), - tsxSmoke("artifact-reference-normalization-smoke"), - tsxSmoke("artifact-diagnostics-normalizer-smoke"), - tsxSmoke("typed-artifacts-smoke"), - tsxSmoke("tool-call-artifacts-smoke"), - tsxSmoke("artifact-browser-error-collection-smoke"), - tsxSmoke("browser-artifact-persistence-idempotency-smoke"), - tsxSmoke("executable-browser-dto-smoke"), - tsxSmoke("partial-artifact-discovery-smoke"), - tsxSmoke("mounted-workspace-diff-smoke"), - npmScript("test:mount-artifact-capture-policy"), - tsxSmoke("replay-export-blueprint-smoke"), - tsxSmoke("replay-export-manifest-integrity-smoke"), - tsxSmoke("materialize-replay-package-smoke"), - ], - }, - runtime: { - description: "Runtime state, action, reference, and WordPress command smoke checks.", - commands: [ - tsxSmoke("run-registry-smoke"), - tsxSmoke("wordpress-state-contract-smoke"), - tsxSmoke("playground-command-errors-smoke"), - tsxSmoke("runtime-command-result-envelope-smoke"), - tsxSmoke("playground-command-timeout-smoke"), - tsxSmoke("replay-export-snapshot-scoping-smoke"), - tsxSmoke("runtime-overlay-validation-smoke"), - npmScript("test:runtime-php-snippets"), - npmScript("test:wp-cli-temporary-script"), - npmScript("test:php-runtime-provider-registry"), - tsxSmoke("composer-backed-source-hydration-smoke"), - tsxSmoke("composer-package-overlay-autoload-layout-smoke"), - tsxSmoke("recipe-run-composer-autoload-extra-plugin-smoke"), - tsxSmoke("runtime-component-lifecycle-replay-smoke"), - ], - }, - package: { - description: "Package build contract smoke checks.", + declared: { + description: "Build and typecheck work that file discovery cannot express.", commands: [ + // tsc -b for runtime-core/runtime-playground/cli, plus the CLI bin + // permission and build-provenance steps. npmScript("build"), - npmScript("test:cli-build-freshness"), - npmScript("test:runtime-services"), - npmScript("test:runtime-services-lifecycle"), - npmScript("test:disposable-mysql-mysqli-e2e"), - ], - }, - agent: { - description: "Agent task, fanout, and delegation contract smoke checks.", - commands: [ - tsxSmoke("agent-runtime-signal-smoke"), - tsxSmoke("agent-runtime-ability-lifecycle-smoke"), - tsxSmoke("agent-runtime-ability-tools-smoke"), - tsxSmoke("agent-sandbox-incomplete-scope-smoke"), - phpSmoke("php-public-api-facade-smoke"), - npmScript("test:agent-no-data-machine-loop"), - tsxSmoke("recipe-run-summary-smoke"), - tsxSmoke("fanout-contract-smoke"), - phpSmoke("php-agents-api-execution-targets-smoke"), - npmScript("test:php-agents-api-adapter-contract"), - npmScript("test:php-sandbox-workspace-executor"), - phpSmoke("php-browser-runtime-agent-substrate-smoke"), - phpSmoke("php-browser-runtime-url-policy-smoke"), - phpSmoke("php-run-plan-contract-smoke"), - tsxSmoke("host-delegation-contract-smoke"), - tsxSmoke("component-contracts-agent-task-smoke"), - npmScript("test:fanout-aggregation-contract-parity"), - tsxSmoke("agent-fanout-execution-smoke"), - ], - }, - "wordpress-plugin": { - description: "WordPress plugin PHP contract smoke checks.", - commands: [ - npmScript("test:php-wasm-extension-manifests"), - npmScript("test:php-host-run-result-normalizer"), - npmScript("test:php-agent-outcome-classifier"), - npmScript("test:php-agent-runtime-execution"), - npmScript("test:php-browser-provider-auth-strategy"), - npmScript("test:php-browser-preview-only-session"), - npmScript("test:php-managed-host-command"), - npmScript("test:php-worker-runner"), - npmScript("test:php-tool-policy-normalization"), - npmScript("test:php-runner-workspace-backend-contract"), - npmScript("test:php-browser-callback-contracts"), - npmScript("test:php-runtime-package-public-contract"), - npmScript("test:php-runtime-package-canonical-importer"), - npmScript("test:php-runtime-task-runner"), - npmScript("test:php-fuzz-suite-runner"), - npmScript("test:php-browser-contained-site-contract"), - npmScript("test:php-artifact-import-idempotency"), - npmScript("test:php-browser-runtime-local-package"), - npmScript("test:php-fanout-aggregation-contract"), - npmScript("test:php-cli-command"), - npmScript("test:php-patch-approval-filter"), - npmScript("test:php-path-policy-parity"), - npmScript("test:php-provider-credential-boundary"), - ], - }, - cloudflare: { - description: "Cloudflare runtime contract smoke checks and package typecheck.", - commands: [ - // test:cloudflare-runtime also typechecks the package and is a superset of - // test:cloudflare-{queue,administrator-claim,principal-credentials, - // remote-principal-credential-gate}, which stay as narrow debug entrypoints. + // Carries `tsc -p packages/runtime-cloudflare --noEmit`. The test files it + // chains are also discovered; the typecheck is the reason it stays. npmScript("test:cloudflare-runtime"), - npmScript("test:cloudflare-wordpress-auth"), - npmScript("test:cloudflare-wordpress-archive-corpus"), ], }, } satisfies Record @@ -210,7 +42,7 @@ export const smokeGroups = { export const smokeManifest = { groups: smokeGroups, aggregateGroups: { - check: ["core", "policy", "artifact", "runtime", "package", "agent", "wordpress-plugin", "cloudflare"], + check: ["declared"], }, } as const From 412b1f51d69bfba78d491a406620d0ea4f6ddb8e Mon Sep 17 00:00:00 2001 From: Chris Huber Date: Sat, 29 Aug 2026 10:25:24 -0400 Subject: [PATCH 2/4] fix(smoke): isolate destructive and CPU-starved tests from the parallel phase CI surfaced two failure classes the local run did not. tests/prepare-declaration-rebuild.test.ts deletes packages/runtime-core/dist and runs npm install at the repository root. Under concurrency that removed @automattic/wp-codebox-core/dist/index.js while 17 other files were importing it, producing a burst of ERR_MODULE_NOT_FOUND. It is destructive to shared state rather than merely order-sensitive, so it is excluded; the Homeboy gate still runs it through test:prepare-declaration-rebuild. tests/browser-actions-navigation-capture.browser.test.ts and tests/editor-actions-save.integration.test.ts each boot a browser and a WordPress runtime. On a two-core runner at concurrency 8 they were starved and timed out, one after 75 seconds on savePost. Both move to the serial phase. Local runs passed because the machine has enough cores to absorb the oversubscription and the dist deletion window stayed narrow. --- scripts/smoke-discovery.ts | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/scripts/smoke-discovery.ts b/scripts/smoke-discovery.ts index 0324950f4..4a18526d1 100644 --- a/scripts/smoke-discovery.ts +++ b/scripts/smoke-discovery.ts @@ -24,6 +24,11 @@ export const DISCOVERY_EXCLUSIONS: readonly Exclusion[] = [ { file: "tests/mysqli-poll.integration.test.ts", reason: "requires Docker; runs in the agent-task-contracts workflow" }, { file: "tests/runtime-sources-playground-integration.test.ts", reason: "exceeds the per-file budget; runs in the agent-task-contracts workflow" }, { file: "tests/release-package-coverage.test.ts", reason: "needs the 427 MB plugin zip from package:wordpress-plugin; runs in the Homeboy gate" }, + { + file: "tests/prepare-declaration-rebuild.test.ts", + reason: + "destructive to shared state: deletes packages/runtime-core/dist and runs npm install at the repository root, which breaks every concurrent import of @automattic/wp-codebox-core; runs in the Homeboy gate", + }, // Known-failing and unmaintained. Tracked for triage; see the discovery audit // in issue #2402. These were added in June 2026, never wired to a gate, and @@ -41,9 +46,10 @@ export const DISCOVERY_EXCLUSIONS: readonly Exclusion[] = [ ] /* - * These contend on the shared Playground WordPress archive cache and fail when - * run alongside each other. They are correct in isolation, so they run in a - * serial phase after the parallel one rather than being excluded. + * These contend on the shared Playground WordPress archive cache, or boot a + * full browser and WordPress runtime and time out when starved of CPU. They are + * correct in isolation, so they run in a serial phase after the parallel one + * rather than being excluded. */ export const DISCOVERY_SERIAL: readonly string[] = [ "scripts/doctor-command-smoke.ts", @@ -51,6 +57,8 @@ export const DISCOVERY_SERIAL: readonly string[] = [ "tests/phpunit-runtime-rejection.test.ts", "tests/playground-readonly-mounts.test.ts", "tests/playground-phpunit-bootstrap-failure.integration.test.ts", + "tests/browser-actions-navigation-capture.browser.test.ts", + "tests/editor-actions-save.integration.test.ts", ] const excluded = new Set(DISCOVERY_EXCLUSIONS.map((entry) => entry.file)) From 1d943c19ce4a08d89bc4e8c1686314941baacc9b Mon Sep 17 00:00:00 2001 From: Chris Huber Date: Sat, 29 Aug 2026 10:42:57 -0400 Subject: [PATCH 3/4] fix(smoke): serialize the adaptive exploration cancellation test tests/browser-adaptive-exploration.test.ts asserts cancellation timing, so CPU starvation makes it fail rather than merely run slower. Same class as the two tests serialized in the previous commit. --- scripts/smoke-discovery.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/smoke-discovery.ts b/scripts/smoke-discovery.ts index 4a18526d1..e100d2674 100644 --- a/scripts/smoke-discovery.ts +++ b/scripts/smoke-discovery.ts @@ -59,6 +59,8 @@ export const DISCOVERY_SERIAL: readonly string[] = [ "tests/playground-phpunit-bootstrap-failure.integration.test.ts", "tests/browser-actions-navigation-capture.browser.test.ts", "tests/editor-actions-save.integration.test.ts", + // Asserts cancellation timing, so it fails when starved rather than slowed. + "tests/browser-adaptive-exploration.test.ts", ] const excluded = new Set(DISCOVERY_EXCLUSIONS.map((entry) => entry.file)) From bc2bebe725be654aa8b569ab56aa00efa1cff377 Mon Sep 17 00:00:00 2001 From: Chris Huber Date: Sat, 29 Aug 2026 11:00:10 -0400 Subject: [PATCH 4/4] fix(smoke): keep the ordered chains and exclude their member files tests/native-mariadb-runtime-service.test.ts failed on CI because dissolving test:runtime-services lost the ordering its members rely on. Being individually discoverable is not the same as being safe to run standalone or concurrently. Restore test:generic-primitives and test:runtime-services alongside build and test:cloudflare-runtime as declared commands, and exclude the 22 files those chains own from discovery so nothing runs twice. This is what the original must-keep analysis indicated; I overrode it for two chains on the grounds that their files were discoverable, which was the wrong test. Coverage unchanged: 145 of the 147 files the old manifest executed are covered, and the remaining two are build helpers that 'build' still runs. Discovery now contributes 334 files, 326 parallel and 8 serial. --- scripts/smoke-discovery.ts | 32 +++++++++++++++++++++++++++++++- scripts/smoke-manifest.ts | 9 +++++++-- 2 files changed, 38 insertions(+), 3 deletions(-) diff --git a/scripts/smoke-discovery.ts b/scripts/smoke-discovery.ts index e100d2674..5da09bde3 100644 --- a/scripts/smoke-discovery.ts +++ b/scripts/smoke-discovery.ts @@ -63,7 +63,37 @@ export const DISCOVERY_SERIAL: readonly string[] = [ "tests/browser-adaptive-exploration.test.ts", ] -const excluded = new Set(DISCOVERY_EXCLUSIONS.map((entry) => entry.file)) +/* + * Files owned by the declared chains in smoke-manifest.ts. The chains order + * them deliberately and some depend on an earlier member having run, so they are + * executed there rather than discovered independently. + */ +export const CHAIN_OWNED_FILES: readonly string[] = [ + "tests/artifact-path-primitives.test.ts", + "tests/bench-command-step-behavior.test.ts", + "tests/browser-callback-materialization-contracts.test.ts", + "tests/browser-canonical-preview-origin.test.ts", + "tests/cloudflare-allocation-lifecycle.test.ts", + "tests/cloudflare-coordinator-site-partitioning.test.ts", + "tests/cloudflare-d1-operation-repository.test.ts", + "tests/cloudflare-d1-provisioner.test.mjs", + "tests/cloudflare-phase-trace.test.ts", + "tests/cloudflare-principal-credential-operator.test.mjs", + "tests/cloudflare-principal-credential-repository.test.ts", + "tests/cloudflare-provisioning-api.test.ts", + "tests/cloudflare-public-reader.test.ts", + "tests/cloudflare-queue-batch.test.ts", + "tests/cloudflare-remote-principal-credential-gate.test.mjs", + "tests/cloudflare-runtime.test.ts", + "tests/cloudflare-site-context.test.ts", + "tests/external-mysql-runtime-service.test.ts", + "tests/generic-ability-runtime-run.test.ts", + "tests/native-mariadb-runtime-service.test.ts", + "tests/runtime-services.test.ts", + "tests/source-package-compiler-primitives.test.ts", +] + +const excluded = new Set([...DISCOVERY_EXCLUSIONS.map((entry) => entry.file), ...CHAIN_OWNED_FILES]) const serial = new Set(DISCOVERY_SERIAL) function listFiles(root: string, directory: string, pattern: RegExp): string[] { diff --git a/scripts/smoke-manifest.ts b/scripts/smoke-manifest.ts index bc2602d8b..2099b50ab 100644 --- a/scripts/smoke-manifest.ts +++ b/scripts/smoke-manifest.ts @@ -32,8 +32,13 @@ export const smokeGroups = { // tsc -b for runtime-core/runtime-playground/cli, plus the CLI bin // permission and build-provenance steps. npmScript("build"), - // Carries `tsc -p packages/runtime-cloudflare --noEmit`. The test files it - // chains are also discovered; the typecheck is the reason it stays. + // These chains order their member files deliberately, and some of those + // files depend on earlier ones having run. Discovery cannot express that, + // so the chains stay and their members are excluded from discovery via + // CHAIN_OWNED_FILES in smoke-discovery.ts. + npmScript("test:generic-primitives"), + npmScript("test:runtime-services"), + // Also carries `tsc -p packages/runtime-cloudflare --noEmit`. npmScript("test:cloudflare-runtime"), ], },