diff --git a/docs/browser-runtime-dependency-audit.md b/docs/browser-runtime-dependency-audit.md index 496cd5b8..da49ebc2 100644 --- a/docs/browser-runtime-dependency-audit.md +++ b/docs/browser-runtime-dependency-audit.md @@ -69,7 +69,7 @@ Packaging coverage is intentional: 1. `npm run build` compiles the runtime packages and CLI package. 2. `npm pack --workspace @automattic/wp-codebox-cli --dry-run --json` proves the CLI package includes compiled `dist` files and omits TypeScript source. 3. `npm run package:wordpress-plugin` builds the plugin zip from source files, checked-in plugin assets, and the staged CLI release bundle. -4. `npm run smoke -- --group package` asserts the CLI pack shape and plugin zip shape, including the browser runtime asset and vendored CLI/Node runtime path. +4. `npm run check` asserts the CLI pack shape and plugin zip shape, including the browser runtime asset and vendored CLI/Node runtime path. 6. `npm run smoke -- --command wordpress-plugin-smoke` verifies the browser Playground session ability and runtime dependency metadata shape. ## Provenance And Transfer Review diff --git a/docs/transfer-namespace-plan.md b/docs/transfer-namespace-plan.md index c9a43613..fc093042 100644 --- a/docs/transfer-namespace-plan.md +++ b/docs/transfer-namespace-plan.md @@ -158,7 +158,7 @@ After the target npm scope and repository URL are confirmed: - Rebuild generated `dist` output with `npm run build` so compiled imports match the source package names. - Rebuild the WordPress plugin zip with `npm run package:wordpress-plugin`. -- Run `npm run smoke -- --group package` to verify npm packing and plugin zip +- Run `npm run check` to verify npm packing and plugin zip contents after the rename. - Run `npm run check` before publishing or tagging a release candidate. diff --git a/package.json b/package.json index 485e93d4..0c015f67 100644 --- a/package.json +++ b/package.json @@ -332,7 +332,7 @@ "test:wp-cli-recipe-result": "tsx tests/wp-cli-recipe-result.test.ts", "test:browser-sdk-facade": "tsx tests/browser-sdk-facade.test.ts", "test:browser-viewport-replay": "node ./node_modules/typescript/bin/tsc -b packages/runtime-core --force && node ./node_modules/typescript/bin/tsc -b packages/runtime-playground packages/cli --force && node scripts/ensure-cli-bin-executable.mjs && tsx tests/browser-viewport-replay.test.ts", - "check": "npm run test:production-boundary-enforcement && npm run smoke -- --group=check" + "check": "npm run smoke -- --group=check" }, "workspaces": [ "packages/*" diff --git a/scripts/run-smoke.ts b/scripts/run-smoke.ts index f5df7d68..90179ac6 100644 --- a/scripts/run-smoke.ts +++ b/scripts/run-smoke.ts @@ -1,5 +1,10 @@ import { spawn } from "node:child_process" +import { fileURLToPath } from "node:url" import { smokeGroups, smokeManifest, type SmokeCommand } from "./smoke-manifest.ts" +import { discoveredParallelCommands, discoveredSerialCommands } from "./smoke-discovery.ts" + +const repositoryRoot = fileURLToPath(new URL("..", import.meta.url)) +const DEFAULT_CONCURRENCY = 8 type ResolvedGroup = { name: string @@ -20,15 +25,21 @@ function usage(): string { ].join("\n") } -function parseArgs(args: string[]): { group?: string; command?: string; all: boolean; list: boolean } { +function parseArgs(args: string[]): { group?: string; command?: string; all: boolean; list: boolean; concurrency: number } { let group: string | undefined let command: string | undefined let all = false let list = false + let concurrency = DEFAULT_CONCURRENCY for (let index = 0; index < args.length; index += 1) { const arg = args[index] - if (arg === "--all") { + if (arg.startsWith("--concurrency=")) { + concurrency = Number(arg.slice("--concurrency=".length)) + if (!Number.isInteger(concurrency) || concurrency < 1) { + throw new Error("--concurrency requires a positive integer") + } + } else if (arg === "--all") { all = true } else if (arg === "--list") { list = true @@ -55,7 +66,7 @@ function parseArgs(args: string[]): { group?: string; command?: string; all: boo } } - return { group, command, all, list } + return { group, command, all, list, concurrency } } function resolveGroup(groupName: string): ResolvedGroup { @@ -108,6 +119,63 @@ function runCommand(command: SmokeCommand): Promise { }) } +type CommandOutcome = { command: SmokeCommand; error?: Error; output: string } + +/* + * Discovered files run as independent processes, so the parallel phase captures + * output per command and reports every failure instead of stopping at the first. + */ +function runCapturedCommand(command: SmokeCommand): Promise { + return new Promise((resolve) => { + let output = "" + const child = spawn(command.command, command.args, { + cwd: new URL("..", import.meta.url), + stdio: ["ignore", "pipe", "pipe"], + shell: process.platform === "win32", + }) + + child.stdout?.on("data", (chunk) => { output += String(chunk) }) + child.stderr?.on("data", (chunk) => { output += String(chunk) }) + child.on("error", (error) => resolve({ command, error, output })) + child.on("exit", (code, signal) => { + if (code === 0) { + resolve({ command, output }) + return + } + resolve({ + command, + error: new Error(`${command.name} failed${signal ? ` with signal ${signal}` : ` with exit code ${code ?? "unknown"}`}`), + output, + }) + }) + }) +} + +async function runInParallel(commands: SmokeCommand[], concurrency: number): Promise { + const failures: Error[] = [] + let cursor = 0 + let finished = 0 + + async function worker(): Promise { + while (cursor < commands.length) { + const command = commands[cursor] + cursor += 1 + const outcome = await runCapturedCommand(command) + finished += 1 + if (outcome.error) { + failures.push(outcome.error) + console.log(`\n[smoke] FAIL (${finished}/${commands.length}) ${command.name}`) + console.log(outcome.output.trimEnd()) + } else if (finished % 25 === 0) { + console.log(`[smoke] ${finished}/${commands.length} complete`) + } + } + } + + await Promise.all(Array.from({ length: Math.max(1, concurrency) }, worker)) + return failures +} + async function main(): Promise { const options = parseArgs(process.argv.slice(2)) @@ -121,12 +189,51 @@ async function main(): Promise { throw new Error("Use only one of --all, --group, or --command.") } - const group = options.command ? resolveCommand(options.command) : resolveGroup(options.all ? "check" : options.group ?? "check") - console.log(`[smoke] Running ${group.commands.length} command(s) from ${group.name}`) + if (options.command) { + const group = resolveCommand(options.command) + for (const command of group.commands) await runCommand(command) + return + } + + const name = options.all ? "check" : options.group ?? "check" + + if (name !== "check") { + const group = resolveGroup(name) + console.log(`[smoke] Running ${group.commands.length} command(s) from ${group.name}`) + for (const command of group.commands) await runCommand(command) + return + } + + // The aggregate: declared commands first (they build artifacts the discovered + // files rely on), then discovered files in parallel, then the serial tail. + const declared = resolveGroup("check") + const parallel = discoveredParallelCommands(repositoryRoot) + const serial = discoveredSerialCommands(repositoryRoot) + + console.log( + `[smoke] check: ${declared.commands.length} declared, ${parallel.length} discovered (concurrency ${options.concurrency}), ${serial.length} serial`, + ) + + for (const command of declared.commands) await runCommand(command) + + console.log(`\n[smoke] discovered phase: ${parallel.length} files at concurrency ${options.concurrency}`) + const failures = await runInParallel(parallel, options.concurrency) - for (const command of group.commands) { - await runCommand(command) + console.log(`\n[smoke] serial phase: ${serial.length} files`) + for (const command of serial) { + const outcome = await runCapturedCommand(command) + if (outcome.error) { + failures.push(outcome.error) + console.log(`\n[smoke] FAIL ${command.name}`) + console.log(outcome.output.trimEnd()) + } + } + + if (failures.length > 0) { + throw new Error(`${failures.length} smoke command(s) failed:\n` + failures.map((f) => ` - ${f.message}`).join("\n")) } + + console.log(`\n[smoke] check passed: ${declared.commands.length + parallel.length + serial.length} command(s)`) } main().catch((error: unknown) => { diff --git a/scripts/smoke-discovery.ts b/scripts/smoke-discovery.ts new file mode 100644 index 00000000..5da09bde --- /dev/null +++ b/scripts/smoke-discovery.ts @@ -0,0 +1,135 @@ +import { readdirSync, statSync } from "node:fs" +import { join } from "node:path" + +import type { SmokeCommand } from "./smoke-manifest.js" + +/* + * Test files are discovered by convention rather than registered by hand. + * Adding tests/.test.ts or scripts/-smoke.ts is enough to make it + * run. Anything that must not run has to be listed below with a reason, so the + * exclusions stay short, visible, and reviewable. + */ + +export const DISCOVERY_PATTERNS = { + tests: /^[^/]+\.test\.(ts|mjs)$/, + scripts: /^[^/]+-smoke\.(ts|php)$/, +} as const + +type Exclusion = { file: string; reason: string } + +export const DISCOVERY_EXCLUSIONS: readonly Exclusion[] = [ + { file: "scripts/run-smoke.ts", reason: "the runner itself; discovering it would recurse" }, + + // Require an environment the aggregate does not provision. + { file: "tests/mysqli-poll.integration.test.ts", reason: "requires Docker; runs in the agent-task-contracts workflow" }, + { file: "tests/runtime-sources-playground-integration.test.ts", reason: "exceeds the per-file budget; runs in the agent-task-contracts workflow" }, + { file: "tests/release-package-coverage.test.ts", reason: "needs the 427 MB plugin zip from package:wordpress-plugin; runs in the Homeboy gate" }, + { + file: "tests/prepare-declaration-rebuild.test.ts", + reason: + "destructive to shared state: deletes packages/runtime-core/dist and runs npm install at the repository root, which breaks every concurrent import of @automattic/wp-codebox-core; runs in the Homeboy gate", + }, + + // Known-failing and unmaintained. Tracked for triage; see the discovery audit + // in issue #2402. These were added in June 2026, never wired to a gate, and + // have not been touched since. + { file: "tests/artifact-reference-dtos.test.ts", reason: "failing and unmaintained; pending triage" }, + { file: "tests/browser-blueprint-ref-permission.test.ts", reason: "failing and unmaintained; pending triage" }, + { file: "tests/command-diagnostics.test.ts", reason: "failing and unmaintained; pending triage" }, + { file: "tests/docs-boundary-language.test.ts", reason: "failing and unmaintained; pending triage" }, + { file: "tests/performance-observation-contracts.test.ts", reason: "blocked on raw NUL in generated PHP; pending triage" }, + { file: "tests/rest-request-query-params.test.ts", reason: "blocked on raw NUL in generated PHP; pending triage" }, + { file: "tests/wordpress-crud-contracts.test.ts", reason: "blocked on raw NUL in generated PHP; pending triage" }, + { file: "tests/temp-runtime-cleanup.test.ts", reason: "failing and unmaintained; pending triage" }, + { file: "tests/wordpress-runtime-discovery-coverage-plan.test.ts", reason: "failing and unmaintained; pending triage" }, + { file: "scripts/agent-runtime-task-ability-smoke.ts", reason: "failing and unmaintained; pending triage" }, +] + +/* + * These contend on the shared Playground WordPress archive cache, or boot a + * full browser and WordPress runtime and time out when starved of CPU. They are + * correct in isolation, so they run in a serial phase after the parallel one + * rather than being excluded. + */ +export const DISCOVERY_SERIAL: readonly string[] = [ + "scripts/doctor-command-smoke.ts", + "tests/bounded-recipe-plan.integration.test.ts", + "tests/phpunit-runtime-rejection.test.ts", + "tests/playground-readonly-mounts.test.ts", + "tests/playground-phpunit-bootstrap-failure.integration.test.ts", + "tests/browser-actions-navigation-capture.browser.test.ts", + "tests/editor-actions-save.integration.test.ts", + // Asserts cancellation timing, so it fails when starved rather than slowed. + "tests/browser-adaptive-exploration.test.ts", +] + +/* + * Files owned by the declared chains in smoke-manifest.ts. The chains order + * them deliberately and some depend on an earlier member having run, so they are + * executed there rather than discovered independently. + */ +export const CHAIN_OWNED_FILES: readonly string[] = [ + "tests/artifact-path-primitives.test.ts", + "tests/bench-command-step-behavior.test.ts", + "tests/browser-callback-materialization-contracts.test.ts", + "tests/browser-canonical-preview-origin.test.ts", + "tests/cloudflare-allocation-lifecycle.test.ts", + "tests/cloudflare-coordinator-site-partitioning.test.ts", + "tests/cloudflare-d1-operation-repository.test.ts", + "tests/cloudflare-d1-provisioner.test.mjs", + "tests/cloudflare-phase-trace.test.ts", + "tests/cloudflare-principal-credential-operator.test.mjs", + "tests/cloudflare-principal-credential-repository.test.ts", + "tests/cloudflare-provisioning-api.test.ts", + "tests/cloudflare-public-reader.test.ts", + "tests/cloudflare-queue-batch.test.ts", + "tests/cloudflare-remote-principal-credential-gate.test.mjs", + "tests/cloudflare-runtime.test.ts", + "tests/cloudflare-site-context.test.ts", + "tests/external-mysql-runtime-service.test.ts", + "tests/generic-ability-runtime-run.test.ts", + "tests/native-mariadb-runtime-service.test.ts", + "tests/runtime-services.test.ts", + "tests/source-package-compiler-primitives.test.ts", +] + +const excluded = new Set([...DISCOVERY_EXCLUSIONS.map((entry) => entry.file), ...CHAIN_OWNED_FILES]) +const serial = new Set(DISCOVERY_SERIAL) + +function listFiles(root: string, directory: string, pattern: RegExp): string[] { + return readdirSync(join(root, directory)) + .filter((entry) => pattern.test(entry)) + .filter((entry) => statSync(join(root, directory, entry)).isFile()) + .map((entry) => `${directory}/${entry}`) +} + +export function discoverSmokeFiles(root = process.cwd()): string[] { + const files = [ + ...listFiles(root, "tests", DISCOVERY_PATTERNS.tests), + ...listFiles(root, "scripts", DISCOVERY_PATTERNS.scripts), + ] + return files.filter((file) => !excluded.has(file)).sort() +} + +function toCommand(file: string): SmokeCommand { + return { + name: file, + command: file.endsWith(".php") ? "php" : file.endsWith(".mjs") ? "node" : "tsx", + args: [file], + } +} + +export function discoveredCommands(root = process.cwd()): SmokeCommand[] { + return discoverSmokeFiles(root).map(toCommand) +} + +/** Files safe to run concurrently. */ +export function discoveredParallelCommands(root = process.cwd()): SmokeCommand[] { + return discoverSmokeFiles(root).filter((file) => !serial.has(file)).map(toCommand) +} + +/** Files that must run one at a time, after the parallel phase. */ +export function discoveredSerialCommands(root = process.cwd()): SmokeCommand[] { + const found = new Set(discoverSmokeFiles(root)) + return DISCOVERY_SERIAL.filter((file) => found.has(file)).map(toCommand) +} diff --git a/scripts/smoke-manifest.ts b/scripts/smoke-manifest.ts index 83277142..2099b50a 100644 --- a/scripts/smoke-manifest.ts +++ b/scripts/smoke-manifest.ts @@ -17,192 +17,29 @@ function npmScript(name: string): SmokeCommand { } } -function tsxSmoke(name: string, script = name): SmokeCommand { - return { - name, - command: "tsx", - args: [`scripts/${script}.ts`], - } -} - -function phpSmoke(name: string, script = name): SmokeCommand { - return { - name, - command: "php", - args: [`scripts/${script}.php`], - } -} - +/* + * Test files are not registered here. `scripts/smoke-discovery.ts` finds + * tests/*.test.{ts,mjs} and scripts/*-smoke.{ts,php} by convention, and + * `npm run check` runs them after the commands below. + * + * This group is only for work that is not a single test file: compilation and + * typechecking. Everything else belongs in a discovered file. + */ export const smokeGroups = { - core: { - description: "Build and core command contract smoke checks.", + declared: { + description: "Build and typecheck work that file discovery cannot express.", commands: [ + // tsc -b for runtime-core/runtime-playground/cli, plus the CLI bin + // permission and build-provenance steps. npmScript("build"), + // These chains order their member files deliberately, and some of those + // files depend on earlier ones having run. Discovery cannot express that, + // so the chains stay and their members are excluded from discovery via + // CHAIN_OWNED_FILES in smoke-discovery.ts. npmScript("test:generic-primitives"), - npmScript("test:php-json-codec"), - npmScript("test:primitive-contract-parity"), - npmScript("test:php-primitive-contract-parity"), - npmScript("test:browser-task-builder"), - npmScript("test:host-recipe-builder"), - npmScript("test:browser-runner-template"), - npmScript("test:browser-runtime-file-ops"), - npmScript("test:browser-prepared-runtime-filesystem-overlays"), - npmScript("test:browser-provider-bridge-inheritance"), - npmScript("test:host-http-transport"), - npmScript("test:browser-preview-routing"), - npmScript("test:browser-routed-command-security"), - tsxSmoke("runtime-backend-registry-smoke"), - tsxSmoke("backend-package-adapter-registry-smoke"), - tsxSmoke("command-registry-smoke"), - tsxSmoke("browser-probe-contract-smoke"), - tsxSmoke("command-codecs-smoke"), - tsxSmoke("command-args-smoke"), - tsxSmoke("host-tool-registry-smoke"), - tsxSmoke("host-command-tool-smoke"), - tsxSmoke("runtime-env-smoke"), - tsxSmoke("task-input-contract-smoke"), - tsxSmoke("status-taxonomy-smoke"), - npmScript("test:schema-parity"), - npmScript("test:recipe-validation-descriptors"), - npmScript("test:recipe-runtime-backend-normalization"), - npmScript("test:runtime-preset-registry"), - npmScript("test:provider-runtime-contracts"), - tsxSmoke("discovery-command-smoke"), - npmScript("test:doctor-archive-inspection"), - tsxSmoke("doctor-command-smoke"), - tsxSmoke("cli-json-failure-smoke"), - tsxSmoke("source-checkout-entrypoint-smoke"), - tsxSmoke("cli-unsettled-command-smoke"), - tsxSmoke("php-snippets-smoke"), - tsxSmoke("agent-runtime-failure-smoke"), - tsxSmoke("recipe-run-terminal-phase-failure-smoke"), - ], - }, - policy: { - description: "Workspace and runtime policy smoke checks.", - commands: [ - tsxSmoke("file-tree-policy-smoke"), - tsxSmoke("policy-validation-smoke"), - tsxSmoke("workspace-policy-smoke"), - phpSmoke("php-runner-workspace-tools-smoke"), - phpSmoke("php-runner-workspace-executor-dispatch-smoke"), - tsxSmoke("source-policy-smoke"), - tsxSmoke("overlay-preparer-registry-smoke"), - ], - }, - artifact: { - description: "Artifact contract and normalization smoke checks.", - commands: [ - tsxSmoke("artifact-bundle-verifier-smoke"), - tsxSmoke("artifact-layout-writer-smoke"), - tsxSmoke("artifact-apply-adapter-smoke"), - tsxSmoke("transfer-proof-smoke"), - tsxSmoke("artifact-redaction-smoke"), - tsxSmoke("artifact-patch-git-apply-smoke"), - tsxSmoke("artifact-reference-normalization-smoke"), - tsxSmoke("artifact-diagnostics-normalizer-smoke"), - tsxSmoke("typed-artifacts-smoke"), - tsxSmoke("tool-call-artifacts-smoke"), - tsxSmoke("artifact-browser-error-collection-smoke"), - tsxSmoke("browser-artifact-persistence-idempotency-smoke"), - tsxSmoke("executable-browser-dto-smoke"), - tsxSmoke("partial-artifact-discovery-smoke"), - tsxSmoke("mounted-workspace-diff-smoke"), - npmScript("test:mount-artifact-capture-policy"), - tsxSmoke("replay-export-blueprint-smoke"), - tsxSmoke("replay-export-manifest-integrity-smoke"), - tsxSmoke("materialize-replay-package-smoke"), - ], - }, - runtime: { - description: "Runtime state, action, reference, and WordPress command smoke checks.", - commands: [ - tsxSmoke("run-registry-smoke"), - tsxSmoke("wordpress-state-contract-smoke"), - tsxSmoke("playground-command-errors-smoke"), - tsxSmoke("runtime-command-result-envelope-smoke"), - tsxSmoke("playground-command-timeout-smoke"), - tsxSmoke("replay-export-snapshot-scoping-smoke"), - tsxSmoke("runtime-overlay-validation-smoke"), - npmScript("test:runtime-php-snippets"), - npmScript("test:wp-cli-temporary-script"), - npmScript("test:php-runtime-provider-registry"), - tsxSmoke("composer-backed-source-hydration-smoke"), - tsxSmoke("composer-package-overlay-autoload-layout-smoke"), - tsxSmoke("recipe-run-composer-autoload-extra-plugin-smoke"), - tsxSmoke("runtime-component-lifecycle-replay-smoke"), - ], - }, - package: { - description: "Package build contract smoke checks.", - commands: [ - npmScript("build"), - npmScript("test:cli-build-freshness"), npmScript("test:runtime-services"), - npmScript("test:runtime-services-lifecycle"), - npmScript("test:disposable-mysql-mysqli-e2e"), - ], - }, - agent: { - description: "Agent task, fanout, and delegation contract smoke checks.", - commands: [ - tsxSmoke("agent-runtime-signal-smoke"), - tsxSmoke("agent-runtime-ability-lifecycle-smoke"), - tsxSmoke("agent-runtime-ability-tools-smoke"), - tsxSmoke("agent-sandbox-incomplete-scope-smoke"), - phpSmoke("php-public-api-facade-smoke"), - npmScript("test:agent-no-data-machine-loop"), - tsxSmoke("recipe-run-summary-smoke"), - tsxSmoke("fanout-contract-smoke"), - phpSmoke("php-agents-api-execution-targets-smoke"), - npmScript("test:php-agents-api-adapter-contract"), - npmScript("test:php-sandbox-workspace-executor"), - phpSmoke("php-browser-runtime-agent-substrate-smoke"), - phpSmoke("php-browser-runtime-url-policy-smoke"), - phpSmoke("php-run-plan-contract-smoke"), - tsxSmoke("host-delegation-contract-smoke"), - tsxSmoke("component-contracts-agent-task-smoke"), - npmScript("test:fanout-aggregation-contract-parity"), - tsxSmoke("agent-fanout-execution-smoke"), - ], - }, - "wordpress-plugin": { - description: "WordPress plugin PHP contract smoke checks.", - commands: [ - npmScript("test:php-wasm-extension-manifests"), - npmScript("test:php-host-run-result-normalizer"), - npmScript("test:php-agent-outcome-classifier"), - npmScript("test:php-agent-runtime-execution"), - npmScript("test:php-browser-provider-auth-strategy"), - npmScript("test:php-browser-preview-only-session"), - npmScript("test:php-managed-host-command"), - npmScript("test:php-worker-runner"), - npmScript("test:php-tool-policy-normalization"), - npmScript("test:php-runner-workspace-backend-contract"), - npmScript("test:php-browser-callback-contracts"), - npmScript("test:php-runtime-package-public-contract"), - npmScript("test:php-runtime-package-canonical-importer"), - npmScript("test:php-runtime-task-runner"), - npmScript("test:php-fuzz-suite-runner"), - npmScript("test:php-browser-contained-site-contract"), - npmScript("test:php-artifact-import-idempotency"), - npmScript("test:php-browser-runtime-local-package"), - npmScript("test:php-fanout-aggregation-contract"), - npmScript("test:php-cli-command"), - npmScript("test:php-patch-approval-filter"), - npmScript("test:php-path-policy-parity"), - npmScript("test:php-provider-credential-boundary"), - ], - }, - cloudflare: { - description: "Cloudflare runtime contract smoke checks and package typecheck.", - commands: [ - // test:cloudflare-runtime also typechecks the package and is a superset of - // test:cloudflare-{queue,administrator-claim,principal-credentials, - // remote-principal-credential-gate}, which stay as narrow debug entrypoints. + // Also carries `tsc -p packages/runtime-cloudflare --noEmit`. npmScript("test:cloudflare-runtime"), - npmScript("test:cloudflare-wordpress-auth"), - npmScript("test:cloudflare-wordpress-archive-corpus"), ], }, } satisfies Record @@ -210,7 +47,7 @@ export const smokeGroups = { export const smokeManifest = { groups: smokeGroups, aggregateGroups: { - check: ["core", "policy", "artifact", "runtime", "package", "agent", "wordpress-plugin", "cloudflare"], + check: ["declared"], }, } as const