compute: judge the CLI against the same toolchains as the runtimes

Every result file records the CLI it was built with, but the compute model
only used that to credit the CLI with a Node.js major. A cell's verdict
ignored which CLI built it, so an older CLI failing where a newer one passed
read as verified, and ns doctor had no way to ask about its own CLI.

Each build is now evidence about both packages it pinned. The runtime and the
CLI each see the other as a companion pin, so a failure lands on the CLI's row
when a newer CLI builds the same runtime and toolchain, and stays on both
until something distinguishes them. The CLI row gains Xcode, CocoaPods and the
Android keys next to Node.js.

The per-release endpoint accepts cli=<version> and answers with the same
verified, failed, suspect and inferred lists for that CLI release, or
tracked: false when the document does not carry it.

Author: rigor789

README.md

@@ -75,18 +75,23 @@ result file for a verification done outside CI.
 
 ## How a cell gets its state
 
-A failed build pins several toolchains at once and cannot say which one is to
-blame, so a failure only implicates toolchain versions that no successful
-build of the same release used; when one of those versions is already the
+A failed build pins several things at once and cannot say which one is to
+blame: the toolchains, and the other package it was built with (the CLI for a
+runtime, the runtime for the CLI). A failure only implicates pins that no
+successful build of the same release used; when one of them is already the
 sole suspect of another failure, it explains the failure and the rest stay
-unjudged.
+unjudged. So when a newer CLI builds the same runtime and toolchain, the
+failure lands on the older CLI's row and the runtime's cell stays verified.
+Until then both rows show the failure.
 
 For a release and a toolchain version, in this order:
 
 1. **Unsupported / Advisory**: a maintainer advisory matches the release and
    the toolchain version.
 2. **Verified**: a recorded CI build succeeded with that toolchain version.
-   Any success outranks failures. A failure only marks the cell unsupported
+   A build is evidence about the runtime and about the CLI it was built with
+   alike, so the same result verifies the toolchain for both, and the Node.js
+   major for the CLI. Any success outranks failures. A failure only marks the cell unsupported
    once a second, independent run has confirmed it; a single failure shows as
    unverified with a pending second attempt. A confirmed failure on a newer
    release also marks older releases without a result of their own as
@@ -115,9 +120,12 @@ than the declaration. Clicking a cell shows the same timeline over releases.
 
 - `/` the app.
 - `/v1/compatibility.json` the full document (`$schema` points at its schema).
-- `/v1/packages/<name>/<version>?xcode=26.2&jdk=21` one release: its
-  effective requirements, the toolchain versions verified, failed, suspect or
-  inferred for it, and the advisories matching the given toolchain versions.
+- `/v1/packages/<name>/<version>?xcode=26.2&jdk=21&cli=9.1.1` one release:
+  its effective requirements, the toolchain versions verified, failed, suspect
+  or inferred for it, and the advisories matching the given toolchain
+  versions. With `cli`, the response carries the same for that CLI release
+  under `cli`, so a project's CLI is judged against the same toolchains; a
+  CLI release the document does not track answers with `tracked: false`.
 - `/v1/schemas/{compatibility,requirements,package,overrides,verified}.json`.
 - `/health`.
 

shared/compute.ts

@@ -20,11 +20,13 @@ export interface TrackedPackageSpec {
 	keep: number;
 }
 
+export const CLI_PACKAGE = "nativescript";
+
 export const TRACKED_PACKAGES: TrackedPackageSpec[] = [
 	{ name: "@nativescript/ios", toolchains: ["xcode", "cocoapods"], keep: 12 },
 	{ name: "@nativescript/android", toolchains: ["compileSdk", "buildTools", "jdk"], keep: 12 },
 	{ name: "@nativescript/visionos", toolchains: ["xcode"], keep: 6 },
-	{ name: "nativescript", toolchains: ["node"], keep: 8 },
+	{ name: CLI_PACKAGE, toolchains: ["node", "xcode", "cocoapods", "compileSdk", "buildTools", "jdk"], keep: 8 },
 ];
 
 /**
@@ -167,90 +169,102 @@ export function classify(entry: VerificationResult): ResultClass {
 	return (entry.attempts ?? 1) >= 2 ? "confirmed" : "suspect";
 }
 
+interface PinnedBuild {
+	package: string;
+	version: string;
+	toolchains: Partial<Record<ToolchainKey, string>>;
+	/** The other package the build pinned: the CLI for a runtime, the runtime for the CLI. */
+	companion: { package: string; version: string };
+	outcome: ResultClass;
+}
+
 /**
- * A successful result proves every toolchain it pinned for the runtime it
- * built, and the Node.js major for the CLI it built with. A failure only
- * speaks about the runtime's toolchains.
+ * One CI build pins a runtime release and a CLI release together, so it is
+ * evidence about both: each toolchain for the runtime, and each toolchain plus
+ * the Node.js major for the CLI.
  */
+function pinnedBuilds(entry: VerificationResult): PinnedBuild[] {
+	const outcome = classify(entry);
+	const runtime = { package: entry.package, version: entry.version };
+	const cli = { package: CLI_PACKAGE, version: entry.with.nativescript };
+	return [
+		{ ...runtime, toolchains: entry.toolchains, companion: cli, outcome },
+		{ ...cli, toolchains: { ...entry.toolchains, node: entry.with.node }, companion: runtime, outcome },
+	];
+}
+
 export function verifiedPairs(
 	verified: VerificationResult[],
 	outcome: ResultClass = "success",
 ): Array<{ package: string; version: string; toolchain: ToolchainKey; toolchainVersion: string }> {
 	return verified
-		.filter((entry) => classify(entry) === outcome)
-		.flatMap((entry) => [
-			...Object.entries(entry.toolchains).map(([toolchain, toolchainVersion]) => ({
-				package: entry.package,
-				version: entry.version,
-				toolchain: toolchain as ToolchainKey,
-				toolchainVersion: toolchainVersion!,
+		.flatMap(pinnedBuilds)
+		.filter((build) => build.outcome === outcome)
+		.flatMap((build) =>
+			(Object.entries(build.toolchains) as Array<[ToolchainKey, string]>).map(([toolchain, toolchainVersion]) => ({
+				package: build.package,
+				version: build.version,
+				toolchain,
+				toolchainVersion,
 			})),
-			// A failed runtime build says nothing about the CLI's Node.js support.
-			...(outcome === "success"
-				? [
-						{
-							package: "nativescript",
-							version: entry.with.nativescript,
-							toolchain: "node" as ToolchainKey,
-							toolchainVersion: entry.with.node,
-						},
-					]
-				: []),
-		]);
+		);
+}
+
+interface Pin {
+	key: string;
+	version: string;
+	/** Companion-package pins take part in blame but are never reported as a toolchain. */
+	toolchain: boolean;
 }
 
 /**
- * A failed build pins several toolchains at once and cannot say which one is
- * to blame. A failure therefore implicates only toolchain versions that no
- * successful build of the same release used, and once one of its pinned
- * versions is the sole suspect of another failure, that version explains the
- * failure and the rest stay unjudged.
+ * A failed build pins several things at once and cannot say which one is to
+ * blame: its toolchains and the other package it was built with. A failure
+ * therefore implicates only pins that no successful build of the same release
+ * used, and once one of its pins is the sole suspect of another failure, that
+ * pin explains the failure and the rest stay unjudged. A failure whose only
+ * suspect is the companion package says nothing about any toolchain.
  */
 function verifiedFor(
 	packageName: string,
 	version: string,
 	verified: VerificationResult[],
 	outcome: ResultClass,
 ): VersionCompatibility["verified"] {
-	const forThisVersion = (entries: VerificationResult[]) =>
-		entries.filter((entry) => entry.package === packageName && entry.version === version);
-	const successes = forThisVersion(verified).filter((entry) => classify(entry) === "success");
-	const provenToWork = (toolchain: ToolchainKey, toolchainVersion: string) =>
-		successes.some((ok) => {
-			const okVersion = ok.toolchains[toolchain as Exclude<ToolchainKey, "node">];
-			return okVersion !== undefined && sameLine(okVersion, toolchainVersion);
-		});
+	const builds = verified
+		.flatMap(pinnedBuilds)
+		.filter((build) => build.package === packageName && build.version === version);
+	const successes = builds.filter((build) => build.outcome === "success");
+	const pinsOf = (build: PinnedBuild): Pin[] => [
+		...Object.entries(build.toolchains).map(([key, pinned]) => ({ key, version: pinned!, toolchain: true })),
+		{ key: build.companion.package, version: build.companion.version, toolchain: false },
+	];
+	const samePin = (a: Pin, b: Pin) => a.key === b.key && (a.toolchain ? sameLine(a.version, b.version) : a.version === b.version);
+	const provenToWork = (pin: Pin) => successes.some((ok) => pinsOf(ok).some((used) => samePin(used, pin)));
 
 	const result: VersionCompatibility["verified"] = {};
-	const add = (toolchain: ToolchainKey, toolchainVersion: string) => {
-		const list = (result[toolchain] ??= []);
-		if (!list.includes(toolchainVersion)) {
-			list.push(toolchainVersion);
+	const add = (pin: Pin) => {
+		if (!pin.toolchain) {
+			return;
+		}
+		const list = (result[pin.key as ToolchainKey] ??= []);
+		if (!list.includes(pin.version)) {
+			list.push(pin.version);
 		}
 	};
 
 	if (outcome === "success") {
-		for (const pair of verifiedPairs(verified, "success")) {
-			if (pair.package === packageName && pair.version === version) {
-				add(pair.toolchain, pair.toolchainVersion);
-			}
+		for (const ok of successes) {
+			pinsOf(ok).forEach(add);
 		}
 	} else {
-		const suspects = forThisVersion(verified)
-			.filter((entry) => classify(entry) === outcome)
-			.map((entry) =>
-				(Object.entries(entry.toolchains) as Array<[ToolchainKey, string]>).filter(
-					([toolchain, toolchainVersion]) => !provenToWork(toolchain, toolchainVersion),
-				),
-			);
+		const suspects = builds
+			.filter((build) => build.outcome === outcome)
+			.map((build) => pinsOf(build).filter((pin) => !provenToWork(pin)));
 		const culprits = suspects.filter((pins) => pins.length === 1).map(([pin]) => pin);
 		for (const pins of suspects) {
-			const explained = culprits.find(([toolchain, toolchainVersion]) =>
-				pins.some(([key, value]) => key === toolchain && sameLine(value, toolchainVersion)),
-			);
-			for (const [toolchain, toolchainVersion] of explained ? [explained] : pins) {
-				add(toolchain, toolchainVersion);
-			}
+			const explained = culprits.find((culprit) => pins.some((pin) => samePin(pin, culprit)));
+			(explained ? [explained] : pins).forEach(add);
 		}
 	}
 

shared/types.ts

@@ -104,10 +104,10 @@ export interface Overrides {
 }
 
 /**
- * One CI-proven build of a runtime version with every relevant version
- * pinned, stored as its own file under data/verified/. It verifies each
- * toolchain in `toolchains` for the runtime, and the Node.js major for the
- * CLI version it was built with.
+ * One CI build of a runtime version with every relevant version pinned,
+ * stored as its own file under data/verified/. It is evidence about the
+ * runtime and about the CLI it was built with alike: each toolchain in
+ * `toolchains` for both, and the Node.js major for the CLI.
  */
 export interface VerificationResult {
 	$schema?: string;

test/compute.test.ts

@@ -135,9 +135,10 @@ describe("summarizeCell", () => {
 import { verifiedPairs } from "../shared/compute";
 
 describe("verifiedPairs", () => {
-	it("derives runtime toolchain pairs and the CLI's Node.js pair from one build", () => {
+	it("credits one build to the runtime and to the CLI it was built with", () => {
 		expect(verifiedPairs(verified)).toEqual([
 			{ package: "@nativescript/ios", version: "9.1.0", toolchain: "xcode", toolchainVersion: "26.2" },
+			{ package: "nativescript", version: "9.1.1", toolchain: "xcode", toolchainVersion: "26.2" },
 			{ package: "nativescript", version: "9.1.1", toolchain: "node", toolchainVersion: "22" },
 		]);
 	});
@@ -188,9 +189,9 @@ describe("failed builds", () => {
 		expect(cellFor(doc(), "@nativescript/android", "9.1.1", "jdk", "21")).toMatchObject({ state: "unverified", reason: "one CI build failed; a second attempt is pending" });
 	});
 
-	it("a success outranks a sibling failure and only successes prove the CLI's Node.js support", () => {
+	it("a success outranks a sibling failure and proves the CLI's toolchains too", () => {
 		expect(cellFor(doc(), "@nativescript/android", "9.1.1", "jdk", "17")).toMatchObject({ state: "verified", reason: "verified by CI" });
-		expect(doc().packages["nativescript"].versions["9.1.1"].verified).toEqual({ node: ["24"] });
+		expect(doc().packages["nativescript"].versions["9.1.1"].verified).toEqual({ jdk: ["17"], node: ["24"] });
 		expect(doc().packages["@nativescript/android"].versions["9.1.1"].failed).toEqual({ jdk: ["25"] });
 		// The JDK 17 failure is not attributed to JDK 17: another build of the release succeeded with it.
 		expect(doc().packages["@nativescript/android"].versions["9.1.1"].suspect).toEqual({ jdk: ["21"] });
@@ -309,6 +310,45 @@ describe("failure attribution", () => {
 	});
 });
 
+describe("CLI attribution", () => {
+	const build = (cli: string, xcode: string, extra: Partial<VerificationResult> = {}): VerificationResult => ({
+		package: "@nativescript/ios", version: "9.0.2", toolchains: { xcode }, with: { nativescript: cli, node: "24" }, recordedAt: "2026-09-09T00:00:00Z", ...extra,
+	});
+	const failed = { outcome: "failure" as const, attempts: 2 };
+	const doc = (verified: VerificationResult[]) =>
+		buildDocument({
+			generatedAt: "2026-09-09T00:00:00.000Z",
+			toolchains: { xcode: [{ version: "27.0" }, { version: "26.4" }], cocoapods: [], compileSdk: [], buildTools: [], jdk: [], node: [{ version: "24.0.0" }] },
+			packages: [
+				{
+					spec: { name: "@nativescript/ios", toolchains: ["xcode"], keep: 5 },
+					document: { distTags: {}, manifests: [{ version: "9.0.2", requirements: { xcode: ">=16" } }] },
+				},
+				{
+					spec: { name: "nativescript", toolchains: ["node", "xcode"], keep: 5 },
+					document: { distTags: {}, manifests: [{ version: "9.1.1", requirements: { node: ">=20" } }, { version: "9.1.0", requirements: { node: ">=20" } }] },
+				},
+			],
+			overrides: { requirements: [], advisories: [] },
+			verified,
+		});
+
+	it("blames the CLI, not the toolchain, when a newer CLI builds the same combination", () => {
+		const d = doc([build("9.1.0", "27.0", failed), build("9.1.1", "27.0"), build("9.1.0", "26.4")]);
+		expect(cellFor(d, "@nativescript/ios", "9.0.2", "xcode", "27.0").state).toBe("verified");
+		expect(d.packages["@nativescript/ios"].versions["9.0.2"].failed).toBeUndefined();
+		expect(cellFor(d, "nativescript", "9.1.0", "xcode", "27.0")).toMatchObject({ state: "unsupported" });
+		expect(cellFor(d, "nativescript", "9.1.1", "xcode", "27.0").state).toBe("verified");
+		expect(d.packages["nativescript"].versions["9.1.0"].verified).toEqual({ xcode: ["26.4"], node: ["24"] });
+	});
+
+	it("blames both until another CLI proves the toolchain", () => {
+		const d = doc([build("9.1.0", "27.0", failed), build("9.1.0", "26.4")]);
+		expect(cellFor(d, "@nativescript/ios", "9.0.2", "xcode", "27.0").state).toBe("unsupported");
+		expect(cellFor(d, "nativescript", "9.1.0", "xcode", "27.0").state).toBe("unsupported");
+	});
+});
+
 describe("summarizeToolchain", () => {
 	it("names the run of releases that support a toolchain version", () => {
 		const doc = document();

worker/index.ts

@@ -3,7 +3,7 @@ import overridesSchema from "../schemas/overrides.json";
 import packageSchema from "../schemas/package.json";
 import requirementsSchema from "../schemas/requirements.json";
 import verifiedSchema from "../schemas/verified.json";
-import { buildDocument, collectOverrides, matchingAdvisories, TRACKED_PACKAGES } from "../shared/compute";
+import { buildDocument, CLI_PACKAGE, collectOverrides, matchingAdvisories, TRACKED_PACKAGES } from "../shared/compute";
 import { fetchPackage } from "./sources/npm";
 import { fetchToolchains } from "./sources/toolchains";
 import type { CompatibilityDocument, OverrideEntry, ToolchainKey, VerificationResult } from "../shared/types";
@@ -61,7 +61,8 @@ export default {
 				return Response.json(document, { headers: cacheHeaders() });
 			}
 
-			// /v1/packages/@scope/name/1.2.3?xcode=26.2&jdk=21 → effective requirements plus matching advisories
+			// /v1/packages/@scope/name/1.2.3?xcode=26.2&jdk=21&cli=9.1.1 → effective requirements plus
+			// matching advisories, and the same for the CLI release when one is given
 			const match = url.pathname.match(/^\/v1\/packages\/((?:@[^/]+\/)?[^/]+)\/([^/]+)$/);
 			if (match) {
 				const [, name, version] = match.map(decodeURIComponent);
@@ -70,15 +71,28 @@ export default {
 					return Response.json({ error: "unknown package version" }, { status: 404 });
 				}
 				const toolchain: Partial<Record<ToolchainKey, string>> = {};
+				let cli: string | undefined;
 				for (const [key, value] of url.searchParams) {
-					toolchain[key as ToolchainKey] = value;
+					if (key === "cli") {
+						cli = value;
+					} else {
+						toolchain[key as ToolchainKey] = value;
+					}
 				}
+				const describe = (pkg: string, release: string) => {
+					const known = document.packages[pkg]?.versions[release];
+					return {
+						package: pkg,
+						version: release,
+						...(known
+							? { ...known, advisories: matchingAdvisories(document.advisories, pkg, release, toolchain) }
+							: { tracked: false }),
+					};
+				};
 				return Response.json(
 					{
-						package: name,
-						version,
-						...entry,
-						advisories: matchingAdvisories(document.advisories, name, version, toolchain),
+						...describe(name, version),
+						...(cli ? { cli: describe(CLI_PACKAGE, cli) } : {}),
 					},
 					{ headers: cacheHeaders() },
 				);