Add runtime-driven canary and unstable SDK publishing

docs/developer-docs/secrets.md

@@ -10,7 +10,7 @@ This document covers secrets management for the github/copilot-sdk repository. I
 These secrets are used by the per-language SDK test workflows and the canary workflow.
 
 * **`COPILOT_DEVELOPER_CLI_INTEGRATION_HMAC_KEY`**: HMAC key used to authenticate with the Copilot Developer CLI integration endpoint during tests. Injected as `COPILOT_HMAC_KEY` in test environments.
-  * Workflows: `nodejs-sdk-tests.yml`, `python-sdk-tests.yml`, `go-sdk-tests.yml`, `dotnet-sdk-tests.yml`, `rust-sdk-tests.yml`, `sdk-canary.yml`
+  * Workflows: `nodejs-sdk-tests.yml`, `python-sdk-tests.yml`, `go-sdk-tests.yml`, `dotnet-sdk-tests.yml`, `rust-sdk-tests.yml`, `publish.yml`
 
 ## Agentic workflow secrets
 
@@ -58,6 +58,8 @@ These secrets support Java SDK Maven Central publishing, snapshot publishing, an
 ## Secrets not managed in this repository
 
 * **`GITHUB_TOKEN`**: Automatically provided by GitHub Actions. No manual management required.
+  The runtime-driven Node SDK workflow grants it `packages: read` only while acquiring
+  private runtime packages from GitHub Packages.
 
 ## Further reading
 

nodejs/README.md

@@ -21,6 +21,10 @@ release's `SHA256SUMS.txt`.
 
 `npm run pack:release` builds the main package and all platform packages. Set
 `COPILOT_CLI_DOWNLOAD_BASE_URL` to use a release mirror while packaging.
+Release workflows instead set `COPILOT_SDK_RUNTIME_PACKAGE_DIR` to a directory
+containing validated runtime npm package roots named for all eight platforms.
+This keeps `COPILOT_CLI_USE_NPM_PACKAGE` false and embeds those runtime files in
+the self-contained SDK platform packages.
 
 ## Installation
 

nodejs/package.json

@@ -36,8 +36,10 @@
         "auth:refresh": "node ../scripts/npm-auth-refresh.mjs --run",
         "clean": "rimraf --glob dist *.tgz",
         "build": "tsx esbuild-copilotsdk-nodejs.ts",
+        "acquire:runtime-packages": "tsx scripts/runtime-package-acquisition.ts",
         "pack:release": "tsx scripts/package-sdk.ts",
         "verify:release-packages": "tsx scripts/verify-release-packages.ts",
+        "release:manifest": "tsx scripts/release-manifest.ts",
         "prepare:runtime": "tsx scripts/prepare-runtime.ts",
         "test": "vitest run",
         "test:watch": "vitest",

nodejs/scripts/npm-release.js

@@ -1,13 +1,15 @@
 import { spawn } from "node:child_process";
+import { readFileSync } from "node:fs";
+import { resolve } from "node:path";
 import { pathToFileURL } from "node:url";
-
+import { verifyPackageSetManifestFiles } from "./package-set-manifest.js";
 const PUBLIC_CONFLICT =
     /^(?:npm (?:error|ERR!) code EPUBLISHCONFLICT|npm (?:error|ERR!) (?:403 [^\r\n]* - )?(?:You )?cannot publish over (?:the )?previously published versions(?:: [^\r\n]+)?\.?)\r?$/im;
 const AZURE_CONFLICT =
     /^npm (?:error|ERR!) (?:403 [^\r\n]* - )?(?:The feed '[^'\r\n]+' )?already contains file '[^'\r\n]+\.tgz' in package '[^'\r\n]+'\.?\r?$/im;
 
 export function runCommand(command, args, { stream = false } = {}) {
-    return new Promise((resolve, reject) => {
+    return new Promise((resolveResult, reject) => {
         const child = spawn(command, args, { shell: false });
         let stdout = "";
         let stderr = "";
@@ -21,11 +23,22 @@ export function runCommand(command, args, { stream = false } = {}) {
             if (stream) process.stderr.write(chunk);
         });
         child.on("error", reject);
-        child.on("close", (status) => resolve({ status: status ?? 1, stdout, stderr }));
+        child.on("close", (status) => resolveResult({ status: status ?? 1, stdout, stderr }));
     });
 }
 
-export async function assertVersionAbsent(packageName, version, registry, runner = runCommand) {
+function parseNpmJson(result) {
+    for (const output of [result.stdout, result.stderr]) {
+        try {
+            return JSON.parse(output);
+        } catch {
+            // The caller reports the complete npm output if neither stream is JSON.
+        }
+    }
+    return undefined;
+}
+
+export async function getRegistryVersion(packageName, version, registry, runner = runCommand) {
     const result = await runner("npm", [
         "view",
         `${packageName}@${version}`,
@@ -34,52 +47,126 @@ export async function assertVersionAbsent(packageName, version, registry, runner
         "--registry",
         registry,
     ]);
-
-    if (result.status === 0) {
-        throw new Error(`${packageName}@${version} already exists on public npm.`);
+    const parsed = parseNpmJson(result);
+    if (result.status === 0 && typeof parsed === "string") {
+        return parsed;
     }
-
-    try {
-        if (JSON.parse(result.stdout)?.error?.code === "E404") return;
-    } catch {
-        // The failure below includes npm's output for diagnosis.
+    if (result.status !== 0 && parsed?.error?.code === "E404") {
+        return undefined;
     }
-
     const output = `${result.stdout}\n${result.stderr}`.trim();
     throw new Error(
-        `Could not confirm that ${packageName}@${version} is absent from public npm (npm exited ${result.status}).${output ? `\n${output}` : ""}`
+        `Could not read ${packageName}@${version} from ${registry} (npm exited ${result.status}).${output ? `\n${output}` : ""}`
     );
 }
 
-export async function publishTarball(tarball, tag, registry, mode, runner = runCommand) {
+export async function assertVersionAbsent(packageName, version, registry, runner = runCommand) {
+    const existing = await getRegistryVersion(packageName, version, registry, runner);
+    if (existing !== undefined) {
+        throw new Error(`${packageName}@${version} already exists on ${registry}.`);
+    }
+}
+
+export async function publishTarball(tarball, tag, registry, mode, runner = runCommand, identity) {
     const args = ["publish", tarball, "--tag", tag, "--registry", registry];
     if (mode === "public") args.push("--access", "public");
     if (mode !== "public" && mode !== "azure") throw new Error(`Unknown publish mode: ${mode}`);
 
     const result = await runner("npm", args, { stream: true });
-    if (result.status === 0) return;
+    if (result.status === 0) {
+        return;
+    }
 
     const output = `${result.stdout}\n${result.stderr}`;
     if (PUBLIC_CONFLICT.test(output) || (mode === "azure" && AZURE_CONFLICT.test(output))) {
-        console.log(
-            "Version already published; treating the immutable-version conflict as success."
-        );
+        const subject =
+            identity?.name && identity?.version
+                ? `${identity.name}@${identity.version}`
+                : "Version";
+        console.log(`${subject} is already published; treating the conflict as success.`);
         return;
     }
 
     throw new Error(`npm publish failed with exit code ${result.status}.`);
 }
 
+function readReleaseManifest(manifestPath, packageDirectory) {
+    const manifest = JSON.parse(readFileSync(manifestPath, "utf8"));
+    verifyPackageSetManifestFiles(manifest, packageDirectory);
+    return manifest;
+}
+
+export async function publishManifest(
+    manifestPath,
+    packageDirectory,
+    tag,
+    registry,
+    mode,
+    runner = runCommand
+) {
+    const manifest = readReleaseManifest(manifestPath, packageDirectory);
+    const packages = manifest.packages
+        .map((packed) => ({
+            ...packed,
+            version: manifest.sdk.version,
+            tarball: resolve(packageDirectory, packed.filename),
+        }))
+        .sort((left, right) => {
+            if (left.name === "@github/copilot-sdk") return 1;
+            if (right.name === "@github/copilot-sdk") return -1;
+            return left.name.localeCompare(right.name);
+        });
+
+    const semver = await import("semver");
+    for (const packed of packages) {
+        const taggedVersion = await getRegistryVersion(packed.name, tag, registry, runner);
+        if (taggedVersion !== undefined && semver.gt(taggedVersion, packed.version)) {
+            throw new Error(
+                `${packed.name}@${tag} already points to newer version ${taggedVersion}; refusing to rewind it to ${packed.version}.`
+            );
+        }
+    }
+    for (const packed of packages) {
+        await publishTarball(packed.tarball, tag, registry, mode, runner, packed);
+    }
+    for (const packed of packages) {
+        const taggedVersion = await getRegistryVersion(packed.name, tag, registry, runner);
+        if (taggedVersion === packed.version) {
+            continue;
+        }
+        if (mode === "public") {
+            throw new Error(
+                `${packed.name}@${tag} resolves to ${taggedVersion ?? "no version"}, expected ${packed.version}. Public trusted publishing cannot repair dist-tags.`
+            );
+        }
+        if (taggedVersion !== undefined && semver.gt(taggedVersion, packed.version)) {
+            throw new Error(
+                `${packed.name}@${tag} advanced to newer version ${taggedVersion}; refusing to rewind it to ${packed.version}.`
+            );
+        }
+        const result = await runner(
+            "npm",
+            ["dist-tag", "add", `${packed.name}@${packed.version}`, tag, "--registry", registry],
+            { stream: true }
+        );
+        if (result.status !== 0) {
+            throw new Error(`Failed to set ${packed.name}@${packed.version} dist-tag ${tag}.`);
+        }
+    }
+}
+
 async function main() {
     const [command, ...args] = process.argv.slice(2);
     if (command === "preflight" && args.length === 3) {
         await assertVersionAbsent(...args);
-        console.log(`${args[0]}@${args[1]} is available on public npm.`);
+        console.log(`${args[0]}@${args[1]} is available on ${args[2]}.`);
     } else if (command === "publish" && args.length === 4) {
         await publishTarball(...args);
+    } else if (command === "publish-manifest" && args.length === 5) {
+        await publishManifest(...args);
     } else {
         throw new Error(
-            "Usage: npm-release.js preflight <package> <version> <registry> | publish <tarball> <tag> <registry> <public|azure>"
+            "Usage: npm-release.js preflight <package> <version> <registry> | publish <tarball> <tag> <registry> <public|azure> | publish-manifest <manifest> <package-directory> <tag> <registry> <public|azure>"
         );
     }
 }

nodejs/scripts/package-set-manifest.js

@@ -0,0 +1,65 @@
+import assert from "node:assert/strict";
+import { createHash } from "node:crypto";
+import { readFileSync, statSync } from "node:fs";
+import { basename, dirname, resolve } from "node:path";
+
+export const SDK_PACKAGE_NAMES = [
+    "@github/copilot-sdk",
+    "@github/copilot-sdk-darwin-arm64",
+    "@github/copilot-sdk-darwin-x64",
+    "@github/copilot-sdk-linux-arm64",
+    "@github/copilot-sdk-linux-x64",
+    "@github/copilot-sdk-linuxmusl-arm64",
+    "@github/copilot-sdk-linuxmusl-x64",
+    "@github/copilot-sdk-win32-arm64",
+    "@github/copilot-sdk-win32-x64",
+];
+
+export function packageIntegrity(bytes) {
+    return `sha512-${createHash("sha512").update(bytes).digest("base64")}`;
+}
+
+export function verifyPackageSetManifestFiles(manifest, packageDirectory) {
+    assert.equal(manifest?.schemaVersion, 1, "Unsupported release manifest schema");
+    assert.equal(typeof manifest.sdk?.version, "string", "Invalid SDK version");
+    assert(Array.isArray(manifest.packages), "Release manifest packages must be an array");
+    assert.equal(manifest.packages.length, 9, "Release manifest must contain nine packages");
+
+    const packageNames = new Set();
+    for (const packed of manifest.packages) {
+        assert.equal(typeof packed?.name, "string", "Invalid release package name");
+        assert.equal(typeof packed.filename, "string", "Invalid release package filename");
+        assert.equal(typeof packed.integrity, "string", "Invalid release package integrity");
+        assert.equal(typeof packed.size, "number", "Invalid release package size");
+        assert(
+            !packageNames.has(packed.name),
+            `Duplicate package in release manifest: ${packed.name}`
+        );
+        packageNames.add(packed.name);
+
+        const archive = resolve(packageDirectory, packed.filename);
+        assert.equal(
+            dirname(archive),
+            resolve(packageDirectory),
+            `Unsafe release filename: ${packed.filename}`
+        );
+        assert.equal(
+            basename(archive),
+            packed.filename,
+            `Unsafe release filename: ${packed.filename}`
+        );
+        const bytes = readFileSync(archive);
+        assert.equal(statSync(archive).size, packed.size, `Size mismatch for ${packed.filename}`);
+        assert.equal(
+            packageIntegrity(bytes),
+            packed.integrity,
+            `Integrity mismatch for ${packed.filename}`
+        );
+    }
+
+    assert.deepEqual(
+        [...packageNames].sort(),
+        [...SDK_PACKAGE_NAMES].sort(),
+        "Release manifest package names do not match the expected package set"
+    );
+}