feat(server): checksum-verified downloads + pinned whisper runtime/model manifest (WSP-1)

packages/server/src/ai/download.test.ts

@@ -0,0 +1,74 @@
+import {createHash} from 'node:crypto';
+import {existsSync, readFileSync} from 'node:fs';
+import {mkdtemp, readFile, rm, writeFile} from 'node:fs/promises';
+import {tmpdir} from 'node:os';
+import path from 'node:path';
+import {afterEach, beforeEach, describe, expect, it, vi} from 'vitest';
+import {downloadFile, downloadVerified} from './download';
+
+const bytes = Buffer.from('verified model');
+const pin = {sha256: createHash('sha256').update(bytes).digest('hex'), size: bytes.length};
+let dir: string;
+let dest: string;
+beforeEach(async () => {
+  dir = await mkdtemp(path.join(tmpdir(), 'verified-download-'));
+  dest = path.join(dir, 'model');
+});
+afterEach(async () => {
+  vi.unstubAllGlobals();
+  await rm(dir, {recursive: true, force: true});
+});
+
+function respond(body: Uint8Array) {
+  vi.stubGlobal('fetch', vi.fn(async () => new Response(new Uint8Array(body))));
+}
+
+describe('downloadVerified', () => {
+  it('verifies before atomic replacement and reports progress without content-length', async () => {
+    await writeFile(dest, 'old');
+    respond(bytes);
+    const progress = vi.fn();
+    await downloadVerified('https://example.test/model', dest, pin, (state) => {
+      expect(readFileSync(dest, 'utf8')).toBe('old');
+      progress(state);
+    });
+    expect(await readFile(dest)).toEqual(bytes);
+    expect(progress).toHaveBeenLastCalledWith({received: bytes.length, total: bytes.length});
+    expect(existsSync(`${dest}.part`)).toBe(false);
+  });
+
+  it.each(['corrupt', 'truncated', 'oversized'] as const)('rejects %s content and cleans partial, preserving the old file', async (kind) => {
+    await writeFile(dest, 'old');
+    respond(kind === 'corrupt' ? Buffer.alloc(bytes.length) : kind === 'truncated' ? bytes.subarray(0, 3) : Buffer.concat([bytes, bytes]));
+    await expect(downloadVerified('https://example.test/model', dest, pin)).rejects.toThrow(kind === 'corrupt' ? 'SHA-256 mismatch' : 'size mismatch');
+    expect(await readFile(dest, 'utf8')).toBe('old');
+    expect(existsSync(`${dest}.part`)).toBe(false);
+  });
+
+  it('cleans partial files after a transport failure', async () => {
+    vi.stubGlobal('fetch', vi.fn(async () => new Response(new ReadableStream({
+      start(controller) { controller.enqueue(bytes.subarray(0, 3)); },
+      pull(controller) { controller.error(new Error('connection terminated')); },
+    }))));
+    await expect(downloadVerified('https://example.test/model', dest, pin)).rejects.toThrow('connection terminated');
+    expect(existsSync(dest)).toBe(false);
+    expect(existsSync(`${dest}.part`)).toBe(false);
+  });
+
+  it('handles HTTP and filesystem errors without publishing a destination', async () => {
+    vi.stubGlobal('fetch', vi.fn(async () => new Response('missing', {status: 404})));
+    await expect(downloadVerified('https://example.test/model', dest, pin)).rejects.toThrow('HTTP 404');
+    respond(bytes);
+    await expect(downloadVerified('https://example.test/model', path.join(dir, 'missing', 'model'), pin)).rejects.toThrow();
+    expect(existsSync(dest)).toBe(false);
+    expect(existsSync(`${dest}.part`)).toBe(false);
+  });
+
+  it('retains unverified arbitrary llama bytes and progress', async () => {
+    vi.stubGlobal('fetch', vi.fn(async () => new Response(bytes, {headers: {'content-length': String(bytes.length)}})));
+    const progress = vi.fn();
+    await downloadFile('https://example.test/custom.gguf', dest, progress);
+    expect(await readFile(dest)).toEqual(bytes);
+    expect(progress).toHaveBeenLastCalledWith({received: bytes.length, total: bytes.length});
+  });
+});

packages/server/src/ai/download.ts

@@ -0,0 +1,77 @@
+import {createHash} from 'node:crypto';
+import {createWriteStream} from 'node:fs';
+import {open, rename, unlink} from 'node:fs/promises';
+import {Readable} from 'node:stream';
+import {pipeline} from 'node:stream/promises';
+
+export interface DownloadProgress {
+  received: number;
+  total: number | null;
+}
+
+export interface DownloadIntegrity {
+  sha256: string;
+  size: number;
+}
+
+/** Shared streamer for arbitrary llama URLs and pinned, verified artifacts.
+ * The caller owns serialization of downloads to the same destination. */
+export async function downloadFile(
+  url: string,
+  dest: string,
+  onProgress?: (progress: DownloadProgress) => void,
+  integrity?: DownloadIntegrity,
+): Promise<void> {
+  const partial = `${dest}.part`;
+  try {
+    const res = await fetch(url, {redirect: 'follow'});
+    if (!res.ok || !res.body) {
+      await res.body?.cancel();
+      throw new Error(`HTTP ${res.status}`);
+    }
+    const total = integrity?.size ?? (Number(res.headers.get('content-length')) || null);
+    const hash = integrity ? createHash('sha256') : null;
+    let received = 0;
+    onProgress?.({received, total});
+    const reader = res.body.getReader();
+    async function* chunks() {
+      try {
+        for (;;) {
+          const {done, value} = await reader.read();
+          if (done) break;
+          received += value.byteLength;
+          if (integrity && received > integrity.size) throw new Error(`Download size mismatch: expected ${integrity.size} bytes, received at least ${received}`);
+          hash?.update(value);
+          onProgress?.({received, total});
+          yield value;
+        }
+      } finally {
+        await reader.cancel().catch(() => undefined);
+        reader.releaseLock();
+      }
+    }
+    // pipeline propagates disk/read errors and closes the file before cleanup or rename.
+    await pipeline(Readable.from(chunks()), createWriteStream(partial));
+    if (integrity) {
+      if (received !== integrity.size) throw new Error(`Download size mismatch: expected ${integrity.size} bytes, received ${received}`);
+      if (hash!.digest('hex') !== integrity.sha256) throw new Error('Download SHA-256 mismatch');
+    }
+    const fh = await open(partial, 'r+');
+    await fh.sync();
+    await fh.close();
+    await rename(partial, dest);
+  } catch (error) {
+    await unlink(partial).catch(() => undefined);
+    throw error;
+  }
+}
+
+/** Stream → verify size and SHA-256 → atomically replace the destination. */
+export async function downloadVerified(
+  url: string,
+  dest: string,
+  integrity: DownloadIntegrity,
+  onProgress?: (progress: DownloadProgress) => void,
+): Promise<void> {
+  await downloadFile(url, dest, onProgress, integrity);
+}

packages/server/src/ai/pinnedDownload.test.ts

@@ -0,0 +1,56 @@
+import {createHash} from 'node:crypto';
+import {existsSync} from 'node:fs';
+import {mkdtemp, readFile, rm, writeFile} from 'node:fs/promises';
+import {tmpdir} from 'node:os';
+import path from 'node:path';
+import {afterEach, beforeEach, expect, it, vi} from 'vitest';
+import {downloadPinned} from './pinnedDownload';
+
+const bytes = 'model bytes';
+const pin = {version: 'v1', url: 'https://example.test/model.bin', sha256: createHash('sha256').update(bytes).digest('hex'), size: Buffer.byteLength(bytes)};
+let dir: string;
+let dest: string;
+const fetchMock = vi.fn(async () => new Response(bytes));
+beforeEach(async () => {
+  dir = await mkdtemp(path.join(tmpdir(), 'pinned-download-'));
+  dest = path.join(dir, 'model.bin');
+  fetchMock.mockClear();
+  vi.stubGlobal('fetch', fetchMock);
+});
+afterEach(async () => {
+  vi.unstubAllGlobals();
+  await rm(dir, {recursive: true, force: true});
+});
+
+it('skips the same file and version, then downloads on a version-only pin bump', async () => {
+  await downloadPinned(pin, dest);
+  const progress = vi.fn();
+  await downloadPinned(pin, dest, progress);
+  expect(fetchMock).toHaveBeenCalledTimes(1);
+  expect(progress).toHaveBeenLastCalledWith({received: pin.size, total: pin.size});
+  await downloadPinned({...pin, version: 'v2'}, dest);
+  expect(fetchMock).toHaveBeenCalledTimes(2);
+  expect(JSON.parse(await readFile(`${dest}.verified.json`, 'utf8')).version).toBe('v2');
+});
+
+it.each(['missing', 'legacy', 'truncated', 'bad-receipt'])('downloads %s installations', async (kind) => {
+  await downloadPinned(pin, dest);
+  if (kind === 'missing') await rm(dest);
+  if (kind === 'legacy') await rm(`${dest}.verified.json`);
+  if (kind === 'truncated') await writeFile(dest, 'short');
+  if (kind === 'bad-receipt') await writeFile(`${dest}.verified.json`, '{');
+  await downloadPinned(pin, dest);
+  expect(fetchMock).toHaveBeenCalledTimes(2);
+  expect(await readFile(dest, 'utf8')).toBe(bytes);
+});
+
+it('does not leave a valid receipt or partial after a failed upgrade, and allows retry', async () => {
+  await downloadPinned(pin, dest);
+  const next = {...pin, version: 'v2'};
+  fetchMock.mockResolvedValueOnce(new Response('corrupt'));
+  await expect(downloadPinned(next, dest)).rejects.toThrow('size mismatch');
+  expect(await readFile(dest, 'utf8')).toBe(bytes);
+  for (const suffix of ['.part', '.verified.json', '.verified.json.part']) expect(existsSync(`${dest}${suffix}`)).toBe(false);
+  await downloadPinned(next, dest);
+  expect(fetchMock).toHaveBeenCalledTimes(3);
+});

packages/server/src/ai/pinnedDownload.ts

@@ -0,0 +1,33 @@
+import {readFile, rename, stat, unlink, writeFile} from 'node:fs/promises';
+import {downloadVerified, type DownloadProgress} from './download';
+import type {ArtifactPin} from './runtimeManifest';
+
+/** A receipt is written only after verification. Legacy files without a receipt
+ * are re-downloaded. Compare the entire pin so hash/URL changes also invalidate.
+ * Callers serialize writes to dest; extraction and runtime installs are separate. */
+export async function downloadPinned(
+  pin: ArtifactPin,
+  dest: string,
+  onProgress?: (progress: DownloadProgress) => void,
+): Promise<void> {
+  const receipt = `${dest}.verified.json`;
+  const identity = JSON.stringify({version: pin.version, url: pin.url, sha256: pin.sha256, size: pin.size});
+  const current = await readFile(receipt, 'utf8').catch(() => null);
+  const file = await stat(dest).catch(() => null);
+  if (current === identity && file?.isFile() && file.size === pin.size) {
+    onProgress?.({received: pin.size, total: pin.size});
+    return;
+  }
+  // Invalidate first: an interrupted replacement must never inherit an old receipt.
+  await unlink(receipt).catch((error: NodeJS.ErrnoException) => {
+    if (error.code !== 'ENOENT') throw error;
+  });
+  try {
+    await downloadVerified(pin.url, dest, pin, onProgress);
+    await writeFile(`${receipt}.part`, identity);
+    await rename(`${receipt}.part`, receipt);
+  } catch (error) {
+    await unlink(`${receipt}.part`).catch(() => undefined);
+    throw error;
+  }
+}

packages/server/src/ai/runtimeManifest.test.ts

@@ -0,0 +1,47 @@
+import {describe, expect, it} from 'vitest';
+import {RELEASE_TARGETS, RUNTIME_MANIFEST, WHISPER_MODEL_PIN, type ArtifactPin} from './runtimeManifest';
+
+function checkPin(pin: ArtifactPin) {
+  expect(new URL(pin.url).protocol).toBe('https:');
+  expect(pin.url).not.toMatch(/latest|\/main\//i);
+  expect(pin.version.length).toBeGreaterThan(0);
+  expect(pin.url).toContain(pin.version);
+  expect(pin.sha256).toMatch(/^[a-f0-9]{64}$/);
+  expect(Number.isSafeInteger(pin.size) && pin.size > 0).toBe(true);
+}
+
+describe('pinned runtime manifest', () => {
+  it('represents all four release targets and both tools with pins or typed reasons', () => {
+    expect(Object.keys(RUNTIME_MANIFEST).sort()).toEqual([...RELEASE_TARGETS].sort());
+    expect(RELEASE_TARGETS).toHaveLength(4);
+    for (const target of RELEASE_TARGETS) {
+      expect(Object.keys(RUNTIME_MANIFEST[target]).sort()).toEqual(['ffmpeg', 'whisper-cli']);
+      for (const artifact of Object.values(RUNTIME_MANIFEST[target])) {
+        if (artifact.status === 'unsupported') {
+          expect(['no-upstream-cli', 'no-native-prebuilt']).toContain(artifact.reason);
+          expect(artifact.detail.length).toBeGreaterThan(0);
+        } else {
+          expect(artifact.status).toBe('supported');
+          checkPin(artifact);
+          expect(['zip', 'tar.xz']).toContain(artifact.archive);
+          expect(artifact.binaryPath).not.toMatch(/^\/|\\|(^|\/)\.\.(\/|$)/);
+          expect(artifact.binaryPath).toMatch(/(?:ffmpeg|whisper-cli)(?:\.exe)?$/);
+        }
+      }
+    }
+  });
+
+  it('includes the Windows whisper companion DLL directory', () => {
+    const artifact = RUNTIME_MANIFEST['x86_64-pc-windows-msvc']['whisper-cli'];
+    expect(artifact.status).toBe('supported');
+    if (artifact.status !== 'supported') throw new Error('Windows whisper must be supported');
+    expect(artifact.extractDir).toBe('Release');
+    expect(artifact.binaryPath.startsWith(`${artifact.extractDir}/`)).toBe(true);
+  });
+
+  it('pins the base model to an immutable repository revision', () => {
+    checkPin(WHISPER_MODEL_PIN);
+    expect(WHISPER_MODEL_PIN.fileName).toBe('ggml-base.bin');
+    expect(WHISPER_MODEL_PIN.version).toMatch(/^[a-f0-9]{40}$/);
+  });
+});

packages/server/src/ai/runtimeManifest.ts

@@ -0,0 +1,88 @@
+/** Pins verified by downloading and hashing the actual artifacts (WSP-1).
+ * Unsupported is per tool: consumers must require BOTH tools for a ready runtime.
+ * Windows whisper needs the DLLs alongside the executable from the same archive.
+ * Extraction/provisioning belongs to WSP-2. */
+export const RELEASE_TARGETS = [
+  'aarch64-apple-darwin',
+  'x86_64-apple-darwin',
+  'x86_64-unknown-linux-gnu',
+  'x86_64-pc-windows-msvc',
+] as const;
+export type ReleaseTarget = typeof RELEASE_TARGETS[number];
+export type RuntimeTool = 'whisper-cli' | 'ffmpeg';
+
+export interface ArtifactPin {
+  version: string;
+  url: string;
+  sha256: string;
+  size: number;
+}
+export type RuntimeArtifact = (ArtifactPin & {
+  status: 'supported';
+  archive: 'zip' | 'tar.xz';
+  binaryPath: string;
+  extractDir?: string;
+}) | {
+  status: 'unsupported';
+  reason: 'no-upstream-cli' | 'no-native-prebuilt';
+  detail: string;
+};
+
+const noWhisperCli: RuntimeArtifact = {
+  status: 'unsupported',
+  reason: 'no-upstream-cli',
+  detail: 'whisper.cpp v1.8.2 publishes Windows CLI archives and an Apple XCFramework, but no macOS/Linux whisper-cli prebuilt. Owner must select a trusted builder or publish builds.',
+};
+
+export const RUNTIME_MANIFEST: Record<ReleaseTarget, Record<RuntimeTool, RuntimeArtifact>> = {
+  'aarch64-apple-darwin': {
+    'whisper-cli': noWhisperCli,
+    ffmpeg: {
+      status: 'unsupported',
+      reason: 'no-native-prebuilt',
+      detail: 'The selected macOS provider (evermeet.cx) explicitly supplies Intel binaries only. A trusted native ARM provider is an owner decision; do not silently require Rosetta.',
+    },
+  },
+  'x86_64-apple-darwin': {
+    'whisper-cli': noWhisperCli,
+    ffmpeg: {
+      status: 'supported', version: '7.1.1',
+      url: 'https://evermeet.cx/ffmpeg/ffmpeg-7.1.1.zip',
+      sha256: '8d7917c1cebd7a29e68c0a0a6cc4ecc3fe05c7fffed958636c7018b319afdda4',
+      size: 25458015, archive: 'zip', binaryPath: 'ffmpeg',
+    },
+  },
+  'x86_64-unknown-linux-gnu': {
+    'whisper-cli': noWhisperCli,
+    ffmpeg: {
+      status: 'supported', version: '7.0.2',
+      url: 'https://johnvansickle.com/ffmpeg/releases/ffmpeg-7.0.2-amd64-static.tar.xz',
+      sha256: 'abda8d77ce8309141f83ab8edf0596834087c52467f6badf376a6a2a4c87cf67',
+      size: 41888096, archive: 'tar.xz', binaryPath: 'ffmpeg-7.0.2-amd64-static/ffmpeg',
+    },
+  },
+  'x86_64-pc-windows-msvc': {
+    'whisper-cli': {
+      status: 'supported', version: '1.8.2',
+      url: 'https://github.com/ggml-org/whisper.cpp/releases/download/v1.8.2/whisper-bin-x64.zip',
+      sha256: 'b1514ebc099765e39fa37eb780b92a140a94c86bb0b3b3d98226b38825979732',
+      size: 3832432, archive: 'zip', binaryPath: 'Release/whisper-cli.exe',
+      /** Install the ENTIRE directory beside the executable, including companion DLLs. */
+      extractDir: 'Release',
+    },
+    ffmpeg: {
+      status: 'supported', version: '8.1.2',
+      url: 'https://www.gyan.dev/ffmpeg/builds/packages/ffmpeg-8.1.2-essentials_build.zip',
+      sha256: 'db580001caa24ac104c8cb856cd113a87b0a443f7bdf47d8c12b1d740584a2ec',
+      size: 109728040, archive: 'zip', binaryPath: 'ffmpeg-8.1.2-essentials_build/bin/ffmpeg.exe',
+    },
+  },
+};
+
+export const WHISPER_MODEL_PIN: ArtifactPin & {fileName: string} = {
+  fileName: 'ggml-base.bin',
+  version: '5359861c739e955e79d9a303bcbc70fb988958b1',
+  url: 'https://huggingface.co/ggerganov/whisper.cpp/resolve/5359861c739e955e79d9a303bcbc70fb988958b1/ggml-base.bin',
+  sha256: '60ed5bc3dd14eea856493d334349b405782ddcaf0028d4b5df4088345fba2efe',
+  size: 147951465,
+};