feat(server,sdk): asset transcription service + OpenAI-compatible backend (MEET-2)

packages/sdk/src/ai.ts

@@ -63,7 +63,26 @@ export interface AiProviderSettings {
   autoStart?: boolean;
 }
 
+/** Audio configuration is independent of the chat engine. Omitted means local.
+ * Keys follow AiProviderSettings.apiKey preserve/set/clear semantics. */
+export interface AiTranscriptionConfig {
+  provider: 'off' | 'local' | 'openai-compat';
+  baseUrl?: string;
+  model?: string;
+  apiKey?: string | null;
+  apiKeySet?: boolean;
+}
+
+export interface AiTranscriptionResult {
+  text: string;
+  /** Segment offsets are seconds from the start of the recording. */
+  segments?: Array<{start: number; end: number; text: string}>;
+  /** Audio duration in milliseconds (0 if the backend does not report it). */
+  durationMs: number;
+}
+
 export interface AiConfig {
+  transcription?: AiTranscriptionConfig;
   /** The default provider — used unless an agent run overrides it. */
   provider: AiProvider;
   /** Per-provider settings, so every provider can be configured at once. */

packages/sdk/src/client.ts

@@ -5,6 +5,7 @@ import type {
   AgentChatMessage,
   AgentChatOptions,
   AiConfig,
+  AiTranscriptionResult,
   AiPricingResponse,
   AiPricingTable,
   AiUsageResponse,
@@ -208,6 +209,7 @@ export interface DataClient {
   aiSearch(query: string, limit?: number): Promise<AiSearchResponse>;
   aiTasks(goal: string, context?: string): Promise<AiTasksResponse>;
   aiDownloadModel(url?: string): Promise<AiStatus['download']>;
+  transcribeAsset(assetId: string, pageId: string): Promise<AiTranscriptionResult>;
   aiComplete(text: string, onToken: (token: string) => void, opts?: {instruction?: string; signal?: AbortSignal}): Promise<string>;
   aiGenerate(prompt: string, onToken: (token: string) => void, opts?: {system?: string; maxTokens?: number; signal?: AbortSignal}): Promise<string>;
   /**
@@ -2146,6 +2148,10 @@ export class HttpDataClient implements DataClient {
     return this.request<AiTasksResponse>('POST', API.aiTasks, {goal, context});
   }
 
+  async transcribeAsset(assetId: string, pageId: string): Promise<AiTranscriptionResult> {
+    return this.request<AiTranscriptionResult>('POST', API.aiTranscribe, {assetId, pageId});
+  }
+
   async aiDownloadModel(url?: string): Promise<AiStatus['download']> {
     return this.request<AiStatus['download']>('POST', API.aiModelDownload, {url});
   }

packages/sdk/src/index.ts

@@ -692,6 +692,8 @@ export type {
   InterviewStep,
   AiProvider,
   AiConfig,
+  AiTranscriptionConfig,
+  AiTranscriptionResult,
   AiProviderSettings,
   AiEffort,
   AiSkill,

packages/sdk/src/routes.ts

@@ -137,6 +137,7 @@ export const API = {
   aiTasks: '/api/ai/tasks',
   /** Continue/complete document text: `POST` `{text, instruction?}` → SSE. */
   aiComplete: '/api/ai/complete',
+  aiTranscribe: '/api/ai/transcribe',
   /** Download a model file for the in-process engine: `POST` `{url?}`. */
   aiModelDownload: '/api/ai/models/download',
   /** The agent harness: `POST` `{messages, effort?, thinking?, skills?}` → SSE tool/reasoning/proposal/final events. */

packages/server/src/ableOidc.test.ts

@@ -568,7 +568,7 @@ describe('able OIDC relying party', () => {
       issuer: 'local',
       scope: 'read',
       createdBy: 'test',
-      // PAT authentication uses wall-clock time, independently of the OIDC clock.
+      // PAT validation uses the database clock, not the injected OIDC clock.
       expiresAt: new Date(Date.now() + 60_000),
     });
     const app = createApp(store, undefined, new PageHub(), {

packages/server/src/ai/providers.ts

@@ -1,7 +1,7 @@
 import {spawn, type ChildProcess} from 'node:child_process';
 import {existsSync} from 'node:fs';
 import path from 'node:path';
-import {providerSettings, type AiConfig, type AiProvider} from '@book.dev/sdk';
+import {providerSettings, type AiConfig, type AiProvider, type AiTranscriptionResult} from '@book.dev/sdk';
 
 /**
  * Inference engines behind one interface. Generation streams tokens;
@@ -84,7 +84,19 @@ export interface GenerateOptions {
   signal?: AbortSignal;
 }
 
+export interface TranscribeOptions {
+  filename?: string;
+  mime?: string;
+  signal?: AbortSignal;
+}
+
+/** MEET-3 implements this capability; no chat/embedding engine is required. */
+export interface TranscriptionEngine {
+  transcribe(bytes: Uint8Array, opts?: TranscribeOptions): Promise<AiTranscriptionResult>;
+}
+
 export interface AiEngine {
+  transcribe?: TranscriptionEngine['transcribe'];
   readonly kind: string;
   /** Throws (with a user-readable message) when the engine can't run. */
   ensureReady(): Promise<void>;
@@ -108,6 +120,12 @@ export interface AiEngine {
 export class MockEngine implements AiEngine {
   readonly kind = 'mock';
 
+  async transcribe(bytes: Uint8Array, opts: TranscribeOptions = {}): Promise<AiTranscriptionResult> {
+    opts.signal?.throwIfAborted();
+    const text = `Mock transcription (${bytes.byteLength} bytes).`;
+    return {text, segments: [{start: 0, end: 1, text}], durationMs: 1000};
+  }
+
   async ensureReady(): Promise<void> {
     // always ready
   }
@@ -199,10 +217,40 @@ export class OpenAiCompatEngine implements AiEngine {
     private readonly baseUrl: string,
     private readonly model: string,
     kind = 'openai',
+    private readonly apiKey?: string | null,
   ) {
     this.kind = kind;
   }
 
+  async transcribe(bytes: Uint8Array, opts: TranscribeOptions = {}): Promise<AiTranscriptionResult> {
+    const form = new FormData();
+    const extensions: Record<string, string> = {'audio/webm': 'webm', 'video/webm': 'webm', 'audio/wav': 'wav', 'audio/x-wav': 'wav', 'audio/mpeg': 'mp3', 'audio/mp4': 'm4a', 'audio/ogg': 'ogg', 'audio/flac': 'flac'};
+    const filename = opts.filename || `audio.${extensions[opts.mime?.split(';')[0] ?? ''] ?? 'bin'}`;
+    form.append('file', new Blob([new Uint8Array(bytes)], {type: opts.mime || 'application/octet-stream'}), filename);
+    form.append('model', this.model || 'whisper-1');
+    form.append('response_format', 'verbose_json');
+    form.append('timestamp_granularities[]', 'segment');
+    const base = this.baseUrl.replace(/\/+$/, '').replace(/\/v1$/, '');
+    const res = await fetch(`${base}/v1/audio/transcriptions`, {
+      method: 'POST',
+      headers: this.apiKey ? {Authorization: `Bearer ${this.apiKey}`} : {},
+      body: form,
+      signal: opts.signal ? AbortSignal.any([opts.signal, AbortSignal.timeout(300_000)]) : AbortSignal.timeout(300_000),
+    });
+    // Do not echo upstream bodies: they can contain credentials or recording text.
+    if (!res.ok) throw new Error(`Transcription provider returned HTTP ${res.status}`);
+    const data = await res.json() as {text?: unknown; duration?: unknown; segments?: unknown};
+    if (typeof data.text !== 'string') throw new Error('Invalid transcription provider response');
+    const segments: AiTranscriptionResult['segments'] = Array.isArray(data.segments)
+      ? data.segments.filter((s): s is {start: number; end: number; text: string} =>
+        s && Number.isFinite(s.start) && s.start >= 0 && Number.isFinite(s.end) && s.end >= s.start && typeof s.text === 'string')
+        .map(({start, end, text}) => ({start, end, text}))
+      : undefined;
+    const duration = typeof data.duration === 'number' && Number.isFinite(data.duration) && data.duration >= 0
+      ? data.duration : segments?.reduce((end, s) => Math.max(end, s.end), 0) ?? 0;
+    return {text: data.text, ...(segments ? {segments} : {}), durationMs: Math.round(duration * 1000)};
+  }
+
   async ensureReady(): Promise<void> {
     const res = await fetch(`${this.baseUrl}/v1/models`, {signal: AbortSignal.timeout(3000)}).catch(() => null);
     if (!res?.ok) {

packages/server/src/ai/routes.ts

@@ -6,7 +6,7 @@ import type {PageStore} from '../store';
 import type {AppEnv} from '../appEnv';
 import {isLocalInstanceOwner, requireAuthenticatedRead, requireCreate, requireInstanceAdmin, requireInstanceOwner} from '../access';
 import {AgentRunner, type AgentMessage} from './agent';
-import type {AiService} from './service';
+import {TranscriptionConfigError, type AiService} from './service';
 import {McpConfigError, type ExternalAgentTool, type McpClientManager} from './mcpClients';
 import type {TokenUsage} from './providers';
 import type {AiUsageLog, UsageKind} from './usage';
@@ -61,6 +61,22 @@ export function mountAiRoutes(app: Hono<AppEnv>, ai: AiService, store: PageStore
     if (!['off', 'mock', 'llama', 'mlx', 'openai', 'claude'].includes(body.provider)) {
       return c.json({error: `Unknown provider: ${String(body.provider)}`}, 400);
     }
+    if (body.transcription !== undefined) {
+      const audio = body.transcription;
+      if (!audio || !['off', 'local', 'openai-compat'].includes(audio.provider)
+        || [audio.baseUrl, audio.model].some((v) => v !== undefined && typeof v !== 'string')
+        || (audio.apiKey !== undefined && audio.apiKey !== null && typeof audio.apiKey !== 'string')) {
+        return c.json({error: 'Invalid transcription configuration'}, 400);
+      }
+      if (audio.baseUrl) {
+        try {
+          const url = new URL(audio.baseUrl);
+          if (!['http:', 'https:'].includes(url.protocol) || url.username || url.password) throw new Error();
+        } catch {
+          return c.json({error: 'Transcription baseUrl must be an HTTP(S) URL without embedded credentials'}, 400);
+        }
+      }
+    }
     // Redact the echoed config too: a blank-on-save PRESERVES the stored key
     // (see `AiService.setConfig`), so returning the saved config raw would hand a
     // previously-stored key back to the writer that just blanked the field. The key
@@ -160,6 +176,36 @@ export function mountAiRoutes(app: Hono<AppEnv>, ai: AiService, store: PageStore
     });
   });
 
+  app.post(API.aiTranscribe, async (c) => {
+    const body = await c.req.json().catch(() => null) as {assetId?: unknown; pageId?: unknown} | null;
+    if (typeof body?.assetId !== 'string' || typeof body.pageId !== 'string' || !body.assetId || !body.pageId) {
+      return c.json({error: 'assetId and pageId are required'}, 400);
+    }
+    const {assetId, pageId} = body;
+    if (!/^[0-9a-f]{64}$/.test(assetId) || !/^[0-9a-f]{8}(?:-[0-9a-f]{4}){3}-[0-9a-f]{12}$/i.test(pageId)) {
+      return c.json({error: 'asset not found'}, 404);
+    }
+    const principal = c.get('principal');
+    if (!await store.getPageFor(principal, pageId)
+      || !(await store.pagesReferencingAsset(assetId)).includes(pageId)) {
+      return c.json({error: 'asset not found'}, 404);
+    }
+    const asset = await store.getAssetFor(principal, assetId);
+    if (!asset) return c.json({error: 'asset not found'}, 404);
+    try {
+      const {engine, provider, model} = await ai.transcriptionBackend();
+      await requirePaidInferenceAccess(c, store, provider === 'openai-compat' ? 'openai' : 'off');
+      const result = await engine.transcribe(asset.bytes, {mime: asset.mime, signal: c.req.raw.signal});
+      // Audio backends do not report token counts. Keep unknown cloud cost null.
+      await aiUsage?.log({provider, model, kind: 'transcribe', principal, usage: {inputTokens: 0, outputTokens: 0}});
+      return c.json(result);
+    } catch (err) {
+      if (err instanceof HTTPException) throw err;
+      if (err instanceof TranscriptionConfigError) return c.json({error: err.message}, 400);
+      return c.json({error: 'Transcription failed. Check the provider in Settings → AI and retry.'}, 502);
+    }
+  });
+
   app.post(API.aiModelDownload, async (c) => {
     // Fetches a caller-supplied URL onto the server's disk (SSRF + disk-fill
     // surface) — only the trusted instance owner may supply it.
@@ -378,6 +424,12 @@ function redactAiConfig(config: AiConfig): AiConfig {
   delete redacted.apiKey;
   if (hasKey(config.apiKey)) redacted.apiKeySet = true;
   else delete redacted.apiKeySet;
+  if (config.transcription) {
+    redacted.transcription = {...config.transcription};
+    delete redacted.transcription.apiKey;
+    if (hasKey(config.transcription.apiKey)) redacted.transcription.apiKeySet = true;
+    else delete redacted.transcription.apiKeySet;
+  }
   if (config.providers) {
     redacted.providers = Object.fromEntries(
       Object.entries(config.providers).map(([p, settings]) => {

packages/server/src/ai/service.ts

@@ -3,7 +3,7 @@ import {rename, unlink} from 'node:fs/promises';
 import path from 'node:path';
 import {providerSettings, type AiConfig, type AiProvider, type AiProviderSettings, type AiSearchResponse, type AiStatus, type AiTasksResponse} from '@book.dev/sdk';
 import type {Db} from '../db';
-import {createEngine, type AiEngine, type GenerateOptions} from './providers';
+import {createEngine, MockEngine, OpenAiCompatEngine, type TranscriptionEngine, type AiEngine, type GenerateOptions} from './providers';
 import {assembleSearchResults, bm25Scores, buildIndex, cosine, pageRowsToDocs, parseTaskList, type Bm25Index} from './search';
 import {SkillStore} from './skills';
 
@@ -73,6 +73,13 @@ function mergeStoredKeys(prev: AiConfig, next: AiConfig): AiConfig {
     }
     merged.providers = out as AiConfig['providers'];
   }
+  if (next.transcription || prev.transcription) {
+    merged.transcription = {...(next.transcription ?? prev.transcription!)};
+    delete merged.transcription.apiKeySet;
+    const key = resolveKey(prev.transcription?.apiKey, next.transcription?.apiKey);
+    if (key === undefined) delete merged.transcription.apiKey;
+    else merged.transcription.apiKey = key;
+  }
   return merged;
 }
 
@@ -88,6 +95,8 @@ interface DownloadState {
   error?: string;
 }
 
+export class TranscriptionConfigError extends Error {}
+
 export class AiService {
   private config: AiConfig = DEFAULT_CONFIG;
   private engine: AiEngine | null = null;
@@ -103,6 +112,8 @@ export class AiService {
   constructor(
     private readonly db: Db,
     private readonly modelsDir: string,
+    /** MEET-3: lazily resolve the managed local audio backend. */
+    private readonly localTranscription?: () => Promise<TranscriptionEngine | null>,
   ) {
     this.skills = new SkillStore(db);
   }
@@ -140,6 +151,22 @@ export class AiService {
     return this.config;
   }
 
+  /** Resolve once per request so config changes cannot bypass the paid gate. */
+  async transcriptionBackend(): Promise<{engine: TranscriptionEngine; provider: 'local' | 'mock' | 'openai-compat'; model: string}> {
+    const config = await this.getConfig();
+    const audio = config.transcription;
+    if (audio?.provider === 'off') {
+      throw new TranscriptionConfigError('Transcription is off. Enable it in Settings → AI.');
+    }
+    if (audio?.provider === 'openai-compat') {
+      return {engine: new OpenAiCompatEngine(audio.baseUrl?.trim() || 'https://api.openai.com', audio.model?.trim() || 'whisper-1', 'openai', audio.apiKey), provider: 'openai-compat', model: audio.model?.trim() || 'whisper-1'};
+    }
+    const local = await this.localTranscription?.();
+    if (local) return {engine: local, provider: 'local', model: audio?.model ?? 'local'};
+    if (config.provider === 'mock') return {engine: new MockEngine(), provider: 'mock', model: 'mock'};
+    throw new TranscriptionConfigError('Local transcription is unavailable. Configure transcription in Settings → AI.');
+  }
+
   async status(): Promise<AiStatus> {
     await this.loadConfig();
     let ready = false;

packages/server/src/ai/usage.ts

@@ -68,11 +68,11 @@ const PROP = {
 } as const;
 
 /** The kinds of model call we attribute. */
-export type UsageKind = 'agent' | 'complete' | 'generate';
+export type UsageKind = 'agent' | 'complete' | 'generate' | 'transcribe';
 
 /** One model call to attribute: what ran, how many tokens, and for whom. */
 export interface UsageEvent {
-  provider: AiProvider;
+  provider: AiProvider | 'local' | 'openai-compat';
   model: string;
   kind: UsageKind;
   usage: TokenUsage;
@@ -364,7 +364,7 @@ export class AiUsageLog {
       // A claude call with no configured model runs on the engine's default — price
       // (and log) against that so cost isn't spuriously null.
       const model = event.model || (event.provider === 'claude' ? AnthropicEngine.DEFAULT_MODEL : '');
-      const cost = this.computeCost(event.provider, model, event.usage, effective);
+      const cost = event.provider === 'openai-compat' ? null : event.provider === 'local' ? 0 : this.computeCost(event.provider, model, event.usage, effective);
       const properties: Record<string, unknown> = {
         [PROP.time]: new Date().toISOString(),
         [PROP.user]: formatUser(event.principal),

packages/server/src/localClient.ts

@@ -5,6 +5,7 @@ import type {
   AgentTokenList,
   CreatedAgentToken,
   AiConfig,
+  AiTranscriptionResult,
   AiPricingResponse,
   AiPricingTable,
   AiUsageResponse,
@@ -920,6 +921,10 @@ export class LocalDataClient implements DataClient {
     return Promise.reject(this.aiUnavailable());
   }
 
+  transcribeAsset(): Promise<AiTranscriptionResult> {
+    return Promise.reject(this.aiUnavailable());
+  }
+
   aiComplete(): Promise<string> {
     return Promise.reject(this.aiUnavailable());
   }