From 35b81ce9dee2c56162b2cf38c22e792f4513515a Mon Sep 17 00:00:00 2001
From: Dhruva Reddy <dhruva@vapi.ai>
Date: Wed, 13 May 2026 12:16:02 -0700
Subject: [PATCH 1/4] feat: npm run audit command for orphan/duplicate/drift
 detection
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Adds `npm run audit -- <org>` — single-command audit for the state-vs-dashboard
drift conditions that have been accumulating cruft in customer-fork repos.

Detects (read-only):
- orphan local YAML files (no state entry — Scenario B leftovers)
- state ghosts (state UUID missing on dashboard)
- state UUID collisions (cascade-duplicate fingerprint)
- content-identical resources (same lastPulledHash)
- sibling base-slug clusters (cascade-risk warning)
- dashboard orphans (UUID not in state; suppressed by .vapi-ignore)
- assistants with inline model.tools (suspected duplicate-spawn surface)

Exit code: 0 if clean, 1 if any findings.

Designed for DI: state loader, local file lister, remote fetcher are all
injectable, making tests filesystem-free and network-free.

Promotes `listExistingResourceIds` in src/pull.ts from `function` to
`export function` (one-word edit) to avoid duplicating the directory walker.

Tests in tests/audit.test.ts will be added in a follow-up commit on this
branch.
---
 package.json     |   1 +
 src/audit-cmd.ts | 104 +++++++++++
 src/audit.ts     | 457 +++++++++++++++++++++++++++++++++++++++++++++++
 src/pull.ts      |   2 +-
 4 files changed, 563 insertions(+), 1 deletion(-)
 create mode 100644 src/audit-cmd.ts
 create mode 100644 src/audit.ts

diff --git a/package.json b/package.json
index 8476770..3779c13 100644
--- a/package.json
+++ b/package.json
@@ -13,6 +13,7 @@
     "call": "bash -c 'exec tsx src/call-cmd.ts \"$@\" 2> >(grep --line-buffered -v \"buffer underflow\" >&2)' --",
     "cleanup": "tsx src/cleanup-cmd.ts",
     "validate": "tsx src/validate-cmd.ts",
+    "audit": "tsx src/audit-cmd.ts",
     "sim": "tsx src/sim-cmd.ts",
     "rollback": "tsx src/rollback-cmd.ts",
     "build": "tsc --noEmit",
diff --git a/src/audit-cmd.ts b/src/audit-cmd.ts
new file mode 100644
index 0000000..559661d
--- /dev/null
+++ b/src/audit-cmd.ts
@@ -0,0 +1,104 @@
+// CLI entry: `npm run audit -- <org>`
+//
+// Read-only audit for the state-vs-dashboard drift conditions that have been
+// accumulating cruft in customer-fork repos. Mirrors `src/validate-cmd.ts` for
+// argument parsing and env banner so the operator experience is consistent
+// across the engine.
+//
+// Exit code: 0 if no findings, 1 if any (warn or error). No `--strict` flag in
+// v1 — a single severity bar keeps the surface small while we observe what
+// shows up in real customer state.
+
+import { resolve } from "path";
+import { fileURLToPath } from "url";
+import {
+  type AuditFinding,
+  formatFinding,
+  runAudit,
+  summarizeFindings,
+} from "./audit.ts";
+import { APPLY_FILTER, VAPI_BASE_URL, VAPI_ENV } from "./config.ts";
+import type { ResourceType } from "./types.ts";
+import { VALID_RESOURCE_TYPES } from "./types.ts";
+
+function groupFindings(
+  findings: AuditFinding[],
+): Map<ResourceType, AuditFinding[]> {
+  const grouped = new Map<ResourceType, AuditFinding[]>();
+  for (const f of findings) {
+    const arr = grouped.get(f.type) ?? [];
+    arr.push(f);
+    grouped.set(f.type, arr);
+  }
+  // Stable inner ordering: by rule, then by first resourceId.
+  for (const arr of grouped.values()) {
+    arr.sort((a, b) => {
+      if (a.rule !== b.rule) return a.rule.localeCompare(b.rule);
+      const aFirst = a.resourceIds[0] ?? "";
+      const bFirst = b.resourceIds[0] ?? "";
+      return aFirst.localeCompare(bFirst);
+    });
+  }
+  return grouped;
+}
+
+async function main(): Promise<void> {
+  console.log(
+    "═══════════════════════════════════════════════════════════════",
+  );
+  console.log(`🔎 Vapi GitOps Audit - Environment: ${VAPI_ENV}`);
+  console.log(`   API: ${VAPI_BASE_URL}`);
+  console.log(
+    "═══════════════════════════════════════════════════════════════\n",
+  );
+
+  // Respect --type filter (parsed by config.ts into APPLY_FILTER). When the
+  // operator passes one or more --type flags we audit only those types; the
+  // default sweep covers every entry in VALID_RESOURCE_TYPES.
+  const types: ResourceType[] = APPLY_FILTER.resourceTypes?.length
+    ? APPLY_FILTER.resourceTypes
+    : [...VALID_RESOURCE_TYPES];
+
+  if (APPLY_FILTER.resourceTypes?.length) {
+    console.log(`🔧 Type filter: ${types.join(", ")}\n`);
+  }
+
+  const findings = await runAudit({ types });
+
+  console.log(summarizeFindings(findings));
+
+  if (findings.length === 0) {
+    process.exit(0);
+  }
+
+  // Group findings by resource type → rule for human-readable output.
+  const grouped = groupFindings(findings);
+
+  // Iterate types in the configured filter order so the operator can scan top-down.
+  for (const type of types) {
+    const arr = grouped.get(type);
+    if (!arr?.length) continue;
+    console.log(`\n${type} (${arr.length} finding(s)):`);
+    for (const f of arr) {
+      console.log(formatFinding(f));
+    }
+  }
+
+  // Any finding → exit 1. v1 has no --strict gate; a warning still indicates
+  // operator-actionable drift.
+  process.exit(1);
+}
+
+const isMainModule =
+  process.argv[1] !== undefined &&
+  resolve(process.argv[1]) === fileURLToPath(import.meta.url);
+
+if (isMainModule) {
+  main().catch((error) => {
+    console.error(
+      "\n❌ Audit failed:",
+      error instanceof Error ? error.message : error,
+    );
+    process.exit(1);
+  });
+}
diff --git a/src/audit.ts b/src/audit.ts
new file mode 100644
index 0000000..b077430
--- /dev/null
+++ b/src/audit.ts
@@ -0,0 +1,457 @@
+// ─────────────────────────────────────────────────────────────────────────────
+// Read-only audit — detects orphan / duplicate / drift conditions in gitops
+// state that have been accumulating cruft in customer-fork repos.
+//
+// Surfaces seven conditions (none mutate state or filesystem):
+//   1. orphan-yaml          local YAML with no state entry
+//   2. state-ghost          state UUID missing from dashboard
+//   3. state-uuid-collision multiple slugs pointing at the same UUID
+//   4. content-identical    multiple slugs sharing the same lastPulledHash
+//   5. sibling-base-slug    multiple slugs sharing the same base-slug
+//   6. dashboard-orphan     dashboard UUID not in state (.vapi-ignore suppresses)
+//   7. inline-tools         assistant with non-empty model.tools (use toolIds)
+//
+// Designed for dependency injection: state loader, local file lister, remote
+// fetcher, and per-assistant tool reader are all swappable so tests stay
+// filesystem-free and network-free.
+//
+// CLI entry: `npm run audit -- <org>`. Exit code 0 if clean, 1 if any finding.
+// ─────────────────────────────────────────────────────────────────────────────
+
+import { readFile } from "fs/promises";
+import { join } from "path";
+import { matchesIgnore, RESOURCES_DIR } from "./config.ts";
+import {
+  extractBaseSlug,
+  fetchAllResources,
+  listExistingResourceIds,
+  type VapiResource,
+} from "./pull.ts";
+import { FOLDER_MAP } from "./resources.ts";
+import { loadState } from "./state.ts";
+import type { ResourceType, StateFile } from "./types.ts";
+import { VALID_RESOURCE_TYPES } from "./types.ts";
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Public types
+// ─────────────────────────────────────────────────────────────────────────────
+
+export type AuditRule =
+  | "orphan-yaml"
+  | "state-ghost"
+  | "state-uuid-collision"
+  | "content-identical"
+  | "sibling-base-slug"
+  | "dashboard-orphan"
+  | "inline-tools";
+
+export type AuditSeverity = "warn" | "error";
+
+export interface AuditFinding {
+  severity: AuditSeverity;
+  type: ResourceType;
+  rule: AuditRule;
+  resourceIds: string[]; // 1+ slugs
+  uuid?: string; // for state-ghost / dashboard-orphan / collision
+  message: string;
+  suggestedAction?: string;
+}
+
+export interface AuditOptions {
+  // Subset of resource types to audit. Defaults to VALID_RESOURCE_TYPES.
+  types?: ResourceType[];
+  // When false, skips fetchAllResources calls — used by tests AND for a
+  // fully-offline mode if we ever want it. Default true.
+  fetchRemote?: boolean;
+  // DI seam: swap the network call. When supplied, `fetchRemote` is implied
+  // true and the default `fetchAllResources` is not used.
+  remoteFetcher?: (t: ResourceType) => Promise<VapiResource[]>;
+  // DI seam: swap state loading (default reads .vapi-state.<env>.json).
+  stateLoader?: () => StateFile;
+  // DI seam: swap local-file enumeration (default walks resources/<env>/<type>/).
+  listLocalIds?: (t: ResourceType) => string[];
+  // DI seam: swap per-assistant inline-tool reader. Returning a non-empty
+  // array marks the assistant as carrying inline tools. Defaults to YAML/MD
+  // frontmatter parsing of the assistant file on disk. Sync OR async return
+  // is accepted so tests can keep their fixtures plain-object.
+  readAssistantTools?: (resourceId: string) => unknown[] | Promise<unknown[]>;
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Default DI implementations
+// ─────────────────────────────────────────────────────────────────────────────
+
+// Read assistants/<resourceId>.{md,yml,yaml} and pull out `model.tools` if it's
+// an array. Returns [] when the file is missing, malformed, or the field is
+// absent. Designed to be tolerant — audit is read-only and should not throw
+// on unparseable assistant files (push/validate already surface those).
+async function defaultReadAssistantTools(
+  resourceId: string,
+): Promise<unknown[]> {
+  const baseDir = join(RESOURCES_DIR, FOLDER_MAP.assistants);
+  const candidates = [
+    join(baseDir, `${resourceId}.md`),
+    join(baseDir, `${resourceId}.yml`),
+    join(baseDir, `${resourceId}.yaml`),
+  ];
+
+  for (const path of candidates) {
+    let raw: string;
+    try {
+      raw = await readFile(path, "utf-8");
+    } catch {
+      continue;
+    }
+
+    // For .md we only need the frontmatter; for .yml/.yaml the whole file.
+    let yamlSource = raw;
+    if (path.endsWith(".md")) {
+      const match = raw.match(/^---\r?\n([\s\S]*?)\r?\n---/);
+      if (!match) return [];
+      yamlSource = match[1] ?? "";
+    }
+
+    // Lazy import yaml to keep the module light when DI is used in tests.
+    const { parse: parseYaml } = await import("yaml");
+    let parsed: unknown;
+    try {
+      parsed = parseYaml(yamlSource);
+    } catch {
+      return [];
+    }
+    if (!parsed || typeof parsed !== "object") return [];
+    const model = (parsed as { model?: unknown }).model;
+    if (!model || typeof model !== "object") return [];
+    const tools = (model as { tools?: unknown }).tools;
+    return Array.isArray(tools) ? tools : [];
+  }
+
+  return [];
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Slug helpers (kept local; mirror src/pull.ts conventions)
+// ─────────────────────────────────────────────────────────────────────────────
+
+function slugify(name: string): string {
+  return name
+    .toLowerCase()
+    .replace(/[^a-z0-9]+/g, "-")
+    .replace(/^-+|-+$/g, "")
+    .replace(/-+/g, "-");
+}
+
+function extractRemoteName(resource: VapiResource): string | undefined {
+  if (typeof resource.name === "string" && resource.name) return resource.name;
+  // Tools store their name under function.name
+  const fn = (resource as { function?: unknown }).function;
+  if (fn && typeof fn === "object") {
+    const fname = (fn as { name?: unknown }).name;
+    if (typeof fname === "string" && fname) return fname;
+  }
+  return undefined;
+}
+
+// Build the candidate resourceId(s) that a dashboard-orphan UUID would map to,
+// so we can check them against `.vapi-ignore`. Two shapes are produced because
+// real customer .vapi-ignore patterns target either form:
+//   - the bare name-slug (e.g. `assistants/iform-triage-classifier`)
+//   - the `<name>-<uuid8>` form pull.ts emits (`assistants/iform-...-d98136d9`)
+function candidateResourceIdsForRemote(resource: VapiResource): string[] {
+  const name = extractRemoteName(resource);
+  const shortId = resource.id.slice(0, 8);
+  if (!name) return [`resource-${shortId}`];
+  const baseSlug = slugify(name);
+  return [baseSlug, `${baseSlug}-${shortId}`];
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Individual check implementations
+//
+// Each takes the per-type inputs it needs and returns AuditFinding[]. Keeping
+// them small and pure makes the test surface obvious and avoids the "one giant
+// function with seven nested loops" antipattern.
+// ─────────────────────────────────────────────────────────────────────────────
+
+function checkOrphanYaml(
+  type: ResourceType,
+  localIds: string[],
+  state: StateFile,
+): AuditFinding[] {
+  const stateKeys = new Set(Object.keys(state[type]));
+  const findings: AuditFinding[] = [];
+  for (const localId of localIds) {
+    if (stateKeys.has(localId)) continue;
+    findings.push({
+      severity: "warn",
+      type,
+      rule: "orphan-yaml",
+      resourceIds: [localId],
+      message: `local file ${type}/${localId} has no state entry`,
+      suggestedAction:
+        "delete file OR run `npm run pull` to re-key it into state",
+    });
+  }
+  return findings;
+}
+
+function checkStateGhosts(
+  type: ResourceType,
+  state: StateFile,
+  remoteUuids: Set<string>,
+): AuditFinding[] {
+  const findings: AuditFinding[] = [];
+  for (const [resourceId, entry] of Object.entries(state[type])) {
+    if (remoteUuids.has(entry.uuid)) continue;
+    findings.push({
+      severity: "warn",
+      type,
+      rule: "state-ghost",
+      resourceIds: [resourceId],
+      uuid: entry.uuid,
+      message: `state entry ${type}/${resourceId} points at UUID ${entry.uuid} which is not on the dashboard`,
+      suggestedAction:
+        "remove state entry; dashboard resource no longer exists",
+    });
+  }
+  return findings;
+}
+
+function checkStateUuidCollisions(
+  type: ResourceType,
+  state: StateFile,
+): AuditFinding[] {
+  const byUuid = new Map<string, string[]>();
+  for (const [resourceId, entry] of Object.entries(state[type])) {
+    const slugs = byUuid.get(entry.uuid) ?? [];
+    slugs.push(resourceId);
+    byUuid.set(entry.uuid, slugs);
+  }
+
+  const findings: AuditFinding[] = [];
+  for (const [uuid, slugs] of byUuid) {
+    if (slugs.length < 2) continue;
+    findings.push({
+      severity: "error",
+      type,
+      rule: "state-uuid-collision",
+      resourceIds: slugs.slice().sort(),
+      uuid,
+      message: `${slugs.length} state slugs for ${type} share UUID ${uuid}: ${slugs.join(", ")}`,
+      suggestedAction:
+        "manual state edit — pick one slug and delete the others",
+    });
+  }
+  return findings;
+}
+
+function checkContentIdentical(
+  type: ResourceType,
+  state: StateFile,
+): { findings: AuditFinding[]; identicalSlugs: Set<string> } {
+  const byHash = new Map<string, string[]>();
+  for (const [resourceId, entry] of Object.entries(state[type])) {
+    const hash = entry.lastPulledHash;
+    if (!hash) continue;
+    const slugs = byHash.get(hash) ?? [];
+    slugs.push(resourceId);
+    byHash.set(hash, slugs);
+  }
+
+  const findings: AuditFinding[] = [];
+  const identicalSlugs = new Set<string>();
+  for (const [, slugs] of byHash) {
+    if (slugs.length < 2) continue;
+    const sorted = slugs.slice().sort();
+    for (const s of sorted) identicalSlugs.add(s);
+    findings.push({
+      severity: "warn",
+      type,
+      rule: "content-identical",
+      resourceIds: sorted,
+      message: `${slugs.length} ${type} share the same lastPulledHash (content-identical): ${sorted.join(", ")}`,
+      suggestedAction:
+        "consolidate references onto a single canonical slug, then retire others via delete-files → push → API-delete → pull",
+    });
+  }
+  return { findings, identicalSlugs };
+}
+
+function checkSiblingBaseSlug(
+  type: ResourceType,
+  state: StateFile,
+  identicalSlugs: Set<string>,
+): AuditFinding[] {
+  const byBase = new Map<string, string[]>();
+  for (const resourceId of Object.keys(state[type])) {
+    const base = extractBaseSlug(resourceId);
+    // A resourceId without the `-<8hex>` suffix returns itself — clustering
+    // bare slugs against suffixed ones is the whole point, so keep them.
+    const slugs = byBase.get(base) ?? [];
+    slugs.push(resourceId);
+    byBase.set(base, slugs);
+  }
+
+  const findings: AuditFinding[] = [];
+  for (const [base, slugs] of byBase) {
+    if (slugs.length < 2) continue;
+    const sorted = slugs.slice().sort();
+    const overlapsIdentical = sorted.some((s) => identicalSlugs.has(s));
+    const crossRef = overlapsIdentical
+      ? " — overlaps with content-identical cluster (cascade-duplicate risk)"
+      : "";
+    findings.push({
+      severity: "warn",
+      type,
+      rule: "sibling-base-slug",
+      resourceIds: sorted,
+      message: `${slugs.length} ${type} share base slug "${base}": ${sorted.join(", ")}${crossRef}`,
+      suggestedAction: "investigate cascade-duplicate risk",
+    });
+  }
+  return findings;
+}
+
+function checkDashboardOrphans(
+  type: ResourceType,
+  state: StateFile,
+  remote: VapiResource[],
+): AuditFinding[] {
+  const knownUuids = new Set<string>();
+  for (const entry of Object.values(state[type])) knownUuids.add(entry.uuid);
+
+  const findings: AuditFinding[] = [];
+  const folderPath = FOLDER_MAP[type];
+
+  for (const resource of remote) {
+    if (knownUuids.has(resource.id)) continue;
+
+    // .vapi-ignore suppression: a customer can intentionally ignore a remote
+    // resource (e.g. dashboard-managed test artifact). Try the candidate
+    // resourceIds the pull engine would have generated.
+    const candidates = candidateResourceIdsForRemote(resource);
+    let matchedPattern: string | null = null;
+    for (const cand of candidates) {
+      matchedPattern = matchesIgnore(folderPath, cand);
+      if (matchedPattern) break;
+    }
+    if (matchedPattern) continue;
+
+    const displayName = extractRemoteName(resource) ?? "(unnamed)";
+    findings.push({
+      severity: "warn",
+      type,
+      rule: "dashboard-orphan",
+      resourceIds: candidates,
+      uuid: resource.id,
+      message: `dashboard ${type} "${displayName}" (UUID ${resource.id}) is not tracked in state`,
+      suggestedAction:
+        "add to `.vapi-ignore` if intentional, or delete from dashboard",
+    });
+  }
+  return findings;
+}
+
+async function checkInlineTools(
+  state: StateFile,
+  readAssistantTools: (resourceId: string) => unknown[] | Promise<unknown[]>,
+): Promise<AuditFinding[]> {
+  const findings: AuditFinding[] = [];
+  for (const resourceId of Object.keys(state.assistants)) {
+    const tools = await readAssistantTools(resourceId);
+    if (!Array.isArray(tools) || tools.length === 0) continue;
+    findings.push({
+      severity: "warn",
+      type: "assistants",
+      rule: "inline-tools",
+      resourceIds: [resourceId],
+      message: `assistant ${resourceId} declares ${tools.length} inline model.tools (not toolIds references)`,
+      suggestedAction:
+        "migrate inline tools to `toolIds` references — inline blocks are a suspected duplicate-spawn surface",
+    });
+  }
+  return findings;
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Public entry: runAudit
+// ─────────────────────────────────────────────────────────────────────────────
+
+export async function runAudit(
+  opts: AuditOptions = {},
+): Promise<AuditFinding[]> {
+  const types = opts.types ?? [...VALID_RESOURCE_TYPES];
+  const fetchRemote = opts.fetchRemote ?? true;
+
+  const stateLoader = opts.stateLoader ?? loadState;
+  const listLocalIds = opts.listLocalIds ?? listExistingResourceIds;
+  const remoteFetcher = opts.remoteFetcher ?? fetchAllResources;
+  const readAssistantTools =
+    opts.readAssistantTools ?? defaultReadAssistantTools;
+
+  const state = stateLoader();
+
+  // Parallelize dashboard fetches per type — the API is per-resource-type and
+  // the audit is read-only, so concurrency is safe and noticeably faster on
+  // an org with many types.
+  const remoteByType = new Map<ResourceType, VapiResource[]>();
+  if (fetchRemote) {
+    const fetched = await Promise.all(
+      types.map(async (t) => [t, await remoteFetcher(t)] as const),
+    );
+    for (const [t, list] of fetched) remoteByType.set(t, list);
+  }
+
+  const findings: AuditFinding[] = [];
+
+  for (const type of types) {
+    const localIds = listLocalIds(type);
+
+    findings.push(...checkOrphanYaml(type, localIds, state));
+
+    if (fetchRemote) {
+      const remote = remoteByType.get(type) ?? [];
+      const remoteUuids = new Set(remote.map((r) => r.id));
+      findings.push(...checkStateGhosts(type, state, remoteUuids));
+      findings.push(...checkDashboardOrphans(type, state, remote));
+    }
+
+    findings.push(...checkStateUuidCollisions(type, state));
+
+    const { findings: identicalFindings, identicalSlugs } =
+      checkContentIdentical(type, state);
+    findings.push(...identicalFindings);
+
+    findings.push(...checkSiblingBaseSlug(type, state, identicalSlugs));
+
+    if (type === "assistants") {
+      findings.push(...(await checkInlineTools(state, readAssistantTools)));
+    }
+  }
+
+  return findings;
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Formatting helpers (used by the CLI handler and exported for tests)
+// ─────────────────────────────────────────────────────────────────────────────
+
+export function formatFinding(f: AuditFinding): string {
+  const icon = f.severity === "error" ? "❌" : "⚠️ ";
+  const uuidSuffix = f.uuid ? ` [uuid=${f.uuid}]` : "";
+  const lines = [
+    `  ${icon} [${f.rule}] ${f.type}/${f.resourceIds.join(", ")}${uuidSuffix}`,
+    `      ${f.message}`,
+  ];
+  if (f.suggestedAction) {
+    lines.push(`      → ${f.suggestedAction}`);
+  }
+  return lines.join("\n");
+}
+
+export function summarizeFindings(findings: AuditFinding[]): string {
+  if (findings.length === 0) return "✅ No audit findings.";
+  const errors = findings.filter((f) => f.severity === "error").length;
+  const warns = findings.filter((f) => f.severity === "warn").length;
+  return `📋 Audit: ${findings.length} finding(s) — ${errors} error(s), ${warns} warning(s)`;
+}
diff --git a/src/pull.ts b/src/pull.ts
index ac298b9..48dfb04 100644
--- a/src/pull.ts
+++ b/src/pull.ts
@@ -286,7 +286,7 @@ export function resourceIdMatchesName(
   return extractBaseSlug(resourceId) === slugify(name);
 }
 
-function listExistingResourceIds(resourceType: ResourceType): string[] {
+export function listExistingResourceIds(resourceType: ResourceType): string[] {
   const dir = join(RESOURCES_DIR, FOLDER_MAP[resourceType]);
   if (!existsSync(dir)) return [];
 

From 69d87689d5940cd2d15fd79ce4bf553ef83e4604 Mon Sep 17 00:00:00 2001
From: Dhruva Reddy <dhruva@vapi.ai>
Date: Wed, 13 May 2026 12:21:13 -0700
Subject: [PATCH 2/4] test: add coverage for npm run audit (7 checks +
 formatter + integration)

Covers all 7 audit checks via DI fixtures (no filesystem, no network):
- orphan-yaml (3 cases)
- state-ghost (3 cases inc. fetchRemote=false short-circuit)
- state-uuid-collision (2 cases)
- content-identical (3 cases inc. missing-hash safety)
- sibling-base-slug (3 cases inc. cross-ref overlap)
- dashboard-orphan (4 cases inc. .vapi-ignore suppression)
- inline-tools (4 cases inc. async-Promise branch)

Plus: 1 integration test combining multiple checks, 1 exit-code mapping
test, 3 formatter tests.
---
 tests/audit.test.ts | 583 ++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 583 insertions(+)
 create mode 100644 tests/audit.test.ts

diff --git a/tests/audit.test.ts b/tests/audit.test.ts
new file mode 100644
index 0000000..293f9b0
--- /dev/null
+++ b/tests/audit.test.ts
@@ -0,0 +1,583 @@
+import assert from "node:assert/strict";
+import { mkdirSync, rmSync, writeFileSync } from "node:fs";
+import { dirname, join } from "node:path";
+import test from "node:test";
+import { fileURLToPath } from "node:url";
+
+// audit.ts → config.ts: config.ts asserts argv[2] / VAPI_TOKEN at module load
+// time. Prime both before importing the module under test — same trick used in
+// tests/validate.test.ts and tests/path-matching.test.ts.
+process.argv = ["node", "test", "test-fixture-org"];
+process.env.VAPI_TOKEN = process.env.VAPI_TOKEN || "test-token-not-used";
+
+const { _resetIgnoreCache } = await import("../src/config.ts");
+const { formatFinding, runAudit, summarizeFindings } = await import(
+  "../src/audit.ts"
+);
+
+import type { AuditFinding } from "../src/audit.ts";
+import type { VapiResource } from "../src/pull.ts";
+import type { ResourceState, ResourceType, StateFile } from "../src/types.ts";
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Helpers — keep fixtures DI-friendly and avoid filesystem / network.
+// ─────────────────────────────────────────────────────────────────────────────
+
+function makeStateEntry(uuid: string, hash?: string): ResourceState {
+  return hash ? { uuid, lastPulledHash: hash } : { uuid };
+}
+
+// All sections start empty so callers only populate the type(s) under test.
+// StateFile requires every key — partial objects would not type-check.
+function makeStateFile(overrides: Partial<StateFile> = {}): StateFile {
+  return {
+    credentials: {},
+    assistants: {},
+    structuredOutputs: {},
+    tools: {},
+    squads: {},
+    personalities: {},
+    scenarios: {},
+    simulations: {},
+    simulationSuites: {},
+    evals: {},
+    ...overrides,
+  };
+}
+
+// Default no-op DI bag so each test only overrides the surface it cares about.
+function baseOpts(state: StateFile) {
+  return {
+    types: ["assistants"] as ResourceType[],
+    fetchRemote: false as const,
+    stateLoader: () => state,
+    listLocalIds: (_t: ResourceType) => [] as string[],
+    readAssistantTools: (_id: string) => [] as unknown[],
+  };
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Rule: orphan-yaml
+// ─────────────────────────────────────────────────────────────────────────────
+
+test("orphan-yaml: local file with no state entry produces 1 warn finding", async () => {
+  const state = makeStateFile();
+  const findings = await runAudit({
+    ...baseOpts(state),
+    listLocalIds: (t) => (t === "assistants" ? ["orphan-bot"] : []),
+  });
+  const orphans = findings.filter((f) => f.rule === "orphan-yaml");
+  assert.equal(orphans.length, 1);
+  assert.equal(orphans[0]!.severity, "warn");
+  assert.equal(orphans[0]!.type, "assistants");
+  assert.deepEqual(orphans[0]!.resourceIds, ["orphan-bot"]);
+});
+
+test("orphan-yaml: local file that IS in state produces no orphan-yaml finding", async () => {
+  const state = makeStateFile({
+    assistants: { "known-bot": makeStateEntry("uuid-1") },
+  });
+  const findings = await runAudit({
+    ...baseOpts(state),
+    listLocalIds: (t) => (t === "assistants" ? ["known-bot"] : []),
+  });
+  const orphans = findings.filter((f) => f.rule === "orphan-yaml");
+  assert.equal(orphans.length, 0);
+});
+
+test("orphan-yaml: state has entry but no local file → 0 orphan-yaml findings (inverse is state-ghost)", async () => {
+  const state = makeStateFile({
+    assistants: { "ghost-bot": makeStateEntry("uuid-1") },
+  });
+  // listLocalIds returns []; the state entry only contributes to state-ghost
+  // (which is gated on fetchRemote=true) — not to orphan-yaml.
+  const findings = await runAudit({
+    ...baseOpts(state),
+    listLocalIds: () => [],
+  });
+  const orphans = findings.filter((f) => f.rule === "orphan-yaml");
+  assert.equal(orphans.length, 0);
+});
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Rule: state-ghost
+// ─────────────────────────────────────────────────────────────────────────────
+
+test("state-ghost: state uuid present on dashboard → 0 findings", async () => {
+  const state = makeStateFile({
+    assistants: { "live-bot": makeStateEntry("uuid-live") },
+  });
+  const findings = await runAudit({
+    ...baseOpts(state),
+    fetchRemote: true,
+    remoteFetcher: async (t) =>
+      t === "assistants" ? [{ id: "uuid-live", name: "live-bot" }] : [],
+  });
+  const ghosts = findings.filter((f) => f.rule === "state-ghost");
+  assert.equal(ghosts.length, 0);
+});
+
+test("state-ghost: state uuid missing from dashboard → 1 warn finding", async () => {
+  const state = makeStateFile({
+    assistants: { "dead-bot": makeStateEntry("uuid-dead") },
+  });
+  const findings = await runAudit({
+    ...baseOpts(state),
+    fetchRemote: true,
+    remoteFetcher: async () => [], // dashboard has nothing
+  });
+  const ghosts = findings.filter((f) => f.rule === "state-ghost");
+  assert.equal(ghosts.length, 1);
+  assert.equal(ghosts[0]!.severity, "warn");
+  assert.equal(ghosts[0]!.uuid, "uuid-dead");
+  assert.deepEqual(ghosts[0]!.resourceIds, ["dead-bot"]);
+});
+
+test("state-ghost: fetchRemote=false short-circuits the check (0 findings)", async () => {
+  const state = makeStateFile({
+    assistants: { "dead-bot": makeStateEntry("uuid-dead") },
+  });
+  const findings = await runAudit({
+    ...baseOpts(state),
+    fetchRemote: false,
+  });
+  const ghosts = findings.filter((f) => f.rule === "state-ghost");
+  assert.equal(ghosts.length, 0);
+});
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Rule: state-uuid-collision
+// ─────────────────────────────────────────────────────────────────────────────
+
+test("state-uuid-collision: 2 slugs pointing at same uuid → 1 error finding", async () => {
+  const state = makeStateFile({
+    assistants: {
+      "slug-a": makeStateEntry("uuid-shared"),
+      "slug-b": makeStateEntry("uuid-shared"),
+    },
+  });
+  const findings = await runAudit(baseOpts(state));
+  const collisions = findings.filter((f) => f.rule === "state-uuid-collision");
+  assert.equal(collisions.length, 1);
+  assert.equal(collisions[0]!.severity, "error");
+  assert.equal(collisions[0]!.uuid, "uuid-shared");
+  assert.equal(collisions[0]!.resourceIds.length, 2);
+  // Sorted output is part of the contract — tests rely on stable diffs.
+  assert.deepEqual(collisions[0]!.resourceIds, ["slug-a", "slug-b"]);
+});
+
+test("state-uuid-collision: 1 slug per uuid → 0 findings", async () => {
+  const state = makeStateFile({
+    assistants: {
+      "slug-a": makeStateEntry("uuid-1"),
+      "slug-b": makeStateEntry("uuid-2"),
+    },
+  });
+  const findings = await runAudit(baseOpts(state));
+  const collisions = findings.filter((f) => f.rule === "state-uuid-collision");
+  assert.equal(collisions.length, 0);
+});
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Rule: content-identical
+// ─────────────────────────────────────────────────────────────────────────────
+
+test("content-identical: 4 entries with same lastPulledHash (riley fixture) → 1 warn, 4 slugs", async () => {
+  const state = makeStateFile({
+    assistants: {
+      "riley-1": makeStateEntry("uuid-r1", "hash-shared"),
+      "riley-2": makeStateEntry("uuid-r2", "hash-shared"),
+      "riley-3": makeStateEntry("uuid-r3", "hash-shared"),
+      "riley-4": makeStateEntry("uuid-r4", "hash-shared"),
+    },
+  });
+  const findings = await runAudit(baseOpts(state));
+  const identicals = findings.filter((f) => f.rule === "content-identical");
+  assert.equal(identicals.length, 1);
+  assert.equal(identicals[0]!.severity, "warn");
+  assert.equal(identicals[0]!.resourceIds.length, 4);
+  assert.deepEqual(identicals[0]!.resourceIds, [
+    "riley-1",
+    "riley-2",
+    "riley-3",
+    "riley-4",
+  ]);
+});
+
+test("content-identical: entries missing lastPulledHash are silently skipped (no crash, 0 findings)", async () => {
+  const state = makeStateFile({
+    assistants: {
+      "no-hash-a": makeStateEntry("uuid-a"),
+      "no-hash-b": makeStateEntry("uuid-b"),
+    },
+  });
+  const findings = await runAudit(baseOpts(state));
+  const identicals = findings.filter((f) => f.rule === "content-identical");
+  assert.equal(identicals.length, 0);
+});
+
+test("content-identical: 2 entries share hash, 1 entry has distinct hash → 1 finding for the 2, the third absent", async () => {
+  const state = makeStateFile({
+    assistants: {
+      "twin-a": makeStateEntry("uuid-a", "hash-twin"),
+      "twin-b": makeStateEntry("uuid-b", "hash-twin"),
+      lone: makeStateEntry("uuid-c", "hash-lone"),
+    },
+  });
+  const findings = await runAudit(baseOpts(state));
+  const identicals = findings.filter((f) => f.rule === "content-identical");
+  assert.equal(identicals.length, 1);
+  assert.deepEqual(identicals[0]!.resourceIds, ["twin-a", "twin-b"]);
+  // The lone entry must not appear in any content-identical finding.
+  assert.equal(identicals[0]!.resourceIds.includes("lone"), false);
+});
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Rule: sibling-base-slug
+// ─────────────────────────────────────────────────────────────────────────────
+
+test("sibling-base-slug: bare + 2 suffixed entries cluster under same base → 1 finding, 3 slugs", async () => {
+  const state = makeStateFile({
+    assistants: {
+      "iform-barge": makeStateEntry("uuid-1"),
+      "iform-barge-d98136d9": makeStateEntry("uuid-2"),
+      "iform-barge-f6b53e27": makeStateEntry("uuid-3"),
+    },
+  });
+  const findings = await runAudit(baseOpts(state));
+  const siblings = findings.filter((f) => f.rule === "sibling-base-slug");
+  assert.equal(siblings.length, 1);
+  assert.equal(siblings[0]!.resourceIds.length, 3);
+  assert.deepEqual(siblings[0]!.resourceIds, [
+    "iform-barge",
+    "iform-barge-d98136d9",
+    "iform-barge-f6b53e27",
+  ]);
+  // No content-identical overlap here → message does NOT contain cross-ref.
+  assert.equal(
+    siblings[0]!.message.includes("overlaps with content-identical"),
+    false,
+  );
+});
+
+test("sibling-base-slug: siblings that share a hash get cross-reference to content-identical", async () => {
+  const state = makeStateFile({
+    assistants: {
+      "iform-barge": makeStateEntry("uuid-1", "hash-shared"),
+      "iform-barge-d98136d9": makeStateEntry("uuid-2", "hash-shared"),
+    },
+  });
+  const findings = await runAudit(baseOpts(state));
+  const siblings = findings.filter((f) => f.rule === "sibling-base-slug");
+  assert.equal(siblings.length, 1);
+  // Cross-ref token confirms the cascade-duplicate-risk hint fires when
+  // sibling cluster overlaps content-identical cluster.
+  assert.ok(
+    siblings[0]!.message.includes("overlaps with content-identical"),
+    `expected cross-ref in sibling message, got: ${siblings[0]!.message}`,
+  );
+});
+
+test("sibling-base-slug: only one entry (no siblings) → 0 findings", async () => {
+  const state = makeStateFile({
+    assistants: {
+      "iform-barge": makeStateEntry("uuid-1"),
+    },
+  });
+  const findings = await runAudit(baseOpts(state));
+  const siblings = findings.filter((f) => f.rule === "sibling-base-slug");
+  assert.equal(siblings.length, 0);
+});
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Rule: dashboard-orphan (incl. .vapi-ignore suppression)
+// ─────────────────────────────────────────────────────────────────────────────
+
+test("dashboard-orphan: remote uuid present in state → 0 findings", async () => {
+  const state = makeStateFile({
+    assistants: { "live-bot": makeStateEntry("uuid-live") },
+  });
+  const findings = await runAudit({
+    ...baseOpts(state),
+    fetchRemote: true,
+    remoteFetcher: async () => [{ id: "uuid-live", name: "live-bot" }],
+  });
+  const orphans = findings.filter((f) => f.rule === "dashboard-orphan");
+  assert.equal(orphans.length, 0);
+});
+
+test("dashboard-orphan: remote uuid NOT in state, no .vapi-ignore match → 1 warn finding", async () => {
+  const state = makeStateFile();
+  const findings = await runAudit({
+    ...baseOpts(state),
+    fetchRemote: true,
+    remoteFetcher: async () => [{ id: "uuid-unknown", name: "wild-bot" }],
+  });
+  const orphans = findings.filter((f) => f.rule === "dashboard-orphan");
+  assert.equal(orphans.length, 1);
+  assert.equal(orphans[0]!.severity, "warn");
+  assert.equal(orphans[0]!.uuid, "uuid-unknown");
+  // Both candidate ids the suppression logic considers must be surfaced —
+  // the bare slug and the `<slug>-<uuid8>` form pull.ts emits.
+  assert.deepEqual(orphans[0]!.resourceIds, ["wild-bot", "wild-bot-uuid-unk"]);
+});
+
+test("dashboard-orphan: fetchRemote=false short-circuits the check (0 findings)", async () => {
+  const state = makeStateFile();
+  const findings = await runAudit({
+    ...baseOpts(state),
+    fetchRemote: false,
+  });
+  const orphans = findings.filter((f) => f.rule === "dashboard-orphan");
+  assert.equal(orphans.length, 0);
+});
+
+test("dashboard-orphan: .vapi-ignore matching the candidate slug suppresses the finding", async () => {
+  // matchesIgnore reads patterns from resources/<env>/.vapi-ignore via a
+  // module-private cache. There is no DI seam for patterns on the audit
+  // path, so prime the cache by writing a real fixture and resetting
+  // between/around the test. Cleanup runs in finally so a thrown assertion
+  // still scrubs the filesystem.
+  const __dirname = dirname(fileURLToPath(import.meta.url));
+  const repoRoot = join(__dirname, "..");
+  const ignoreDir = join(repoRoot, "resources", "test-fixture-org");
+  const ignorePath = join(ignoreDir, ".vapi-ignore");
+
+  mkdirSync(ignoreDir, { recursive: true });
+  writeFileSync(ignorePath, "assistants/wild-bot\n", "utf-8");
+  _resetIgnoreCache();
+  try {
+    const state = makeStateFile();
+    const remote: VapiResource[] = [{ id: "uuid-unknown", name: "wild-bot" }];
+    const findings = await runAudit({
+      ...baseOpts(state),
+      fetchRemote: true,
+      remoteFetcher: async () => remote,
+    });
+    const orphans = findings.filter((f) => f.rule === "dashboard-orphan");
+    assert.equal(orphans.length, 0);
+  } finally {
+    rmSync(ignorePath, { force: true });
+    // Best-effort dir cleanup; ignore failure if other tests populated it.
+    try {
+      rmSync(ignoreDir, { recursive: false });
+    } catch {
+      /* dir not empty — fine */
+    }
+    _resetIgnoreCache();
+  }
+});
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Rule: inline-tools
+// ─────────────────────────────────────────────────────────────────────────────
+
+test("inline-tools: assistant with non-empty model.tools array → 1 warn, message includes count", async () => {
+  const state = makeStateFile({
+    assistants: { "tool-laden": makeStateEntry("uuid-1") },
+  });
+  const findings = await runAudit({
+    ...baseOpts(state),
+    readAssistantTools: (id) =>
+      id === "tool-laden" ? [{ type: "endCall" }, { type: "transfer" }] : [],
+  });
+  const inline = findings.filter((f) => f.rule === "inline-tools");
+  assert.equal(inline.length, 1);
+  assert.equal(inline[0]!.severity, "warn");
+  assert.deepEqual(inline[0]!.resourceIds, ["tool-laden"]);
+  assert.ok(
+    inline[0]!.message.includes("2"),
+    `expected tool count in message, got: ${inline[0]!.message}`,
+  );
+});
+
+test("inline-tools: assistant with empty model.tools array → 0 findings", async () => {
+  const state = makeStateFile({
+    assistants: { clean: makeStateEntry("uuid-1") },
+  });
+  const findings = await runAudit({
+    ...baseOpts(state),
+    readAssistantTools: () => [],
+  });
+  const inline = findings.filter((f) => f.rule === "inline-tools");
+  assert.equal(inline.length, 0);
+});
+
+test("inline-tools: readAssistantTools returns non-array (treated as no inline tools) → 0 findings", async () => {
+  // The readAssistantTools DI surface is typed `unknown[] | Promise<unknown[]>`.
+  // The default impl returns [] when model.tools is undefined, so this test
+  // pins the contract for "no tools" by returning [] (the canonical empty).
+  const state = makeStateFile({
+    assistants: { "no-tools-field": makeStateEntry("uuid-1") },
+  });
+  const findings = await runAudit({
+    ...baseOpts(state),
+    readAssistantTools: () => [],
+  });
+  const inline = findings.filter((f) => f.rule === "inline-tools");
+  assert.equal(inline.length, 0);
+});
+
+test("inline-tools: async readAssistantTools (Promise) still fires the finding", async () => {
+  const state = makeStateFile({
+    assistants: { "async-tool-bot": makeStateEntry("uuid-1") },
+  });
+  const findings = await runAudit({
+    ...baseOpts(state),
+    readAssistantTools: async (id) =>
+      id === "async-tool-bot" ? [{ type: "endCall" }] : [],
+  });
+  const inline = findings.filter((f) => f.rule === "inline-tools");
+  assert.equal(inline.length, 1);
+  assert.deepEqual(inline[0]!.resourceIds, ["async-tool-bot"]);
+});
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Integration — multiple checks coexist
+// ─────────────────────────────────────────────────────────────────────────────
+
+test("integration: orphan-yaml + collision + content-identical(4) + sibling-base(3-overlap) coexist", async () => {
+  const state = makeStateFile({
+    assistants: {
+      // 2 distinct slugs sharing a uuid → 1 collision (error)
+      "coll-a": makeStateEntry("dup-uuid"),
+      "coll-b": makeStateEntry("dup-uuid"),
+      // 4 distinct slugs sharing hash H1 → 1 content-identical (warn, 4 slugs)
+      "riley-1": makeStateEntry("uuid-r1", "H1"),
+      "riley-2": makeStateEntry("uuid-r2", "H1"),
+      "riley-3": makeStateEntry("uuid-r3", "H1"),
+      "riley-4": makeStateEntry("uuid-r4", "H1"),
+      // 3 slugs sharing base "iform-barge"; 2 of them share hash H2 so
+      // sibling-base-slug message picks up the cross-ref AND we get one
+      // extra content-identical finding for those 2.
+      "iform-barge": makeStateEntry("uuid-s1", "H2"),
+      "iform-barge-d98136d9": makeStateEntry("uuid-s2", "H2"),
+      "iform-barge-f6b53e27": makeStateEntry("uuid-s3"),
+    },
+  });
+
+  const findings = await runAudit({
+    types: ["assistants"],
+    fetchRemote: false,
+    stateLoader: () => state,
+    // 1 orphan-yaml: a local file with no state entry.
+    listLocalIds: (t) => (t === "assistants" ? ["stray-local"] : []),
+    readAssistantTools: () => [],
+  });
+
+  // Total: 1 orphan-yaml + 1 collision + 2 content-identical + 1 sibling
+  // = 5 findings, 1 error, 4 warns.
+  assert.equal(findings.length, 5);
+  const errors = findings.filter((f) => f.severity === "error");
+  const warns = findings.filter((f) => f.severity === "warn");
+  assert.equal(errors.length, 1);
+  assert.equal(warns.length, 4);
+
+  // Exact ordering: assistants is the only type, so insertion order follows
+  // the check ordering inside runAudit:
+  //   orphan-yaml → state-uuid-collision → content-identical → sibling-base-slug
+  const rules = findings.map((f) => f.rule);
+  assert.deepEqual(rules, [
+    "orphan-yaml",
+    "state-uuid-collision",
+    "content-identical",
+    "content-identical",
+    "sibling-base-slug",
+  ]);
+
+  // The sibling finding must carry the cross-ref token because 2 of the 3
+  // siblings (iform-barge, iform-barge-d98136d9) also appear in a
+  // content-identical cluster.
+  const sibling = findings.find((f) => f.rule === "sibling-base-slug")!;
+  assert.ok(sibling.message.includes("overlaps with content-identical"));
+});
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Exit-code mapping — spec'd via the same severity-bar logic audit-cmd uses.
+//
+// GAP: audit-cmd.ts inlines `process.exit(findings.length === 0 ? 0 : 1)`
+// rather than delegating to an exported helper. Spec'ing the contract here
+// (rather than a subprocess test) is the project pattern for CLI exit codes.
+// ─────────────────────────────────────────────────────────────────────────────
+
+function computeExitCode(findings: AuditFinding[]): 0 | 1 {
+  return findings.length === 0 ? 0 : 1;
+}
+
+test("exit-code: 0 findings → exit 0; ≥1 finding → exit 1 (mirrors audit-cmd severity bar)", () => {
+  assert.equal(computeExitCode([]), 0);
+  const oneWarn: AuditFinding = {
+    severity: "warn",
+    type: "assistants",
+    rule: "orphan-yaml",
+    resourceIds: ["x"],
+    message: "msg",
+  };
+  assert.equal(computeExitCode([oneWarn]), 1);
+  const oneError: AuditFinding = { ...oneWarn, severity: "error" };
+  assert.equal(computeExitCode([oneError]), 1);
+});
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Formatter helpers
+// ─────────────────────────────────────────────────────────────────────────────
+
+test("formatFinding: multi-line output includes rule, severity icon, slugs, message, suggested action", () => {
+  const f: AuditFinding = {
+    severity: "error",
+    type: "assistants",
+    rule: "state-uuid-collision",
+    resourceIds: ["slug-a", "slug-b"],
+    uuid: "uuid-shared",
+    message: "2 state slugs share UUID uuid-shared",
+    suggestedAction: "manual state edit",
+  };
+  const out = formatFinding(f);
+  const lines = out.split("\n");
+  assert.ok(lines.length >= 3, `expected ≥3 lines, got ${lines.length}`);
+  assert.ok(out.includes("[state-uuid-collision]"));
+  assert.ok(out.includes("assistants/slug-a, slug-b"));
+  assert.ok(out.includes("uuid=uuid-shared"));
+  assert.ok(out.includes("2 state slugs share UUID uuid-shared"));
+  assert.ok(out.includes("manual state edit"));
+  // Error severity uses the ❌ icon (warn uses ⚠️).
+  assert.ok(out.includes("❌"));
+});
+
+test("summarizeFindings: empty array → clean-pass message", () => {
+  const out = summarizeFindings([]);
+  assert.ok(
+    out.includes("No audit findings") || out.includes("0 finding"),
+    `expected clean-pass copy, got: ${out}`,
+  );
+});
+
+test("summarizeFindings: 1 error + 5 warns → '6 finding(s)' with both counts", () => {
+  const mk = (severity: "warn" | "error"): AuditFinding => ({
+    severity,
+    type: "assistants",
+    rule: "orphan-yaml",
+    resourceIds: ["x"],
+    message: "m",
+  });
+  const findings: AuditFinding[] = [
+    mk("error"),
+    mk("warn"),
+    mk("warn"),
+    mk("warn"),
+    mk("warn"),
+    mk("warn"),
+  ];
+  const out = summarizeFindings(findings);
+  assert.ok(out.includes("6"), `expected total count 6, got: ${out}`);
+  assert.ok(out.includes("1"), `expected error count 1, got: ${out}`);
+  assert.ok(out.includes("5"), `expected warn count 5, got: ${out}`);
+  assert.ok(
+    out.toLowerCase().includes("error"),
+    `expected 'error' label, got: ${out}`,
+  );
+  assert.ok(
+    out.toLowerCase().includes("warning"),
+    `expected 'warning' label, got: ${out}`,
+  );
+});

From 90ae2b79e973c960410385cc1ea247f96997ecb9 Mon Sep 17 00:00:00 2001
From: Dhruva Reddy <dhruva@vapi.ai>
Date: Wed, 13 May 2026 12:23:03 -0700
Subject: [PATCH 3/4] refactor: extract exitCodeForFindings helper, pin test to
 it

Closes the gap surfaced by the test-writer phase: audit-cmd.ts had inlined
`findings.length === 0 ? 0 : 1` at every exit-code call site, so the
exit-code test in tests/audit.test.ts could only assert on a parallel
re-derivation rather than the real CLI behavior.

Extracts a tiny exported `exitCodeForFindings(findings)` helper and routes
both exit sites through it. Test imports and pins to the helper, so future
changes to the severity bar (e.g. a `--strict` flag in v2) will surface in
the existing assertion instead of silently drifting.

No behavior change. 155/155 tests pass.
---
 src/audit-cmd.ts    | 10 ++++++++--
 tests/audit.test.ts | 20 +++++++-------------
 2 files changed, 15 insertions(+), 15 deletions(-)

diff --git a/src/audit-cmd.ts b/src/audit-cmd.ts
index 559661d..a4aa23e 100644
--- a/src/audit-cmd.ts
+++ b/src/audit-cmd.ts
@@ -21,6 +21,12 @@ import { APPLY_FILTER, VAPI_BASE_URL, VAPI_ENV } from "./config.ts";
 import type { ResourceType } from "./types.ts";
 import { VALID_RESOURCE_TYPES } from "./types.ts";
 
+// Single source of truth for the exit-code contract. Exported so tests can pin
+// behavior without duplicating the predicate.
+export function exitCodeForFindings(findings: AuditFinding[]): 0 | 1 {
+  return findings.length === 0 ? 0 : 1;
+}
+
 function groupFindings(
   findings: AuditFinding[],
 ): Map<ResourceType, AuditFinding[]> {
@@ -67,7 +73,7 @@ async function main(): Promise<void> {
 
   console.log(summarizeFindings(findings));
 
-  if (findings.length === 0) {
+  if (exitCodeForFindings(findings) === 0) {
     process.exit(0);
   }
 
@@ -86,7 +92,7 @@ async function main(): Promise<void> {
 
   // Any finding → exit 1. v1 has no --strict gate; a warning still indicates
   // operator-actionable drift.
-  process.exit(1);
+  process.exit(exitCodeForFindings(findings));
 }
 
 const isMainModule =
diff --git a/tests/audit.test.ts b/tests/audit.test.ts
index 293f9b0..6f1e771 100644
--- a/tests/audit.test.ts
+++ b/tests/audit.test.ts
@@ -14,6 +14,7 @@ const { _resetIgnoreCache } = await import("../src/config.ts");
 const { formatFinding, runAudit, summarizeFindings } = await import(
   "../src/audit.ts"
 );
+const { exitCodeForFindings } = await import("../src/audit-cmd.ts");
 
 import type { AuditFinding } from "../src/audit.ts";
 import type { VapiResource } from "../src/pull.ts";
@@ -493,19 +494,12 @@ test("integration: orphan-yaml + collision + content-identical(4) + sibling-base
 });
 
 // ─────────────────────────────────────────────────────────────────────────────
-// Exit-code mapping — spec'd via the same severity-bar logic audit-cmd uses.
-//
-// GAP: audit-cmd.ts inlines `process.exit(findings.length === 0 ? 0 : 1)`
-// rather than delegating to an exported helper. Spec'ing the contract here
-// (rather than a subprocess test) is the project pattern for CLI exit codes.
+// Exit-code mapping — pinned to the exported helper in audit-cmd.ts so the test
+// catches drift if the CLI's severity bar ever changes.
 // ─────────────────────────────────────────────────────────────────────────────
 
-function computeExitCode(findings: AuditFinding[]): 0 | 1 {
-  return findings.length === 0 ? 0 : 1;
-}
-
-test("exit-code: 0 findings → exit 0; ≥1 finding → exit 1 (mirrors audit-cmd severity bar)", () => {
-  assert.equal(computeExitCode([]), 0);
+test("exit-code: 0 findings → exit 0; ≥1 finding → exit 1 (via exported helper)", () => {
+  assert.equal(exitCodeForFindings([]), 0);
   const oneWarn: AuditFinding = {
     severity: "warn",
     type: "assistants",
@@ -513,9 +507,9 @@ test("exit-code: 0 findings → exit 0; ≥1 finding → exit 1 (mirrors audit-c
     resourceIds: ["x"],
     message: "msg",
   };
-  assert.equal(computeExitCode([oneWarn]), 1);
+  assert.equal(exitCodeForFindings([oneWarn]), 1);
   const oneError: AuditFinding = { ...oneWarn, severity: "error" };
-  assert.equal(computeExitCode([oneError]), 1);
+  assert.equal(exitCodeForFindings([oneError]), 1);
 });
 
 // ─────────────────────────────────────────────────────────────────────────────

From 2033ebacc844823b01634b42787cf78109b5b367 Mon Sep 17 00:00:00 2001
From: Dhruva Reddy <dhruva@vapi.ai>
Date: Wed, 13 May 2026 12:27:39 -0700
Subject: [PATCH 4/4] fix(audit): use Promise.allSettled for per-type fetches +
 README entry
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Addresses two non-blocking code-review findings before opening the PR:

1. **Fail-fast → fail-graceful for dashboard fetches.** Switched the
   parallel per-type API calls from `Promise.all` to `Promise.allSettled`.
   A transient 500 / 429 / network blip on one resource type used to abort
   the entire audit, leaving the operator with zero findings instead of
   findings-for-the-types-that-succeeded.

   Now: each failed fetch emits a `fetch-failed` finding (severity: warn,
   message includes the underlying error). The per-type loop checks
   `remoteByType.has(type)` before running state-ghost and dashboard-orphan
   checks — preventing the would-be false-positive where an empty-array
   fallback marks every state entry as a ghost.

   New rule: `AuditRule = ... | "fetch-failed"`.

2. **README command table missing `audit`.** Added a row under the
   `validate` entry so operators discover the command from the same
   surface that lists `pull`/`push`/`cleanup`/`rollback`/etc.

New test pinning the fail-graceful path: one type's `remoteFetcher`
throws → exactly 1 `fetch-failed` finding for that type, 0 false-positive
state-ghost findings for any state entry of that type, and other types'
checks proceed normally.

Suite: 156/156 pass (+1 test).
---
 README.md           |  1 +
 src/audit.ts        | 40 ++++++++++++++++++++++++++++++++++------
 tests/audit.test.ts | 41 +++++++++++++++++++++++++++++++++++++++++
 3 files changed, 76 insertions(+), 6 deletions(-)

diff --git a/README.md b/README.md
index 082dc27..e869490 100644
--- a/README.md
+++ b/README.md
@@ -72,6 +72,7 @@ Every command works in two modes:
 | --- | --- | --- | --- |
 | `npm run setup` | ✅ | — | First-time org wizard — creates `.env.<org>` and `resources/<org>/`. |
 | `npm run validate` | — | `npm run validate -- <org>` | Schema-check local YAML/MD with no network call. **Run before every `apply`.** |
+| `npm run audit` | — | `npm run audit -- <org> [--type <t>]` | Read-only drift detector — orphan local YAML, state ghosts, UUID collisions, content-identical clusters, sibling base-slug clusters, dashboard orphans, assistants with inline `model.tools`. Exit 1 on any finding; safe to wire into CI. |
 | `npm run apply` | ✅ | `npm run apply -- <org> [--force]` | **Default deploy verb.** Pull → merge → push in one safe pass; resilient against dashboard drift. |
 | `npm run pull` | ✅ | `npm run pull -- <org> [flags]` | Fetch remote state into local files / state file. Local-first by default — won't clobber local edits. |
 | `npm run push` | ✅ | `npm run push -- <org> [flags]` | Raw push without a pre-pull. **Skip unless you just ran `pull` and are certain state is fresh** — otherwise prefer `apply`. |
diff --git a/src/audit.ts b/src/audit.ts
index b077430..2a1d1d2 100644
--- a/src/audit.ts
+++ b/src/audit.ts
@@ -43,7 +43,8 @@ export type AuditRule =
   | "content-identical"
   | "sibling-base-slug"
   | "dashboard-orphan"
-  | "inline-tools";
+  | "inline-tools"
+  | "fetch-failed";
 
 export type AuditSeverity = "warn" | "error";
 
@@ -393,23 +394,50 @@ export async function runAudit(
 
   // Parallelize dashboard fetches per type — the API is per-resource-type and
   // the audit is read-only, so concurrency is safe and noticeably faster on
-  // an org with many types.
+  // an org with many types. Use `allSettled` so a transient failure on one
+  // type doesn't abort the whole audit; emit a `fetch-failed` finding per
+  // failed type so the operator sees exactly which data is missing.
   const remoteByType = new Map<ResourceType, VapiResource[]>();
+  const fetchFailures: AuditFinding[] = [];
   if (fetchRemote) {
-    const fetched = await Promise.all(
+    const settled = await Promise.allSettled(
       types.map(async (t) => [t, await remoteFetcher(t)] as const),
     );
-    for (const [t, list] of fetched) remoteByType.set(t, list);
+    for (let i = 0; i < settled.length; i++) {
+      const result = settled[i];
+      const type = types[i];
+      if (!result || !type) continue;
+      if (result.status === "fulfilled") {
+        const [t, list] = result.value;
+        remoteByType.set(t, list);
+      } else {
+        const reason = result.reason;
+        const message =
+          reason instanceof Error ? reason.message : String(reason);
+        fetchFailures.push({
+          severity: "warn",
+          type,
+          rule: "fetch-failed",
+          resourceIds: [],
+          message: `Dashboard fetch failed for ${type}: ${message}. State-ghost and dashboard-orphan checks skipped for this type.`,
+          suggestedAction:
+            "Re-run audit after the API recovers, or inspect network connectivity / API token scope.",
+        });
+      }
+    }
   }
 
-  const findings: AuditFinding[] = [];
+  const findings: AuditFinding[] = [...fetchFailures];
 
   for (const type of types) {
     const localIds = listLocalIds(type);
 
     findings.push(...checkOrphanYaml(type, localIds, state));
 
-    if (fetchRemote) {
+    if (fetchRemote && remoteByType.has(type)) {
+      // Only run remote-coupled checks when the fetch actually succeeded —
+      // skip silently for types that failed (operator already saw a
+      // `fetch-failed` finding for them above).
       const remote = remoteByType.get(type) ?? [];
       const remoteUuids = new Set(remote.map((r) => r.id));
       findings.push(...checkStateGhosts(type, state, remoteUuids));
diff --git a/tests/audit.test.ts b/tests/audit.test.ts
index 6f1e771..3cbb682 100644
--- a/tests/audit.test.ts
+++ b/tests/audit.test.ts
@@ -146,6 +146,47 @@ test("state-ghost: fetchRemote=false short-circuits the check (0 findings)", asy
   assert.equal(ghosts.length, 0);
 });
 
+// ─────────────────────────────────────────────────────────────────────────────
+// Rule: fetch-failed — one type's API call throws, the audit emits a
+// `fetch-failed` finding for that type and skips state-ghost /
+// dashboard-orphan checks for it. Other types proceed normally.
+// ─────────────────────────────────────────────────────────────────────────────
+
+test("fetch-failed: one type throws → 1 warn finding for that type; no false state-ghost positives", async () => {
+  const state = makeStateFile({
+    assistants: { "bot-a": makeStateEntry("uuid-a") },
+    tools: { "tool-x": makeStateEntry("uuid-x") },
+  });
+  const findings = await runAudit({
+    ...baseOpts(state),
+    types: ["assistants", "tools"] as ResourceType[],
+    fetchRemote: true,
+    remoteFetcher: async (t: ResourceType) => {
+      if (t === "assistants")
+        throw new Error("simulated 503 from dashboard API");
+      return [{ id: "uuid-x", name: "tool-x" }];
+    },
+  });
+  const fetchFails = findings.filter((f) => f.rule === "fetch-failed");
+  assert.equal(fetchFails.length, 1);
+  assert.equal(fetchFails[0]?.type, "assistants");
+  assert.equal(fetchFails[0]?.severity, "warn");
+  assert.match(fetchFails[0]?.message ?? "", /simulated 503/);
+
+  // Critical regression guard: the failed `assistants` fetch must NOT cause
+  // bot-a to be reported as a state-ghost (which would be a false positive
+  // born of treating an empty remote-list fallback as truth).
+  const ghosts = findings.filter((f) => f.rule === "state-ghost");
+  assert.equal(ghosts.length, 0);
+
+  // The tools type fetched cleanly, so its state-ghost check ran and found
+  // tool-x present on the dashboard. No finding either.
+  const toolGhosts = findings.filter(
+    (f) => f.rule === "state-ghost" && f.type === "tools",
+  );
+  assert.equal(toolGhosts.length, 0);
+});
+
 // ─────────────────────────────────────────────────────────────────────────────
 // Rule: state-uuid-collision
 // ─────────────────────────────────────────────────────────────────────────────