diff --git a/src/act/plans.ts b/src/act/plans.ts index 861d4c8b..b3ee4e39 100644 --- a/src/act/plans.ts +++ b/src/act/plans.ts @@ -1,8 +1,16 @@ import { existsSync, readFileSync } from 'fs' +import { execFileSync } from 'child_process' import { isAbsolute, join } from 'path' import { homedir } from 'os' import type { ActionKind, ActionPlan, PlannedChange } from './types.js' import { sha256 } from './backup.js' +import { + ALWAYSLOAD_MIN_VERSION, + ALWAYSLOAD_STARTUP_CAP_SECONDS, + ENABLE_TOOL_SEARCH_VAR, + parseVersion, + versionPredates, +} from '../optimize.js' import type { WasteFinding } from '../optimize.js' // Turns an optimize finding into a concrete, journaled file-mutation plan. @@ -14,6 +22,9 @@ export type PlanContext = { homeDir?: string cwd?: string shell?: string + // Installed Claude Code version (null when undeterminable). Injectable so + // tests never shell out; production defaults to probing `claude --version`. + claudeVersion?: () => string | null } export type BuiltPlan = { @@ -34,11 +45,32 @@ type ResolvedPaths = { projectSettings: string projectSettingsLocal: string userClaudeJson: string + userSettings: string skillsDir: string agentsDir: string commandsDir: string projectClaudeMd: string shellRc: string + // Not a path, but resolved from the same context: the injectable installed- + // version probe the defer-alwaysload version gate consults. + claudeVersion: () => string | null +} + +// `claude --version` prints e.g. "2.1.130 (Claude Code)"; any failure (binary +// missing, timeout, non-zero exit) yields null and version-gated plans +// degrade to a manual note instead of guessing. +const CLAUDE_VERSION_PROBE_TIMEOUT_MS = 3000 + +function probeClaudeVersion(): string | null { + try { + return execFileSync('claude', ['--version'], { + encoding: 'utf-8', + timeout: CLAUDE_VERSION_PROBE_TIMEOUT_MS, + stdio: ['ignore', 'pipe', 'ignore'], + }).trim() + } catch { + return null + } } function resolvePaths(ctx: PlanContext): ResolvedPaths { @@ -52,11 +84,13 @@ function resolvePaths(ctx: PlanContext): ResolvedPaths { projectSettings: join(cwd, '.claude', 'settings.json'), projectSettingsLocal: join(cwd, '.claude', 'settings.local.json'), userClaudeJson: join(homeDir, '.claude.json'), + userSettings: join(homeDir, '.claude', 'settings.json'), skillsDir: join(homeDir, '.claude', 'skills'), agentsDir: join(homeDir, '.claude', 'agents'), commandsDir: join(homeDir, '.claude', 'commands'), projectClaudeMd: join(cwd, 'CLAUDE.md'), shellRc: join(homeDir, /zsh/.test(shell) ? '.zshrc' : '.bashrc'), + claudeVersion: ctx.claudeVersion ?? probeClaudeVersion, } } @@ -74,6 +108,9 @@ function buildPlan(finding: WasteFinding, r: ResolvedPaths): BuiltPlan { case 'mcp-low-coverage': return buildMcpRemove(finding, r) case 'unused-mcp': return buildMcpRemove(finding, r) case 'mcp-project-scope': return buildMcpProjectScope(finding, r) + case 'mcp-deferral-off': return buildDeferEnable(finding, r) + case 'mcp-alwaysload-hygiene': return buildDeferAlwaysLoad(finding, r) + case 'mcp-defer-threshold': return buildDeferThreshold(finding, r) case 'unused-skills': return buildArchive(finding, r, 'skill') case 'unused-agents': return buildArchive(finding, r, 'agent') case 'unused-commands': return buildArchive(finding, r, 'command') @@ -230,15 +267,20 @@ function projectRemovalNote(server: string, entries: string[], homeDir: string): return `removes ${server} from ${entries.length} project ${noun}: ${entries.map(e => shortPath(e, homeDir)).join(', ')}` } +// Accumulates per-file preview annotations; repeats on a path join with ";". +function pathNoteAdder(pathNotes: Record): (path: string, note: string) => void { + return (path, note) => { + pathNotes[path] = pathNotes[path] ? `${pathNotes[path]}; ${note}` : note + } +} + function buildMcpRemove(finding: WasteFinding, r: ResolvedPaths): BuiltPlan { const servers = finding.apply?.kind === 'mcp-remove' ? finding.apply.servers : [] const searchPaths = [r.projectMcpJson, r.projectSettings, r.projectSettingsLocal, r.userClaudeJson] const docs = new ConfigDocs(r.homeDir) const skips: string[] = [] const pathNotes: Record = {} - const addPathNote = (path: string, note: string): void => { - pathNotes[path] = pathNotes[path] ? `${pathNotes[path]}; ${note}` : note - } + const addPathNote = pathNoteAdder(pathNotes) for (const server of servers) { let removed = false @@ -268,9 +310,7 @@ function buildMcpProjectScope(finding: WasteFinding, r: ResolvedPaths): BuiltPla const docs = new ConfigDocs(r.homeDir) const skips: string[] = [] const pathNotes: Record = {} - const addPathNote = (path: string, note: string): void => { - pathNotes[path] = pathNotes[path] ? `${pathNotes[path]}; ${note}` : note - } + const addPathNote = pathNoteAdder(pathNotes) for (const { server, keepProjects, removeProjects } of entries) { const keepers = keepProjects.filter(p => isAbsolute(p)) @@ -335,6 +375,238 @@ function mcpPlan(kind: ActionKind, findingId: string, description: string, chang return { kind, findingId, description, changes } } +// --------------------------------------------------------------------------- +// MCP deferral plans — defer-enable / defer-alwaysload / defer-threshold (#614) +// --------------------------------------------------------------------------- + +// ENABLE_TOOL_SEARCH and mcpServers config are read once at Claude Code +// process start, so an applied plan changes nothing for sessions already +// running. Stated on every deferral plan. +const NEXT_SESSION_NOTE = 'takes effect on the next session (this config is read at Claude Code start)' + +// findDeferralEnvSetting (src/optimize.ts) reports shell-profile hits with +// exactly this scope string; the plan layer keys its refusal on it. +const SHELL_PROFILE_SCOPE = 'shell profile' + +const SHELL_TOOL_SEARCH_LINE = new RegExp(`^\\s*(?:export\\s+)?${ENABLE_TOOL_SEARCH_VAR}\\s*=.*$`, 'm') + +function envContainer(state: DocState): Record | null { + const env = state.doc.env + return env && typeof env === 'object' ? env as Record : null +} + +// An emptied env object stays in place, matching deleteServer's convention +// for emptied mcpServers containers. +function deleteEnvKey(state: DocState, key: string): boolean { + const env = envContainer(state) + if (!env || !(key in env)) return false + delete env[key] + state.dirty = true + return true +} + +function setEnvKey(state: DocState, key: string, value: string): void { + const env = envContainer(state) ?? (state.doc.env = {}) as Record + env[key] = value + state.dirty = true +} + +// Shell rc files only ever receive marker-block APPENDS (markerChange); +// deleting or rewriting arbitrary user lines is out. Deferral overrides +// found in a profile get precise by-hand instructions naming the exact file +// and line instead of a plan. `replacement` switches the instruction from +// "delete the line" to "change it to ". +function shellOverrideManualNotes(path: string, homeDir: string, replacement?: string): string[] { + const shown = shortPath(path, homeDir) + let content: string | null = null + try { + content = readFileSync(path, 'utf-8') + } catch { + content = null + } + const line = content?.match(SHELL_TOOL_SEARCH_LINE)?.[0]?.trim() + if (!line) { + return [`manual: ${ENABLE_TOOL_SEARCH_VAR} was reported in ${shown} but no such line is there now; nothing to change`] + } + // Keep the original line's `export ` prefix in the suggested replacement — + // a user following the note verbatim would otherwise lose it. + const replacementLine = replacement !== undefined && line.startsWith('export ') && !replacement.startsWith('export ') + ? `export ${replacement}` + : replacement + const action = replacementLine === undefined + ? `delete the line \`${line}\` from ${shown}` + : `in ${shown}, change the line \`${line}\` to \`${replacementLine}\`` + return [`manual: ${action} yourself — codeburn only appends marker blocks to shell files and never edits user lines`] +} + +// mcp-deferral-off -> defer-enable. Only two causes are auto-appliable: +// removing a stale ENABLE_TOOL_SEARCH=false from a settings file, and (for +// the future part-3 verifier) forcing =true once a proxy is verified. The +// rest refuse with instructions: an unknown proxy because an explicit +// override makes requests FAIL outright on proxies that don't forward +// tool_reference blocks (live-docs fact), Vertex because default-off there +// is a platform property, old-version because the fix is `claude update`. +function buildDeferEnable(finding: WasteFinding, r: ResolvedPaths): BuiltPlan { + const payload = finding.apply?.kind === 'defer-enable' ? finding.apply : null + if (!payload) return { plan: null, notes: [] } + + switch (payload.cause) { + case 'env-false': { + if (!payload.settingPath) return { plan: null, notes: [] } + if (payload.settingScope === SHELL_PROFILE_SCOPE) { + return { plan: null, notes: shellOverrideManualNotes(payload.settingPath, r.homeDir) } + } + const docs = new ConfigDocs(r.homeDir) + const state = docs.load(payload.settingPath) + if (!state) return { plan: null, notes: docs.errorNotes() } + if (!deleteEnvKey(state, ENABLE_TOOL_SEARCH_VAR)) { + return { plan: null, notes: [`skipped: ${ENABLE_TOOL_SEARCH_VAR} is no longer set in ${shortPath(payload.settingPath, r.homeDir)}`] } + } + return { + plan: mcpPlan( + 'defer-enable', + finding.id, + `Remove the ${ENABLE_TOOL_SEARCH_VAR}=${payload.value ?? 'false'} override from ${shortPath(payload.settingPath, r.homeDir)}`, + docs.changes(), + ), + notes: [`restores default-on MCP tool deferral; ${NEXT_SESSION_NOTE}`], + } + } + case 'proxy-unknown': { + const where = payload.settingPath ? ` configured in ${payload.settingScope ?? 'settings'} (${shortPath(payload.settingPath, r.homeDir)})` : '' + return { + plan: null, + notes: [ + `not auto-applied: setting ${ENABLE_TOOL_SEARCH_VAR}=true would force deferral back on, but requests fail outright on proxies that don't forward tool_reference blocks. ` + + `Verify that the proxy${where} forwards them before setting the override.`, + ], + } + } + case 'proxy-verified': { + // Part-3 verifier confirmed the proxy forwards tool_reference blocks, + // so the explicit opt-in is safe. User settings env is the target: it + // covers every project without touching shell files. + const docs = new ConfigDocs(r.homeDir) + const state = docs.load(r.userSettings) + if (!state) return { plan: null, notes: docs.errorNotes() } + setEnvKey(state, ENABLE_TOOL_SEARCH_VAR, 'true') + return { + plan: mcpPlan( + 'defer-enable', + finding.id, + `Set ${ENABLE_TOOL_SEARCH_VAR}=true in ${shortPath(r.userSettings, r.homeDir)} (proxy verified to forward tool_reference blocks)`, + docs.changes(), + ), + notes: [`enables MCP tool deferral through the verified proxy; ${NEXT_SESSION_NOTE}`], + } + } + case 'vertex': + return { + plan: null, + notes: [ + `manual: tool search is disabled by default on Vertex AI — a platform property, not a config error. ` + + `Opt in yourself with \`export ${ENABLE_TOOL_SEARCH_VAR}=true\` if your Vertex setup supports it.`, + ], + } + case 'old-version': + return { + plan: null, + notes: ['manual: every observed Claude Code version predates default-on tool search; run `claude update`'], + } + } +} + +// mcp-alwaysload-hygiene -> defer-alwaysload: drop `"alwaysLoad": true` from +// the flagged servers in the exact config files the finding recorded. +function buildDeferAlwaysLoad(finding: WasteFinding, r: ResolvedPaths): BuiltPlan { + const entries = finding.apply?.kind === 'defer-alwaysload' ? finding.apply.servers : [] + if (entries.length === 0) return { plan: null, notes: [] } + + // Version gate: server-level alwaysLoad shipped in v2.1.121. Below that + // (or undeterminable) the key is inert — tools defer normally and there is + // no startup block — so "removing the cost" would be a false claim. + const installed = r.claudeVersion() + const parsed = installed === null ? null : parseVersion(installed) + if (parsed === null) { + return { plan: null, notes: [`skipped: could not determine the installed Claude Code version; removing alwaysLoad is only meaningful on v${ALWAYSLOAD_MIN_VERSION}+`] } + } + if (versionPredates(installed!, ALWAYSLOAD_MIN_VERSION)) { + return { plan: null, notes: [`skipped: installed Claude Code v${parsed.join('.')} predates v${ALWAYSLOAD_MIN_VERSION}, where server-level alwaysLoad shipped; the pin is inert there`] } + } + + const docs = new ConfigDocs(r.homeDir) + const skips: string[] = [] + const pathNotes: Record = {} + const addPathNote = pathNoteAdder(pathNotes) + + for (const { server, paths } of entries) { + let removed = false + for (const path of paths) { + const state = docs.load(path) + if (!state) continue + const container = state.doc.mcpServers + if (!container || typeof container !== 'object') continue + const key = findServerKey(container as Record, server) + if (!key) continue + const entry = (container as Record)[key] + if (!entry || typeof entry !== 'object' || (entry as Record)['alwaysLoad'] !== true) continue + delete (entry as Record)['alwaysLoad'] + state.dirty = true + removed = true + // alwaysLoad also blocks session startup on the server's connection + // (capped at 5s), so unpinning removes that startup cost too. + addPathNote(path, `unpins ${server}: its schema defers on demand and session startup no longer blocks up to ${ALWAYSLOAD_STARTUP_CAP_SECONDS}s on its connection`) + } + if (!removed) skips.push(`skipped ${server}: no "alwaysLoad": true entry found in its config files`) + } + + const changes = docs.changes() + const notes = [...docs.errorNotes(), ...skips] + if (changes.length === 0) return { plan: null, notes } + return { + plan: mcpPlan('defer-alwaysload', finding.id, `Unpin ${entries.length === 1 ? 'an alwaysLoad MCP server' : 'alwaysLoad MCP servers'}`, changes), + notes: [...notes, NEXT_SESSION_NOTE], + ...(Object.keys(pathNotes).length > 0 ? { pathNotes } : {}), + } +} + +// mcp-defer-threshold -> defer-threshold: retune the ENABLE_TOOL_SEARCH auto +// override in place. The detector found the key in config, so this is always +// a value rewrite (or removal), never an env-object creation. +function buildDeferThreshold(finding: WasteFinding, r: ResolvedPaths): BuiltPlan { + const payload = finding.apply?.kind === 'defer-threshold' ? finding.apply : null + if (!payload) return { plan: null, notes: [] } + if (payload.settingScope === SHELL_PROFILE_SCOPE) { + const replacement = payload.removeOverride ? undefined : `${ENABLE_TOOL_SEARCH_VAR}=auto:${payload.recommendedPercent}` + return { plan: null, notes: shellOverrideManualNotes(payload.settingPath, r.homeDir, replacement) } + } + + const docs = new ConfigDocs(r.homeDir) + const state = docs.load(payload.settingPath) + if (!state) return { plan: null, notes: docs.errorNotes() } + const env = envContainer(state) + if (!env || !(ENABLE_TOOL_SEARCH_VAR in env)) { + return { plan: null, notes: [`skipped: ${ENABLE_TOOL_SEARCH_VAR} is no longer set in ${shortPath(payload.settingPath, r.homeDir)}`] } + } + if (payload.removeOverride) { + // Defs already exceed the default auto threshold: the override is pure + // downside, so deleting it restores default auto behavior, which defers. + delete env[ENABLE_TOOL_SEARCH_VAR] + } else { + env[ENABLE_TOOL_SEARCH_VAR] = `auto:${payload.recommendedPercent}` + } + state.dirty = true + + const shown = shortPath(payload.settingPath, r.homeDir) + const description = payload.removeOverride + ? `Remove the ${ENABLE_TOOL_SEARCH_VAR}=${payload.value} override from ${shown} (the default auto threshold already defers this volume)` + : `Tighten ${ENABLE_TOOL_SEARCH_VAR} to auto:${payload.recommendedPercent} in ${shown}` + return { + plan: mcpPlan('defer-threshold', finding.id, description, docs.changes()), + notes: [NEXT_SESSION_NOTE], + } +} + // --------------------------------------------------------------------------- // Archive unused skills / agents / commands // --------------------------------------------------------------------------- diff --git a/src/act/types.ts b/src/act/types.ts index 043908e6..4142abe4 100644 --- a/src/act/types.ts +++ b/src/act/types.ts @@ -1,5 +1,6 @@ export type ActionKind = | 'mcp-remove' | 'mcp-project-scope' + | 'defer-enable' | 'defer-alwaysload' | 'defer-threshold' | 'archive-skill' | 'archive-agent' | 'archive-command' | 'claude-md-rule' | 'shell-config' | 'guard-install' | 'guard-uninstall' diff --git a/src/optimize.ts b/src/optimize.ts index e1f1aeda..7d281bdd 100644 --- a/src/optimize.ts +++ b/src/optimize.ts @@ -7,7 +7,7 @@ import { homedir } from 'os' import { readSessionLines, readSessionFileSync } from './fs-utils.js' import { discoverAllSessions } from './providers/index.js' import { parseJsonlLine, shouldSkipLine } from './parser.js' -import type { DateRange, ProjectSummary } from './types.js' +import type { DateRange, ProjectSummary, SessionSummary } from './types.js' import { formatCost } from './currency.js' import { formatTokens } from './format.js' import { recommendModelDefault, type ModelDefaultRecommendation } from './act/model-defaults.js' @@ -122,6 +122,50 @@ const CAPABILITY_RELIABILITY_LOW_MAX_CANDIDATES = 1 const CAPABILITY_RELIABILITY_LOW_MAX_TOKENS = 50_000 const CAPABILITY_RELIABILITY_HIGH_MIN_CANDIDATES = 5 const CAPABILITY_RELIABILITY_HIGH_IMPACT_TOKENS = 200_000 +// "mcp-deferral-gaps" finding family (#614): MCP tool schemas that sit in the +// prefix because tool search / deferral is off, overridden, or mistuned. +// Claude Code's tool search tool is literally named "ToolSearch"; seeing it +// invoked is direct transcript evidence that deferral was active. +const TOOL_SEARCH_TOOL_NAME = 'ToolSearch' +// Exported: the defer-enable / defer-threshold plan builders in +// src/act/plans.ts edit exactly this settings env key. +export const ENABLE_TOOL_SEARCH_VAR = 'ENABLE_TOOL_SEARCH' +const ANTHROPIC_BASE_URL_VAR = 'ANTHROPIC_BASE_URL' +const CLAUDE_CODE_USE_VERTEX_VAR = 'CLAUDE_CODE_USE_VERTEX' +// The only first-party API host. An unset ANTHROPIC_BASE_URL counts as +// first-party; any other host is an *unknown* proxy — deferral auto-disables +// there because most proxies don't forward tool_reference blocks, but we +// never claim the proxy is incapable, only unknown. +const FIRST_PARTY_API_HOST = 'api.anthropic.com' +// Claude Code changelog v2.1.7: "Enabled MCP tool search auto mode by +// default for all users." Sessions produced entirely by older versions +// could not have had deferral on without explicit opt-in. +const TOOL_SEARCH_DEFAULT_ON_VERSION = '2.1.7' +const DEFERRAL_OFF_MIN_MCP_SESSIONS = 2 +const DEFERRAL_OFF_HIGH_IMPACT_TOKENS = 200_000 +// Conservative: flag alwaysLoad servers only when strictly below one call +// per five sessions. +const ALWAYSLOAD_MAX_CALLS_PER_SESSION = 0.2 +// Server-level alwaysLoad shipped in Claude Code v2.1.121; on older +// versions the key is inert and the server's tools defer normally, so the +// pin costs nothing there. Exported: the defer-alwaysload plan builder +// (src/act/plans.ts) gates on the same boundary against the INSTALLED +// version before offering to remove the pin. +export const ALWAYSLOAD_MIN_VERSION = '2.1.121' +const ALWAYSLOAD_HIGH_IMPACT_TOKENS = 200_000 +// alwaysLoad blocks session startup on the server's connection, capped at 5s. +// Exported so the defer-alwaysload plan preview quotes the same cap. +export const ALWAYSLOAD_STARTUP_CAP_SECONDS = 5 +// ENABLE_TOOL_SEARCH=auto:N defers only when defs exceed N% of the context +// window (default 10). Tuning advice is only worth emitting when the +// upfront-loaded defs are substantial. +const DEFER_THRESHOLD_CONTEXT_WINDOW_TOKENS = 200_000 +const DEFER_THRESHOLD_DEFAULT_PERCENT = 10 +// auto:N is documented for N 0-100; larger values clamp rather than reject +// so a nonsense override still gets tuning advice instead of silence. +const DEFER_THRESHOLD_MAX_PERCENT = 100 +const DEFER_THRESHOLD_MIN_TOKENS_PER_SESSION = 5_000 +const DEFER_THRESHOLD_MEDIUM_IMPACT_TOKENS = 200_000 // ============================================================================ // Scoring constants @@ -205,6 +249,9 @@ export type FindingId = | 'unused-mcp' | 'mcp-low-coverage' | 'mcp-project-scope' + | 'mcp-deferral-off' + | 'mcp-alwaysload-hygiene' + | 'mcp-defer-threshold' | 'retry-heavy-capabilities' | 'low-worth-sessions' | 'context-heavy-sessions' @@ -215,6 +262,13 @@ export type FindingId = | 'unused-skills' | 'unused-commands' +// Cause taxonomy for defer-enable plans (mcp-deferral-off findings). +// 'proxy-verified' is never produced by the detector today: it is reserved +// for the #614 part-3 proxy verifier, which upgrades 'proxy-unknown' once a +// proxy provably forwards tool_reference blocks. The plan layer already +// accepts it and produces a real ENABLE_TOOL_SEARCH=true plan for it. +export type DeferEnableCause = 'env-false' | 'proxy-unknown' | 'proxy-verified' | 'vertex' | 'old-version' + // Machine-readable payload the apply layer needs but the human-facing `fix` // text can't carry losslessly (full lists, per-server keeper paths). Only set // on findings that have an appliable plan; absent otherwise. @@ -224,6 +278,19 @@ export type FindingApply = // only per-project containers a scoped removal may touch. | { kind: 'mcp-project-scope'; servers: Array<{ server: string; keepProjects: string[]; removeProjects: string[] }> } | { kind: 'archive'; names: string[] } + // settingPath/settingScope name the config file carrying the setting the + // cause refers to (the ENABLE_TOOL_SEARCH override for 'env-false', the + // ANTHROPIC_BASE_URL for the proxy causes, CLAUDE_CODE_USE_VERTEX for + // 'vertex'); 'old-version' carries none. value is the observed setting + // value, quoted in plan descriptions. + | { kind: 'defer-enable'; cause: DeferEnableCause; settingPath?: string; settingScope?: string; value?: string } + // One entry per flagged server; paths are the exact config files whose + // entry carries `"alwaysLoad": true` (union across readable scopes). + | { kind: 'defer-alwaysload'; servers: Array<{ server: string; paths: string[] }> } + // recommendedPercent is the tightest auto:N that still defers; when + // removeOverride is set the defs already exceed the default threshold, so + // deleting the override (restoring default auto) is the fix instead. + | { kind: 'defer-threshold'; settingPath: string; settingScope: string; value: string; recommendedPercent: number; removeOverride: boolean } export type WasteFinding = { id: FindingId @@ -520,13 +587,25 @@ function isReadTool(name: string): boolean { return name === 'Read' || name === 'FileReadTool' } -type McpConfigEntry = { normalized: string; original: string; mtime: number } +type McpConfigEntry = { + normalized: string + original: string + mtime: number + // Config files where this server entry carries `"alwaysLoad": true` + // (union across all readable scopes, so the hygiene fix can name every + // file to edit). Server-level only: the per-tool "anthropic/alwaysLoad" + // variant lives in tool _meta served by the MCP server at runtime and is + // not observable in static config. Requires Claude Code v2.1.121+. + alwaysLoadPaths: string[] +} -export function loadMcpConfigs(projectCwds: Iterable): Map { +// `homeDir` is injectable for tests (mirrors src/act/plans.ts PlanContext); +// production callers omit it. +export function loadMcpConfigs(projectCwds: Iterable, homeDir = homedir()): Map { const servers = new Map() const configPaths = [ - join(homedir(), '.claude', 'settings.json'), - join(homedir(), '.claude', 'settings.local.json'), + join(homeDir, '.claude', 'settings.json'), + join(homeDir, '.claude', 'settings.local.json'), ] for (const cwd of projectCwds) { configPaths.push(join(cwd, '.mcp.json')) @@ -541,11 +620,20 @@ export function loadMcpConfigs(projectCwds: Iterable): Map - for (const name of Object.keys(serversObj)) { + for (const [name, rawEntry] of Object.entries(serversObj)) { const normalized = name.replace(/:/g, '_') const existing = servers.get(normalized) if (!existing || existing.mtime < mtime) { - servers.set(normalized, { normalized, original: name, mtime }) + servers.set(normalized, { + normalized, + original: name, + mtime, + alwaysLoadPaths: existing?.alwaysLoadPaths ?? [], + }) + } + const entry = rawEntry as Record | null + if (entry && typeof entry === 'object' && entry.alwaysLoad === true) { + servers.get(normalized)!.alwaysLoadPaths.push(p) } } } @@ -1550,6 +1638,485 @@ export function detectUnusedMcp( } } +// ============================================================================ +// MCP deferral gap detectors — the "mcp-deferral-gaps" family (#614) +// ============================================================================ +// +// Shared invariant: `session.mcpInventory` comes ONLY from +// `deferred_tools_delta` attachments, which Claude Code emits only when tool +// search / deferral is ACTIVE. Inventory presence is therefore direct +// evidence deferral was on; deferral-off sessions still show MCP invocations +// (mcpBreakdown, mcp__ tool calls) but never an inventory. + +type DeferralEnvHit = { + value: string + // Human-readable config scope, e.g. "user settings" / "project local + // settings" / "shell profile". Surfaced verbatim in explanations so the + // user knows which file carries the override. + scope: string + path: string +} + +// Settings scopes are searched most-specific first (project local > +// project > user local > user), matching Claude Code's effective settings +// precedence, so the hit we report is the value that actually takes +// effect. Shell profiles come last: settings "env" entries override the +// inherited environment. `homeDir` is injectable for tests (PlanContext +// style in src/act/plans.ts). +export function findDeferralEnvSetting( + name: string, + projectCwds: Iterable, + homeDir = homedir(), +): DeferralEnvHit | null { + const scopes: Array<{ scope: string; path: string }> = [] + for (const cwd of projectCwds) { + scopes.push({ scope: 'project local settings', path: join(cwd, '.claude', 'settings.local.json') }) + scopes.push({ scope: 'project settings', path: join(cwd, '.claude', 'settings.json') }) + } + scopes.push({ scope: 'user local settings', path: join(homeDir, '.claude', 'settings.local.json') }) + scopes.push({ scope: 'user settings', path: join(homeDir, '.claude', 'settings.json') }) + for (const { scope, path } of scopes) { + if (!existsSync(path)) continue + const config = readJsonFile(path) + const env = config?.env as Record | undefined + const value = env?.[name] + if (typeof value === 'string' || typeof value === 'number' || typeof value === 'boolean') { + return { value: String(value), scope, path } + } + } + // Callers only pass the three *_VAR constants today, but the function is + // exported — escape regex metacharacters so a future caller can't + // silently mis-match. + const escapedName = name.replace(/[.*+?^${}()|[\]\\]/g, '\\$&') + const linePattern = new RegExp(`^\\s*(?:export\\s+)?${escapedName}\\s*=\\s*['"]?([^'"\\s]+)['"]?`, 'm') + for (const profile of SHELL_PROFILES) { + const path = join(homeDir, profile) + if (!existsSync(path)) continue + const content = readSessionFileSync(path) + if (content === null) continue + const match = content.match(linePattern) + if (match) return { value: match[1]!, scope: 'shell profile', path } + } + return null +} + +function isEnvValueFalse(value: string): boolean { + return value.toLowerCase() === 'false' || value === '0' +} + +// Unset counts as first-party; only a URL whose host resolves to +// api.anthropic.com does too. Anything else — including an unparseable +// value — is an unknown proxy and we never assume its capability. +function isFirstPartyBaseUrl(value: string): boolean { + try { + return new URL(value).hostname === FIRST_PARTY_API_HOST + } catch { + return false + } +} + +// Both exported: the defer-alwaysload plan builder (src/act/plans.ts) gates +// on the installed Claude Code version with the same comparison the detector +// uses on observed versions. versionPredates returns false for unparseable +// input, so gates that must fail closed check parseVersion separately. +export function parseVersion(version: string): [number, number, number] | null { + const match = /^(\d+)\.(\d+)\.(\d+)/.exec(version) + if (!match) return null + return [Number(match[1]), Number(match[2]), Number(match[3])] +} + +export function versionPredates(version: string, reference: string): boolean { + const v = parseVersion(version) + const ref = parseVersion(reference) + if (!v || !ref) return false + for (let i = 0; i < 3; i++) { + if (v[i] !== ref[i]) return v[i]! < ref[i]! + } + return false +} + +function anySessionHasMcpInventory(projects: ProjectSummary[]): boolean { + return projects.some(p => p.sessions.some(s => (s.mcpInventory?.length ?? 0) > 0)) +} + +// Deferral is a Claude Code mechanism, and both suppression signals for this +// family (ToolSearch calls, mcpInventory) exist only in Claude Code +// transcripts. Usage evidence must therefore be Claude-scoped too: the +// Codex/Copilot/opencode/Hermes parsers also normalize MCP calls into +// mcpBreakdown, but those sessions can never carry the counter-evidence, so +// counting them would flag a "deferral gap" Claude Code never had. +function isClaudeSession(session: SessionSummary): boolean { + return session.turns.some(t => t.assistantCalls.some(c => c.provider === 'claude')) +} + +/** + * mcp-deferral-off: MCP tool definitions are being loaded upfront in every + * session because tool search / deferral is inactive. + * + * Signal: MCP overhead exists (configured servers and/or observed mcp__ + * invocations), yet across the whole window there is not a single ToolSearch + * invocation and not a single session with an mcpInventory — which + * deferral-active sessions always produce (see family invariant above). + * + * Cause attribution probes config in a fixed order, each producing a + * distinct explanation: ENABLE_TOOL_SEARCH=false override, non-first-party + * ANTHROPIC_BASE_URL (unknown proxy: deferral auto-disables because most + * proxies don't forward tool_reference blocks), CLAUDE_CODE_USE_VERTEX + * (tool search is disabled by default on Vertex AI, changelog v2.1.119), + * every observed Claude Code version predating v2.1.7 (when tool search + * auto mode became on by default), else a generic transcript-evidence + * message. + */ +// Cause attribution for mcp-deferral-off. Probes config in a fixed order — +// false override, unknown proxy, Vertex, stale version, explicit-but- +// ineffective override, generic — each producing a distinct explanation and +// fix. Split out so the detector stays focused on evidence gathering. +function attributeDeferralOffCause( + enableToolSearch: DeferralEnvHit | null, + projectCwds: Set, + apiCalls: ApiCallMeta[], + homeDir: string, +): { cause: string; fix: WasteAction; apply?: FindingApply } { + if (enableToolSearch && isEnvValueFalse(enableToolSearch.value)) { + return { + cause: `Cause: ${ENABLE_TOOL_SEARCH_VAR}=${enableToolSearch.value} is set in ${enableToolSearch.scope} (${shortHomePath(enableToolSearch.path)}), forcing all tool definitions upfront.`, + fix: { + type: 'paste', + destination: 'prompt', + label: 'Ask Claude to remove the stale override:', + text: `Remove the ${ENABLE_TOOL_SEARCH_VAR}=${enableToolSearch.value} setting from ${enableToolSearch.path}. Tool search is on by default on first-party endpoints, so deleting the override re-enables MCP tool deferral.`, + }, + apply: { kind: 'defer-enable', cause: 'env-false', settingPath: enableToolSearch.path, settingScope: enableToolSearch.scope, value: enableToolSearch.value }, + } + } + const baseUrl = findDeferralEnvSetting(ANTHROPIC_BASE_URL_VAR, projectCwds, homeDir) + if (baseUrl && !isFirstPartyBaseUrl(baseUrl.value)) { + return { + cause: `Cause: ${ANTHROPIC_BASE_URL_VAR} points at a non-first-party host in ${baseUrl.scope} (${shortHomePath(baseUrl.path)}). Tool deferral silently auto-disables behind proxies because most don't forward tool_reference blocks; whether this proxy can is unknown.`, + fix: { + type: 'paste', + destination: 'shell-config', + label: `Verify your proxy forwards tool_reference blocks first (an explicit override fails on proxies that don't), then force tool search back on with:`, + text: `export ${ENABLE_TOOL_SEARCH_VAR}=true`, + }, + apply: { kind: 'defer-enable', cause: 'proxy-unknown', settingPath: baseUrl.path, settingScope: baseUrl.scope, value: baseUrl.value }, + } + } + const vertex = findDeferralEnvSetting(CLAUDE_CODE_USE_VERTEX_VAR, projectCwds, homeDir) + if (vertex && !isEnvValueFalse(vertex.value)) { + return { + cause: `Cause: ${CLAUDE_CODE_USE_VERTEX_VAR} is set in ${vertex.scope} (${shortHomePath(vertex.path)}), and tool search is disabled by default on Vertex AI.`, + fix: { + type: 'paste', + destination: 'shell-config', + label: 'Opt in to tool search on Vertex (disabled by default there):', + text: `export ${ENABLE_TOOL_SEARCH_VAR}=true`, + }, + apply: { kind: 'defer-enable', cause: 'vertex', settingPath: vertex.path, settingScope: vertex.scope, value: vertex.value }, + } + } + const versions = apiCalls.map(c => c.version).filter(v => v.length > 0) + if (versions.length > 0 && versions.every(v => versionPredates(v, TOOL_SEARCH_DEFAULT_ON_VERSION))) { + return { + cause: `Cause: every observed Claude Code version in this period predates v${TOOL_SEARCH_DEFAULT_ON_VERSION}, where MCP tool search became on by default.`, + fix: { + type: 'command', + label: 'Update Claude Code to get default-on MCP tool deferral:', + text: 'claude update', + }, + apply: { kind: 'defer-enable', cause: 'old-version' }, + } + } + // false and auto values were handled earlier, so a hit here is an explicit + // truthy override that is already applied — re-suggesting the same export + // would be redundant. + if (enableToolSearch) { + return { + cause: `Cause: none determinable — ${ENABLE_TOOL_SEARCH_VAR}=${enableToolSearch.value} is already set in ${enableToolSearch.scope} (${shortHomePath(enableToolSearch.path)}), yet transcripts show no deferral activity.`, + fix: { + type: 'paste', + destination: 'prompt', + label: 'The override is already on; ask Claude to investigate why deferral is still inactive:', + text: `${ENABLE_TOOL_SEARCH_VAR}=${enableToolSearch.value} is set in ${enableToolSearch.path}, but sessions show no ToolSearch calls and no deferred-tool inventory. Check whether requests pass through a proxy that strips tool_reference blocks and whether the running Claude Code version supports tool search.`, + }, + } + } + return { + cause: `Cause: none determinable from config — no disabling override, proxy, or Vertex setting found.`, + fix: { + type: 'paste', + destination: 'shell-config', + label: 'Deferral is on by default on first-party endpoints; to force it explicitly, add:', + text: `export ${ENABLE_TOOL_SEARCH_VAR}=true`, + }, + } +} + +export function detectMcpDeferralOff( + calls: ToolCall[], + projects: ProjectSummary[], + projectCwds: Set, + apiCalls: ApiCallMeta[], + homeDir = homedir(), +): WasteFinding | null { + // Deferral-active evidence anywhere in the window suppresses the finding. + if (calls.some(c => c.name === TOOL_SEARCH_TOOL_NAME)) return null + if (anySessionHasMcpInventory(projects)) return null + + const configured = loadMcpConfigs(projectCwds, homeDir) + // Servers pinned with alwaysLoad load upfront BY DESIGN: they are never + // deferred (so their missing inventory is not evidence deferral is + // broken), and their schema cost is mcp-alwaysload-hygiene's jurisdiction + // — charging it here too would double-count the same tokens. + const pinnedServers = new Set( + [...configured.values()].filter(e => e.alwaysLoadPaths.length > 0).map(e => e.normalized), + ) + const configuredUnpinned = [...configured.keys()].filter(s => !pinnedServers.has(s)) + + // `calls` comes from scanSessions, which reads Claude Code JSONL only, so + // it is already Claude-scoped (see isClaudeSession for why that matters). + const observedServers = new Set() + for (const call of calls) { + if (!call.name.startsWith('mcp__')) continue + const seg = call.name.split('__')[1] + if (seg && !pinnedServers.has(seg)) observedServers.add(seg) + } + let invocations = 0 + let sessionsWithMcpCalls = 0 + let totalSessions = 0 + for (const project of projects) { + for (const session of project.sessions) { + if (!isClaudeSession(session)) continue + totalSessions++ + let sessionCalls = 0 + for (const [server, data] of Object.entries(session.mcpBreakdown)) { + if (pinnedServers.has(server)) continue + observedServers.add(server) + sessionCalls += data.calls + } + if (sessionCalls > 0) sessionsWithMcpCalls++ + invocations += sessionCalls + } + } + + const servers = new Set([...configuredUnpinned, ...observedServers]) + if (servers.size === 0) return null + + // Configured servers load their schemas into every session; with only + // invocation evidence we can vouch just for the sessions that called MCP. + const affectedSessions = configuredUnpinned.length > 0 ? totalSessions : sessionsWithMcpCalls + if (affectedSessions < DEFERRAL_OFF_MIN_MCP_SESSIONS) return null + + const enableToolSearch = findDeferralEnvSetting(ENABLE_TOOL_SEARCH_VAR, projectCwds, homeDir) + // auto / auto:N overrides — including out-of-range N, which the threshold + // detector clamps — are the defer-threshold detector's jurisdiction (it + // can compute the tuned N); reporting them here too would double-flag the + // same schema overhead. This regex must stay identical to the one in + // detectMcpDeferThreshold or values could slip through both detectors. + if (enableToolSearch && /^auto(?::\d+)?$/.test(enableToolSearch.value)) return null + + // Without deferral there is never an inventory, so per-server tool counts + // are unknown — follow detectUnusedMcp's convention of 5 tools x 400 + // tokens per server. + const perServerSchemaTokens = TOOLS_PER_MCP_SERVER * TOKENS_PER_MCP_TOOL + const perSessionSchemaTokens = servers.size * perServerSchemaTokens + const tokensSaved = perSessionSchemaTokens * affectedSessions + const callRate = affectedSessions > 0 ? invocations / affectedSessions : 0 + // Pluralize on the rendered value so 0.98 and 1.0 read consistently. + const callRateText = callRate.toFixed(1) + + const evidence = + `~${formatTokens(perSessionSchemaTokens)} tokens of MCP tool schema ` + + `(${servers.size} server${servers.size === 1 ? '' : 's'} at ~${formatTokens(perServerSchemaTokens)} tokens/server) sit in the prompt prefix of ` + + `${affectedSessions} session${affectedSessions === 1 ? '' : 's'} at ` + + `${callRateText} MCP call${callRateText === '1.0' ? '' : 's'}/session, with zero ToolSearch calls ` + + `and no deferred-tool inventory observed — tool deferral appears inactive. ` + + `Deferral would move ~all of that schema out of the prefix.` + + const { cause, fix, apply } = attributeDeferralOffCause(enableToolSearch, projectCwds, apiCalls, homeDir) + + return { + id: 'mcp-deferral-off', + title: 'MCP tool deferral appears inactive', + explanation: `${evidence} ${cause}`, + impact: tokensSaved >= DEFERRAL_OFF_HIGH_IMPACT_TOKENS ? 'high' : 'medium', + tokensSaved, + fix, + ...(apply ? { apply } : {}), + } +} + +/** + * mcp-alwaysload-hygiene: servers pinned with `"alwaysLoad": true` (an + * .mcp.json / settings mcpServers sibling of type/url, Claude Code + * v2.1.121+) whose observed call rate doesn't justify exempting them from + * deferral. alwaysLoad puts the server's full tool schema in every + * session's prefix even though deferral is available, and additionally + * blocks session startup on that server's connection (capped at + * ALWAYSLOAD_STARTUP_CAP_SECONDS). + */ +export function detectMcpAlwaysLoadHygiene( + projects: ProjectSummary[], + projectCwds: Set, + apiCalls: ApiCallMeta[] = [], + mcpCoverage = aggregateMcpCoverage(projects), + homeDir = homedir(), +): WasteFinding | null { + const configured = loadMcpConfigs(projectCwds, homeDir) + const pinned = [...configured.values()].filter(e => e.alwaysLoadPaths.length > 0) + if (pinned.length === 0) return null + + // When every observed Claude Code version predates server-level + // alwaysLoad support, the key is inert — the tools defer normally, so + // the pin costs nothing and the finding's claim would be false. + const versions = apiCalls.map(c => c.version).filter(v => v.length > 0) + if (versions.length > 0 && versions.every(v => versionPredates(v, ALWAYSLOAD_MIN_VERSION))) return null + + const totalSessions = projects.reduce((s, p) => s + p.sessions.length, 0) + if (totalSessions === 0) return null + + const coverageByServer = new Map(mcpCoverage.map(c => [c.server, c])) + const invocationsByServer = new Map() + for (const project of projects) { + for (const session of project.sessions) { + for (const [server, data] of Object.entries(session.mcpBreakdown)) { + invocationsByServer.set(server, (invocationsByServer.get(server) ?? 0) + data.calls) + } + } + } + + const lines: string[] = [] + const fixLines: string[] = [] + const applyServers: Array<{ server: string; paths: string[] }> = [] + let tokensSaved = 0 + for (const entry of pinned) { + const invocations = invocationsByServer.get(entry.normalized) ?? 0 + const callRate = invocations / totalSessions + if (callRate >= ALWAYSLOAD_MAX_CALLS_PER_SESSION) continue + // Prefer real inventory numbers when a session observed them; without + // an inventory fall back to the 5 tools x 400 tokens convention, and + // charge every session since alwaysLoad forces the load unconditionally. + const coverage = coverageByServer.get(entry.normalized) + const toolsAvailable = coverage?.toolsAvailable ?? TOOLS_PER_MCP_SERVER + const loadedSessions = coverage?.loadedSessions ?? totalSessions + tokensSaved += toolsAvailable * TOKENS_PER_MCP_TOOL * loadedSessions + lines.push(`${entry.original}: ${invocations} call${invocations === 1 ? '' : 's'} across ${totalSessions} session${totalSessions === 1 ? '' : 's'}`) + fixLines.push(`- Remove "alwaysLoad": true from ${entry.original} in ${entry.alwaysLoadPaths.map(shortHomePath).join(', ')}.`) + // Original (config-key) name plus the exact files carrying the pin, so + // the defer-alwaysload plan edits precisely what was observed. + applyServers.push({ server: entry.original, paths: entry.alwaysLoadPaths }) + } + if (lines.length === 0) return null + + return { + id: 'mcp-alwaysload-hygiene', + title: `${lines.length} alwaysLoad MCP server${lines.length === 1 ? '' : 's'} rarely used`, + explanation: + `These servers are pinned with alwaysLoad, so their tool schemas sit in every session's prefix ` + + `despite deferral being available, and session startup blocks on each server's connection ` + + `(up to ${ALWAYSLOAD_STARTUP_CAP_SECONDS}s). Usage doesn't justify the pin: ${lines.join('; ')}.`, + impact: tokensSaved >= ALWAYSLOAD_HIGH_IMPACT_TOKENS ? 'high' : 'medium', + tokensSaved, + apply: { kind: 'defer-alwaysload', servers: applyServers }, + fix: { + type: 'paste', + destination: 'prompt', + label: 'Ask Claude to unpin the rarely-used servers (tool search still discovers their tools on demand):', + text: fixLines.join('\n'), + }, + } +} + +/** + * mcp-defer-threshold: an explicit ENABLE_TOOL_SEARCH=auto or auto:N + * override whose threshold the configured MCP definitions never reach — + * auto defers only when defs exceed N% of the context window (default 10), + * so everything still loads upfront while carrying substantial per-session + * cost. Lowest priority of the family; emitted only on a substantial gap. + */ +export function detectMcpDeferThreshold( + projects: ProjectSummary[], + projectCwds: Set, + homeDir = homedir(), +): WasteFinding | null { + const setting = findDeferralEnvSetting(ENABLE_TOOL_SEARCH_VAR, projectCwds, homeDir) + if (!setting) return null + // Must accept exactly what detectMcpDeferralOff yields to this detector + // (any auto:N, N unbounded), or an out-of-range N would silently slip + // through both detectors. Oversized values clamp to 100% — a threshold + // that can never trigger, the strongest form of this finding. + const match = /^auto(?::(\d+))?$/.exec(setting.value) + if (!match) return null + const percent = match[1] !== undefined + ? Math.min(DEFER_THRESHOLD_MAX_PERCENT, parseInt(match[1], 10)) + : DEFER_THRESHOLD_DEFAULT_PERCENT + + // Inventory anywhere in the window means the auto threshold did trigger + // and deferral is working; nothing to tune. + if (anySessionHasMcpInventory(projects)) return null + + const configured = loadMcpConfigs(projectCwds, homeDir) + const servers = new Set(configured.keys()) + // Claude-scoped for the same reason as detectMcpDeferralOff: other + // providers' sessions carry MCP breakdowns but not this override's cost. + let totalSessions = 0 + for (const project of projects) { + for (const session of project.sessions) { + if (!isClaudeSession(session)) continue + totalSessions++ + for (const server of Object.keys(session.mcpBreakdown)) servers.add(server) + } + } + if (servers.size === 0) return null + if (totalSessions === 0) return null + + // No inventory (see suppression above) means per-server tool counts are + // unknown — same 5 tools x 400 tokens convention as detectUnusedMcp. + const defsPerSession = servers.size * TOOLS_PER_MCP_SERVER * TOKENS_PER_MCP_TOOL + const onePercent = DEFER_THRESHOLD_CONTEXT_WINDOW_TOKENS / 100 + const thresholdTokens = percent * onePercent + // Deferral kicks in only when defs EXCEED the threshold; at or below it, + // everything loads upfront. + if (defsPerSession > thresholdTokens) return null + if (defsPerSession < DEFER_THRESHOLD_MIN_TOKENS_PER_SESSION) return null + + const tokensSaved = defsPerSession * totalSessions + // Largest integer N such that the defs still exceed N% — i.e. the loosest + // auto:N at which deferral actually kicks in. + const recommendedPercent = Math.max(0, Math.ceil(defsPerSession / onePercent) - 1) + // If the defs already exceed the default 10% threshold, the override is + // pure downside: removing it restores default auto behavior, which defers. + const removeOverride = defsPerSession > DEFER_THRESHOLD_DEFAULT_PERCENT * onePercent + + return { + id: 'mcp-defer-threshold', + title: 'MCP tool search auto threshold never triggers', + explanation: + `${ENABLE_TOOL_SEARCH_VAR}=${setting.value} is set in ${setting.scope} (${shortHomePath(setting.path)}), ` + + `deferring MCP tool definitions only when they exceed ${percent}% of the ${formatTokens(DEFER_THRESHOLD_CONTEXT_WINDOW_TOKENS)}-token context window ` + + `(~${formatTokens(thresholdTokens)} tokens). Your estimated ~${formatTokens(defsPerSession)} tokens of definitions per session fit under that, ` + + `so every tool still loads upfront in all ${totalSessions} session${totalSessions === 1 ? '' : 's'}.`, + impact: tokensSaved >= DEFER_THRESHOLD_MEDIUM_IMPACT_TOKENS ? 'medium' : 'low', + tokensSaved, + apply: { + kind: 'defer-threshold', + settingPath: setting.path, + settingScope: setting.scope, + value: setting.value, + recommendedPercent, + removeOverride, + }, + fix: { + type: 'paste', + destination: 'prompt', + label: 'Ask Claude to tighten the auto threshold:', + text: removeOverride + ? `Remove the ${ENABLE_TOOL_SEARCH_VAR}=${setting.value} override from ${setting.path}; the default auto threshold (${DEFER_THRESHOLD_DEFAULT_PERCENT}%) already defers this volume of tool definitions.` + : `In ${setting.path}, change ${ENABLE_TOOL_SEARCH_VAR}=${setting.value} to ${ENABLE_TOOL_SEARCH_VAR}=auto:${recommendedPercent} so ~${formatTokens(defsPerSession)} tokens of MCP tool definitions per session are deferred instead of loaded upfront.`, + }, + } +} + function expandImports(filePath: string, seen: Set, depth: number): { totalLines: number; importedFiles: number } { if (depth > MAX_IMPORT_DEPTH || seen.has(filePath)) return { totalLines: 0, importedFiles: 0 } seen.add(filePath) @@ -2435,6 +3002,10 @@ export async function scanAndDetect( () => detectUnusedMcp(toolCalls, projects, projectCwds, mcpCoverage), () => detectMcpToolCoverage(projects, mcpCoverage), () => detectMcpProfileAdvisor(projects, mcpCoverage), + // mcp-deferral-gaps family (#614): detection only, no apply plans yet. + () => detectMcpDeferralOff(toolCalls, projects, projectCwds, apiCalls), + () => detectMcpAlwaysLoadHygiene(projects, projectCwds, apiCalls, mcpCoverage), + () => detectMcpDeferThreshold(projects, projectCwds), () => detectCapabilityReliability(projects), () => detectLowWorthSessions(projects), () => detectContextBloat(projects, lowWorthSessionIds), diff --git a/tests/defer-plans.test.ts b/tests/defer-plans.test.ts new file mode 100644 index 00000000..2ba1c6bd --- /dev/null +++ b/tests/defer-plans.test.ts @@ -0,0 +1,555 @@ +import { afterAll, describe, expect, it } from 'vitest' +import { mkdir, mkdtemp, readFile, readdir, rm, writeFile } from 'node:fs/promises' +import { existsSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { createHash } from 'node:crypto' +import { PassThrough, Writable } from 'node:stream' + +import { planFor, planFindings, type PlanContext } from '../src/act/plans.js' +import { runOptimizeApply, type ApplyOptions } from '../src/act/optimize-apply.js' +import { runAction } from '../src/act/apply.js' +import { undoAction } from '../src/act/undo.js' +import { readRecords } from '../src/act/journal.js' +import type { FindingApply, FindingId, WasteAction, WasteFinding } from '../src/optimize.js' + +// Plan-kind tests for the deferral family (defer-enable / defer-alwaysload / +// defer-threshold, #614 commit 2), following tests/optimize-apply.test.ts: +// temp fixture roots, planFor/planFindings, runAction/undoAction, and +// byte-equality assertions on everything the plans must not touch. + +const roots: string[] = [] + +type Fixture = { root: string; home: string; project: string; actionsDir: string } + +async function makeFixture(): Promise { + const root = await mkdtemp(join(tmpdir(), 'codeburn-defer-plans-')) + roots.push(root) + const home = join(root, 'home') + const project = join(root, 'project') + await mkdir(home, { recursive: true }) + await mkdir(project, { recursive: true }) + return { root, home, project, actionsDir: join(root, 'actions') } +} + +afterAll(async () => { + for (const root of roots) await rm(root, { recursive: true, force: true }) +}) + +function makeFinding(id: FindingId, apply?: FindingApply): WasteFinding { + const fix: WasteAction = { type: 'command', label: '', text: '' } + return { id, title: id, explanation: '', impact: 'medium', tokensSaved: 1000, fix, ...(apply ? { apply } : {}) } +} + +// Server-level alwaysLoad shipped in v2.1.121; SUPPORTED sits safely above. +const SUPPORTED_VERSION = '2.1.130' + +function ctx(fx: Fixture, claudeVersion: string | null = SUPPORTED_VERSION): PlanContext { + return { homeDir: fx.home, cwd: fx.project, shell: '/bin/zsh', claudeVersion: () => claudeVersion } +} + +function stringify(doc: unknown): string { + return JSON.stringify(doc, null, 2) + '\n' +} + +async function hashTree(dir: string): Promise { + const h = createHash('sha256') + async function walk(d: string): Promise { + const entries = (await readdir(d, { withFileTypes: true })).sort((a, b) => a.name.localeCompare(b.name)) + for (const entry of entries) { + const full = join(d, entry.name) + if (entry.isDirectory()) { + h.update('D:' + full + '\n') + await walk(full) + } else { + h.update('F:' + full + '\n') + h.update(await readFile(full)) + } + } + } + await walk(dir) + return h.digest('hex') +} + +// --------------------------------------------------------------------------- +// defer-enable +// --------------------------------------------------------------------------- + +describe('defer-enable plan (env-false in a settings file)', () => { + it('removes only ENABLE_TOOL_SEARCH, preserves key order, journals, and undoes byte-identically', async () => { + const fx = await makeFixture() + const settings = join(fx.home, '.claude', 'settings.json') + await mkdir(join(fx.home, '.claude'), { recursive: true }) + // Deliberately odd key order plus unrelated keys: the plan must change + // exactly one env key and keep everything else byte-identical. + const original = stringify({ + zeta: 1, + env: { B: 'x', ENABLE_TOOL_SEARCH: 'false', A: 'y' }, + alpha: { keep: true }, + }) + await writeFile(settings, original) + + const finding = makeFinding('mcp-deferral-off', { + kind: 'defer-enable', cause: 'env-false', settingPath: settings, settingScope: 'user settings', value: 'false', + }) + const [fp] = planFindings([finding], ctx(fx)) + expect(fp!.plan).not.toBeNull() + expect(fp!.plan!.kind).toBe('defer-enable') + expect(fp!.plan!.findingId).toBe('mcp-deferral-off') + expect(fp!.plan!.changes.map(c => c.path)).toEqual([settings]) + expect(fp!.notes.some(n => n.includes('takes effect on the next session'))).toBe(true) + + const rec = await runAction(fp!.plan!, fx.actionsDir) + expect(rec.kind).toBe('defer-enable') + expect(rec.findingId).toBe('mcp-deferral-off') + + // Same serializer, same insertion order: only the target key is gone. + expect(await readFile(settings, 'utf-8')).toBe(stringify({ + zeta: 1, + env: { B: 'x', A: 'y' }, + alpha: { keep: true }, + })) + + await undoAction({ id: rec.id }, { actionsDir: fx.actionsDir }) + expect(await readFile(settings, 'utf-8')).toBe(original) + }) + + it('leaves an emptied env object in place (mcp-remove convention for emptied containers)', async () => { + const fx = await makeFixture() + const settings = join(fx.project, '.claude', 'settings.local.json') + await mkdir(join(fx.project, '.claude'), { recursive: true }) + await writeFile(settings, stringify({ env: { ENABLE_TOOL_SEARCH: '0' }, keep: true })) + + const finding = makeFinding('mcp-deferral-off', { + kind: 'defer-enable', cause: 'env-false', settingPath: settings, settingScope: 'project local settings', value: '0', + }) + const plan = planFor(finding, ctx(fx)) + expect(plan).not.toBeNull() + await runAction(plan!, fx.actionsDir) + expect(await readFile(settings, 'utf-8')).toBe(stringify({ env: {}, keep: true })) + }) + + it('skips with a note when the override drifted away before the plan was built', async () => { + const fx = await makeFixture() + const settings = join(fx.home, '.claude', 'settings.json') + await mkdir(join(fx.home, '.claude'), { recursive: true }) + await writeFile(settings, stringify({ env: { OTHER: '1' } })) + + const finding = makeFinding('mcp-deferral-off', { + kind: 'defer-enable', cause: 'env-false', settingPath: settings, settingScope: 'user settings', value: 'false', + }) + const [fp] = planFindings([finding], ctx(fx)) + expect(fp!.plan).toBeNull() + expect(fp!.notes.some(n => n.includes('ENABLE_TOOL_SEARCH is no longer set'))).toBe(true) + }) + + it('refuses to edit a shell profile, naming the exact file and line', async () => { + const fx = await makeFixture() + const zshrc = join(fx.home, '.zshrc') + await writeFile(zshrc, '# mine\nexport ENABLE_TOOL_SEARCH=false\nalias ll="ls -la"\n') + + const finding = makeFinding('mcp-deferral-off', { + kind: 'defer-enable', cause: 'env-false', settingPath: zshrc, settingScope: 'shell profile', value: 'false', + }) + const [fp] = planFindings([finding], ctx(fx)) + expect(fp!.plan).toBeNull() + const note = fp!.notes.join('\n') + expect(note).toContain('.zshrc') + expect(note).toContain('export ENABLE_TOOL_SEARCH=false') + expect(note).toContain('never edits user lines') + // Nothing on disk changed. + expect(await readFile(zshrc, 'utf-8')).toContain('alias ll') + }) + + it('refuses cause proxy-unknown with verify-the-proxy instructions', async () => { + const fx = await makeFixture() + const settings = join(fx.home, '.claude', 'settings.json') + await mkdir(join(fx.home, '.claude'), { recursive: true }) + const original = stringify({ env: { ANTHROPIC_BASE_URL: 'https://proxy.corp.example' } }) + await writeFile(settings, original) + + const finding = makeFinding('mcp-deferral-off', { + kind: 'defer-enable', cause: 'proxy-unknown', settingPath: settings, settingScope: 'user settings', value: 'https://proxy.corp.example', + }) + const [fp] = planFindings([finding], ctx(fx)) + expect(fp!.plan).toBeNull() + const note = fp!.notes.join('\n') + expect(note).toContain('not auto-applied') + expect(note).toContain('tool_reference') + expect(note).toContain('Verify') + expect(await readFile(settings, 'utf-8')).toBe(original) + }) + + it('cause proxy-verified sets ENABLE_TOOL_SEARCH=true in user settings, creating the file when absent', async () => { + const fx = await makeFixture() + const settings = join(fx.home, '.claude', 'settings.json') + expect(existsSync(settings)).toBe(false) + + const finding = makeFinding('mcp-deferral-off', { kind: 'defer-enable', cause: 'proxy-verified' }) + const [fp] = planFindings([finding], ctx(fx)) + expect(fp!.plan).not.toBeNull() + expect(fp!.plan!.changes[0]).toMatchObject({ op: 'create', path: settings, expectedHash: null }) + + const rec = await runAction(fp!.plan!, fx.actionsDir) + expect(JSON.parse(await readFile(settings, 'utf-8'))).toEqual({ env: { ENABLE_TOOL_SEARCH: 'true' } }) + + await undoAction({ id: rec.id }, { actionsDir: fx.actionsDir }) + expect(existsSync(settings)).toBe(false) + }) + + it('cause proxy-verified preserves an existing user settings file byte-for-byte around the new key', async () => { + const fx = await makeFixture() + const settings = join(fx.home, '.claude', 'settings.json') + await mkdir(join(fx.home, '.claude'), { recursive: true }) + const original = stringify({ theme: 'dark', env: { PATH_EXTRA: '/x' }, hooks: {} }) + await writeFile(settings, original) + + const finding = makeFinding('mcp-deferral-off', { kind: 'defer-enable', cause: 'proxy-verified' }) + const plan = planFor(finding, ctx(fx)) + const rec = await runAction(plan!, fx.actionsDir) + expect(await readFile(settings, 'utf-8')).toBe(stringify({ + theme: 'dark', + env: { PATH_EXTRA: '/x', ENABLE_TOOL_SEARCH: 'true' }, + hooks: {}, + })) + await undoAction({ id: rec.id }, { actionsDir: fx.actionsDir }) + expect(await readFile(settings, 'utf-8')).toBe(original) + }) + + it('causes vertex and old-version are manual with their required instructions', async () => { + const fx = await makeFixture() + const vertex = makeFinding('mcp-deferral-off', { + kind: 'defer-enable', cause: 'vertex', settingPath: join(fx.home, '.zshrc'), settingScope: 'shell profile', value: '1', + }) + const [vp] = planFindings([vertex], ctx(fx)) + expect(vp!.plan).toBeNull() + expect(vp!.notes.join('\n')).toContain('Vertex') + expect(vp!.notes.join('\n')).toContain('ENABLE_TOOL_SEARCH=true') + + const oldVersion = makeFinding('mcp-deferral-off', { kind: 'defer-enable', cause: 'old-version' }) + const [op] = planFindings([oldVersion], ctx(fx)) + expect(op!.plan).toBeNull() + expect(op!.notes.join('\n')).toContain('claude update') + }) + + it('a finding without a payload stays manual with no notes', async () => { + const fx = await makeFixture() + const [fp] = planFindings([makeFinding('mcp-deferral-off')], ctx(fx)) + expect(fp!.plan).toBeNull() + expect(fp!.notes).toEqual([]) + }) +}) + +// --------------------------------------------------------------------------- +// defer-alwaysload +// --------------------------------------------------------------------------- + +describe('defer-alwaysload plan', () => { + async function pinnedFixture(): Promise<{ fx: Fixture; mcpJson: string; settings: string; finding: WasteFinding; mcpOriginal: string; settingsOriginal: string }> { + const fx = await makeFixture() + const mcpJson = join(fx.project, '.mcp.json') + const settings = join(fx.home, '.claude', 'settings.json') + await mkdir(join(fx.home, '.claude'), { recursive: true }) + const mcpOriginal = stringify({ + mcpServers: { + pinned: { command: 'x', alwaysLoad: true, args: ['--a'] }, + keepme: { command: 'y', alwaysLoad: true }, + }, + unrelated: 'z', + }) + const settingsOriginal = stringify({ + env: { X: '1' }, + mcpServers: { pinned: { url: 'https://s.example', alwaysLoad: true } }, + }) + await writeFile(mcpJson, mcpOriginal) + await writeFile(settings, settingsOriginal) + const finding = makeFinding('mcp-alwaysload-hygiene', { + kind: 'defer-alwaysload', + servers: [{ server: 'pinned', paths: [mcpJson, settings] }], + }) + return { fx, mcpJson, settings, finding, mcpOriginal, settingsOriginal } + } + + it('removes alwaysLoad from the named server in the exact recorded files, then undoes byte-identically', async () => { + const { fx, mcpJson, settings, finding, mcpOriginal, settingsOriginal } = await pinnedFixture() + const [fp] = planFindings([finding], ctx(fx)) + expect(fp!.plan).not.toBeNull() + expect(fp!.plan!.kind).toBe('defer-alwaysload') + expect(fp!.plan!.findingId).toBe('mcp-alwaysload-hygiene') + expect(fp!.plan!.changes.map(c => c.path).sort()).toEqual([mcpJson, settings].sort()) + // Preview must surface the startup-block cost the pin also carries. + const pathNoteText = Object.values(fp!.pathNotes ?? {}).join('\n') + expect(pathNoteText).toContain('startup') + expect(pathNoteText).toContain('5s') + expect(fp!.notes.some(n => n.includes('takes effect on the next session'))).toBe(true) + + const rec = await runAction(fp!.plan!, fx.actionsDir) + expect(rec.kind).toBe('defer-alwaysload') + + // Only pinned loses its alwaysLoad; keepme (not in the payload) keeps it. + expect(await readFile(mcpJson, 'utf-8')).toBe(stringify({ + mcpServers: { + pinned: { command: 'x', args: ['--a'] }, + keepme: { command: 'y', alwaysLoad: true }, + }, + unrelated: 'z', + })) + expect(await readFile(settings, 'utf-8')).toBe(stringify({ + env: { X: '1' }, + mcpServers: { pinned: { url: 'https://s.example' } }, + })) + + await undoAction({ id: rec.id }, { actionsDir: fx.actionsDir }) + expect(await readFile(mcpJson, 'utf-8')).toBe(mcpOriginal) + expect(await readFile(settings, 'utf-8')).toBe(settingsOriginal) + }) + + it('applies at exactly the minimum version (v2.1.121) and above it', async () => { + for (const version of ['2.1.121', '2.2.0', '3.0.0 (Claude Code)']) { + const { fx, finding } = await pinnedFixture() + const plan = planFor(finding, ctx(fx, version)) + expect(plan, `version ${version}`).not.toBeNull() + } + }) + + it('refuses below the version gate, naming the required version', async () => { + const { fx, finding, mcpJson, mcpOriginal } = await pinnedFixture() + const [fp] = planFindings([finding], ctx(fx, '2.1.120')) + expect(fp!.plan).toBeNull() + expect(fp!.notes.join('\n')).toContain('2.1.121') + expect(fp!.notes.join('\n')).toContain('v2.1.120') + expect(await readFile(mcpJson, 'utf-8')).toBe(mcpOriginal) + }) + + it('refuses when the version probe fails or returns garbage', async () => { + for (const probed of [null, 'not a version']) { + const { fx, finding } = await pinnedFixture() + const [fp] = planFindings([finding], ctx(fx, probed)) + expect(fp!.plan, `probe ${String(probed)}`).toBeNull() + expect(fp!.notes.join('\n')).toContain('2.1.121') + } + }) + + it('skips a server whose pin drifted away, with a note', async () => { + const fx = await makeFixture() + const mcpJson = join(fx.project, '.mcp.json') + await writeFile(mcpJson, stringify({ mcpServers: { pinned: { command: 'x' } } })) + const finding = makeFinding('mcp-alwaysload-hygiene', { + kind: 'defer-alwaysload', + servers: [{ server: 'pinned', paths: [mcpJson] }], + }) + const [fp] = planFindings([finding], ctx(fx)) + expect(fp!.plan).toBeNull() + expect(fp!.notes.some(n => n.includes('skipped pinned'))).toBe(true) + }) +}) + +// --------------------------------------------------------------------------- +// defer-threshold +// --------------------------------------------------------------------------- + +describe('defer-threshold plan', () => { + it('rewrites the auto value in place, preserving env siblings and key order, and undoes byte-identically', async () => { + const fx = await makeFixture() + const settings = join(fx.project, '.claude', 'settings.local.json') + await mkdir(join(fx.project, '.claude'), { recursive: true }) + const original = stringify({ env: { OTHER: '1', ENABLE_TOOL_SEARCH: 'auto', LAST: '2' }, misc: 3 }) + await writeFile(settings, original) + + const finding = makeFinding('mcp-defer-threshold', { + kind: 'defer-threshold', settingPath: settings, settingScope: 'project local settings', + value: 'auto', recommendedPercent: 2, removeOverride: false, + }) + const [fp] = planFindings([finding], ctx(fx)) + expect(fp!.plan).not.toBeNull() + expect(fp!.plan!.kind).toBe('defer-threshold') + expect(fp!.plan!.findingId).toBe('mcp-defer-threshold') + expect(fp!.plan!.description).toContain('auto:2') + expect(fp!.notes.some(n => n.includes('takes effect on the next session'))).toBe(true) + + const rec = await runAction(fp!.plan!, fx.actionsDir) + expect(rec.kind).toBe('defer-threshold') + expect(await readFile(settings, 'utf-8')).toBe(stringify({ + env: { OTHER: '1', ENABLE_TOOL_SEARCH: 'auto:2', LAST: '2' }, + misc: 3, + })) + + await undoAction({ id: rec.id }, { actionsDir: fx.actionsDir }) + expect(await readFile(settings, 'utf-8')).toBe(original) + }) + + it('removes the override entirely when the default auto threshold already defers', async () => { + const fx = await makeFixture() + const settings = join(fx.home, '.claude', 'settings.json') + await mkdir(join(fx.home, '.claude'), { recursive: true }) + await writeFile(settings, stringify({ env: { ENABLE_TOOL_SEARCH: 'auto:50', KEEP: 'k' } })) + + const finding = makeFinding('mcp-defer-threshold', { + kind: 'defer-threshold', settingPath: settings, settingScope: 'user settings', + value: 'auto:50', recommendedPercent: 14, removeOverride: true, + }) + const plan = planFor(finding, ctx(fx)) + expect(plan).not.toBeNull() + expect(plan!.description).toContain('Remove') + await runAction(plan!, fx.actionsDir) + expect(await readFile(settings, 'utf-8')).toBe(stringify({ env: { KEEP: 'k' } })) + }) + + it('refuses to rewrite a shell profile, quoting the line and the replacement', async () => { + const fx = await makeFixture() + const bashrc = join(fx.home, '.bashrc') + await writeFile(bashrc, '# cfg\nENABLE_TOOL_SEARCH="auto:25"\n') + + const finding = makeFinding('mcp-defer-threshold', { + kind: 'defer-threshold', settingPath: bashrc, settingScope: 'shell profile', + value: 'auto:25', recommendedPercent: 2, removeOverride: false, + }) + const [fp] = planFindings([finding], ctx(fx)) + expect(fp!.plan).toBeNull() + const note = fp!.notes.join('\n') + expect(note).toContain('.bashrc') + expect(note).toContain('ENABLE_TOOL_SEARCH="auto:25"') + expect(note).toContain('ENABLE_TOOL_SEARCH=auto:2') + }) + + it('skips with a note when the override drifted away', async () => { + const fx = await makeFixture() + const settings = join(fx.home, '.claude', 'settings.json') + await mkdir(join(fx.home, '.claude'), { recursive: true }) + await writeFile(settings, stringify({ env: {} })) + + const finding = makeFinding('mcp-defer-threshold', { + kind: 'defer-threshold', settingPath: settings, settingScope: 'user settings', + value: 'auto', recommendedPercent: 2, removeOverride: false, + }) + const [fp] = planFindings([finding], ctx(fx)) + expect(fp!.plan).toBeNull() + expect(fp!.notes.some(n => n.includes('no longer set'))).toBe(true) + }) +}) + +// --------------------------------------------------------------------------- +// runOptimizeApply end-to-end (dry run + apply) for the deferral family +// --------------------------------------------------------------------------- + +const ANSI = /\[[0-9;]*m/g + +function makeIo(): { input: PassThrough; output: Writable; errorOutput: Writable; stdout(): string } { + const input = new PassThrough() + input.end('') + const outChunks: Buffer[] = [] + const output = new Writable({ write(c, _e, cb) { outChunks.push(Buffer.from(c)); cb() } }) + const errorOutput = new Writable({ write(_c, _e, cb) { cb() } }) + return { input, output, errorOutput, stdout: () => Buffer.concat(outChunks).toString('utf-8').replace(ANSI, '') } +} + +async function deferFamilyFixture(): Promise<{ fx: Fixture; findings: WasteFinding[]; settings: string; mcpJson: string }> { + const fx = await makeFixture() + const settings = join(fx.home, '.claude', 'settings.json') + const localSettings = join(fx.home, '.claude', 'settings.local.json') + const mcpJson = join(fx.project, '.mcp.json') + await mkdir(join(fx.home, '.claude'), { recursive: true }) + await writeFile(settings, stringify({ env: { ENABLE_TOOL_SEARCH: 'false' } })) + await writeFile(localSettings, stringify({ env: { ENABLE_TOOL_SEARCH: 'auto' } })) + await writeFile(mcpJson, stringify({ mcpServers: { pinned: { command: 'x', alwaysLoad: true } } })) + const findings: WasteFinding[] = [ + makeFinding('mcp-deferral-off', { + kind: 'defer-enable', cause: 'env-false', settingPath: settings, settingScope: 'user settings', value: 'false', + }), + makeFinding('mcp-alwaysload-hygiene', { + kind: 'defer-alwaysload', servers: [{ server: 'pinned', paths: [mcpJson] }], + }), + makeFinding('mcp-defer-threshold', { + kind: 'defer-threshold', settingPath: localSettings, settingScope: 'user local settings', + value: 'auto', recommendedPercent: 2, removeOverride: false, + }), + ] + return { fx, findings, settings, mcpJson } +} + +function applyOpts(fx: Fixture, io: ReturnType, extra: Partial & { findings: WasteFinding[] }): ApplyOptions { + return { + ctx: ctx(fx), + actionsDir: fx.actionsDir, + input: io.input, + output: io.output, + errorOutput: io.errorOutput, + ...extra, + } +} + +describe('runOptimizeApply with deferral plans', () => { + it('dry run lists the exact file paths and notes and changes nothing on disk', async () => { + const { fx, findings, settings, mcpJson } = await deferFamilyFixture() + const before = await hashTree(fx.root) + const io = makeIo() + await runOptimizeApply([], undefined, applyOpts(fx, io, { findings, dryRun: true })) + + const out = io.stdout() + // The fixture home is outside the real ~, so paths print unabbreviated. + expect(out).toContain(settings) + expect(out).toContain(join(fx.home, '.claude', 'settings.local.json')) + expect(out).toContain(mcpJson) + expect(out).toContain('takes effect on the next session') + expect(out).toContain('Dry run: nothing was changed.') + expect(await hashTree(fx.root)).toBe(before) + expect(await readRecords(fx.actionsDir)).toHaveLength(0) + // Paths asserted against the raw fixture so nothing drifted. + expect(existsSync(settings) && existsSync(mcpJson)).toBe(true) + }) + + it('--yes applies all three kinds and journals one record each', async () => { + const { fx, findings, settings, mcpJson } = await deferFamilyFixture() + const io = makeIo() + await runOptimizeApply([], undefined, applyOpts(fx, io, { findings, yes: true })) + + const records = await readRecords(fx.actionsDir) + expect(records.map(r => r.kind).sort()).toEqual(['defer-alwaysload', 'defer-enable', 'defer-threshold']) + expect(records.map(r => r.findingId).sort()).toEqual(['mcp-alwaysload-hygiene', 'mcp-defer-threshold', 'mcp-deferral-off']) + + expect(JSON.parse(await readFile(settings, 'utf-8'))).toEqual({ env: {} }) + expect(JSON.parse(await readFile(mcpJson, 'utf-8'))).toEqual({ mcpServers: { pinned: { command: 'x' } } }) + expect(JSON.parse(await readFile(join(fx.home, '.claude', 'settings.local.json'), 'utf-8'))).toEqual({ env: { ENABLE_TOOL_SEARCH: 'auto:2' } }) + + // Undo everything, newest first, and verify the tree round-trips. + for (const rec of [...records].reverse()) { + await undoAction({ id: rec.id }, { actionsDir: fx.actionsDir }) + } + expect(JSON.parse(await readFile(settings, 'utf-8'))).toEqual({ env: { ENABLE_TOOL_SEARCH: 'false' } }) + expect(JSON.parse(await readFile(mcpJson, 'utf-8'))).toEqual({ mcpServers: { pinned: { command: 'x', alwaysLoad: true } } }) + }) + + it('refusal causes render as manual findings with their instruction notes', async () => { + const fx = await makeFixture() + const findings = [ + makeFinding('mcp-deferral-off', { + kind: 'defer-enable', cause: 'proxy-unknown', settingPath: join(fx.home, '.claude', 'settings.json'), settingScope: 'user settings', value: 'https://proxy.example', + }), + ] + const io = makeIo() + await runOptimizeApply([], undefined, applyOpts(fx, io, { findings, dryRun: true })) + + const out = io.stdout() + expect(out).toContain('No appliable config-class fixes') + expect(out).toContain('tool_reference') + expect(await readRecords(fx.actionsDir)).toHaveLength(0) + }) + + it('stale-plan guard: an edit after planning is refused, leaving the file as-is', async () => { + const fx = await makeFixture() + const settings = join(fx.home, '.claude', 'settings.json') + await mkdir(join(fx.home, '.claude'), { recursive: true }) + await writeFile(settings, stringify({ env: { ENABLE_TOOL_SEARCH: 'false' } })) + const plan = planFor(makeFinding('mcp-deferral-off', { + kind: 'defer-enable', cause: 'env-false', settingPath: settings, settingScope: 'user settings', value: 'false', + }), ctx(fx)) + expect(plan).not.toBeNull() + + const interim = stringify({ env: { ENABLE_TOOL_SEARCH: 'false', ADDED: '1' } }) + await writeFile(settings, interim) + + await expect(runAction(plan!, fx.actionsDir)).rejects.toThrow(/changed since the plan was built/) + expect(await readFile(settings, 'utf-8')).toBe(interim) + expect(await readRecords(fx.actionsDir)).toHaveLength(0) + }) +}) diff --git a/tests/mcp-deferral.test.ts b/tests/mcp-deferral.test.ts new file mode 100644 index 00000000..a4411669 --- /dev/null +++ b/tests/mcp-deferral.test.ts @@ -0,0 +1,745 @@ +import { describe, it, expect, afterAll } from 'vitest' +import { mkdtempSync, rmSync, mkdirSync, writeFileSync } from 'fs' +import { tmpdir } from 'os' +import { dirname, join } from 'path' + +import { + detectMcpDeferralOff, + detectMcpAlwaysLoadHygiene, + detectMcpDeferThreshold, + findDeferralEnvSetting, + type ApiCallMeta, + type ToolCall, +} from '../src/optimize.js' +import type { + ClassifiedTurn, + ParsedApiCall, + ProjectSummary, + SessionSummary, + TaskCategory, + TokenUsage, +} from '../src/types.js' + +// --------------------------------------------------------------------------- +// Test fixtures (same conventions as tests/mcp-coverage.test.ts) +// --------------------------------------------------------------------------- + +const ZERO_USAGE: TokenUsage = { + inputTokens: 0, + outputTokens: 0, + cacheCreationInputTokens: 0, + cacheReadInputTokens: 0, + cachedInputTokens: 0, + reasoningTokens: 0, + webSearchRequests: 0, +} + +function makeCall(opts: { tools?: string[]; provider?: string } = {}): ParsedApiCall { + const tools = opts.tools ?? [] + return { + provider: opts.provider ?? 'claude', + model: 'Opus 4.7', + usage: { ...ZERO_USAGE }, + costUSD: 0, + tools, + mcpTools: tools.filter(t => t.startsWith('mcp__')), + skills: [], + hasAgentSpawn: false, + hasPlanMode: false, + speed: 'standard', + timestamp: '2026-05-04T00:00:00Z', + bashCommands: [], + deduplicationKey: 'k', + } +} + +function makeTurn(calls: ParsedApiCall[]): ClassifiedTurn { + return { + userMessage: '', + assistantCalls: calls, + timestamp: '2026-05-04T00:00:00Z', + sessionId: 's1', + category: 'coding', + retries: 0, + hasEdits: false, + } +} + +function makeSession(opts: { + sessionId?: string + inventory?: string[] + turns?: ClassifiedTurn[] + mcpBreakdown?: Record +}): SessionSummary { + const turns = opts.turns ?? [] + const apiCalls = turns.reduce((s, t) => s + t.assistantCalls.length, 0) + const emptyCategoryBreakdown = {} as Record + return { + sessionId: opts.sessionId ?? 's1', + project: 'p', + firstTimestamp: '2026-05-04T00:00:00Z', + lastTimestamp: '2026-05-04T00:00:00Z', + totalCostUSD: 0, + totalInputTokens: 0, + totalOutputTokens: 0, + totalCacheReadTokens: 0, + totalCacheWriteTokens: 0, + apiCalls, + turns, + modelBreakdown: {}, + toolBreakdown: {}, + mcpBreakdown: opts.mcpBreakdown ?? {}, + bashBreakdown: {}, + categoryBreakdown: emptyCategoryBreakdown, + skillBreakdown: {}, + ...(opts.inventory ? { mcpInventory: opts.inventory } : {}), + } +} + +function project(sessions: SessionSummary[]): ProjectSummary { + return { + project: 'p', + projectPath: '/tmp/p', + sessions, + totalCostUSD: 0, + totalApiCalls: sessions.reduce((s, ses) => s + ses.apiCalls, 0), + } +} + +function toolCall(name: string): ToolCall { + return { name, input: {}, sessionId: 's1', project: 'p' } +} + +function apiCall(version: string): ApiCallMeta { + return { cacheCreationTokens: 0, version } +} + +// --------------------------------------------------------------------------- +// Filesystem fixtures: every detector call injects a temp home dir and temp +// project cwds so the developer's real ~/.claude config never leaks in. +// --------------------------------------------------------------------------- + +const FIXTURE_ROOTS: string[] = [] + +function makeDir(prefix: string): string { + const dir = mkdtempSync(join(tmpdir(), prefix)) + FIXTURE_ROOTS.push(dir) + return dir +} + +function writeFile(path: string, content: string): void { + mkdirSync(dirname(path), { recursive: true }) + writeFileSync(path, content) +} + +function writeJson(path: string, value: unknown): void { + writeFile(path, JSON.stringify(value)) +} + +afterAll(() => { + for (const dir of FIXTURE_ROOTS) { + rmSync(dir, { recursive: true, force: true }) + } +}) + +// A minimal Claude-provider turn: the deferral detectors count only Claude +// Code sessions (identified by turn provider), since deferral is a Claude +// Code mechanism and other providers can never carry the counter-evidence. +function claudeTurns(): ClassifiedTurn[] { + return [makeTurn([makeCall()])] +} + +// Two sessions that invoked an MCP server but never observed an inventory: +// the canonical deferral-off transcript shape. +function deferralOffSessions(): SessionSummary[] { + const turns = [makeTurn([makeCall({ tools: ['mcp__srv__t1'] })])] + return [ + makeSession({ sessionId: 'a', turns, mcpBreakdown: { srv: { calls: 2 } } }), + makeSession({ sessionId: 'b', turns, mcpBreakdown: { srv: { calls: 1 } } }), + ] +} + +// --------------------------------------------------------------------------- +// findDeferralEnvSetting +// --------------------------------------------------------------------------- + +describe('findDeferralEnvSetting', () => { + it('returns null when nothing is configured', () => { + const home = makeDir('codeburn-home-') + expect(findDeferralEnvSetting('ENABLE_TOOL_SEARCH', [], home)).toBeNull() + }) + + it('reads settings.json env fields before shell profiles', () => { + const home = makeDir('codeburn-home-') + writeJson(join(home, '.claude', 'settings.json'), { env: { ENABLE_TOOL_SEARCH: 'false' } }) + writeFile(join(home, '.zshrc'), 'export ENABLE_TOOL_SEARCH=true\n') + const hit = findDeferralEnvSetting('ENABLE_TOOL_SEARCH', [], home) + expect(hit).not.toBeNull() + expect(hit!.value).toBe('false') + expect(hit!.scope).toBe('user settings') + }) + + it('matches shell profile lines with and without export', () => { + const home = makeDir('codeburn-home-') + writeFile(join(home, '.bashrc'), '# config\nENABLE_TOOL_SEARCH="auto:25"\n') + const hit = findDeferralEnvSetting('ENABLE_TOOL_SEARCH', [], home) + expect(hit).not.toBeNull() + expect(hit!.value).toBe('auto:25') + expect(hit!.scope).toBe('shell profile') + }) + + it('prefers the most-specific scope, matching effective settings precedence', () => { + const home = makeDir('codeburn-home-') + const cwd = makeDir('codeburn-cwd-') + writeJson(join(home, '.claude', 'settings.json'), { env: { ENABLE_TOOL_SEARCH: 'true' } }) + writeJson(join(cwd, '.claude', 'settings.local.json'), { env: { ENABLE_TOOL_SEARCH: 'false' } }) + const hit = findDeferralEnvSetting('ENABLE_TOOL_SEARCH', [cwd], home) + expect(hit).not.toBeNull() + expect(hit!.value).toBe('false') + expect(hit!.scope).toBe('project local settings') + }) +}) + +// --------------------------------------------------------------------------- +// detectMcpDeferralOff +// --------------------------------------------------------------------------- + +describe('detectMcpDeferralOff', () => { + it('returns null for users with no MCP at all', () => { + const home = makeDir('codeburn-home-') + const projects = [project([makeSession({ sessionId: 'a' }), makeSession({ sessionId: 'b' })])] + expect(detectMcpDeferralOff([], projects, new Set(), [], home)).toBeNull() + }) + + it('returns null when ToolSearch calls prove deferral was active', () => { + const home = makeDir('codeburn-home-') + const calls = [toolCall('ToolSearch'), toolCall('mcp__srv__t1')] + const projects = [project(deferralOffSessions())] + expect(detectMcpDeferralOff(calls, projects, new Set(), [], home)).toBeNull() + }) + + it('returns null when any session observed an MCP inventory (deferral active)', () => { + const home = makeDir('codeburn-home-') + const sessions = [ + ...deferralOffSessions(), + makeSession({ sessionId: 'c', inventory: ['mcp__srv__t1'] }), + ] + expect(detectMcpDeferralOff([], [project(sessions)], new Set(), [], home)).toBeNull() + }) + + it('returns null below the minimum MCP-evidence session count', () => { + const home = makeDir('codeburn-home-') + const sessions = [ + makeSession({ sessionId: 'a', turns: claudeTurns(), mcpBreakdown: { srv: { calls: 3 } } }), + makeSession({ sessionId: 'b', turns: claudeTurns() }), + ] + expect(detectMcpDeferralOff([], [project(sessions)], new Set(), [], home)).toBeNull() + }) + + it('ignores MCP usage from non-Claude providers (no counter-evidence exists there)', () => { + const home = makeDir('codeburn-home-') + // Codex/Copilot parsers normalize MCP calls into mcpBreakdown too, but + // those sessions can never show ToolSearch calls or an inventory, so + // they must not count as deferral-off evidence. + const codexTurns = [makeTurn([makeCall({ provider: 'codex', tools: ['mcp__srv__t1'] })])] + const sessions = [ + makeSession({ sessionId: 'a', turns: codexTurns, mcpBreakdown: { srv: { calls: 4 } } }), + makeSession({ sessionId: 'b', turns: codexTurns, mcpBreakdown: { srv: { calls: 4 } } }), + ] + expect(detectMcpDeferralOff([], [project(sessions)], new Set(), [], home)).toBeNull() + }) + + it('emits the generic cause when no config explains the gap', () => { + const home = makeDir('codeburn-home-') + const projects = [project(deferralOffSessions())] + const finding = detectMcpDeferralOff([], projects, new Set(), [], home) + expect(finding).not.toBeNull() + expect(finding!.id).toBe('mcp-deferral-off') + expect(finding!.explanation).toContain('none determinable') + expect(finding!.explanation).toContain('zero ToolSearch calls') + // 1 server x 5 tools x 400 tokens x 2 affected sessions + expect(finding!.tokensSaved).toBe(4000) + // 3 invocations / 2 sessions + expect(finding!.explanation).toContain('1.5 MCP calls/session') + }) + + it('counts every Claude session as affected when servers come from config', () => { + const home = makeDir('codeburn-home-') + const cwd = makeDir('codeburn-cwd-') + writeJson(join(cwd, '.mcp.json'), { mcpServers: { srv: { command: 'x' } } }) + const sessions = [ + makeSession({ sessionId: 'a', turns: claudeTurns() }), + makeSession({ sessionId: 'b', turns: claudeTurns() }), + makeSession({ sessionId: 'c', turns: claudeTurns() }), + // Non-Claude sessions never carry the upfront schema cost. + makeSession({ sessionId: 'd', turns: [makeTurn([makeCall({ provider: 'codex' })])] }), + ] + const finding = detectMcpDeferralOff([], [project(sessions)], new Set([cwd]), [], home) + expect(finding).not.toBeNull() + // 1 configured server x 2000 tokens x 3 Claude sessions + expect(finding!.tokensSaved).toBe(6000) + }) + + it('excludes alwaysLoad-pinned servers: all pinned means no finding', () => { + const home = makeDir('codeburn-home-') + const cwd = makeDir('codeburn-cwd-') + // Deferral working with every server deliberately pinned: pinned tools + // are never deferred, so no inventory and no ToolSearch calls is the + // EXPECTED shape, not evidence of a gap. + writeJson(join(cwd, '.mcp.json'), { mcpServers: { pinned: { command: 'x', alwaysLoad: true } } }) + const sessions = [ + makeSession({ sessionId: 'a', turns: claudeTurns(), mcpBreakdown: { pinned: { calls: 2 } } }), + makeSession({ sessionId: 'b', turns: claudeTurns(), mcpBreakdown: { pinned: { calls: 1 } } }), + ] + expect(detectMcpDeferralOff([], [project(sessions)], new Set([cwd]), [], home)).toBeNull() + }) + + it('charges only unpinned servers when pinned and unpinned coexist', () => { + const home = makeDir('codeburn-home-') + const cwd = makeDir('codeburn-cwd-') + // The pinned server's schema is mcp-alwaysload-hygiene's jurisdiction; + // charging it here too would double-count the same tokens. + writeJson(join(cwd, '.mcp.json'), { + mcpServers: { + pinned: { command: 'x', alwaysLoad: true }, + srv: { command: 'y' }, + }, + }) + const finding = detectMcpDeferralOff([], [project(deferralOffSessions())], new Set([cwd]), [], home) + expect(finding).not.toBeNull() + // 1 unpinned server x 2000 tokens x 2 sessions (not 2 servers x ...) + expect(finding!.tokensSaved).toBe(4000) + }) + + it('attributes ENABLE_TOOL_SEARCH=false in user settings', () => { + const home = makeDir('codeburn-home-') + writeJson(join(home, '.claude', 'settings.json'), { env: { ENABLE_TOOL_SEARCH: 'false' } }) + const finding = detectMcpDeferralOff([], [project(deferralOffSessions())], new Set(), [], home) + expect(finding).not.toBeNull() + expect(finding!.explanation).toContain('ENABLE_TOOL_SEARCH=false') + expect(finding!.explanation).toContain('user settings') + expect(finding!.fix.type).toBe('paste') + expect((finding!.fix as { text: string }).text).toContain('Remove the ENABLE_TOOL_SEARCH=false setting') + }) + + it('attributes ENABLE_TOOL_SEARCH=false in user local settings', () => { + const home = makeDir('codeburn-home-') + writeJson(join(home, '.claude', 'settings.local.json'), { env: { ENABLE_TOOL_SEARCH: 'false' } }) + const finding = detectMcpDeferralOff([], [project(deferralOffSessions())], new Set(), [], home) + expect(finding).not.toBeNull() + expect(finding!.explanation).toContain('user local settings') + }) + + it('attributes ENABLE_TOOL_SEARCH=false in project settings, naming the path', () => { + const home = makeDir('codeburn-home-') + const cwd = makeDir('codeburn-cwd-') + writeJson(join(cwd, '.claude', 'settings.json'), { env: { ENABLE_TOOL_SEARCH: 'false' } }) + const finding = detectMcpDeferralOff([], [project(deferralOffSessions())], new Set([cwd]), [], home) + expect(finding).not.toBeNull() + expect(finding!.explanation).toContain('project settings') + expect(finding!.explanation).toContain(cwd) + }) + + it('attributes ENABLE_TOOL_SEARCH=false in project local settings, naming the path', () => { + const home = makeDir('codeburn-home-') + const cwd = makeDir('codeburn-cwd-') + writeJson(join(cwd, '.claude', 'settings.local.json'), { env: { ENABLE_TOOL_SEARCH: 'false' } }) + const finding = detectMcpDeferralOff([], [project(deferralOffSessions())], new Set([cwd]), [], home) + expect(finding).not.toBeNull() + expect(finding!.explanation).toContain('project local settings') + expect(finding!.explanation).toContain(cwd) + }) + + it('attributes ENABLE_TOOL_SEARCH=false in a shell profile', () => { + const home = makeDir('codeburn-home-') + writeFile(join(home, '.zshrc'), 'export ENABLE_TOOL_SEARCH=false\n') + const finding = detectMcpDeferralOff([], [project(deferralOffSessions())], new Set(), [], home) + expect(finding).not.toBeNull() + expect(finding!.explanation).toContain('shell profile') + }) + + it('attributes a non-first-party ANTHROPIC_BASE_URL with unknown-proxy wording', () => { + const home = makeDir('codeburn-home-') + writeJson(join(home, '.claude', 'settings.json'), { env: { ANTHROPIC_BASE_URL: 'https://llm-proxy.corp.example:8443/v1' } }) + const finding = detectMcpDeferralOff([], [project(deferralOffSessions())], new Set(), [], home) + expect(finding).not.toBeNull() + expect(finding!.explanation).toContain('non-first-party host') + expect(finding!.explanation).toContain('unknown') + // Must not claim the proxy is incapable. + expect(finding!.explanation).not.toContain('incapable') + expect(finding!.fix.label).toContain('tool_reference') + }) + + it('treats api.anthropic.com as first-party and falls through to generic', () => { + const home = makeDir('codeburn-home-') + writeJson(join(home, '.claude', 'settings.json'), { env: { ANTHROPIC_BASE_URL: 'https://api.anthropic.com' } }) + const finding = detectMcpDeferralOff([], [project(deferralOffSessions())], new Set(), [], home) + expect(finding).not.toBeNull() + expect(finding!.explanation).toContain('none determinable') + }) + + it('attributes CLAUDE_CODE_USE_VERTEX', () => { + const home = makeDir('codeburn-home-') + writeFile(join(home, '.bashrc'), 'export CLAUDE_CODE_USE_VERTEX=1\n') + const finding = detectMcpDeferralOff([], [project(deferralOffSessions())], new Set(), [], home) + expect(finding).not.toBeNull() + expect(finding!.explanation).toContain('Vertex') + }) + + it('attributes Claude Code versions predating tool search default-on (v2.1.7)', () => { + const home = makeDir('codeburn-home-') + const apiCalls = [apiCall('2.0.14'), apiCall('2.1.6')] + const finding = detectMcpDeferralOff([], [project(deferralOffSessions())], new Set(), apiCalls, home) + expect(finding).not.toBeNull() + expect(finding!.explanation).toContain('predates v2.1.7') + expect(finding!.fix.type).toBe('command') + expect((finding!.fix as { text: string }).text).toBe('claude update') + }) + + it('does not blame the version when any observed version has default-on tool search', () => { + const home = makeDir('codeburn-home-') + const apiCalls = [apiCall('2.0.14'), apiCall('2.1.30')] + const finding = detectMcpDeferralOff([], [project(deferralOffSessions())], new Set(), apiCalls, home) + expect(finding).not.toBeNull() + expect(finding!.explanation).toContain('none determinable') + }) + + it('yields to the defer-threshold detector when an auto override is set', () => { + const home = makeDir('codeburn-home-') + writeJson(join(home, '.claude', 'settings.json'), { env: { ENABLE_TOOL_SEARCH: 'auto:50' } }) + expect(detectMcpDeferralOff([], [project(deferralOffSessions())], new Set(), [], home)).toBeNull() + }) + + it('does not re-suggest the export when ENABLE_TOOL_SEARCH=true is already set', () => { + const home = makeDir('codeburn-home-') + writeJson(join(home, '.claude', 'settings.json'), { env: { ENABLE_TOOL_SEARCH: 'true' } }) + const finding = detectMcpDeferralOff([], [project(deferralOffSessions())], new Set(), [], home) + expect(finding).not.toBeNull() + expect(finding!.explanation).toContain('none determinable') + expect(finding!.explanation).toContain('ENABLE_TOOL_SEARCH=true is already set') + expect((finding!.fix as { text: string }).text).not.toContain('export ENABLE_TOOL_SEARCH=true') + }) +}) + +// --------------------------------------------------------------------------- +// Family handoff: every auto value must land in exactly one detector +// --------------------------------------------------------------------------- + +describe('deferral-off / defer-threshold handoff', () => { + it('an out-of-range auto:N produces exactly one finding from the family', () => { + const home = makeDir('codeburn-home-') + const cwd = makeDir('codeburn-cwd-') + // auto:1234 is nonsense-but-reachable: the threshold can never trigger. + // deferral-off must yield AND defer-threshold must accept (clamping to + // 100%), or the value would silently suppress both findings. + writeJson(join(home, '.claude', 'settings.json'), { env: { ENABLE_TOOL_SEARCH: 'auto:1234' } }) + const mcpServers: Record = { srv0: { command: 'x' }, srv1: { command: 'x' }, srv2: { command: 'x' } } + writeJson(join(cwd, '.mcp.json'), { mcpServers }) + const projects = [project(deferralOffSessions())] + + const offFinding = detectMcpDeferralOff([], projects, new Set([cwd]), [], home) + const thresholdFinding = detectMcpDeferThreshold(projects, new Set([cwd]), home) + expect(offFinding).toBeNull() + expect(thresholdFinding).not.toBeNull() + expect(thresholdFinding!.id).toBe('mcp-defer-threshold') + // Clamped to the 100% ceiling of the 200k window. + expect(thresholdFinding!.explanation).toContain('100%') + }) +}) + +// --------------------------------------------------------------------------- +// detectMcpAlwaysLoadHygiene +// --------------------------------------------------------------------------- + +describe('detectMcpAlwaysLoadHygiene', () => { + function fiveSessions(callsForPinned: number): SessionSummary[] { + return Array.from({ length: 5 }, (_, i) => makeSession({ + sessionId: `s${i}`, + mcpBreakdown: i === 0 && callsForPinned > 0 ? { pinned: { calls: callsForPinned } } : {}, + })) + } + + it('returns null when no server sets alwaysLoad', () => { + const home = makeDir('codeburn-home-') + const cwd = makeDir('codeburn-cwd-') + writeJson(join(cwd, '.mcp.json'), { mcpServers: { pinned: { command: 'x' } } }) + expect(detectMcpAlwaysLoadHygiene([project(fiveSessions(0))], new Set([cwd]), [], undefined, home)).toBeNull() + }) + + it('flags an alwaysLoad server with usage below one call per five sessions', () => { + const home = makeDir('codeburn-home-') + const cwd = makeDir('codeburn-cwd-') + writeJson(join(cwd, '.mcp.json'), { mcpServers: { pinned: { command: 'x', alwaysLoad: true } } }) + const finding = detectMcpAlwaysLoadHygiene([project(fiveSessions(0))], new Set([cwd]), [], undefined, home) + expect(finding).not.toBeNull() + expect(finding!.id).toBe('mcp-alwaysload-hygiene') + expect(finding!.explanation).toContain('pinned: 0 calls across 5 sessions') + // No inventory -> 5 tools x 400 tokens fallback, charged in all 5 sessions + expect(finding!.tokensSaved).toBe(10_000) + expect((finding!.fix as { text: string }).text).toContain('"alwaysLoad": true') + expect((finding!.fix as { text: string }).text).toContain('.mcp.json') + }) + + it('does not flag an alwaysLoad server at or above the call-rate threshold', () => { + const home = makeDir('codeburn-home-') + const cwd = makeDir('codeburn-cwd-') + writeJson(join(cwd, '.mcp.json'), { mcpServers: { pinned: { command: 'x', alwaysLoad: true } } }) + // 1 call / 5 sessions = 0.2 exactly; the threshold flags only strictly below. + expect(detectMcpAlwaysLoadHygiene([project(fiveSessions(1))], new Set([cwd]), [], undefined, home)).toBeNull() + }) + + it('prefers observed inventory tool counts and loaded sessions when available', () => { + const home = makeDir('codeburn-home-') + const cwd = makeDir('codeburn-cwd-') + writeJson(join(cwd, '.mcp.json'), { mcpServers: { pinned: { command: 'x', alwaysLoad: true } } }) + const inventory = Array.from({ length: 8 }, (_, i) => `mcp__pinned__t${i}`) + const sessions = [ + makeSession({ sessionId: 'a', inventory }), + makeSession({ sessionId: 'b', inventory }), + makeSession({ sessionId: 'c' }), + makeSession({ sessionId: 'd' }), + makeSession({ sessionId: 'e' }), + ] + const finding = detectMcpAlwaysLoadHygiene([project(sessions)], new Set([cwd]), [], undefined, home) + expect(finding).not.toBeNull() + // 8 tools x 400 tokens x 2 loaded sessions + expect(finding!.tokensSaved).toBe(6400) + }) + + it('does not flag an alwaysLoad server clearly over the call-rate threshold', () => { + const home = makeDir('codeburn-home-') + const cwd = makeDir('codeburn-cwd-') + writeJson(join(cwd, '.mcp.json'), { mcpServers: { pinned: { command: 'x', alwaysLoad: true } } }) + // 5 calls / 5 sessions = 1.0, well over the 0.2 threshold. + expect(detectMcpAlwaysLoadHygiene([project(fiveSessions(5))], new Set([cwd]), [], undefined, home)).toBeNull() + }) + + it('returns null when there are no sessions in the window', () => { + const home = makeDir('codeburn-home-') + const cwd = makeDir('codeburn-cwd-') + writeJson(join(cwd, '.mcp.json'), { mcpServers: { pinned: { command: 'x', alwaysLoad: true } } }) + expect(detectMcpAlwaysLoadHygiene([], new Set([cwd]), [], undefined, home)).toBeNull() + }) + + it('returns null when every observed version predates server-level alwaysLoad (v2.1.121)', () => { + const home = makeDir('codeburn-home-') + const cwd = makeDir('codeburn-cwd-') + writeJson(join(cwd, '.mcp.json'), { mcpServers: { pinned: { command: 'x', alwaysLoad: true } } }) + // On these versions the key is inert: tools defer normally, no cost. + const apiCalls = [apiCall('2.1.100'), apiCall('2.1.120')] + expect(detectMcpAlwaysLoadHygiene([project(fiveSessions(0))], new Set([cwd]), apiCalls, undefined, home)).toBeNull() + }) + + it('still flags when any observed version supports alwaysLoad', () => { + const home = makeDir('codeburn-home-') + const cwd = makeDir('codeburn-cwd-') + writeJson(join(cwd, '.mcp.json'), { mcpServers: { pinned: { command: 'x', alwaysLoad: true } } }) + const apiCalls = [apiCall('2.1.100'), apiCall('2.1.121')] + expect(detectMcpAlwaysLoadHygiene([project(fiveSessions(0))], new Set([cwd]), apiCalls, undefined, home)).not.toBeNull() + }) +}) + +// --------------------------------------------------------------------------- +// detectMcpDeferThreshold +// --------------------------------------------------------------------------- + +describe('detectMcpDeferThreshold', () => { + function mcpJsonWithServers(count: number): Record { + const mcpServers: Record = {} + for (let i = 0; i < count; i++) mcpServers[`srv${i}`] = { command: 'x' } + return { mcpServers } + } + + it('returns null when ENABLE_TOOL_SEARCH is not configured', () => { + const home = makeDir('codeburn-home-') + const cwd = makeDir('codeburn-cwd-') + writeJson(join(cwd, '.mcp.json'), mcpJsonWithServers(3)) + expect(detectMcpDeferThreshold([project([makeSession({ turns: claudeTurns() })])], new Set([cwd]), home)).toBeNull() + }) + + it('returns null for non-auto values like true or false', () => { + const home = makeDir('codeburn-home-') + const cwd = makeDir('codeburn-cwd-') + writeJson(join(cwd, '.mcp.json'), mcpJsonWithServers(3)) + writeJson(join(home, '.claude', 'settings.json'), { env: { ENABLE_TOOL_SEARCH: 'true' } }) + expect(detectMcpDeferThreshold([project([makeSession({ turns: claudeTurns() })])], new Set([cwd]), home)).toBeNull() + }) + + it('flags an auto threshold that the estimated defs fit under', () => { + const home = makeDir('codeburn-home-') + const cwd = makeDir('codeburn-cwd-') + // 3 servers x 5 tools x 400 tokens = 6k defs/session, under the default + // 10% of 200k (20k) but over the 5k substantial-cost floor. + writeJson(join(cwd, '.mcp.json'), mcpJsonWithServers(3)) + writeJson(join(home, '.claude', 'settings.json'), { env: { ENABLE_TOOL_SEARCH: 'auto' } }) + const sessions = [ + makeSession({ sessionId: 'a', turns: claudeTurns() }), + makeSession({ sessionId: 'b', turns: claudeTurns() }), + ] + const finding = detectMcpDeferThreshold([project(sessions)], new Set([cwd]), home) + expect(finding).not.toBeNull() + expect(finding!.id).toBe('mcp-defer-threshold') + expect(finding!.impact).toBe('low') + // 6k defs/session x 2 sessions + expect(finding!.tokensSaved).toBe(12_000) + // Largest N where 6k defs still exceed N% of 200k: N=2 (6000 > 4000) + expect((finding!.fix as { text: string }).text).toContain('ENABLE_TOOL_SEARCH=auto:2') + }) + + it('returns null when defs exceed the configured threshold (auto already defers)', () => { + const home = makeDir('codeburn-home-') + const cwd = makeDir('codeburn-cwd-') + // 6k defs/session vs auto:2 threshold of 4k -> deferral kicks in. + writeJson(join(cwd, '.mcp.json'), mcpJsonWithServers(3)) + writeJson(join(home, '.claude', 'settings.json'), { env: { ENABLE_TOOL_SEARCH: 'auto:2' } }) + expect(detectMcpDeferThreshold([project([makeSession({ turns: claudeTurns() })])], new Set([cwd]), home)).toBeNull() + }) + + it('returns null when the upfront defs are not substantial', () => { + const home = makeDir('codeburn-home-') + const cwd = makeDir('codeburn-cwd-') + // 2 servers = 4k defs/session, under the 5k floor. + writeJson(join(cwd, '.mcp.json'), mcpJsonWithServers(2)) + writeJson(join(home, '.claude', 'settings.json'), { env: { ENABLE_TOOL_SEARCH: 'auto' } }) + expect(detectMcpDeferThreshold([project([makeSession({ turns: claudeTurns() })])], new Set([cwd]), home)).toBeNull() + }) + + it('returns null when an inventory shows deferral already working', () => { + const home = makeDir('codeburn-home-') + const cwd = makeDir('codeburn-cwd-') + writeJson(join(cwd, '.mcp.json'), mcpJsonWithServers(3)) + writeJson(join(home, '.claude', 'settings.json'), { env: { ENABLE_TOOL_SEARCH: 'auto' } }) + const sessions = [makeSession({ inventory: ['mcp__srv0__t1'] })] + expect(detectMcpDeferThreshold([project(sessions)], new Set([cwd]), home)).toBeNull() + }) + + it('recommends removing the override when defs already exceed the default 10%', () => { + const home = makeDir('codeburn-home-') + const cwd = makeDir('codeburn-cwd-') + // 15 servers = 30k defs/session: fits under auto:50 (100k) but exceeds + // the default 10% (20k), so dropping the override is enough. + writeJson(join(cwd, '.mcp.json'), mcpJsonWithServers(15)) + writeJson(join(home, '.claude', 'settings.json'), { env: { ENABLE_TOOL_SEARCH: 'auto:50' } }) + const finding = detectMcpDeferThreshold([project([makeSession({ turns: claudeTurns() })])], new Set([cwd]), home) + expect(finding).not.toBeNull() + expect((finding!.fix as { text: string }).text).toContain('Remove the ENABLE_TOOL_SEARCH=auto:50 override') + }) + + it('returns null for users with no MCP servers at all', () => { + const home = makeDir('codeburn-home-') + writeJson(join(home, '.claude', 'settings.json'), { env: { ENABLE_TOOL_SEARCH: 'auto' } }) + expect(detectMcpDeferThreshold([project([makeSession({ turns: claudeTurns() })])], new Set(), home)).toBeNull() + }) + + it('returns null when there are no sessions in the window', () => { + const home = makeDir('codeburn-home-') + const cwd = makeDir('codeburn-cwd-') + writeJson(join(cwd, '.mcp.json'), mcpJsonWithServers(3)) + writeJson(join(home, '.claude', 'settings.json'), { env: { ENABLE_TOOL_SEARCH: 'auto' } }) + // Configured servers but zero sessions: no phantom "all 0 sessions" + // finding and no fabricated tokensSaved. + expect(detectMcpDeferThreshold([project([])], new Set([cwd]), home)).toBeNull() + }) +}) + +// --------------------------------------------------------------------------- +// Apply payloads: the machine data src/act/plans.ts turns into defer-enable / +// defer-alwaysload / defer-threshold plans (#614 commit 2). Human-facing +// explanation/fix text is covered above and must not change. +// --------------------------------------------------------------------------- + +describe('deferral finding apply payloads', () => { + it('env-false carries a defer-enable payload naming the settings file, scope, and value', () => { + const home = makeDir('codeburn-home-') + writeJson(join(home, '.claude', 'settings.json'), { env: { ENABLE_TOOL_SEARCH: 'false' } }) + const finding = detectMcpDeferralOff([], [project(deferralOffSessions())], new Set(), [], home) + expect(finding!.apply).toEqual({ + kind: 'defer-enable', + cause: 'env-false', + settingPath: join(home, '.claude', 'settings.json'), + settingScope: 'user settings', + value: 'false', + }) + }) + + it('env-false in a shell profile keeps the shell-profile scope so the plan layer can refuse', () => { + const home = makeDir('codeburn-home-') + writeFile(join(home, '.zshrc'), 'export ENABLE_TOOL_SEARCH=false\n') + const finding = detectMcpDeferralOff([], [project(deferralOffSessions())], new Set(), [], home) + expect(finding!.apply).toMatchObject({ kind: 'defer-enable', cause: 'env-false', settingScope: 'shell profile' }) + }) + + it('an unknown proxy carries cause proxy-unknown pointing at the base-URL setting', () => { + const home = makeDir('codeburn-home-') + writeJson(join(home, '.claude', 'settings.json'), { env: { ANTHROPIC_BASE_URL: 'https://llm-proxy.corp.example' } }) + const finding = detectMcpDeferralOff([], [project(deferralOffSessions())], new Set(), [], home) + expect(finding!.apply).toMatchObject({ + kind: 'defer-enable', + cause: 'proxy-unknown', + settingPath: join(home, '.claude', 'settings.json'), + }) + }) + + it('Vertex and old-version carry their refusal causes', () => { + const homeVertex = makeDir('codeburn-home-') + writeFile(join(homeVertex, '.bashrc'), 'export CLAUDE_CODE_USE_VERTEX=1\n') + const vertexFinding = detectMcpDeferralOff([], [project(deferralOffSessions())], new Set(), [], homeVertex) + expect(vertexFinding!.apply).toMatchObject({ kind: 'defer-enable', cause: 'vertex' }) + + const homeOld = makeDir('codeburn-home-') + const oldFinding = detectMcpDeferralOff([], [project(deferralOffSessions())], new Set(), [apiCall('2.0.14')], homeOld) + expect(oldFinding!.apply).toEqual({ kind: 'defer-enable', cause: 'old-version' }) + }) + + it('the generic none-determinable causes carry no payload (nothing appliable)', () => { + const home = makeDir('codeburn-home-') + const generic = detectMcpDeferralOff([], [project(deferralOffSessions())], new Set(), [], home) + expect(generic!.apply).toBeUndefined() + + const homeTrue = makeDir('codeburn-home-') + writeJson(join(homeTrue, '.claude', 'settings.json'), { env: { ENABLE_TOOL_SEARCH: 'true' } }) + const alreadyOn = detectMcpDeferralOff([], [project(deferralOffSessions())], new Set(), [], homeTrue) + expect(alreadyOn!.apply).toBeUndefined() + }) + + it('alwaysload-hygiene payload names each flagged server and the exact files carrying the pin', () => { + const home = makeDir('codeburn-home-') + const cwd = makeDir('codeburn-cwd-') + writeJson(join(cwd, '.mcp.json'), { mcpServers: { pinned: { command: 'x', alwaysLoad: true } } }) + const sessions = Array.from({ length: 5 }, (_, i) => makeSession({ sessionId: `s${i}` })) + const finding = detectMcpAlwaysLoadHygiene([project(sessions)], new Set([cwd]), [], undefined, home) + expect(finding!.apply).toEqual({ + kind: 'defer-alwaysload', + servers: [{ server: 'pinned', paths: [join(cwd, '.mcp.json')] }], + }) + }) + + it('defer-threshold payload carries the override location, tightened N, and removal flag', () => { + const home = makeDir('codeburn-home-') + const cwd = makeDir('codeburn-cwd-') + const mcpServers: Record = {} + for (let i = 0; i < 3; i++) mcpServers[`srv${i}`] = { command: 'x' } + writeJson(join(cwd, '.mcp.json'), { mcpServers }) + writeJson(join(home, '.claude', 'settings.json'), { env: { ENABLE_TOOL_SEARCH: 'auto' } }) + const finding = detectMcpDeferThreshold([project([makeSession({ turns: claudeTurns() })])], new Set([cwd]), home) + expect(finding!.apply).toEqual({ + kind: 'defer-threshold', + settingPath: join(home, '.claude', 'settings.json'), + settingScope: 'user settings', + value: 'auto', + recommendedPercent: 2, + removeOverride: false, + }) + }) + + it('defer-threshold payload sets removeOverride when the default threshold already defers', () => { + const home = makeDir('codeburn-home-') + const cwd = makeDir('codeburn-cwd-') + const mcpServers: Record = {} + for (let i = 0; i < 15; i++) mcpServers[`srv${i}`] = { command: 'x' } + writeJson(join(cwd, '.mcp.json'), { mcpServers }) + writeJson(join(home, '.claude', 'settings.json'), { env: { ENABLE_TOOL_SEARCH: 'auto:50' } }) + const finding = detectMcpDeferThreshold([project([makeSession({ turns: claudeTurns() })])], new Set([cwd]), home) + expect(finding!.apply).toMatchObject({ kind: 'defer-threshold', value: 'auto:50', removeOverride: true }) + }) +})