fix(review): skip RAG pruning for truncated trees

src/review/rag-index.ts

@@ -49,6 +49,7 @@ import {
  *  blob SHA (#4365) — git's own content hash, free on the tree response — used to skip re-embedding a file
  *  whose content hasn't changed since the last full index). */
 type TreeEntry = { path: string; size?: number | undefined; sha?: string | undefined };
+type RepoTree = { entries: TreeEntry[]; truncated: boolean };
 
 /**
  * Sort key that puts small, high-value manifest/config files (package.json, tsconfig*.json,
@@ -101,11 +102,11 @@ function ghHeaders(token: string | undefined, accept: string): Record<string, st
 
 /**
  * Fetch the FULL recursive git tree for a repo at `ref` and return only the blob (file) entries. Uses the
- * Git Trees API (`?recursive=1`) — one call yields the whole tree. Returns [] on any non-OK / error response
- * (fail-safe: a tree we can't read = nothing to index). `truncated` is honored (GitHub truncates very large
- * trees) — we index whatever it returned; the MAX_CHUNKS cap is the real bound anyway.
+ * Git Trees API (`?recursive=1`) — one call yields the whole tree. Returns null on any non-OK / error response
+ * (fail-safe: a tree we can't read = nothing to index). `truncated` is surfaced with the returned entries so
+ * callers can index the partial positive results without treating absent paths as evidence that files were removed.
  */
-async function fetchRepoTree(_env: Env, repoFullName: string, ref: string, token: string | undefined, admissionKey: GitHubRateLimitAdmissionKey | undefined): Promise<TreeEntry[] | null> {
+async function fetchRepoTree(_env: Env, repoFullName: string, ref: string, token: string | undefined, admissionKey: GitHubRateLimitAdmissionKey | undefined): Promise<RepoTree | null> {
   try {
     const { owner, name } = repoParts(repoFullName);
     const url = `https://api.github.com/repos/${encodeURIComponent(owner)}/${encodeURIComponent(name)}/git/trees/${encodeURIComponent(ref)}?recursive=1`;
@@ -116,7 +117,7 @@ async function fetchRepoTree(_env: Env, repoFullName: string, ref: string, token
       ...(admissionKey ? { githubRateLimitAdmissionKey: admissionKey } : {}),
     });
     if (!response.ok) return null;
-    const body = (await response.json()) as { tree?: Array<{ path?: string; type?: string; size?: number; sha?: string }> } | null;
+    const body = (await response.json()) as { tree?: Array<{ path?: string; type?: string; size?: number; sha?: string }>; truncated?: boolean } | null;
     const entries: TreeEntry[] = [];
     for (const node of body?.tree ?? []) {
       if (node.type !== "blob" || typeof node.path !== "string" || node.path.length === 0) continue;
@@ -126,7 +127,7 @@ async function fetchRepoTree(_env: Env, repoFullName: string, ref: string, token
         ...(typeof node.sha === "string" && node.sha.length > 0 ? { sha: node.sha } : {}),
       });
     }
-    return entries;
+    return { entries, truncated: body?.truncated === true };
   } catch (error) {
     console.error(JSON.stringify({ level: "error", event: "rag_index_tree_error", repo: repoFullName, message: String(error).slice(0, 200) }));
     return null;
@@ -299,14 +300,16 @@ export async function indexRepo(
     const ref = indexRef(repo.defaultBranch);
 
     // 1. Fetch the tree, filter to indexable code/docs, and prune retained chunks for files that disappeared
-    //    or moved to a non-indexable path. If the tree fetch fails (null), skip pruning to avoid deleting good
-    //    chunks during a transient GitHub/API failure.
-    const rawTree = await fetchRepoTree(env, repoFullName, ref, token, admissionKey);
-    if (rawTree === null) return empty;
-    const tree = rawTree
+    //    or moved to a non-indexable path. If the tree fetch fails (null) or is truncated, skip pruning to avoid
+    //    deleting good chunks without a complete view of the repository.
+    const repoTree = await fetchRepoTree(env, repoFullName, ref, token, admissionKey);
+    if (repoTree === null) return empty;
+    const tree = repoTree.entries
       .filter((entry) => isIndexablePath(entry.path, entry.size))
       .sort((a, b) => manifestPriority(a.path) - manifestPriority(b.path) || a.path.localeCompare(b.path));
-    await pruneMissingPaths(infra, project, repoName, new Set(tree.map((entry) => entry.path)));
+    // A truncated tree is valid positive evidence for every returned path, but absent paths may live in the
+    // unreturned tail. Skip destructive reconciliation while still indexing every usable entry GitHub returned.
+    if (!repoTree.truncated) await pruneMissingPaths(infra, project, repoName, new Set(tree.map((entry) => entry.path)));
     if (tree.length === 0) return empty;
 
     // 2. Fetch + chunk + upsert, stopping once the per-repo vector cap is reached. `stored` seeds from the

test/unit/rag-index.test.ts

@@ -75,13 +75,14 @@ function stubGithub(opts: {
   tree?: Array<{ path: string; type?: string; size?: number; sha?: string }>;
   files?: Record<string, string>;
   treeStatus?: number;
+  treeTruncated?: boolean;
 }) {
   const files = opts.files ?? {};
   vi.stubGlobal("fetch", async (input: RequestInfo | URL) => {
     const url = input.toString();
     if (url.includes("/git/trees/")) {
       if (opts.treeStatus && opts.treeStatus !== 200) return new Response("err", { status: opts.treeStatus });
-      return Response.json({ tree: (opts.tree ?? []).map((n) => ({ type: "blob", ...n })), truncated: false });
+      return Response.json({ tree: (opts.tree ?? []).map((n) => ({ type: "blob", ...n })), truncated: opts.treeTruncated ?? false });
     }
     if (url.includes("/contents/")) {
       // Decode the path back out of the URL to look up the canned file body.
@@ -159,7 +160,7 @@ describe("indexRepo: full repo index (tree → chunk → embed → upsert)", ()
     expect(vec.upserted).toContain(`${ns}|src/a.ts::0`);
   });
 
-  it("prunes chunks for paths missing from the current full tree before returning retrieved context", async () => {
+  it("prunes chunks for paths missing from a complete tree before returning retrieved context", async () => {
     const { env, vec } = indexEnv();
     const ns = ragNamespace(PROJECT, "gittensory");
     await env.DB.prepare("INSERT INTO repo_chunks (id, project, repo, path, chunk_index, kind, text) VALUES (?,?,?,?,?,?,?)")
@@ -178,6 +179,27 @@ describe("indexRepo: full repo index (tree → chunk → embed → upsert)", ()
     expect(vec.deleted).toContain(`${ns}|src/deleted-secret.ts::0`);
   });
 
+  it("indexes entries from a truncated tree without pruning paths that may be in the unreturned tail", async () => {
+    const { env, vec } = indexEnv();
+    const ns = ragNamespace(PROJECT, "gittensory");
+    await env.DB.prepare("INSERT INTO repo_chunks (id, project, repo, path, chunk_index, kind, text) VALUES (?,?,?,?,?,?,?)")
+      .bind(`${ns}|src/unreturned-tail.ts::0`, PROJECT, "gittensory", "src/unreturned-tail.ts", 0, "code", "still present beyond the truncated response")
+      .run();
+
+    stubGithub({
+      tree: [{ path: "src/current.ts", size: 30 }],
+      files: { "src/current.ts": "export const current = 1;\n" },
+      treeTruncated: true,
+    });
+
+    const result = await indexRepo(env, PROJECT, REPO);
+
+    expect(result.files).toBe(1);
+    expect(result.indexed).toBe(1);
+    expect(await pathsFor(env, PROJECT, "gittensory")).toEqual(["src/current.ts", "src/unreturned-tail.ts"]);
+    expect(vec.deleted).not.toContain(`${ns}|src/unreturned-tail.ts::0`);
+  });
+
   it("skips a file that fails to fetch (404) and indexes the rest (fail-safe)", async () => {
     const { env } = indexEnv();
     stubGithub({

test/unit/routes-gate-outcome-breakdown.test.ts

@@ -1,10 +1,12 @@
-import { afterEach, describe, expect, it, vi } from "vitest";
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
 import { createApp } from "../../src/api/routes";
 import { createSessionForGitHubUser } from "../../src/auth/security";
 import { recordAuditEvent, upsertInstallation, upsertRepositoryFromGitHub } from "../../src/db/repositories";
 import { GATE_OUTCOME_BREAKDOWN_WINDOW_DAYS, buildGateOutcomeBreakdown, classifyGateOutcomeAuditBucket } from "../../src/services/gate-outcome-breakdown";
 import { createTestEnv } from "../helpers/d1";
 
+const TEST_NOW = "2026-07-11T12:00:00.000Z";
+
 function stubMinerDetection(): void {
   vi.stubGlobal("fetch", async (input: RequestInfo | URL) => {
     if (input.toString().includes("gittensor.io")) return Response.json([]);
@@ -27,14 +29,22 @@ async function seedOwnedRepo(env: Env, owner: string, name: string, installation
 }
 
 describe("GET /v1/app/maintainer-dashboard gateOutcomeBreakdown (#2203)", () => {
-  afterEach(() => vi.unstubAllGlobals());
+  beforeEach(() => {
+    vi.useFakeTimers({ toFake: ["Date"] });
+    vi.setSystemTime(new Date(TEST_NOW));
+  });
+
+  afterEach(() => {
+    vi.useRealTimers();
+    vi.unstubAllGlobals();
+  });
 
   it("surfaces repo-scoped gate-outcome counts on qualityDashboard for an owner session", async () => {
     const app = createApp();
     const env = createTestEnv({ ADMIN_GITHUB_LOGINS: "" });
     await seedOwnedRepo(env, "owner", "repo", 101);
     stubMinerDetection();
-    const now = "2026-07-11T12:00:00.000Z";
+    const now = TEST_NOW;
     await recordAuditEvent(env, {
       eventType: "agent.action.merge",
       actor: "loopover",
@@ -90,7 +100,7 @@ describe("GET /v1/app/maintainer-dashboard gateOutcomeBreakdown (#2203)", () =>
     const env = createTestEnv({ ADMIN_GITHUB_LOGINS: "" });
     await seedOwnedRepo(env, "owner", "repo", 101);
     stubMinerDetection();
-    const now = "2026-07-11T12:00:00.000Z";
+    const now = TEST_NOW;
     await recordAuditEvent(env, {
       eventType: "agent.action.merge",
       actor: "loopover",