diff --git a/src/review/rag-index.ts b/src/review/rag-index.ts index 69df3a881..b1f025a70 100644 --- a/src/review/rag-index.ts +++ b/src/review/rag-index.ts @@ -49,6 +49,7 @@ import { * blob SHA (#4365) — git's own content hash, free on the tree response — used to skip re-embedding a file * whose content hasn't changed since the last full index). */ type TreeEntry = { path: string; size?: number | undefined; sha?: string | undefined }; +type RepoTree = { entries: TreeEntry[]; truncated: boolean }; /** * Sort key that puts small, high-value manifest/config files (package.json, tsconfig*.json, @@ -101,11 +102,11 @@ function ghHeaders(token: string | undefined, accept: string): Record { +async function fetchRepoTree(_env: Env, repoFullName: string, ref: string, token: string | undefined, admissionKey: GitHubRateLimitAdmissionKey | undefined): Promise { try { const { owner, name } = repoParts(repoFullName); const url = `https://api.github.com/repos/${encodeURIComponent(owner)}/${encodeURIComponent(name)}/git/trees/${encodeURIComponent(ref)}?recursive=1`; @@ -116,7 +117,7 @@ async function fetchRepoTree(_env: Env, repoFullName: string, ref: string, token ...(admissionKey ? { githubRateLimitAdmissionKey: admissionKey } : {}), }); if (!response.ok) return null; - const body = (await response.json()) as { tree?: Array<{ path?: string; type?: string; size?: number; sha?: string }> } | null; + const body = (await response.json()) as { tree?: Array<{ path?: string; type?: string; size?: number; sha?: string }>; truncated?: boolean } | null; const entries: TreeEntry[] = []; for (const node of body?.tree ?? []) { if (node.type !== "blob" || typeof node.path !== "string" || node.path.length === 0) continue; @@ -126,7 +127,7 @@ async function fetchRepoTree(_env: Env, repoFullName: string, ref: string, token ...(typeof node.sha === "string" && node.sha.length > 0 ? { sha: node.sha } : {}), }); } - return entries; + return { entries, truncated: body?.truncated === true }; } catch (error) { console.error(JSON.stringify({ level: "error", event: "rag_index_tree_error", repo: repoFullName, message: String(error).slice(0, 200) })); return null; @@ -299,14 +300,16 @@ export async function indexRepo( const ref = indexRef(repo.defaultBranch); // 1. Fetch the tree, filter to indexable code/docs, and prune retained chunks for files that disappeared - // or moved to a non-indexable path. If the tree fetch fails (null), skip pruning to avoid deleting good - // chunks during a transient GitHub/API failure. - const rawTree = await fetchRepoTree(env, repoFullName, ref, token, admissionKey); - if (rawTree === null) return empty; - const tree = rawTree + // or moved to a non-indexable path. If the tree fetch fails (null) or is truncated, skip pruning to avoid + // deleting good chunks without a complete view of the repository. + const repoTree = await fetchRepoTree(env, repoFullName, ref, token, admissionKey); + if (repoTree === null) return empty; + const tree = repoTree.entries .filter((entry) => isIndexablePath(entry.path, entry.size)) .sort((a, b) => manifestPriority(a.path) - manifestPriority(b.path) || a.path.localeCompare(b.path)); - await pruneMissingPaths(infra, project, repoName, new Set(tree.map((entry) => entry.path))); + // A truncated tree is valid positive evidence for every returned path, but absent paths may live in the + // unreturned tail. Skip destructive reconciliation while still indexing every usable entry GitHub returned. + if (!repoTree.truncated) await pruneMissingPaths(infra, project, repoName, new Set(tree.map((entry) => entry.path))); if (tree.length === 0) return empty; // 2. Fetch + chunk + upsert, stopping once the per-repo vector cap is reached. `stored` seeds from the diff --git a/test/unit/rag-index.test.ts b/test/unit/rag-index.test.ts index 09e16c774..a4fd9d161 100644 --- a/test/unit/rag-index.test.ts +++ b/test/unit/rag-index.test.ts @@ -75,13 +75,14 @@ function stubGithub(opts: { tree?: Array<{ path: string; type?: string; size?: number; sha?: string }>; files?: Record; treeStatus?: number; + treeTruncated?: boolean; }) { const files = opts.files ?? {}; vi.stubGlobal("fetch", async (input: RequestInfo | URL) => { const url = input.toString(); if (url.includes("/git/trees/")) { if (opts.treeStatus && opts.treeStatus !== 200) return new Response("err", { status: opts.treeStatus }); - return Response.json({ tree: (opts.tree ?? []).map((n) => ({ type: "blob", ...n })), truncated: false }); + return Response.json({ tree: (opts.tree ?? []).map((n) => ({ type: "blob", ...n })), truncated: opts.treeTruncated ?? false }); } if (url.includes("/contents/")) { // Decode the path back out of the URL to look up the canned file body. @@ -159,7 +160,7 @@ describe("indexRepo: full repo index (tree → chunk → embed → upsert)", () expect(vec.upserted).toContain(`${ns}|src/a.ts::0`); }); - it("prunes chunks for paths missing from the current full tree before returning retrieved context", async () => { + it("prunes chunks for paths missing from a complete tree before returning retrieved context", async () => { const { env, vec } = indexEnv(); const ns = ragNamespace(PROJECT, "gittensory"); await env.DB.prepare("INSERT INTO repo_chunks (id, project, repo, path, chunk_index, kind, text) VALUES (?,?,?,?,?,?,?)") @@ -178,6 +179,27 @@ describe("indexRepo: full repo index (tree → chunk → embed → upsert)", () expect(vec.deleted).toContain(`${ns}|src/deleted-secret.ts::0`); }); + it("indexes entries from a truncated tree without pruning paths that may be in the unreturned tail", async () => { + const { env, vec } = indexEnv(); + const ns = ragNamespace(PROJECT, "gittensory"); + await env.DB.prepare("INSERT INTO repo_chunks (id, project, repo, path, chunk_index, kind, text) VALUES (?,?,?,?,?,?,?)") + .bind(`${ns}|src/unreturned-tail.ts::0`, PROJECT, "gittensory", "src/unreturned-tail.ts", 0, "code", "still present beyond the truncated response") + .run(); + + stubGithub({ + tree: [{ path: "src/current.ts", size: 30 }], + files: { "src/current.ts": "export const current = 1;\n" }, + treeTruncated: true, + }); + + const result = await indexRepo(env, PROJECT, REPO); + + expect(result.files).toBe(1); + expect(result.indexed).toBe(1); + expect(await pathsFor(env, PROJECT, "gittensory")).toEqual(["src/current.ts", "src/unreturned-tail.ts"]); + expect(vec.deleted).not.toContain(`${ns}|src/unreturned-tail.ts::0`); + }); + it("skips a file that fails to fetch (404) and indexes the rest (fail-safe)", async () => { const { env } = indexEnv(); stubGithub({ diff --git a/test/unit/routes-gate-outcome-breakdown.test.ts b/test/unit/routes-gate-outcome-breakdown.test.ts index 210705e93..2f481b501 100644 --- a/test/unit/routes-gate-outcome-breakdown.test.ts +++ b/test/unit/routes-gate-outcome-breakdown.test.ts @@ -1,10 +1,12 @@ -import { afterEach, describe, expect, it, vi } from "vitest"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { createApp } from "../../src/api/routes"; import { createSessionForGitHubUser } from "../../src/auth/security"; import { recordAuditEvent, upsertInstallation, upsertRepositoryFromGitHub } from "../../src/db/repositories"; import { GATE_OUTCOME_BREAKDOWN_WINDOW_DAYS, buildGateOutcomeBreakdown, classifyGateOutcomeAuditBucket } from "../../src/services/gate-outcome-breakdown"; import { createTestEnv } from "../helpers/d1"; +const TEST_NOW = "2026-07-11T12:00:00.000Z"; + function stubMinerDetection(): void { vi.stubGlobal("fetch", async (input: RequestInfo | URL) => { if (input.toString().includes("gittensor.io")) return Response.json([]); @@ -27,14 +29,22 @@ async function seedOwnedRepo(env: Env, owner: string, name: string, installation } describe("GET /v1/app/maintainer-dashboard gateOutcomeBreakdown (#2203)", () => { - afterEach(() => vi.unstubAllGlobals()); + beforeEach(() => { + vi.useFakeTimers({ toFake: ["Date"] }); + vi.setSystemTime(new Date(TEST_NOW)); + }); + + afterEach(() => { + vi.useRealTimers(); + vi.unstubAllGlobals(); + }); it("surfaces repo-scoped gate-outcome counts on qualityDashboard for an owner session", async () => { const app = createApp(); const env = createTestEnv({ ADMIN_GITHUB_LOGINS: "" }); await seedOwnedRepo(env, "owner", "repo", 101); stubMinerDetection(); - const now = "2026-07-11T12:00:00.000Z"; + const now = TEST_NOW; await recordAuditEvent(env, { eventType: "agent.action.merge", actor: "loopover", @@ -90,7 +100,7 @@ describe("GET /v1/app/maintainer-dashboard gateOutcomeBreakdown (#2203)", () => const env = createTestEnv({ ADMIN_GITHUB_LOGINS: "" }); await seedOwnedRepo(env, "owner", "repo", 101); stubMinerDetection(); - const now = "2026-07-11T12:00:00.000Z"; + const now = TEST_NOW; await recordAuditEvent(env, { eventType: "agent.action.merge", actor: "loopover",