From a9f1b02f34fe985db3cf1612b91d11215d9af57b Mon Sep 17 00:00:00 2001 From: Dean Valentine Date: Thu, 5 Mar 2026 21:25:07 -0800 Subject: [PATCH 01/15] fix(public,frontend): tighten search filter and install URL --- .../api/src/lib/services/public-read-model.ts | 110 +++++++++++------- ...points.public-and-auth.integration.test.ts | 67 +++++++++++ .../frontend/src/lib/extension-install-url.ts | 2 + .../frontend/src/routes/+layout.svelte | 3 +- .../frontend/src/routes/+page.svelte | 6 +- .../frontend/test/e2e/smoke.spec.ts | 15 +++ 6 files changed, 160 insertions(+), 43 deletions(-) create mode 100644 src/typescript/frontend/src/lib/extension-install-url.ts diff --git a/src/typescript/api/src/lib/services/public-read-model.ts b/src/typescript/api/src/lib/services/public-read-model.ts index 1560129..6c798f8 100644 --- a/src/typescript/api/src/lib/services/public-read-model.ts +++ b/src/typescript/api/src/lib/services/public-read-model.ts @@ -80,6 +80,10 @@ interface PublicSearchInvestigationsResult { hasMore: boolean; } +interface SearchInvestigationPageRow { + id: string; +} + interface PublicMetricsResult { totalInvestigatedPosts: number; investigatedPostsWithFlags: number; @@ -105,6 +109,10 @@ function parsePlatform(value: string): Platform { return platformSchema.parse(value); } +function escapeLikePattern(query: string): string { + return query.replace(/[\\%_]/g, (char) => `\\${char}`); +} + export class PublicReadModelInvariantError extends Error { constructor(message: string) { super(message); @@ -190,6 +198,36 @@ function publicMetricsConditions(input: PublicMetricsInput): Prisma.Sql[] { return conditions; } +async function loadSearchInvestigationPageRows( + prisma: PrismaClient, + input: PublicSearchInvestigationsInput, +): Promise { + const platformFilter = + input.platform === undefined ? Prisma.empty : Prisma.sql`AND p."platform" = ${input.platform}`; + const textFilter = + input.query === undefined + ? Prisma.empty + : Prisma.sql`AND cb."contentText" ILIKE ${`%${escapeLikePattern(input.query)}%`} ESCAPE '\\'`; + + const minimumClaimCount = input.minClaimCount ?? 0; + + return prisma.$queryRaw` + SELECT i."id" + FROM "Investigation" i + JOIN "PostVersion" pv ON pv."id" = i."postVersionId" + JOIN "Post" p ON p."id" = pv."postId" + JOIN "ContentBlob" cb ON cb."id" = pv."contentBlobId" + LEFT JOIN "Claim" c ON c."investigationId" = i."id" + WHERE i."status" = 'COMPLETE' + ${platformFilter} + ${textFilter} + GROUP BY i."id", i."checkedAt" + HAVING COUNT(c."id") >= ${minimumClaimCount} + ORDER BY i."checkedAt" DESC NULLS LAST, i."id" DESC + LIMIT ${input.limit + 1} OFFSET ${input.offset} + `; +} + export async function getPublicInvestigationById( prisma: PrismaClient, investigationId: string, @@ -304,6 +342,9 @@ export async function getPublicPostInvestigations( }, input: true, claims: { + orderBy: { + id: "asc", + }, select: { id: true, summary: true, @@ -351,47 +392,23 @@ export async function searchPublicInvestigations( prisma: PrismaClient, input: PublicSearchInvestigationsInput, ): Promise { + const pageRows = await loadSearchInvestigationPageRows(prisma, input); + const hasMore = pageRows.length > input.limit; + const pageIds = pageRows.slice(0, input.limit).map((row) => row.id); + + if (pageIds.length === 0) { + return { + investigations: [], + hasMore, + }; + } + const investigations = await prisma.investigation.findMany({ where: { - status: "COMPLETE", - ...(input.minClaimCount !== undefined && input.minClaimCount > 0 - ? { claims: { some: {} } } - : {}), - ...(input.platform === undefined && input.query === undefined - ? {} - : { - postVersion: { - ...(input.platform === undefined - ? {} - : { - post: { - platform: input.platform, - }, - }), - ...(input.query === undefined - ? {} - : { - contentBlob: { - contentText: { - contains: input.query, - mode: "insensitive", - }, - }, - }), - }, - }), - }, - orderBy: [ - { - checkedAt: { - sort: "desc", - nulls: "last", - }, + id: { + in: pageIds, }, - { id: "desc" }, - ], - skip: input.offset, - take: input.limit + 1, + }, include: { postVersion: { select: { @@ -412,6 +429,9 @@ export async function searchPublicInvestigations( }, input: true, claims: { + orderBy: { + id: "asc", + }, select: { id: true, summary: true, @@ -426,11 +446,19 @@ export async function searchPublicInvestigations( }, }); - const hasMore = investigations.length > input.limit; - const page = hasMore ? investigations.slice(0, input.limit) : investigations; + const investigationsById = new Map( + investigations.map((investigation) => [investigation.id, investigation]), + ); return { - investigations: page.map((investigation) => { + investigations: pageIds.map((investigationId) => { + const investigation = investigationsById.get(investigationId); + if (investigation === undefined) { + invariantViolation( + `searchPublicInvestigations loaded page id ${investigationId} but could not hydrate it`, + ); + } + const lifecycle = parsePublicLifecycle({ investigationId: investigation.id, provenance: investigation.input.provenance, diff --git a/src/typescript/api/test/integration/api-endpoints.public-and-auth.integration.test.ts b/src/typescript/api/test/integration/api-endpoints.public-and-auth.integration.test.ts index 11d8f8f..cee6b54 100644 --- a/src/typescript/api/test/integration/api-endpoints.public-and-auth.integration.test.ts +++ b/src/typescript/api/test/integration/api-endpoints.public-and-auth.integration.test.ts @@ -782,6 +782,21 @@ void test("public.searchInvestigations filters by query/platform and includes fa }); await seedClaimWithSource(moonInvestigation.id, 1); + const multiClaimMoonPost = await seedPost({ + platform: "LESSWRONG", + externalId: "public-search-moon-2", + url: "https://www.lesswrong.com/posts/public-search-moon-2", + contentText: `${moonMarker} gamma`, + }); + const multiClaimMoonInvestigation = await seedCompleteInvestigation({ + postId: multiClaimMoonPost.id, + contentHash: multiClaimMoonPost.contentHash, + contentText: multiClaimMoonPost.contentText, + provenance: "SERVER_VERIFIED", + }); + await seedClaimWithSource(multiClaimMoonInvestigation.id, 2); + await seedClaimWithSource(multiClaimMoonInvestigation.id, 3); + const xPost = await seedPost({ platform: "X", externalId: "public-search-x-1", @@ -816,12 +831,25 @@ void test("public.searchInvestigations filters by query/platform and includes fa const queryIds = new Set(queryResult.investigations.map((item) => item.id)); assert.equal(queryIds.has(moonInvestigation.id), true); + assert.equal(queryIds.has(multiClaimMoonInvestigation.id), true); assert.equal(queryIds.has(fallbackMoonInvestigation.id), true); assert.equal( queryResult.investigations.every((item) => item.platform === "LESSWRONG"), true, ); + const minClaimCountResult = await caller.public.searchInvestigations({ + query: moonMarker, + minClaimCount: 2, + limit: 20, + offset: 0, + }); + + assert.deepEqual( + minClaimCountResult.investigations.map((item) => item.id), + [multiClaimMoonInvestigation.id], + ); + const platformResult = await caller.public.searchInvestigations({ platform: "X", limit: 20, @@ -866,7 +894,46 @@ void test("public.searchInvestigations filters by query/platform and includes fa graphqlResult.searchInvestigations.investigations.map((item) => item.id), ); assert.equal(graphqlIds.has(moonInvestigation.id), true); + assert.equal(graphqlIds.has(multiClaimMoonInvestigation.id), true); assert.equal(graphqlIds.has(fallbackMoonInvestigation.id), true); + + const minClaimCountGraphqlResult = await queryPublicGraphql<{ + searchInvestigations: { + investigations: { + id: string; + }[]; + }; + }>( + ` + query SearchInvestigationsWithMinClaimCount( + $query: String! + $minClaimCount: Int! + $limit: Int! + $offset: Int! + ) { + searchInvestigations( + query: $query + minClaimCount: $minClaimCount + limit: $limit + offset: $offset + ) { + investigations { + id + } + } + } + `, + { + query: moonMarker, + minClaimCount: 2, + limit: 20, + offset: 0, + }, + ); + assert.deepEqual( + minClaimCountGraphqlResult.searchInvestigations.investigations.map((item) => item.id), + [multiClaimMoonInvestigation.id], + ); }); void test("public.getMetrics counts all complete investigations and honors filters", async () => { diff --git a/src/typescript/frontend/src/lib/extension-install-url.ts b/src/typescript/frontend/src/lib/extension-install-url.ts new file mode 100644 index 0000000..43323e4 --- /dev/null +++ b/src/typescript/frontend/src/lib/extension-install-url.ts @@ -0,0 +1,2 @@ +export const CHROME_WEB_STORE_URL = + "https://chromewebstore.google.com/detail/openerrata/iflopmpcoifkihfimncdjkokibdfkkbd"; diff --git a/src/typescript/frontend/src/routes/+layout.svelte b/src/typescript/frontend/src/routes/+layout.svelte index 10f5e25..a7bb634 100644 --- a/src/typescript/frontend/src/routes/+layout.svelte +++ b/src/typescript/frontend/src/routes/+layout.svelte @@ -1,5 +1,6 @@ @@ -21,7 +22,7 @@ >GitHub Install Extension + import { CHROME_WEB_STORE_URL } from "$lib/extension-install-url"; + + OpenErrata - AI-Powered Fact-Checking for the Web @@ -13,7 +17,7 @@

Install for Chrome { test("renders hero and key sections", async ({ page }) => { await page.goto("/"); @@ -48,6 +51,14 @@ test.describe("Landing page", () => { await expect(nav.getByText("Corrections")).toBeVisible(); await expect(nav.getByText("GitHub")).toBeVisible(); await expect(nav.getByText("Install Extension")).toBeVisible(); + await expect(nav.getByRole("link", { name: "Install Extension" })).toHaveAttribute( + "href", + EXPECTED_EXTENSION_URL, + ); + await expect(page.getByRole("link", { name: "Install for Chrome" })).toHaveAttribute( + "href", + EXPECTED_EXTENSION_URL, + ); }); }); @@ -74,6 +85,10 @@ test.describe("Corrections page", () => { await expect(nav.getByText("OpenErrata")).toBeVisible(); await expect(nav.getByText("Corrections")).toBeVisible(); await expect(nav.getByText("Install Extension")).toBeVisible(); + await expect(nav.getByRole("link", { name: "Install Extension" })).toHaveAttribute( + "href", + EXPECTED_EXTENSION_URL, + ); }); }); From db77f75083a131e186b6f63912b191c8e2b05c6c Mon Sep 17 00:00:00 2001 From: Dean Valentine Date: Thu, 5 Mar 2026 21:58:32 -0800 Subject: [PATCH 02/15] chore(extension): bump version to 0.3.3 --- src/typescript/extension/package.json | 2 +- src/typescript/extension/src/manifest.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/typescript/extension/package.json b/src/typescript/extension/package.json index 360e4f2..3a24bbc 100644 --- a/src/typescript/extension/package.json +++ b/src/typescript/extension/package.json @@ -1,6 +1,6 @@ { "name": "@openerrata/extension", - "version": "0.3.2", + "version": "0.3.3", "type": "module", "scripts": { "dev": "vite build --watch --mode development", diff --git a/src/typescript/extension/src/manifest.json b/src/typescript/extension/src/manifest.json index b835b69..936385c 100644 --- a/src/typescript/extension/src/manifest.json +++ b/src/typescript/extension/src/manifest.json @@ -1,7 +1,7 @@ { "manifest_version": 3, "name": "OpenErrata", - "version": "0.3.2", + "version": "0.3.3", "description": "Crowdsourced, inline LLM investigations of the things you're reading.", "permissions": ["activeTab", "storage", "scripting", "webNavigation", "alarms"], "host_permissions": ["https://*/*", "http://*/*"], From b8e7acf53ae78dba77176a82c59dd8df85590459 Mon Sep 17 00:00:00 2001 From: Dean Valentine Date: Thu, 5 Mar 2026 22:11:56 -0800 Subject: [PATCH 03/15] fix(api): validate configured OpenAI model --- src/typescript/api/src/lib/config/env.ts | 2 +- src/typescript/api/src/lib/config/startup.ts | 8 ++++++-- .../lib/services/openai-key-validation-core.ts | 11 +++++++++-- .../api/src/lib/services/openai-key-validation.ts | 10 +++++++++- src/typescript/api/test/unit/env-config.test.ts | 1 + .../test/unit/openai-key-validation-core.test.ts | 15 +++++++++++++++ 6 files changed, 41 insertions(+), 6 deletions(-) diff --git a/src/typescript/api/src/lib/config/env.ts b/src/typescript/api/src/lib/config/env.ts index 7488e16..5f4c521 100644 --- a/src/typescript/api/src/lib/config/env.ts +++ b/src/typescript/api/src/lib/config/env.ts @@ -42,7 +42,7 @@ const baseEnvironmentSchema = z.object({ "DATABASE_URL must use postgres:// or postgresql://", ), OPENAI_API_KEY: z.string().trim().min(1).optional(), - OPENAI_MODEL_ID: z.string().trim().min(1).default("gpt-5.4-thinking"), + OPENAI_MODEL_ID: z.string().trim().min(1).default("gpt-5.4"), OPENAI_MAX_RESPONSE_TOOL_ROUNDS: positiveIntegerFromEnv.default(150), WORKER_CONCURRENCY: positiveIntegerFromEnv.default(250), HMAC_SECRET: z.string().trim().min(1, "HMAC_SECRET is required"), diff --git a/src/typescript/api/src/lib/config/startup.ts b/src/typescript/api/src/lib/config/startup.ts index aa48a72..6e55791 100644 --- a/src/typescript/api/src/lib/config/startup.ts +++ b/src/typescript/api/src/lib/config/startup.ts @@ -1,6 +1,6 @@ import OpenAI from "openai"; import { getPrisma } from "$lib/db/client"; -import { requireOpenAiApiKey } from "./env.js"; +import { getEnv, requireOpenAiApiKey } from "./env.js"; type StartupComponent = "api" | "worker" | "selector"; @@ -36,7 +36,11 @@ async function assertDatabaseCredentials(component: StartupComponent): Promise { try { const client = new OpenAI({ apiKey: requireOpenAiApiKey() }); - await client.models.list(); + await client.responses.create({ + model: getEnv().OPENAI_MODEL_ID, + input: "Reply with the single word pong.", + max_output_tokens: 1, + }); } catch (error) { throw new Error(`[startup:${component}] OpenAI credential check failed`, { cause: error }); } diff --git a/src/typescript/api/src/lib/services/openai-key-validation-core.ts b/src/typescript/api/src/lib/services/openai-key-validation-core.ts index 5e2ed51..f7f58a9 100644 --- a/src/typescript/api/src/lib/services/openai-key-validation-core.ts +++ b/src/typescript/api/src/lib/services/openai-key-validation-core.ts @@ -47,14 +47,21 @@ export async function validateOpenAiApiKeyForSettingsWithReachability( } const statusOutcome = classifyOpenAiKeyValidationStatus(readOpenAiStatusCode(error)); - if (statusOutcome) { + if (statusOutcome && statusOutcome.openaiApiKeyStatus !== "error") { return statusOutcome; } + const specificMessage = readErrorMessage(error); + if (specificMessage !== null) { + return { + openaiApiKeyStatus: "error", + openaiApiKeyMessage: specificMessage, + }; + } + return { openaiApiKeyStatus: "error", openaiApiKeyMessage: - readErrorMessage(error) ?? "Could not validate this key with OpenAI. Check outbound network access and retry.", }; } diff --git a/src/typescript/api/src/lib/services/openai-key-validation.ts b/src/typescript/api/src/lib/services/openai-key-validation.ts index afbfda5..d9c8fd5 100644 --- a/src/typescript/api/src/lib/services/openai-key-validation.ts +++ b/src/typescript/api/src/lib/services/openai-key-validation.ts @@ -1,5 +1,6 @@ import { OPENAI_KEY_VALIDATION_TIMEOUT_MS } from "@openerrata/shared"; import OpenAI from "openai"; +import { getEnv } from "$lib/config/env.js"; import { validateOpenAiApiKeyForSettingsWithReachability, type OpenAiKeyValidationStatusOutcome, @@ -13,7 +14,14 @@ async function validateOpenAiApiKeyReachability(openAiApiKey: string): Promise { assert.equal(environment.BLOB_STORAGE_REGION, "us-west-2"); assert.equal(environment.BLOB_STORAGE_ENDPOINT, undefined); assert.equal(environment.WORKER_CONCURRENCY, 250); + assert.equal(environment.OPENAI_MODEL_ID, "gpt-5.4"); }); test("accepts a custom worker concurrency", () => { diff --git a/src/typescript/api/test/unit/openai-key-validation-core.test.ts b/src/typescript/api/test/unit/openai-key-validation-core.test.ts index 7ec8a4b..73ea8f8 100644 --- a/src/typescript/api/test/unit/openai-key-validation-core.test.ts +++ b/src/typescript/api/test/unit/openai-key-validation-core.test.ts @@ -57,6 +57,21 @@ test("validateOpenAiApiKeyForSettingsWithReachability maps known OpenAI status c openaiApiKeyMessage: "OpenAI authenticated this key, but access is restricted for validation checks.", }); + + const modelNotFoundResult = await validateOpenAiApiKeyForSettingsWithReachability( + "sk-model-test-key-abcdefghijklmnopqrstuvwxyz", + async () => { + const error = new Error("The requested model 'gpt-5.4-thinking' does not exist.") as Error & { + status: number; + }; + error.status = 400; + throw error; + }, + ); + assert.deepEqual(modelNotFoundResult, { + openaiApiKeyStatus: "error", + openaiApiKeyMessage: "The requested model 'gpt-5.4-thinking' does not exist.", + }); }); test("validateOpenAiApiKeyForSettingsWithReachability handles timeout and generic failures", async () => { From 94672c8ab1142d8b5a66937220f90b44e5e344f2 Mon Sep 17 00:00:00 2001 From: Dean Valentine Date: Thu, 5 Mar 2026 22:17:13 -0800 Subject: [PATCH 04/15] fix(api): use valid output budget in OpenAI probe --- src/typescript/api/src/lib/config/startup.ts | 2 +- src/typescript/api/src/lib/services/openai-key-validation.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/typescript/api/src/lib/config/startup.ts b/src/typescript/api/src/lib/config/startup.ts index 6e55791..ed9b9fd 100644 --- a/src/typescript/api/src/lib/config/startup.ts +++ b/src/typescript/api/src/lib/config/startup.ts @@ -39,7 +39,7 @@ async function assertOpenAiCredentials(component: StartupComponent): Promise Date: Thu, 5 Mar 2026 22:49:09 -0800 Subject: [PATCH 05/15] fix(frontend): render investigation reasoning as markdown --- src/typescript/frontend/package.json | 2 + .../frontend/src/lib/claim-markdown.ts | 22 ++++++ .../src/routes/corrections/[id]/+page.svelte | 77 ++++++++++++++++++- .../frontend/test/unit/claim-markdown.test.ts | 24 ++++++ src/typescript/pnpm-lock.yaml | 6 ++ 5 files changed, 129 insertions(+), 2 deletions(-) create mode 100644 src/typescript/frontend/src/lib/claim-markdown.ts create mode 100644 src/typescript/frontend/test/unit/claim-markdown.test.ts diff --git a/src/typescript/frontend/package.json b/src/typescript/frontend/package.json index 28dab55..727c6ff 100644 --- a/src/typescript/frontend/package.json +++ b/src/typescript/frontend/package.json @@ -15,11 +15,13 @@ "dependencies": { "@sveltejs/adapter-node": "^5.5.3", "@sveltejs/kit": "^2.52.2", + "markdown-it": "^14.1.1", "svelte": "^5.53.0" }, "devDependencies": { "@playwright/test": "^1.56.1", "@sveltejs/vite-plugin-svelte": "^6.2.4", + "@types/markdown-it": "^14.1.2", "svelte-check": "^4.2.1", "tsx": "^4.21.0", "typescript": "^5.9.3", diff --git a/src/typescript/frontend/src/lib/claim-markdown.ts b/src/typescript/frontend/src/lib/claim-markdown.ts new file mode 100644 index 0000000..c1075e3 --- /dev/null +++ b/src/typescript/frontend/src/lib/claim-markdown.ts @@ -0,0 +1,22 @@ +import MarkdownIt from "markdown-it"; + +const markdownRenderer = new MarkdownIt({ + html: false, + linkify: true, + breaks: true, +}); + +const defaultLinkOpenRenderer = + markdownRenderer.renderer.rules["link_open"] ?? + ((tokens, index, options, _env, self) => self.renderToken(tokens, index, options)); + +markdownRenderer.renderer.rules["link_open"] = (tokens, index, options, env, self) => { + const token = tokens[index]; + token?.attrSet("target", "_blank"); + token?.attrSet("rel", "noopener noreferrer"); + return defaultLinkOpenRenderer(tokens, index, options, env, self); +}; + +export function renderClaimReasoningHtml(markdown: string): string { + return markdownRenderer.render(markdown); +} diff --git a/src/typescript/frontend/src/routes/corrections/[id]/+page.svelte b/src/typescript/frontend/src/routes/corrections/[id]/+page.svelte index 9131867..ab116a7 100644 --- a/src/typescript/frontend/src/routes/corrections/[id]/+page.svelte +++ b/src/typescript/frontend/src/routes/corrections/[id]/+page.svelte @@ -1,6 +1,7 @@ '); + + assert.equal(html.includes(" @@ -111,7 +115,7 @@ {/if}
0}> - {investigation.claimCount} correction{investigation.claimCount !== 1 ? "s" : ""} + {correctionCountLabel(investigation.claimCount)} →
diff --git a/src/typescript/frontend/src/routes/corrections/[id]/+page.svelte b/src/typescript/frontend/src/routes/corrections/[id]/+page.svelte index ab116a7..fca8a42 100644 --- a/src/typescript/frontend/src/routes/corrections/[id]/+page.svelte +++ b/src/typescript/frontend/src/routes/corrections/[id]/+page.svelte @@ -36,6 +36,20 @@ } return withoutProtocol.substring(0, maxLength) + "..."; } + + function correctionsFoundLabel(count: number): string { + return count === 1 ? "Correction found" : `${count} corrections found`; + } + + function correctionsMetaDescription(count: number, platformLabel: string): string { + return count === 1 + ? `OpenErrata found a correction for this ${platformLabel} post.` + : `OpenErrata found ${count} corrections for this ${platformLabel} post.`; + } + + function platformLabelFor(platform: PublicInvestigationResult["post"]["platform"]): string { + return platformLabels[platform] ?? platform; + } @@ -43,9 +57,10 @@ Corrections for {truncateUrl(result.post.url, 40)} - OpenErrata {:else} Investigation Not Found - OpenErrata @@ -72,7 +87,7 @@
- {platformLabels[result.post.platform]} + {platformLabelFor(result.post.platform)} {formatDate(result.investigation.checkedAt)} at {formatTime( @@ -98,7 +113,7 @@ {#if result.claims.length === 0} No corrections found. This post passed fact-checking with no issues. {:else} - {result.claims.length} correction{result.claims.length !== 1 ? "s" : ""} found + {correctionsFoundLabel(result.claims.length)} {/if}

From 9ee2cae636ea526d3dc7681efa4c8773bc39c4de Mon Sep 17 00:00:00 2001 From: Dean Valentine Date: Fri, 6 Mar 2026 11:36:04 -0800 Subject: [PATCH 08/15] fix(frontend): remove detail correction summary line --- .../src/routes/corrections/[id]/+page.svelte | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/src/typescript/frontend/src/routes/corrections/[id]/+page.svelte b/src/typescript/frontend/src/routes/corrections/[id]/+page.svelte index fca8a42..b7ba69d 100644 --- a/src/typescript/frontend/src/routes/corrections/[id]/+page.svelte +++ b/src/typescript/frontend/src/routes/corrections/[id]/+page.svelte @@ -37,10 +37,6 @@ return withoutProtocol.substring(0, maxLength) + "..."; } - function correctionsFoundLabel(count: number): string { - return count === 1 ? "Correction found" : `${count} corrections found`; - } - function correctionsMetaDescription(count: number, platformLabel: string): string { return count === 1 ? `OpenErrata found a correction for this ${platformLabel} post.` @@ -109,13 +105,6 @@ -

- {#if result.claims.length === 0} - No corrections found. This post passed fact-checking with no issues. - {:else} - {correctionsFoundLabel(result.claims.length)} - {/if} -

{#if result.claims.length > 0} @@ -280,11 +269,6 @@ flex-shrink: 0; } - .correction-summary { - font-size: 1rem; - color: var(--color-text-muted); - } - /* Claims */ .claims { display: flex; From 68701d8a66f9a3aa80d125f667b0da83c83b4ba0 Mon Sep 17 00:00:00 2001 From: Dean Valentine Date: Fri, 2 Oct 2026 17:06:28 -0700 Subject: [PATCH 09/15] shared: tighten the extension/API/public contract - http(s)-only URL schema for every link field - View statuses carry investigationId; image URLs travel only as occurrences; per-platform external ids; Wikipedia URL identity as a PAGE_ID | TITLE union - Typed content-script/background message maps; drop the protocol `v` field - Public output schemas: provenance-discriminated origin, typed provider - Drop the InvestigationModel enum (model is recorded as the provider id) - Normalization:
/
separate words; Wikipedia boilerplate excluded by structural markers and localized section titles Co-Authored-By: Claude Opus 5.5 --- src/typescript/shared/src/constants.ts | 4 +- src/typescript/shared/src/enums.ts | 8 - src/typescript/shared/src/index.ts | 1 + src/typescript/shared/src/normalize.ts | 24 +- src/typescript/shared/src/observed-images.ts | 22 + src/typescript/shared/src/schemas.ts | 29 +- src/typescript/shared/src/schemas/common.ts | 34 +- .../shared/src/schemas/extension-protocol.ts | 474 ++++++++++-------- .../shared/src/schemas/investigation.ts | 44 +- .../shared/src/schemas/public-api.ts | 73 ++- src/typescript/shared/src/types.ts | 56 +-- src/typescript/shared/src/version-identity.ts | 2 +- .../shared/src/wikipedia-canonicalization.ts | 159 +++++- .../shared/src/wikipedia-identity.ts | 73 ++- .../block-separator-exhaustiveness.test.ts | 41 +- .../test/unit/claim-payload-schema.test.ts | 48 ++ .../observed-image-occurrence-schema.test.ts | 73 ++- .../unit/wikipedia-canonicalization.test.ts | 85 +++- .../test/unit/wikipedia-identity.test.ts | 40 +- 19 files changed, 821 insertions(+), 469 deletions(-) create mode 100644 src/typescript/shared/src/observed-images.ts create mode 100644 src/typescript/shared/test/unit/claim-payload-schema.test.ts diff --git a/src/typescript/shared/src/constants.ts b/src/typescript/shared/src/constants.ts index 6f6970d..db14e33 100644 --- a/src/typescript/shared/src/constants.ts +++ b/src/typescript/shared/src/constants.ts @@ -1,8 +1,7 @@ -import type { InvestigationModel, InvestigationProvider } from "./enums.js"; +import type { InvestigationProvider } from "./enums.js"; export const WORD_COUNT_LIMIT = 10000; export const POLL_INTERVAL_MS = 5000; -export const EXTENSION_MESSAGE_PROTOCOL_VERSION = 1; export const MAX_BATCH_STATUS_POSTS = 100; export const MAX_IMAGES_PER_INVESTIGATION = 10; export const MAX_IMAGE_BYTES = 20_000_000; @@ -13,4 +12,3 @@ export const MAX_OBSERVED_CONTENT_TEXT_UTF8_BYTES = 500_000; export const OPENAI_KEY_VALIDATION_TIMEOUT_MS = 8_000; export const DEFAULT_INVESTIGATION_PROVIDER: InvestigationProvider = "OPENAI"; -export const DEFAULT_INVESTIGATION_MODEL: InvestigationModel = "OPENAI_GPT_5"; diff --git a/src/typescript/shared/src/enums.ts b/src/typescript/shared/src/enums.ts index 0300467..31ab858 100644 --- a/src/typescript/shared/src/enums.ts +++ b/src/typescript/shared/src/enums.ts @@ -13,14 +13,6 @@ export type MarkdownSource = (typeof MARKDOWN_SOURCE_VALUES)[number]; export const INVESTIGATION_PROVIDER_VALUES = ["OPENAI", "ANTHROPIC"] as const; export type InvestigationProvider = (typeof INVESTIGATION_PROVIDER_VALUES)[number]; -export const INVESTIGATION_MODEL_VALUES = [ - "OPENAI_GPT_5", - "OPENAI_GPT_5_MINI", - "ANTHROPIC_CLAUDE_SONNET", - "ANTHROPIC_CLAUDE_OPUS", -] as const; -export type InvestigationModel = (typeof INVESTIGATION_MODEL_VALUES)[number]; - export const SUPPORTED_IMAGE_MIME_TYPE_VALUES = [ "image/jpeg", "image/png", diff --git a/src/typescript/shared/src/index.ts b/src/typescript/shared/src/index.ts index a05f390..c19439b 100644 --- a/src/typescript/shared/src/index.ts +++ b/src/typescript/shared/src/index.ts @@ -5,6 +5,7 @@ export * from "./normalize.js"; export * from "./wikipedia-canonicalization.js"; export * from "./wikipedia-identity.js"; export * from "./image-occurrence-validation.js"; +export * from "./observed-images.js"; export * from "./optional-non-empty.js"; export * from "./constants.js"; export * from "./trpc-paths.js"; diff --git a/src/typescript/shared/src/normalize.ts b/src/typescript/shared/src/normalize.ts index 2de570f..1a39668 100644 --- a/src/typescript/shared/src/normalize.ts +++ b/src/typescript/shared/src/normalize.ts @@ -1,12 +1,3 @@ -/** - * Block-level HTML elements whose boundaries are treated as word separators - * during content normalization. Used by both the extension (DOM TreeWalker) - * and API (parse5 traversal) to ensure identical output on compact HTML where - * no whitespace text nodes exist between adjacent block elements. - * - * Must be kept in sync between client and server to prevent canonicalization drift. - * Spec §3.8. - */ /** * HTML tags that never contain article prose on any platform. * Text inside these elements is excluded unconditionally during @@ -14,7 +5,18 @@ */ export const NON_CONTENT_TAGS = new Set(["script", "style", "noscript"]); -export const CONTENT_BLOCK_SEPARATOR_TAGS = new Set([ +/** + * HTML elements whose boundaries separate words in extracted text: block-level + * elements, and the line-breaking void elements `br` and `hr`, which render + * the text either side of them on separate lines ("hard.
This" reads + * "hard. This", not "hard.This"). Used by both the extension (DOM TreeWalker) + * and API (parse5 traversal) to ensure identical output on compact HTML where + * no whitespace text nodes exist between such elements and their neighbours. + * + * Must be kept in sync between client and server to prevent canonicalization drift. + * Spec §3.8. + */ +export const WORD_SEPARATOR_TAGS = new Set([ "p", "li", "h1", @@ -29,6 +31,8 @@ export const CONTENT_BLOCK_SEPARATOR_TAGS = new Set([ "td", "th", "div", + "br", + "hr", ]); /** diff --git a/src/typescript/shared/src/observed-images.ts b/src/typescript/shared/src/observed-images.ts new file mode 100644 index 0000000..819a6e7 --- /dev/null +++ b/src/typescript/shared/src/observed-images.ts @@ -0,0 +1,22 @@ +interface ObservedImageSource { + originalIndex: number; + sourceUrl: string; +} + +/** + * The distinct image URLs of a post version, in page order. Image occurrences + * are the single wire representation of a post's images; this list is always + * derived from them rather than transmitted separately, so the two can never + * disagree. An absent occurrence list means no images were observed. + */ +export function observedImageUrlsFromOccurrences( + occurrences: readonly ObservedImageSource[] | undefined, +): string[] { + if (occurrences === undefined) { + return []; + } + const inPageOrder = [...occurrences].sort( + (left, right) => left.originalIndex - right.originalIndex, + ); + return Array.from(new Set(inPageOrder.map((occurrence) => occurrence.sourceUrl))); +} diff --git a/src/typescript/shared/src/schemas.ts b/src/typescript/shared/src/schemas.ts index 088476b..837dc43 100644 --- a/src/typescript/shared/src/schemas.ts +++ b/src/typescript/shared/src/schemas.ts @@ -1,9 +1,15 @@ export { + httpUrlSchema, platformSchema, - contentProvenanceSchema, utf8ByteLength, investigationIdSchema, claimIdSchema, + postVersionIdSchema, + tabSessionIdSchema, + lesswrongExternalIdSchema, + xExternalIdSchema, + substackExternalIdSchema, + wikipediaExternalIdSchema, investigationClaimPayloadSchema, investigationResultSchema, WIKIPEDIA_LANGUAGE_CODE_REGEX, @@ -13,9 +19,7 @@ export { viewPostInputSchema, registerObservedVersionInputSchema, registerObservedVersionOutputSchema, - priorInvestigationResultSchema, viewPostOutputSchema, - investigationStatusOutputSchema, getInvestigationInputSchema, getInvestigationOutputSchema, recordViewAndGetStatusInputSchema, @@ -31,16 +35,19 @@ export { } from "./schemas/settings.js"; export { + platformContentSchema, extensionPostStatusSchema, - extensionSkippedStatusSchema, extensionPageStatusSchema, - requestInvestigateResponseSchema, - focusClaimResponseSchema, - annotationVisibilityResponseSchema, extensionRuntimeErrorCodeSchema, - extensionRuntimeErrorResponseSchema, - contentControlMessageSchema, - extensionMessageSchema, + BACKGROUND_REQUESTS, + CONTENT_REQUESTS, + parseBackgroundRequestPayload, + parseBackgroundResponseEnvelope, + parseContentRequestPayload, + parseContentResponseEnvelope, + type ExtensionRuntimeErrorResponse, + type ProtocolParseResult, + type ProtocolResponseEnvelope, } from "./schemas/extension-protocol.js"; export { @@ -49,7 +56,5 @@ export { searchInvestigationsInputSchema, getMetricsInputSchema, publicGetInvestigationOutputSchema, - publicGetPostInvestigationsOutputSchema, publicSearchInvestigationsOutputSchema, - publicGetMetricsOutputSchema, } from "./schemas/public-api.js"; diff --git a/src/typescript/shared/src/schemas/common.ts b/src/typescript/shared/src/schemas/common.ts index c7ec188..64f2ebc 100644 --- a/src/typescript/shared/src/schemas/common.ts +++ b/src/typescript/shared/src/schemas/common.ts @@ -1,15 +1,15 @@ import { z } from "zod"; import { CONTENT_PROVENANCE_VALUES, PLATFORM_VALUES } from "../enums.js"; import { - EXTENSION_MESSAGE_PROTOCOL_VERSION, MAX_OBSERVED_CONTENT_TEXT_CHARS, MAX_OBSERVED_CONTENT_TEXT_UTF8_BYTES, MAX_OBSERVED_IMAGE_OCCURRENCES, } from "../constants.js"; +export const httpUrlSchema = z.url({ protocol: /^https?$/ }); + export const platformSchema = z.enum(PLATFORM_VALUES); export const contentProvenanceSchema = z.enum(CONTENT_PROVENANCE_VALUES); -export const postMediaStateSchema = z.enum(["text_only", "has_images", "has_video"]); const utf8Encoder = new TextEncoder(); @@ -29,7 +29,7 @@ const observedImageOccurrenceSchema = z .object({ originalIndex: z.number().int().nonnegative(), normalizedTextOffset: z.number().int().nonnegative(), - sourceUrl: z.url(), + sourceUrl: httpUrlSchema, captionText: z.string().min(1).optional(), }) .strict(); @@ -39,16 +39,36 @@ export const observedImageOccurrencesSchema = z .max(MAX_OBSERVED_IMAGE_OCCURRENCES); export const postIdSchema = z.string().min(1).brand<"PostId">(); + +// Per-platform external ID formats. Each is a `PostId`, additionally validated +// against the shape the platform actually issues, so a client can never report +// e.g. a Substack slug where the numeric Substack post ID belongs. +export const lesswrongExternalIdSchema = z + .string() + .regex(/^[A-Za-z0-9]+$/) + .brand<"PostId">(); +export const xExternalIdSchema = z.string().regex(/^\d+$/).brand<"PostId">(); +export const substackExternalIdSchema = z.string().regex(/^\d+$/).brand<"PostId">(); +/** `{language}:{pageId}` — see `wikipediaExternalIdFromPageId`. */ +export const wikipediaExternalIdSchema = z + .string() + .regex(/^[a-z][a-z0-9-]*:\d+$/) + .brand<"PostId">(); + export const postVersionIdSchema = z.string().min(1).brand<"PostVersionId">(); -export const sessionIdSchema = z.number().int().nonnegative().brand<"SessionId">(); +/** + * Identifies one content-script page session (one observed page state in one + * tab). Random UUIDs keep ids unique across content-script instances, so a + * re-injected or orphaned script can never collide with the live one. + */ +export const tabSessionIdSchema = z.uuid().brand<"TabSessionId">(); export const investigationIdSchema = z.string().min(1).brand<"InvestigationId">(); export const claimIdSchema = z.string().min(1).brand<"ClaimId">(); export const versionHashSchema = z.string().regex(/^[a-f0-9]{64}$/i); -export const extensionMessageProtocolVersionSchema = z.literal(EXTENSION_MESSAGE_PROTOCOL_VERSION); const claimSourceSchema = z .object({ - url: z.url(), + url: httpUrlSchema, title: z.string().min(1), snippet: z.string().min(1), }) @@ -95,7 +115,7 @@ export const xMetadataSchema = z authorHandle: z.string().min(1), authorDisplayName: z.string().min(1).nullable().optional(), text: observedContentTextSchema, - mediaUrls: z.array(z.url()), + mediaUrls: z.array(httpUrlSchema), likeCount: z.number().int().nonnegative().optional(), retweetCount: z.number().int().nonnegative().optional(), postedAt: z.iso.datetime().optional(), diff --git a/src/typescript/shared/src/schemas/extension-protocol.ts b/src/typescript/shared/src/schemas/extension-protocol.ts index 0c1d3e6..c30806f 100644 --- a/src/typescript/shared/src/schemas/extension-protocol.ts +++ b/src/typescript/shared/src/schemas/extension-protocol.ts @@ -1,141 +1,147 @@ +/** + * Message protocol between the extension's contexts (spec §3.8.1). Every + * message is `{ type, payload }`; every reply is a response envelope + * (`{ ok: true, value }` or an `{ ok: false, error }` runtime error). Each + * direction has one request map from message type to payload and response + * schemas — the single table both senders and handlers are typed from, and + * that receivers validate incoming payloads against. + * + * - `BACKGROUND_REQUESTS`: content script / popup → background + * (`runtime.sendMessage`). + * - `CONTENT_REQUESTS`: background / popup → a tab's content script + * (`tabs.sendMessage`). + * + * All contexts ship in one extension bundle, so the protocol is not versioned: + * a content script orphaned by an extension update can no longer reach the new + * background at all. API compatibility is versioned separately, over HTTP + * (`x-openerrata-extension-version`, spec §3.3). + */ import { z } from "zod"; import { claimIdSchema, contentProvenanceSchema, - extensionMessageProtocolVersionSchema, investigationClaimPayloadSchema, investigationClaimSchema, investigationIdSchema, + lesswrongExternalIdSchema, + lesswrongMetadataSchema, observedContentTextSchema, observedImageOccurrencesSchema, platformSchema, postIdSchema, - postMediaStateSchema, - sessionIdSchema, - lesswrongMetadataSchema, - xMetadataSchema, + substackExternalIdSchema, substackMetadataSchema, + tabSessionIdSchema, + wikipediaExternalIdSchema, wikipediaMetadataSchema, + xExternalIdSchema, + xMetadataSchema, } from "./common.js"; -import { - getInvestigationInputSchema, - investigationStatusOutputSchema, - priorInvestigationResultSchema, - viewPostInputSchema, -} from "./investigation.js"; +import { priorInvestigationResultSchema } from "./investigation.js"; -const focusClaimPayloadSchema = z - .object({ - claimId: claimIdSchema, - }) - .strict(); +// ── Observed page content ───────────────────────────────────────────────── const platformContentBaseSchema = z .object({ - externalId: z.string().min(1), url: z.url(), - // Normalized plain text as observed by the client. - // Textless content is currently treated as unsupported. + // Normalized plain text as observed by the client. Textless content is + // skipped (`no_text`) before it is ever sent. contentText: observedContentTextSchema, - mediaState: postMediaStateSchema, - imageUrls: z.array(z.url()), - imageOccurrences: observedImageOccurrencesSchema.optional(), - }) - .strict(); - -const lesswrongPlatformContentSchema = platformContentBaseSchema - .extend({ - platform: z.literal("LESSWRONG"), - metadata: lesswrongMetadataSchema, - }) - .strict(); - -const xPlatformContentSchema = platformContentBaseSchema - .extend({ - platform: z.literal("X"), - metadata: xMetadataSchema, - }) - .strict(); - -const substackPlatformContentSchema = platformContentBaseSchema - .extend({ - platform: z.literal("SUBSTACK"), - metadata: substackMetadataSchema, - }) - .strict(); - -const wikipediaPlatformContentSchema = platformContentBaseSchema - .extend({ - platform: z.literal("WIKIPEDIA"), - metadata: wikipediaMetadataSchema, + // Video makes a post non-analyzable; images do not (spec §2.4.2). + hasVideo: z.boolean(), + // Every observed image, in page order. The single source of image data: + // image URLs and "has images" are derived from it. + imageOccurrences: observedImageOccurrencesSchema, }) .strict(); -const platformContentSchema = z.discriminatedUnion("platform", [ - lesswrongPlatformContentSchema, - xPlatformContentSchema, - substackPlatformContentSchema, - wikipediaPlatformContentSchema, +export const platformContentSchema = z.discriminatedUnion("platform", [ + platformContentBaseSchema + .extend({ + platform: z.literal("LESSWRONG"), + externalId: lesswrongExternalIdSchema, + metadata: lesswrongMetadataSchema, + }) + .strict(), + platformContentBaseSchema + .extend({ + platform: z.literal("X"), + externalId: xExternalIdSchema, + metadata: xMetadataSchema, + }) + .strict(), + platformContentBaseSchema + .extend({ + platform: z.literal("SUBSTACK"), + externalId: substackExternalIdSchema, + metadata: substackMetadataSchema, + }) + .strict(), + platformContentBaseSchema + .extend({ + platform: z.literal("WIKIPEDIA"), + externalId: wikipediaExternalIdSchema, + metadata: wikipediaMetadataSchema, + }) + .strict(), ]); +// ── Per-tab page status (background cache, popup, content script) ───────── + const extensionPostStatusBaseSchema = z .object({ kind: z.literal("POST"), - tabSessionId: sessionIdSchema, + tabSessionId: tabSessionIdSchema, platform: platformSchema, externalId: postIdSchema, pageUrl: z.url(), - investigationId: investigationIdSchema.optional(), - }) - .strict(); - -const extensionPostNotInvestigatedSchema = extensionPostStatusBaseSchema - .extend({ - investigationState: z.literal("NOT_INVESTIGATED"), - priorInvestigationResult: priorInvestigationResultSchema.nullable(), - }) - .strict(); - -const extensionPostInvestigatingSchema = extensionPostStatusBaseSchema - .extend({ - investigationState: z.literal("INVESTIGATING"), - status: z.union([z.literal("PENDING"), z.literal("PROCESSING")]), - provenance: contentProvenanceSchema, - pendingClaims: z.array(investigationClaimPayloadSchema), - confirmedClaims: z.array(investigationClaimPayloadSchema), - priorInvestigationResult: priorInvestigationResultSchema.nullable(), - }) - .strict(); - -const extensionPostFailedSchema = extensionPostStatusBaseSchema - .extend({ - investigationState: z.literal("FAILED"), - provenance: contentProvenanceSchema, - }) - .strict(); - -const extensionPostApiErrorSchema = extensionPostStatusBaseSchema - .extend({ - investigationState: z.literal("API_ERROR"), - }) - .strict(); - -const extensionPostInvestigatedSchema = extensionPostStatusBaseSchema - .extend({ - investigationState: z.literal("INVESTIGATED"), - provenance: contentProvenanceSchema, - claims: z.array(investigationClaimSchema), }) .strict(); export const extensionPostStatusSchema = z.discriminatedUnion("investigationState", [ - extensionPostNotInvestigatedSchema, - extensionPostInvestigatingSchema, - extensionPostFailedSchema, - extensionPostApiErrorSchema, - extensionPostInvestigatedSchema, + extensionPostStatusBaseSchema + .extend({ + investigationState: z.literal("NOT_INVESTIGATED"), + priorInvestigationResult: priorInvestigationResultSchema.nullable(), + }) + .strict(), + extensionPostStatusBaseSchema + .extend({ + investigationState: z.literal("INVESTIGATING"), + investigationId: investigationIdSchema, + status: z.union([z.literal("PENDING"), z.literal("PROCESSING")]), + provenance: contentProvenanceSchema, + pendingClaims: z.array(investigationClaimPayloadSchema), + confirmedClaims: z.array(investigationClaimPayloadSchema), + priorInvestigationResult: priorInvestigationResultSchema.nullable(), + }) + .strict(), + extensionPostStatusBaseSchema + .extend({ + investigationState: z.literal("FAILED"), + investigationId: investigationIdSchema, + provenance: contentProvenanceSchema, + }) + .strict(), + // The extension could not obtain a status from the API (network failure, + // incompatible extension version, invalid settings, ...). Unlike FAILED, + // this says nothing about any server-side investigation. + extensionPostStatusBaseSchema + .extend({ + investigationState: z.literal("API_ERROR"), + }) + .strict(), + extensionPostStatusBaseSchema + .extend({ + investigationState: z.literal("INVESTIGATED"), + investigationId: investigationIdSchema, + provenance: contentProvenanceSchema, + claims: z.array(investigationClaimSchema), + }) + .strict(), ]); +/** Skip reasons, spec §3.8 "All skip reasons". */ const extensionSkippedReasonSchema = z.enum([ "has_video", "word_count", @@ -144,49 +150,40 @@ const extensionSkippedReasonSchema = z.enum([ "unsupported_content", ]); -export const extensionSkippedStatusSchema = z +/** + * A supported page the extension deliberately does not send to the API. + * Skipped pages are identified by URL only: a skip can happen before the + * platform's post ID is known (e.g. a Substack paywall or an unrenderable + * tweet), so no external ID is reported for them. + */ +const extensionSkippedStatusSchema = z .object({ kind: z.literal("SKIPPED"), - tabSessionId: sessionIdSchema, + tabSessionId: tabSessionIdSchema, platform: platformSchema, - externalId: postIdSchema, pageUrl: z.url(), reason: extensionSkippedReasonSchema, }) .strict(); -export const extensionPageStatusSchema = z.union([ +export const extensionPageStatusSchema = z.discriminatedUnion("kind", [ extensionPostStatusSchema, extensionSkippedStatusSchema, ]); -export const requestInvestigateResponseSchema = z - .object({ - ok: z.boolean(), - }) - .strict(); - -export const focusClaimResponseSchema = z - .object({ - ok: z.boolean(), - }) - .strict(); - -export const annotationVisibilityResponseSchema = z - .object({ - visible: z.boolean(), - }) - .strict(); +// ── Responses ───────────────────────────────────────────────────────────── export const extensionRuntimeErrorCodeSchema = z.enum([ "PAYLOAD_TOO_LARGE", "UPGRADE_REQUIRED", "MALFORMED_EXTENSION_VERSION", "INVALID_EXTENSION_MESSAGE", - "UNSUPPORTED_PROTOCOL_VERSION", + // Stored settings are unusable (invalid API URL, or no host permission for + // it); retrying cannot help until the user fixes them in the options page. + "INVALID_EXTENSION_SETTINGS", ]); -export const extensionRuntimeErrorResponseSchema = z +const extensionRuntimeErrorResponseSchema = z .object({ ok: z.literal(false), error: z.string().min(1), @@ -194,102 +191,151 @@ export const extensionRuntimeErrorResponseSchema = z }) .strict(); -function extensionMessageWithPayload( - type: TType, - payload: TPayload, -) { - return z - .object({ - v: extensionMessageProtocolVersionSchema, - type: z.literal(type), - payload, - }) - .strict(); +function extensionResponseEnvelopeSchema(value: TValue) { + return z.discriminatedUnion("ok", [ + z.object({ ok: z.literal(true), value }).strict(), + extensionRuntimeErrorResponseSchema, + ]); } -function extensionMessageWithoutPayload(type: TType) { - return z - .object({ - v: extensionMessageProtocolVersionSchema, - type: z.literal(type), - }) - .strict(); -} +const annotationVisibilitySchema = z.object({ visible: z.boolean() }).strict(); -const requestInvestigateMessageSchema = extensionMessageWithoutPayload("REQUEST_INVESTIGATE"); - -const showAnnotationsMessageSchema = extensionMessageWithoutPayload("SHOW_ANNOTATIONS"); +/** Whether the popup may show a status for the tab, or must show an upgrade notice. */ +const tabStatusResponseSchema = z.discriminatedUnion("kind", [ + z.object({ kind: z.literal("UPGRADE_REQUIRED"), message: z.string().min(1) }).strict(), + z.object({ kind: z.literal("STATUS"), status: extensionPageStatusSchema.nullable() }).strict(), +]); -const hideAnnotationsMessageSchema = extensionMessageWithoutPayload("HIDE_ANNOTATIONS"); +// ── Request maps ────────────────────────────────────────────────────────── -const getAnnotationVisibilityMessageSchema = extensionMessageWithoutPayload( - "GET_ANNOTATION_VISIBILITY", -); +interface RequestDefinition { + payload: z.ZodType; + response: z.ZodType; +} -const focusClaimMessageSchema = extensionMessageWithPayload("FOCUS_CLAIM", focusClaimPayloadSchema); +export const BACKGROUND_REQUESTS = { + /** A tracked post's content; returns the status cached for that page session. */ + PAGE_CONTENT: { + payload: z + .object({ tabSessionId: tabSessionIdSchema, content: platformContentSchema }) + .strict(), + response: extensionPostStatusSchema, + }, + PAGE_SKIPPED: { + payload: extensionSkippedStatusSchema.omit({ kind: true }).strict(), + response: z.null(), + }, + /** The page session ended (navigation or new content); its status is discarded. */ + PAGE_RESET: { + payload: z.object({ tabSessionId: tabSessionIdSchema }).strict(), + response: z.null(), + }, + INVESTIGATE_NOW: { + payload: z + .object({ tabSessionId: tabSessionIdSchema, content: platformContentSchema }) + .strict(), + response: extensionPostStatusSchema, + }, + /** Popup read of a tab's cached status. */ + GET_TAB_STATUS: { + payload: z.object({ tabId: z.number().int().nonnegative() }).strict(), + response: tabStatusResponseSchema, + }, +} as const satisfies Record; + +export const CONTENT_REQUESTS = { + /** Liveness probe; must have no side effects. */ + PING: { payload: z.null(), response: z.object({ alive: z.literal(true) }).strict() }, + /** Pure read of highlight visibility; must have no side effects. */ + GET_VISIBILITY: { payload: z.null(), response: annotationVisibilitySchema }, + SHOW_ANNOTATIONS: { payload: z.null(), response: annotationVisibilitySchema }, + HIDE_ANNOTATIONS: { payload: z.null(), response: annotationVisibilitySchema }, + /** Investigate the post in the current page session; `ok: false` when there is none. */ + REQUEST_INVESTIGATE: { payload: z.null(), response: z.object({ ok: z.boolean() }).strict() }, + FOCUS_CLAIM: { + payload: z.object({ claimId: claimIdSchema }).strict(), + response: z.object({ ok: z.boolean() }).strict(), + }, + /** + * The page changed its URL through the History API. Content scripts run in + * an isolated world and cannot observe the page's `pushState` calls, so the + * background relays `webNavigation.onHistoryStateUpdated`. + */ + LOCATION_CHANGED: { payload: z.null(), response: z.null() }, + /** The background cached a new status for this tab. */ + STATUS_CHANGED: { + payload: z.object({ status: extensionPageStatusSchema }).strict(), + response: z.null(), + }, +} as const satisfies Record; + +type BackgroundRequests = typeof BACKGROUND_REQUESTS; +type ContentRequests = typeof CONTENT_REQUESTS; + +/** Outcome of validating an incoming message part against the protocol. */ +export type ProtocolParseResult = + | { success: true; data: Value } + | { success: false; error: string }; + +export type ExtensionRuntimeErrorResponse = z.infer; + +/** A response envelope whose `ok: true` value has been validated. */ +export type ProtocolResponseEnvelope = + | { ok: true; value: Value } + | ExtensionRuntimeErrorResponse; + +// The helpers below validate against the schema registered for `type`. The +// compiler cannot relate the (union-typed) parse result back to the generic +// `Type`, so each states that relation once, right after validation. + +export function parseBackgroundRequestPayload( + type: Type, + payload: unknown, +): ProtocolParseResult> { + const result = BACKGROUND_REQUESTS[type].payload.safeParse(payload); + if (!result.success) return { success: false, error: result.error.message }; + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- validated by the schema registered for `type` + return { success: true, data: result.data as z.output }; +} -const contentControlMessageSchemas = [ - requestInvestigateMessageSchema, - showAnnotationsMessageSchema, - hideAnnotationsMessageSchema, - getAnnotationVisibilityMessageSchema, - focusClaimMessageSchema, -] as const; +export function parseBackgroundResponseEnvelope( + type: Type, + response: unknown, +): ProtocolParseResult>> { + const result = extensionResponseEnvelopeSchema(BACKGROUND_REQUESTS[type].response).safeParse( + response, + ); + if (!result.success) return { success: false, error: result.error.message }; + if (!result.data.ok) return { success: true, data: result.data }; + return { + success: true, + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- validated by the schema registered for `type` + data: { ok: true, value: result.data.value as z.output }, + }; +} -export const contentControlMessageSchema = z.union(contentControlMessageSchemas); +export function parseContentRequestPayload( + type: Type, + payload: unknown, +): ProtocolParseResult> { + const result = CONTENT_REQUESTS[type].payload.safeParse(payload); + if (!result.success) return { success: false, error: result.error.message }; + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- validated by the schema registered for `type` + return { success: true, data: result.data as z.output }; +} -export const extensionMessageSchema = z.discriminatedUnion("type", [ - extensionMessageWithPayload( - "PAGE_CONTENT", - z - .object({ - tabSessionId: sessionIdSchema, - content: platformContentSchema, - }) - .strict(), - ), - extensionMessageWithPayload( - "PAGE_SKIPPED", - z - .object({ - tabSessionId: sessionIdSchema, - platform: platformSchema, - externalId: postIdSchema, - pageUrl: z.url(), - reason: extensionSkippedReasonSchema, - }) - .strict(), - ), - extensionMessageWithPayload( - "PAGE_RESET", - z - .object({ - tabSessionId: sessionIdSchema, - }) - .strict(), - ), - extensionMessageWithPayload( - "GET_STATUS", - getInvestigationInputSchema - .extend({ - tabSessionId: sessionIdSchema.optional(), - }) - .strict(), - ), - extensionMessageWithPayload( - "INVESTIGATE_NOW", - z - .object({ - tabSessionId: sessionIdSchema, - request: viewPostInputSchema, - }) - .strict(), - ), - ...contentControlMessageSchemas, - extensionMessageWithoutPayload("GET_CACHED"), - extensionMessageWithPayload("STATUS_RESPONSE", investigationStatusOutputSchema), - extensionMessageWithPayload( - "ANNOTATIONS", - z.object({ claims: z.array(investigationClaimSchema) }).strict(), - ), -]); +export function parseContentResponseEnvelope( + type: Type, + response: unknown, +): ProtocolParseResult>> { + const result = extensionResponseEnvelopeSchema(CONTENT_REQUESTS[type].response).safeParse( + response, + ); + if (!result.success) return { success: false, error: result.error.message }; + if (!result.data.ok) return { success: true, data: result.data }; + return { + success: true, + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- validated by the schema registered for `type` + data: { ok: true, value: result.data.value as z.output }, + }; +} diff --git a/src/typescript/shared/src/schemas/investigation.ts b/src/typescript/shared/src/schemas/investigation.ts index d6483f1..3eb5fd5 100644 --- a/src/typescript/shared/src/schemas/investigation.ts +++ b/src/typescript/shared/src/schemas/investigation.ts @@ -19,7 +19,9 @@ import { const versionedPostInputSharedSchema = z .object({ url: z.url(), - observedImageUrls: z.array(z.url()).optional(), + // Every image the client observed, in page order. This is the only image + // representation on the wire; the unique image URL list is derived from it + // (`observedImageUrlsFromOccurrences`). Omitted means no images were observed. observedImageOccurrences: observedImageOccurrencesSchema.optional(), }) .strict(); @@ -134,17 +136,19 @@ const investigationStatusInvestigatedSchema = z }) .strict(); +/** + * Status of the investigation (if any) for one post version, as returned by + * `recordViewAndGetStatus`. Every variant that refers to an existing + * investigation carries its id, so a client that did not start the + * investigation itself can still poll `getInvestigation` for progress. + */ export const viewPostOutputSchema = z.discriminatedUnion("investigationState", [ investigationStatusNotInvestigatedSchema, - investigationStatusInvestigatingSchema, - investigationStatusInvestigatedSchema, -]); - -export const investigationStatusOutputSchema = z.discriminatedUnion("investigationState", [ - investigationStatusNotInvestigatedSchema, - investigationStatusInvestigatingSchema, - investigationStatusFailedSchema, - investigationStatusInvestigatedSchema, + investigationStatusInvestigatingSchema + .extend({ investigationId: investigationIdSchema }) + .strict(), + investigationStatusFailedSchema.extend({ investigationId: investigationIdSchema }).strict(), + investigationStatusInvestigatedSchema.extend({ investigationId: investigationIdSchema }).strict(), ]); export const getInvestigationInputSchema = z @@ -153,22 +157,12 @@ export const getInvestigationInputSchema = z }) .strict(); +// `checkedAt` is set iff an investigation is COMPLETE (DB invariant +// INV-INV-CHECKED-AT-CONSISTENCY), so only the INVESTIGATED variant carries it. export const getInvestigationOutputSchema = z.discriminatedUnion("investigationState", [ - investigationStatusNotInvestigatedSchema - .extend({ - checkedAt: z.iso.datetime().optional(), - }) - .strict(), - investigationStatusInvestigatingSchema - .extend({ - checkedAt: z.iso.datetime().optional(), - }) - .strict(), - investigationStatusFailedSchema - .extend({ - checkedAt: z.iso.datetime().optional(), - }) - .strict(), + investigationStatusNotInvestigatedSchema, + investigationStatusInvestigatingSchema, + investigationStatusFailedSchema, investigationStatusInvestigatedSchema .extend({ checkedAt: z.iso.datetime(), diff --git a/src/typescript/shared/src/schemas/public-api.ts b/src/typescript/shared/src/schemas/public-api.ts index ee4af2c..adf0a39 100644 --- a/src/typescript/shared/src/schemas/public-api.ts +++ b/src/typescript/shared/src/schemas/public-api.ts @@ -1,10 +1,13 @@ import { z } from "zod"; +import { INVESTIGATION_PROVIDER_VALUES } from "../enums.js"; import { + claimIdSchema, + contentProvenanceSchema, + httpUrlSchema, investigationIdSchema, platformSchema, postIdSchema, versionHashSchema, - contentProvenanceSchema, investigationClaimSchema, } from "./common.js"; @@ -61,18 +64,38 @@ export const getMetricsInputSchema = z }) .strict(); -const publicInvestigationOriginSchema = z - .object({ - provenance: contentProvenanceSchema, - serverVerifiedAt: z.iso.datetime().optional(), - }) - .strict(); +/* + * Output schemas below describe the public GraphQL wire format: nullable + * fields are present with `null`, never omitted. + */ + +/** + * `provenance` is the immutable snapshot of how the investigated content was + * obtained; `serverVerifiedAt` is the post version's verification latch. A + * SERVER_VERIFIED investigation ran on verified content, so its latch is + * always set. A CLIENT_FALLBACK investigation's latch may be set later, when a + * subsequent server fetch verifies the same content. + */ +const publicInvestigationOriginSchema = z.discriminatedUnion("provenance", [ + z + .object({ + provenance: contentProvenanceSchema.extract(["SERVER_VERIFIED"]), + serverVerifiedAt: z.iso.datetime(), + }) + .strict(), + z + .object({ + provenance: contentProvenanceSchema.extract(["CLIENT_FALLBACK"]), + serverVerifiedAt: z.iso.datetime().nullable(), + }) + .strict(), +]); const publicPostSchema = z .object({ platform: platformSchema, externalId: postIdSchema, - url: z.url(), + url: httpUrlSchema, }) .strict(); @@ -82,7 +105,8 @@ const publicInvestigationMetadataSchema = z corroborationCount: z.number().int().nonnegative(), checkedAt: z.iso.datetime(), promptVersion: z.string().min(1), - provider: z.string().min(1), + provider: z.enum(INVESTIGATION_PROVIDER_VALUES), + /** The provider's model id that produced the result, e.g. "gpt-6.1-sol". */ model: z.string().min(1), origin: publicInvestigationOriginSchema, }) @@ -99,30 +123,11 @@ export const publicGetInvestigationOutputSchema = z const claimSummarySchema = z .object({ - id: z.string().min(1), + id: claimIdSchema, summary: z.string().min(1), }) .strict(); -export const publicGetPostInvestigationsOutputSchema = z - .object({ - post: publicPostSchema.nullable(), - investigations: z.array( - z - .object({ - id: investigationIdSchema, - contentHash: versionHashSchema, - corroborationCount: z.number().int().nonnegative(), - checkedAt: z.iso.datetime(), - claimCount: z.number().int().nonnegative(), - claimSummaries: z.array(claimSummarySchema), - origin: publicInvestigationOriginSchema, - }) - .strict(), - ), - }) - .strict(); - export const publicSearchInvestigationsOutputSchema = z .object({ investigations: z.array( @@ -133,7 +138,7 @@ export const publicSearchInvestigationsOutputSchema = z checkedAt: z.iso.datetime(), platform: platformSchema, externalId: postIdSchema, - url: z.url(), + url: httpUrlSchema, corroborationCount: z.number().int().nonnegative(), claimCount: z.number().int().nonnegative(), claimSummaries: z.array(claimSummarySchema), @@ -144,11 +149,3 @@ export const publicSearchInvestigationsOutputSchema = z hasMore: z.boolean(), }) .strict(); - -export const publicGetMetricsOutputSchema = z - .object({ - totalInvestigatedPosts: z.number().int().nonnegative(), - investigatedPostsWithFlags: z.number().int().nonnegative(), - factCheckIncidence: z.number().nonnegative(), - }) - .strict(); diff --git a/src/typescript/shared/src/types.ts b/src/typescript/shared/src/types.ts index 649b96e..56cdec5 100644 --- a/src/typescript/shared/src/types.ts +++ b/src/typescript/shared/src/types.ts @@ -17,32 +17,13 @@ export type InvestigationClaimPayload = z.infer< export type ClaimId = z.infer; -// ── Platform metadata contracts ─────────────────────────────────────────── -export interface PlatformMetadataByPlatform { - LESSWRONG: Extract< - z.infer, - { platform: "LESSWRONG" } - >["metadata"]; - X: Extract< - z.infer, - { platform: "X" } - >["metadata"]; - SUBSTACK: Extract< - z.infer, - { platform: "SUBSTACK" } - >["metadata"]; - WIKIPEDIA: Extract< - z.infer, - { platform: "WIKIPEDIA" } - >["metadata"]; -} +export type InvestigationId = z.infer; + +export type PostId = z.infer; // ── Platform adapter (spec §3.8) ────────────────────────────────────────── -export type PlatformContent = Extract< - z.infer, - { type: "PAGE_CONTENT" } ->["payload"]["content"]; +export type PlatformContent = z.infer; export type ObservedImageOccurrence = NonNullable< z.infer["observedImageOccurrences"] @@ -71,10 +52,6 @@ export type RecordViewAndGetStatusInputWire = z.input< typeof import("./schemas.js").recordViewAndGetStatusInputSchema >; -export type InvestigationStatusOutput = z.infer< - typeof import("./schemas.js").investigationStatusOutputSchema ->; - export type GetInvestigationInput = z.infer< typeof import("./schemas.js").getInvestigationInputSchema >; @@ -160,9 +137,7 @@ export type ExtensionApiInput

= export type ExtensionPostStatus = z.infer; -export type ExtensionSkippedStatus = z.infer< - typeof import("./schemas.js").extensionSkippedStatusSchema ->; +export type ExtensionSkippedStatus = Extract; export type ExtensionSkippedReason = ExtensionSkippedStatus["reason"]; @@ -172,6 +147,23 @@ export type ExtensionRuntimeErrorCode = z.infer< typeof import("./schemas.js").extensionRuntimeErrorCodeSchema >; -// ── Extension message protocol ──────────────────────────────────────────── +export type TabSessionId = z.infer; + +// ── Extension message protocol (spec §3.8.1) ────────────────────────────── -export type ExtensionMessage = z.infer; +type BackgroundRequests = typeof import("./schemas.js").BACKGROUND_REQUESTS; +type ContentRequests = typeof import("./schemas.js").CONTENT_REQUESTS; + +export type BackgroundRequestType = keyof BackgroundRequests; +export type BackgroundRequestPayload = z.infer< + BackgroundRequests[T]["payload"] +>; +export type BackgroundResponse = z.infer< + BackgroundRequests[T]["response"] +>; + +export type ContentRequestType = keyof ContentRequests; +export type ContentRequestPayload = z.infer< + ContentRequests[T]["payload"] +>; +export type ContentResponse = z.infer; diff --git a/src/typescript/shared/src/version-identity.ts b/src/typescript/shared/src/version-identity.ts index 1e7a4fe..3604a82 100644 --- a/src/typescript/shared/src/version-identity.ts +++ b/src/typescript/shared/src/version-identity.ts @@ -1,6 +1,6 @@ import { validateAndSortImageOccurrences } from "./image-occurrence-validation.js"; -export interface VersionIdentityImageOccurrence { +interface VersionIdentityImageOccurrence { originalIndex: number; normalizedTextOffset: number; sourceUrl: string; diff --git a/src/typescript/shared/src/wikipedia-canonicalization.ts b/src/typescript/shared/src/wikipedia-canonicalization.ts index ace625d..651124d 100644 --- a/src/typescript/shared/src/wikipedia-canonicalization.ts +++ b/src/typescript/shared/src/wikipedia-canonicalization.ts @@ -96,16 +96,112 @@ export function effectiveHeadingText( return node.textContent; } -export const WIKIPEDIA_EXCLUDED_SECTION_TITLES = [ - "references", - "notes", - "further reading", - "external links", - "bibliography", - "sources", - "citations", -] as const; +/** + * Appendix sections that list citations, sources and outbound links rather + * than carry article prose, by title (compared after + * `normalizeWikipediaSectionTitle`). Section titles are the one signal here + * that is per-language: these are the same kinds of section English excludes, + * as titled on the largest Wikipedias, taken from the titles those wikis' own + * articles use. A title matches on any wiki, since some wikis' articles use + * another language's titles. "See also" sections and their equivalents stay: + * English keeps them. + */ +const WIKIPEDIA_EXCLUDED_SECTION_TITLES_BY_LANGUAGE = { + en: [ + "references", + "notes", + "further reading", + "external links", + "bibliography", + "sources", + "citations", + ], + de: [ + "einzelnachweise", + "nachweise", + "belege", + "anmerkungen", + "fußnoten", + "literatur", + "weiterführende literatur", + "weblinks", + "quellen", + ], + fr: ["notes et références", "références", "bibliographie", "liens externes", "lien externe"], + es: [ + "referencias", + "notas", + "bibliografía", + "bibliografía consultada", + "bibliografía básica", + "enlaces externos", + "enlace externo", + "fuentes", + "fuente", + ], + it: ["note", "bibliografia", "collegamenti esterni", "altri progetti", "fonti"], + pt: [ + "referências", + "notas", + "notas explicativas", + "bibliografia", + "leitura adicional", + "ligações externas", + "links externos", + "fontes", + ], + nl: [ + "noten", + "voetnoten", + "referenties", + "bronnen", + "bronvermelding", + "literatuur", + "externe links", + "externe link", + ], + pl: ["przypisy", "uwagi", "bibliografia", "dalsza literatura", "linki zewnętrzne"], + ru: ["примечания", "комментарии", "литература", "библиография", "источники", "ссылки"], + ja: ["脚注", "注釈", "出典", "参考文献", "参考", "読書案内", "外部リンク"], + // zh.wikipedia serves each reader their script variant, so both forms occur. + zh: [ + "注释", + "註釋", + "注解", + "註解", + "脚注", + "腳註", + "参考文献", + "參考文獻", + "参考资料", + "參考資料", + "参考来源", + "參考來源", + "参考", + "參考", + "来源", + "來源", + "延伸阅读", + "延伸閱讀", + "扩展阅读", + "擴展閱讀", + "进阶读物", + "進階讀物", + "外部链接", + "外部鏈接", + "外部連結", + ], +} as const satisfies Record; + +export const WIKIPEDIA_EXCLUDED_SECTION_TITLES: readonly string[] = Object.values( + WIKIPEDIA_EXCLUDED_SECTION_TITLES_BY_LANGUAGE, +).flat(); +/** + * Class tokens of non-prose elements. The conventions MediaWiki and its + * communities apply on every wiki come first; the per-wiki tokens after them + * cover boxes those conventions miss. + */ const WIKIPEDIA_EXCLUDED_CLASS_TOKENS = [ // Navigation / metadata "mw-editsection", @@ -119,15 +215,38 @@ const WIKIPEDIA_EXCLUDED_CLASS_TOKENS = [ "noprint", "navbox", "vertical-navbox", + // Blocks about the article rather than of it: maintenance and quality + // banners, sister-project boxes, person-data tables, French "main article" + // banners. + "metadata", + // What Wikimedia's search index leaves out as navigation: hatnotes ("For + // other uses, see …"), navboxes, authority-control boxes. + "navigation-not-searchable", + // Per-wiki boxes that carry neither convention above: fr.wikipedia's portal + // bar; nl.wikipedia's appendix box (sources, footnotes and external links + // under bold labels rather than section headings) and sister-project boxes. + "bandeau-portail", + "appendix", + "interproject", + "interprojecttemplate", // Interactive UI injected by Wikipedia's JavaScript — not present in the // Wikipedia Parse API response and not article content. "mw-collapsible-toggle", // "show"/"hide" toggle buttons on collapsible infobox rows "mw-tmh-player", // Video/audio player wrapper added by TimedMediaHandler JS // (contains "Duration: N seconds." and time display) + "cachelinks", // fr.wikipedia gadget appending "[archive]" (Wikiwix) after external links ] as const; +/** + * ARIA roles of non-prose landmarks. Navboxes, series sidebars and "main + * article" links declare `role="navigation"` on every wiki, whatever their + * per-wiki class names. + */ +const WIKIPEDIA_EXCLUDED_ROLES = ["navigation"] as const; + const WIKIPEDIA_EXCLUDED_SECTION_TITLE_SET = new Set(WIKIPEDIA_EXCLUDED_SECTION_TITLES); const WIKIPEDIA_EXCLUDED_CLASS_TOKEN_SET = new Set(WIKIPEDIA_EXCLUDED_CLASS_TOKENS); +const WIKIPEDIA_EXCLUDED_ROLE_SET = new Set(WIKIPEDIA_EXCLUDED_ROLES); export function normalizeWikipediaSectionTitle(value: string): string { return normalizeContent(value).toLowerCase(); @@ -152,15 +271,25 @@ function isExcludedWikipediaTag(tagName: string): boolean { return NON_CONTENT_TAGS.has(tagName.toLowerCase()); } +function isExcludedWikipediaRole(role: string | null): boolean { + return role !== null && WIKIPEDIA_EXCLUDED_ROLE_SET.has(role.trim().toLowerCase()); +} + +/** What the exclusion predicate reads of an element, from a DOM Element or a parse5 node alike. */ +interface WikipediaExclusionDescriptor extends WikipediaElementDescriptor { + /** The `role` attribute, or null when the element has none. */ + role: string | null; +} + /** * Shared Wikipedia element exclusion predicate used by both the browser * adapter (DOM traversal) and API canonical fetcher (parse5 traversal). * Keeping this centralized prevents client/server canonicalization drift. + * It reads only markup the Parse API returns and Wikipedia's scripts leave + * alone (tags, classes, roles), never inline styles, which scripts and reader + * interaction change on the live page. */ -export function shouldExcludeWikipediaElement(input: { - tagName: string; - classTokens: readonly string[]; -}): boolean { +export function shouldExcludeWikipediaElement(input: WikipediaExclusionDescriptor): boolean { if (isExcludedWikipediaTag(input.tagName)) { return true; } @@ -169,5 +298,9 @@ export function shouldExcludeWikipediaElement(input: { return true; } + if (isExcludedWikipediaRole(input.role)) { + return true; + } + return input.classTokens.some((token) => isExcludedWikipediaClassToken(token)); } diff --git a/src/typescript/shared/src/wikipedia-identity.ts b/src/typescript/shared/src/wikipedia-identity.ts index 189509d..00f261d 100644 --- a/src/typescript/shared/src/wikipedia-identity.ts +++ b/src/typescript/shared/src/wikipedia-identity.ts @@ -3,14 +3,26 @@ const WIKIPEDIA_ARTICLE_PATH_PREFIX = "/wiki/"; const WIKIPEDIA_INDEX_PATH_REGEX = /^\/w\/index\.php(?:[/?#]|$)/i; const WIKIPEDIA_PAGE_ID_REGEX = /^\d+$/; -const NON_ARTICLE_NAMESPACE_PREFIXES = new Set([ +/** + * Canonical (English) names of MediaWiki's non-article namespaces. MediaWiki + * accepts these canonical names on every language edition, so they are + * recognized regardless of host language. Localized namespace names (e.g. + * German "Diskussion:") cannot be enumerated here; URL-level parsing therefore + * only rules out *known* non-article pages, and callers with access to the + * page itself must treat MediaWiki's `wgNamespaceNumber` as authoritative. + */ +const CANONICAL_NON_ARTICLE_NAMESPACE_PREFIXES = new Set([ "talk", "user", "user talk", "wikipedia", "wikipedia talk", + "project", + "project talk", "file", "file talk", + "image", + "image talk", "mediawiki", "mediawiki talk", "template", @@ -83,7 +95,7 @@ function isArticleNamespace(title: string): boolean { } const namespacePrefix = title.slice(0, separator).replace(/_/g, " ").trim().toLowerCase(); - return !NON_ARTICLE_NAMESPACE_PREFIXES.has(namespacePrefix); + return !CANONICAL_NON_ARTICLE_NAMESPACE_PREFIXES.has(namespacePrefix); } function normalizeWikipediaPageIdToken(rawToken: string | null): string | null { @@ -102,27 +114,31 @@ function readWikipediaPageIdFromQuery(parsedUrl: URL): string | null { return normalizeWikipediaPageIdToken(parsedUrl.searchParams.get("pageid")); } -function wikipediaExternalIdFromTitle(language: string, title: string): string { - return `${language}:${title}`; -} - +/** + * The stored external ID of a Wikipedia article. It is always derived from the + * numeric page ID (never the title), because titles change on page moves while + * page IDs do not. + */ export function wikipediaExternalIdFromPageId(language: string, pageId: string): string { return `${language}:${pageId}`; } -interface ParsedWikipediaIdentity { - language: string; - title: string | null; - pageId: string | null; - identityKind: "TITLE" | "PAGE_ID"; - externalId: string; -} +/** + * What a Wikipedia URL alone says about which article it shows. A URL either + * names the article by numeric page ID (`?curid=` / `?pageid=`, possibly + * alongside a title) or only by title; a title-only URL cannot yield the + * external ID, which needs the page ID from the page itself. + */ +export type WikipediaUrlIdentity = + | { kind: "PAGE_ID"; language: string; pageId: string; title: string | null } + | { kind: "TITLE"; language: string; title: string }; /** - * Parse canonical Wikipedia page identity from URL for both extension and API. - * This function excludes non-article namespaces (e.g. Talk:, File:). + * Parse Wikipedia article identity from a URL, for both extension and API. + * Returns null for non-Wikipedia URLs and for titles in known non-article + * namespaces (see `CANONICAL_NON_ARTICLE_NAMESPACE_PREFIXES`). */ -export function parseWikipediaIdentity(url: string): ParsedWikipediaIdentity | null { +export function parseWikipediaUrlIdentity(url: string): WikipediaUrlIdentity | null { let parsedUrl: URL; try { parsedUrl = new URL(url); @@ -150,28 +166,11 @@ export function parseWikipediaIdentity(url: string): ParsedWikipediaIdentity | n if (title !== null && !isArticleNamespace(title)) { return null; } - if (title === null && pageId === null) { - return null; - } - if (pageId !== null) { - return { - language, - title, - pageId, - identityKind: "PAGE_ID", - externalId: wikipediaExternalIdFromPageId(language, pageId), - }; + return { kind: "PAGE_ID", language, pageId, title }; } - if (title === null) { - return null; + if (title !== null) { + return { kind: "TITLE", language, title }; } - - return { - language, - title, - pageId: null, - identityKind: "TITLE", - externalId: wikipediaExternalIdFromTitle(language, title), - }; + return null; } diff --git a/src/typescript/shared/test/unit/block-separator-exhaustiveness.test.ts b/src/typescript/shared/test/unit/block-separator-exhaustiveness.test.ts index 2cc12a3..70097a0 100644 --- a/src/typescript/shared/test/unit/block-separator-exhaustiveness.test.ts +++ b/src/typescript/shared/test/unit/block-separator-exhaustiveness.test.ts @@ -1,13 +1,14 @@ import assert from "node:assert/strict"; import { test } from "node:test"; -import { CONTENT_BLOCK_SEPARATOR_TAGS } from "../../src/normalize.js"; +import { WORD_SEPARATOR_TAGS } from "../../src/normalize.js"; -// ── Block separator tag exhaustiveness ────────────────────────────────── +// ── Word separator tag exhaustiveness ─────────────────────────────────── // Every HTML spec block-level element must be either in -// CONTENT_BLOCK_SEPARATOR_TAGS or in the explicit exclusion list below with +// WORD_SEPARATOR_TAGS or in the explicit exclusion list below with // a documented rationale. This is a "living documentation" test — when new // block elements become relevant, the test forces a conscious decision -// about inclusion or exclusion. +// about inclusion or exclusion. The only non-block separators are the +// line-breaking void elements. /** * HTML spec block-level elements. This list covers the elements defined as @@ -61,7 +62,7 @@ const HTML_BLOCK_LEVEL_ELEMENTS = [ ] as const; /** - * Block-level elements explicitly excluded from CONTENT_BLOCK_SEPARATOR_TAGS + * Block-level elements explicitly excluded from WORD_SEPARATOR_TAGS * with documented rationale. Each entry must explain why the element does not * need a word-boundary separator. */ @@ -82,7 +83,6 @@ const EXCLUDED_BLOCK_ELEMENTS: Record = { form: "Form container; not article prose", header: "Page/section header; not article prose", hgroup: "Heading group container; child headings already in separator set", - hr: "Horizontal rule; void element with no text content", main: "Structural container; child block elements provide separators", nav: "Navigation container; not article prose", ol: "Ordered list container; child li elements already in separator set", @@ -98,11 +98,17 @@ const EXCLUDED_BLOCK_ELEMENTS: Record = { caption: "Table caption; rare, and table cell separators handle table content", }; -test("every HTML block-level element is either in CONTENT_BLOCK_SEPARATOR_TAGS or explicitly excluded", () => { +/** + * Inline-level elements that still separate words: void elements that end the + * line, so the text either side of them never reads as one word. + */ +const LINE_BREAKING_VOID_ELEMENTS = ["br", "hr"] as const; + +test("every HTML block-level element is either in WORD_SEPARATOR_TAGS or explicitly excluded", () => { const missingElements: string[] = []; for (const tag of HTML_BLOCK_LEVEL_ELEMENTS) { - if (!CONTENT_BLOCK_SEPARATOR_TAGS.has(tag) && !(tag in EXCLUDED_BLOCK_ELEMENTS)) { + if (!WORD_SEPARATOR_TAGS.has(tag) && !(tag in EXCLUDED_BLOCK_ELEMENTS)) { missingElements.push(tag); } } @@ -110,19 +116,26 @@ test("every HTML block-level element is either in CONTENT_BLOCK_SEPARATOR_TAGS o assert.equal( missingElements.length, 0, - `Block-level elements missing from both CONTENT_BLOCK_SEPARATOR_TAGS and EXCLUDED_BLOCK_ELEMENTS: ${missingElements.join(", ")}.\n` + - `Add each to CONTENT_BLOCK_SEPARATOR_TAGS (if it separates prose words) or to EXCLUDED_BLOCK_ELEMENTS (with rationale).`, + `Block-level elements missing from both WORD_SEPARATOR_TAGS and EXCLUDED_BLOCK_ELEMENTS: ${missingElements.join(", ")}.\n` + + `Add each to WORD_SEPARATOR_TAGS (if it separates prose words) or to EXCLUDED_BLOCK_ELEMENTS (with rationale).`, ); }); -test("CONTENT_BLOCK_SEPARATOR_TAGS contains no unrecognized elements", () => { +test("line-breaking void elements are word separators", () => { + for (const tag of LINE_BREAKING_VOID_ELEMENTS) { + assert.ok(WORD_SEPARATOR_TAGS.has(tag), `<${tag}> must separate the words around it`); + } +}); + +test("WORD_SEPARATOR_TAGS contains no unrecognized elements", () => { const allKnown = new Set([ ...HTML_BLOCK_LEVEL_ELEMENTS, ...Object.keys(EXCLUDED_BLOCK_ELEMENTS), + ...LINE_BREAKING_VOID_ELEMENTS, ]); const unrecognized: string[] = []; - for (const tag of CONTENT_BLOCK_SEPARATOR_TAGS) { + for (const tag of WORD_SEPARATOR_TAGS) { if (!allKnown.has(tag)) { unrecognized.push(tag); } @@ -131,7 +144,7 @@ test("CONTENT_BLOCK_SEPARATOR_TAGS contains no unrecognized elements", () => { assert.equal( unrecognized.length, 0, - `CONTENT_BLOCK_SEPARATOR_TAGS contains elements not in the HTML block-level list: ${unrecognized.join(", ")}.\n` + - `Either add them to HTML_BLOCK_LEVEL_ELEMENTS or remove from CONTENT_BLOCK_SEPARATOR_TAGS.`, + `WORD_SEPARATOR_TAGS contains elements that are neither block-level nor line-breaking: ${unrecognized.join(", ")}.\n` + + `Either add them to HTML_BLOCK_LEVEL_ELEMENTS / LINE_BREAKING_VOID_ELEMENTS or remove from WORD_SEPARATOR_TAGS.`, ); }); diff --git a/src/typescript/shared/test/unit/claim-payload-schema.test.ts b/src/typescript/shared/test/unit/claim-payload-schema.test.ts new file mode 100644 index 0000000..6eee204 --- /dev/null +++ b/src/typescript/shared/test/unit/claim-payload-schema.test.ts @@ -0,0 +1,48 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { + httpUrlSchema, + investigationClaimPayloadSchema, + investigationResultSchema, +} from "../../src/index.js"; + +const NON_HTTP_URLS = [ + "data:text/plain,hello", + "mailto:someone@example.com", + "ftp://example.com/file", + "file:///etc/passwd", +]; + +function claimWithSourceUrl(url: string) { + return { + text: "Claim", + context: "Context", + summary: "Summary", + reasoning: "Reasoning", + sources: [{ url, title: "Title", snippet: "Snippet" }], + }; +} + +test("httpUrlSchema accepts only absolute http(s) URLs", () => { + for (const url of ["https://example.com/a?b=c", "http://example.com", "HTTPS://EXAMPLE.COM/"]) { + assert.equal(httpUrlSchema.safeParse(url).success, true, url); + } + for (const url of [...NON_HTTP_URLS, "example.com/page", ""]) { + assert.equal(httpUrlSchema.safeParse(url).success, false, url); + } +}); + +test("claim sources must link to http(s) URLs", () => { + assert.equal( + investigationClaimPayloadSchema.safeParse(claimWithSourceUrl("https://example.com")).success, + true, + ); + for (const url of NON_HTTP_URLS) { + assert.equal(investigationClaimPayloadSchema.safeParse(claimWithSourceUrl(url)).success, false); + assert.equal( + investigationResultSchema.safeParse({ claims: [claimWithSourceUrl(url)] }).success, + false, + url, + ); + } +}); diff --git a/src/typescript/shared/test/unit/observed-image-occurrence-schema.test.ts b/src/typescript/shared/test/unit/observed-image-occurrence-schema.test.ts index 88a95d3..5ef4efd 100644 --- a/src/typescript/shared/test/unit/observed-image-occurrence-schema.test.ts +++ b/src/typescript/shared/test/unit/observed-image-occurrence-schema.test.ts @@ -1,8 +1,9 @@ import assert from "node:assert/strict"; import { test } from "node:test"; import { + BACKGROUND_REQUESTS, MAX_OBSERVED_IMAGE_OCCURRENCES, - extensionMessageSchema, + observedImageUrlsFromOccurrences, viewPostInputSchema, } from "../../src/index.js"; @@ -48,28 +49,60 @@ test("viewPostInputSchema rejects observedImageOccurrences over limit", () => { assert.equal(result.success, false); }); -test("extensionMessageSchema rejects PAGE_CONTENT imageOccurrences over limit", () => { - const result = extensionMessageSchema.safeParse({ - v: 1, - type: "PAGE_CONTENT", - payload: { - tabSessionId: 1, - content: { - platform: "X", - externalId: "1900000000000000000", - url: "https://x.com/example/status/1900000000000000000", - contentText: "Hello world", - mediaState: "has_images", - imageUrls: [], - imageOccurrences: buildObservedOccurrences(MAX_OBSERVED_IMAGE_OCCURRENCES + 1), - metadata: { - authorHandle: "example", - text: "Hello world", - mediaUrls: [], - }, +test("PAGE_CONTENT payload rejects imageOccurrences over limit", () => { + const result = BACKGROUND_REQUESTS.PAGE_CONTENT.payload.safeParse({ + tabSessionId: "5f0b8d0e-7c55-4c1b-9d0a-1e2f3a4b5c6d", + content: { + platform: "X", + externalId: "1900000000000000000", + url: "https://x.com/example/status/1900000000000000000", + contentText: "Hello world", + hasVideo: false, + imageOccurrences: buildObservedOccurrences(MAX_OBSERVED_IMAGE_OCCURRENCES + 1), + metadata: { + authorHandle: "example", + text: "Hello world", + mediaUrls: [], }, }, }); assert.equal(result.success, false); }); + +test("PAGE_CONTENT payload rejects an external ID in the wrong platform format", () => { + const result = BACKGROUND_REQUESTS.PAGE_CONTENT.payload.safeParse({ + tabSessionId: "5f0b8d0e-7c55-4c1b-9d0a-1e2f3a4b5c6d", + content: { + platform: "SUBSTACK", + // A Substack slug is not a Substack post ID. + externalId: "my-post-slug", + url: "https://example.substack.com/p/my-post-slug", + contentText: "Hello world", + hasVideo: false, + imageOccurrences: [], + metadata: { + substackPostId: "123", + publicationSubdomain: "example", + slug: "my-post-slug", + title: "Title", + authorName: "Author", + }, + }, + }); + + assert.equal(result.success, false); +}); + +test("observedImageUrlsFromOccurrences lists distinct URLs in page order", () => { + const occurrences = [ + { originalIndex: 2, normalizedTextOffset: 9, sourceUrl: "https://images.example/a.jpg" }, + { originalIndex: 0, normalizedTextOffset: 0, sourceUrl: "https://images.example/b.jpg" }, + { originalIndex: 1, normalizedTextOffset: 4, sourceUrl: "https://images.example/a.jpg" }, + ]; + assert.deepEqual(observedImageUrlsFromOccurrences(occurrences), [ + "https://images.example/b.jpg", + "https://images.example/a.jpg", + ]); + assert.deepEqual(observedImageUrlsFromOccurrences(undefined), []); +}); diff --git a/src/typescript/shared/test/unit/wikipedia-canonicalization.test.ts b/src/typescript/shared/test/unit/wikipedia-canonicalization.test.ts index e5260e3..3f5f1ee 100644 --- a/src/typescript/shared/test/unit/wikipedia-canonicalization.test.ts +++ b/src/typescript/shared/test/unit/wikipedia-canonicalization.test.ts @@ -1,46 +1,95 @@ import assert from "node:assert/strict"; import { test } from "node:test"; import { + WIKIPEDIA_EXCLUDED_SECTION_TITLES, effectiveHeadingLevel, effectiveHeadingText, headingLevelFromTag, isExcludedWikipediaSectionTitle, + normalizeWikipediaSectionTitle, shouldExcludeWikipediaElement, type WikipediaHeadingLevelDescriptor, type WikipediaNodeDescriptor, } from "../../src/wikipedia-canonicalization.js"; +function element(tagName: string, classTokens: string[] = [], role: string | null = null) { + return { tagName, classTokens, role }; +} + test("isExcludedWikipediaSectionTitle normalizes whitespace and casing", () => { assert.equal(isExcludedWikipediaSectionTitle(" References "), true); assert.equal(isExcludedWikipediaSectionTitle("Further Reading"), true); assert.equal(isExcludedWikipediaSectionTitle("History"), false); }); +test("isExcludedWikipediaSectionTitle matches the appendix titles of the largest wikis", () => { + for (const title of [ + "Einzelnachweise", + "Weblinks", + "Notes et références", + "Liens externes", + "Enlaces externos", + "Collegamenti esterni", + "Ligações externas", + "Externe links", + "Przypisy", + "Примечания", + "脚注", + "外部リンク", + "參考文獻", + "外部链接", + ]) { + assert.equal(isExcludedWikipediaSectionTitle(title), true, title); + } +}); + +test("isExcludedWikipediaSectionTitle keeps See also sections in every language, as English does", () => { + for (const title of [ + "See also", + "Siehe auch", + "Voir aussi", + "Véase también", + "Voci correlate", + "関連項目", + ]) { + assert.equal(isExcludedWikipediaSectionTitle(title), false, title); + } +}); + +test("excluded section titles are stored in their normalized form", () => { + // Matching compares normalized heading text against the list verbatim. + for (const title of WIKIPEDIA_EXCLUDED_SECTION_TITLES) { + assert.equal(normalizeWikipediaSectionTitle(title), title); + } +}); + test("shouldExcludeWikipediaElement excludes references-class blocks", () => { - assert.equal( - shouldExcludeWikipediaElement({ - tagName: "ol", - classTokens: ["references"], - }), - true, - ); + assert.equal(shouldExcludeWikipediaElement(element("ol", ["references"])), true); }); test("shouldExcludeWikipediaElement excludes citation superscripts only", () => { + assert.equal(shouldExcludeWikipediaElement(element("sup", ["reference"])), true); + assert.equal(shouldExcludeWikipediaElement(element("sup")), false); +}); + +test("shouldExcludeWikipediaElement excludes navigation landmarks whatever their classes", () => { + // de "Hauptartikel" links, nl navboxes, en series sidebars. + assert.equal(shouldExcludeWikipediaElement(element("div", ["hauptartikel"], "navigation")), true); + assert.equal(shouldExcludeWikipediaElement(element("table", ["sidebar"], " Navigation ")), true); + assert.equal(shouldExcludeWikipediaElement(element("table", ["infobox"], "presentation")), false); + assert.equal(shouldExcludeWikipediaElement(element("div", [], "note")), false); +}); + +test("shouldExcludeWikipediaElement excludes the cross-wiki non-prose conventions", () => { + // Hatnotes, navboxes and authority control are kept out of search. assert.equal( - shouldExcludeWikipediaElement({ - tagName: "sup", - classTokens: ["reference"], - }), + shouldExcludeWikipediaElement(element("div", ["hatnote", "navigation-not-searchable"], "note")), true, ); - assert.equal( - shouldExcludeWikipediaElement({ - tagName: "sup", - classTokens: [], - }), - false, - ); + // Banners and person-data tables are about the article, not of it. + assert.equal(shouldExcludeWikipediaElement(element("table", ["metadata", "ambox"])), true); + assert.equal(shouldExcludeWikipediaElement(element("p")), false); + assert.equal(shouldExcludeWikipediaElement(element("table", ["wikitable"])), false); }); // --------------------------------------------------------------------------- diff --git a/src/typescript/shared/test/unit/wikipedia-identity.test.ts b/src/typescript/shared/test/unit/wikipedia-identity.test.ts index 58d0a42..f1e7b09 100644 --- a/src/typescript/shared/test/unit/wikipedia-identity.test.ts +++ b/src/typescript/shared/test/unit/wikipedia-identity.test.ts @@ -2,7 +2,7 @@ import assert from "node:assert/strict"; import { test } from "node:test"; import { normalizeWikipediaTitleToken, - parseWikipediaIdentity, + parseWikipediaUrlIdentity, wikipediaExternalIdFromPageId, } from "../../src/wikipedia-identity.js"; @@ -11,33 +11,39 @@ test("normalizeWikipediaTitleToken normalizes spacing and underscores", () => { assert.equal(normalizeWikipediaTitleToken(" "), null); }); -test("parseWikipediaIdentity prefers page ID identity when available", () => { - const parsed = parseWikipediaIdentity("https://en.wikipedia.org/wiki/OpenAI?curid=48795986"); +test("parseWikipediaUrlIdentity prefers page ID identity when available", () => { + const parsed = parseWikipediaUrlIdentity("https://en.wikipedia.org/wiki/OpenAI?curid=48795986"); assert.deepEqual(parsed, { + kind: "PAGE_ID", language: "en", title: "OpenAI", pageId: "48795986", - identityKind: "PAGE_ID", - externalId: "en:48795986", }); }); -test("parseWikipediaIdentity parses title identity from /w/index.php route", () => { - const parsed = parseWikipediaIdentity( +test("parseWikipediaUrlIdentity accepts page-ID-only index.php URLs", () => { + const parsed = parseWikipediaUrlIdentity("https://de.wikipedia.org/w/index.php?curid=736"); + assert.deepEqual(parsed, { kind: "PAGE_ID", language: "de", title: null, pageId: "736" }); +}); + +test("parseWikipediaUrlIdentity parses title identity from /w/index.php route", () => { + const parsed = parseWikipediaUrlIdentity( "https://en.wikipedia.org/w/index.php?title=OpenAI&oldid=1340968511", ); - assert.deepEqual(parsed, { - language: "en", - title: "OpenAI", - pageId: null, - identityKind: "TITLE", - externalId: "en:OpenAI", - }); + assert.deepEqual(parsed, { kind: "TITLE", language: "en", title: "OpenAI" }); +}); + +test("parseWikipediaUrlIdentity rejects canonical non-article namespaces on any language edition", () => { + assert.equal(parseWikipediaUrlIdentity("https://en.wikipedia.org/wiki/Talk:OpenAI"), null); + assert.equal(parseWikipediaUrlIdentity("https://en.wikipedia.org/wiki/File:Example.jpg"), null); + // MediaWiki accepts canonical namespace names on every wiki. + assert.equal(parseWikipediaUrlIdentity("https://de.wikipedia.org/wiki/Talk:OpenAI"), null); }); -test("parseWikipediaIdentity rejects non-article namespaces", () => { - assert.equal(parseWikipediaIdentity("https://en.wikipedia.org/wiki/Talk:OpenAI"), null); - assert.equal(parseWikipediaIdentity("https://en.wikipedia.org/wiki/File:Example.jpg"), null); +test("parseWikipediaUrlIdentity rejects non-Wikipedia hosts and namespace-less paths", () => { + assert.equal(parseWikipediaUrlIdentity("https://example.org/wiki/OpenAI"), null); + assert.equal(parseWikipediaUrlIdentity("https://en.wikipedia.org/"), null); + assert.equal(parseWikipediaUrlIdentity("not a url"), null); }); test("wikipediaExternalIdFromPageId builds deterministic external IDs", () => { From 994afad8b1306e2c98c9f6531662b44968d093ce Mon Sep 17 00:00:00 2001 From: Dean Valentine Date: Fri, 2 Oct 2026 17:06:38 -0700 Subject: [PATCH 10/15] api: gpt-6.1-sol, truthful audit, security and queue fixes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Investigator - gpt-6.1-sol in one request config (web_search with sources, verified reasoning summaries); OPENAI_MODEL_ID removed - Investigation.model is the model that ran, recorded at completion - One audit record per provider request (fact-check rounds, validations); typed SDK responses; incomplete responses fail without retries - One request-shaped probe for worker startup and user-key validation; live smoke script (pnpm smoke:openai) Security and correctness - Post URL/author from server-verified data; client URLs validated - User OpenAI keys verified before use, never take over server-paid runs, and are dropped (not FAILED) when OpenAI rejects them - SSRF-safe fetching (validated, pinned addresses) for images and fetch_url - FAILED is terminal; attempt numbers never reset; audit is insert-only - recordViewAndGetStatus reports INVESTIGATING/FAILED with investigationId - Interim claims carry forward from the latest finished investigation, limited to claims whose text is still in the version Queue - Selector admits at most SELECTOR_DAILY_BUDGET investigations per UTC day - Lease ⇔ PROCESSING enforced by deferred triggers; one recovery path; lease loss aborts the run; update lineage shared with the selector Migrations 0024 (model + per-request audit) and 0025 (admission origin, lease invariant, input snapshot). Removes the unused attestation HMAC and public tRPC router. Co-Authored-By: Claude Opus 5.5 --- src/typescript/api/package.json | 2 + .../migration.sql | 204 ++++ .../migration.sql | 351 +++++++ src/typescript/api/prisma/schema.prisma | 280 ++++-- .../api/scripts/openai-live-smoke.ts | 249 +++++ src/typescript/api/src/lib/config/env.ts | 14 +- src/typescript/api/src/lib/config/runtime.ts | 5 +- src/typescript/api/src/lib/config/startup.ts | 50 +- src/typescript/api/src/lib/date.ts | 7 + src/typescript/api/src/lib/db/client.ts | 5 +- .../api/src/lib/db/prisma-enum-compat.ts | 4 - .../api/src/lib/graphql/public-schema.ts | 6 +- .../api/src/lib/investigators/errors.ts | 66 ++ .../src/lib/investigators/fetch-url-tool.ts | 163 +--- .../api/src/lib/investigators/interface.ts | 285 +++--- .../openai-attempt-audit-builder.ts | 97 -- .../lib/investigators/openai-claim-tools.ts | 139 +++ .../openai-claim-validation-scheduler.ts | 33 +- .../investigators/openai-claim-validator.ts | 187 ++-- .../src/lib/investigators/openai-errors.ts | 11 - .../lib/investigators/openai-input-builder.ts | 98 +- .../openai-investigation-run-state.ts | 14 +- .../api/src/lib/investigators/openai-probe.ts | 26 + .../investigators/openai-request-config.ts | 153 +++ .../investigators/openai-response-audit.ts | 610 +++--------- .../src/lib/investigators/openai-schemas.ts | 35 - .../lib/investigators/openai-tool-dispatch.ts | 174 +--- .../src/lib/investigators/openai-tool-loop.ts | 207 ++-- .../api/src/lib/investigators/openai.ts | 567 ++++------- .../api/src/lib/network/host-safety.ts | 179 +--- src/typescript/api/src/lib/network/ip.ts | 165 +--- .../api/src/lib/network/public-http-fetch.ts | 186 ++++ src/typescript/api/src/lib/openai/errors.ts | 74 +- .../api/src/lib/services/attempt-audit.ts | 296 ++---- .../api/src/lib/services/blob-storage.ts | 9 - .../src/lib/services/canonical-resolution.ts | 2 +- .../api/src/lib/services/content-fetcher.ts | 262 +++-- src/typescript/api/src/lib/services/hmac.ts | 28 - .../api/src/lib/services/html-to-markdown.ts | 72 +- .../api/src/lib/services/image-downloader.ts | 175 +--- .../api/src/lib/services/investigate-now.ts | 172 ++++ .../lib/services/investigation-admission.ts | 106 ++ .../src/lib/services/investigation-input.ts | 234 +++++ .../src/lib/services/investigation-lease.ts | 463 +++++---- .../lib/services/investigation-lifecycle.ts | 420 -------- .../src/lib/services/markdown-resolution.ts | 73 +- .../services/openai-key-validation-core.ts | 105 +- .../src/lib/services/openai-key-validation.ts | 30 +- .../src/lib/services/orchestrator-errors.ts | 24 +- .../api/src/lib/services/orchestrator.ts | 382 +++++--- .../api/src/lib/services/prompt-context.ts | 235 ----- .../api/src/lib/services/public-read-model.ts | 138 ++- .../api/src/lib/services/queue-lifecycle.ts | 195 ---- src/typescript/api/src/lib/services/queue.ts | 48 +- .../api/src/lib/services/request-identity.ts | 92 +- .../src/lib/services/selector-entrypoint.ts | 13 +- .../api/src/lib/services/selector.ts | 293 +++++- .../api/src/lib/services/update-lineage.ts | 87 ++ .../api/src/lib/services/user-key-source.ts | 136 +-- .../api/src/lib/services/view-credit.ts | 9 +- .../lib/services/wikipedia-content-filter.ts | 84 +- .../api/src/lib/services/word-count.ts | 7 + .../api/src/lib/services/worker-runner.ts | 2 + src/typescript/api/src/lib/trpc/context.ts | 23 +- src/typescript/api/src/lib/trpc/router.ts | 3 +- .../api/src/lib/trpc/routes/post.ts | 422 ++++---- .../lib/trpc/routes/post/content-storage.ts | 1 - .../trpc/routes/post/content-storage/blobs.ts | 2 +- .../content-storage/content-preparation.ts | 2 +- .../routes/post/content-storage/metadata.ts | 56 +- .../post/content-storage/observed-url.ts | 82 ++ .../post/content-storage/post-upsert.ts | 240 ++--- .../register-observed-version.ts | 24 +- .../routes/post/content-storage/shared.ts | 48 +- .../trpc/routes/post/investigation-queries.ts | 373 ++----- .../api/src/lib/trpc/routes/post/wikipedia.ts | 42 +- .../api/src/lib/trpc/routes/public.ts | 104 -- .../api/test/helpers/fake-openai.ts | 202 ++++ ...ndpoints.attempt-audit.integration.test.ts | 329 +++++++ ...api-endpoints.integration.attempt-audit.ts | 94 +- .../api-endpoints.integration.shared.ts | 128 ++- ...pi-endpoints.lifecycle.integration.test.ts | 919 +++++++++--------- ...points.public-and-auth.integration.test.ts | 692 +++++++------ ...content-and-versioning.integration.test.ts | 183 +++- ...record-and-investigate.integration.test.ts | 11 +- ...terim-source-selection.integration.test.ts | 130 ++- ...nts.record-view-status.integration.test.ts | 207 ++++ ...ion-identity-and-retry.integration.test.ts | 86 +- .../integration/helpers/external-api-mocks.ts | 59 ++ .../helpers/investigate-now-scenario-dsl.ts | 8 +- .../api/test/integration/integration-env.ts | 6 +- .../test/integration/lesswrong-fixtures.ts | 3 + .../api/test/integration/live-canary.test.ts | 86 +- .../trigger-behavior.integration.test.ts | 129 ++- .../api/test/unit/attempt-audit.test.ts | 188 ---- .../test/unit/canonical-resolution.test.ts | 24 +- .../api/test/unit/content-fetcher.test.ts | 158 ++- .../api/test/unit/env-config.test.ts | 30 +- .../api/test/unit/env-example.test.ts | 12 + .../unit/frontend-graphql-contract.test.ts | 134 +++ src/typescript/api/test/unit/hmac.test.ts | 30 - .../api/test/unit/host-safety.test.ts | 143 ++- .../api/test/unit/html-to-markdown.test.ts | 166 ++-- .../api/test/unit/investigation-input.test.ts | 204 ++++ .../api/test/unit/investigation-lease.test.ts | 78 +- .../test/unit/investigation-lifecycle.test.ts | 98 -- .../api/test/unit/ip-prefix.test.ts | 17 +- .../markdown-resolution-placeholders.test.ts | 65 -- .../api/test/unit/markdown-resolution.test.ts | 7 +- .../unit/openai-attempt-audit-builder.test.ts | 108 -- .../api/test/unit/openai-claim-tools.test.ts | 140 +++ .../openai-claim-validation-scheduler.test.ts | 25 +- .../test/unit/openai-claim-validator.test.ts | 327 ++----- .../test/unit/openai-input-builder.test.ts | 59 +- .../openai-investigate-claim-order.test.ts | 197 ---- .../openai-investigation-run-state.test.ts | 41 +- .../api/test/unit/openai-investigator.test.ts | 403 ++++++++ .../unit/openai-key-validation-core.test.ts | 140 ++- .../test/unit/openai-response-audit.test.ts | 918 +++-------------- .../api/test/unit/openai-schemas.test.ts | 64 -- .../test/unit/openai-tool-dispatch.test.ts | 164 ++-- .../api/test/unit/openai-tool-loop.test.ts | 318 +++--- .../api/test/unit/orchestrator-errors.test.ts | 94 +- .../unit/orchestrator-persist-guard.test.ts | 66 +- .../unit/post-investigation-queries.test.ts | 227 ++--- .../api/test/unit/prompt-context.test.ts | 468 --------- .../api/test/unit/public-read-model.test.ts | 39 +- .../api/test/unit/public-schema.test.ts | 211 ++-- .../api/test/unit/queue-lifecycle.test.ts | 354 ------- .../api/test/unit/request-identity.test.ts | 80 +- .../api/test/unit/startup-config.test.ts | 36 + .../api/test/unit/update-lineage.test.ts | 21 + .../api/test/unit/word-count.test.ts | 37 + src/typescript/api/tsconfig.json | 2 +- 134 files changed, 9659 insertions(+), 9766 deletions(-) create mode 100644 src/typescript/api/prisma/migrations/0024_investigation_model_and_attempt_requests/migration.sql create mode 100644 src/typescript/api/prisma/migrations/0025_investigation_admission_and_lease_invariant/migration.sql create mode 100644 src/typescript/api/scripts/openai-live-smoke.ts create mode 100644 src/typescript/api/src/lib/investigators/errors.ts delete mode 100644 src/typescript/api/src/lib/investigators/openai-attempt-audit-builder.ts create mode 100644 src/typescript/api/src/lib/investigators/openai-claim-tools.ts delete mode 100644 src/typescript/api/src/lib/investigators/openai-errors.ts create mode 100644 src/typescript/api/src/lib/investigators/openai-probe.ts create mode 100644 src/typescript/api/src/lib/investigators/openai-request-config.ts delete mode 100644 src/typescript/api/src/lib/investigators/openai-schemas.ts create mode 100644 src/typescript/api/src/lib/network/public-http-fetch.ts delete mode 100644 src/typescript/api/src/lib/services/hmac.ts create mode 100644 src/typescript/api/src/lib/services/investigate-now.ts create mode 100644 src/typescript/api/src/lib/services/investigation-admission.ts create mode 100644 src/typescript/api/src/lib/services/investigation-input.ts delete mode 100644 src/typescript/api/src/lib/services/investigation-lifecycle.ts delete mode 100644 src/typescript/api/src/lib/services/prompt-context.ts delete mode 100644 src/typescript/api/src/lib/services/queue-lifecycle.ts create mode 100644 src/typescript/api/src/lib/services/update-lineage.ts create mode 100644 src/typescript/api/src/lib/services/word-count.ts create mode 100644 src/typescript/api/src/lib/trpc/routes/post/content-storage/observed-url.ts delete mode 100644 src/typescript/api/src/lib/trpc/routes/public.ts create mode 100644 src/typescript/api/test/helpers/fake-openai.ts create mode 100644 src/typescript/api/test/integration/api-endpoints.attempt-audit.integration.test.ts create mode 100644 src/typescript/api/test/integration/api-endpoints.record-view-status.integration.test.ts create mode 100644 src/typescript/api/test/integration/helpers/external-api-mocks.ts delete mode 100644 src/typescript/api/test/unit/attempt-audit.test.ts create mode 100644 src/typescript/api/test/unit/env-example.test.ts create mode 100644 src/typescript/api/test/unit/frontend-graphql-contract.test.ts delete mode 100644 src/typescript/api/test/unit/hmac.test.ts create mode 100644 src/typescript/api/test/unit/investigation-input.test.ts delete mode 100644 src/typescript/api/test/unit/investigation-lifecycle.test.ts delete mode 100644 src/typescript/api/test/unit/markdown-resolution-placeholders.test.ts delete mode 100644 src/typescript/api/test/unit/openai-attempt-audit-builder.test.ts create mode 100644 src/typescript/api/test/unit/openai-claim-tools.test.ts delete mode 100644 src/typescript/api/test/unit/openai-investigate-claim-order.test.ts create mode 100644 src/typescript/api/test/unit/openai-investigator.test.ts delete mode 100644 src/typescript/api/test/unit/openai-schemas.test.ts delete mode 100644 src/typescript/api/test/unit/prompt-context.test.ts delete mode 100644 src/typescript/api/test/unit/queue-lifecycle.test.ts create mode 100644 src/typescript/api/test/unit/startup-config.test.ts create mode 100644 src/typescript/api/test/unit/update-lineage.test.ts create mode 100644 src/typescript/api/test/unit/word-count.test.ts diff --git a/src/typescript/api/package.json b/src/typescript/api/package.json index 4034900..85958ac 100644 --- a/src/typescript/api/package.json +++ b/src/typescript/api/package.json @@ -14,6 +14,7 @@ "worker": "tsx --tsconfig tsconfig.runtime.json src/lib/services/worker-entrypoint.ts", "selector": "tsx --tsconfig tsconfig.runtime.json src/lib/services/selector-entrypoint.ts", "instance-api-key": "tsx --tsconfig tsconfig.runtime.json src/lib/services/instance-api-key-entrypoint.ts", + "smoke:openai": "tsx --tsconfig tsconfig.runtime.json scripts/openai-live-smoke.ts", "prisma:generate": "svelte-kit sync && prisma generate", "prisma:migrate:dev": "prisma migrate dev", "prisma:migrate:deploy": "prisma migrate deploy", @@ -44,6 +45,7 @@ "parse5": "^8.0.0", "pg": "^8.18.0", "turndown": "^7.2.2", + "undici": "^7.22.0", "zod": "^4.3.6" }, "devDependencies": { diff --git a/src/typescript/api/prisma/migrations/0024_investigation_model_and_attempt_requests/migration.sql b/src/typescript/api/prisma/migrations/0024_investigation_model_and_attempt_requests/migration.sql new file mode 100644 index 0000000..38901c5 --- /dev/null +++ b/src/typescript/api/prisma/migrations/0024_investigation_model_and_attempt_requests/migration.sql @@ -0,0 +1,204 @@ +-- Two audit-truthfulness changes: +-- +-- 1. Investigation.model stops being an InvestigationModel enum value guessed +-- at queue time. It becomes the provider model id the stage-1 fact-check +-- requests were actually sent to, recorded at completion: set iff +-- status = COMPLETE (INV-INV-MODEL-AT-COMPLETION). Existing COMPLETE rows +-- take the requestModel of their SUCCEEDED attempt; every other row becomes +-- NULL. A COMPLETE investigation without exactly one SUCCEEDED attempt has +-- no truthful value, so the migration aborts instead of inventing one. +-- +-- 2. InvestigationAttempt audits move to one InvestigationAttemptRequest row +-- per provider request (fact-check rounds, per-claim validations), each +-- with its own response. Existing attempts squashed every request into one +-- record that cannot be split truthfully, so each becomes a single +-- LEGACY_COMBINED request (with one response when it had one) and keeps its +-- children unchanged. Dropped legacy data: responseOutputText (the SDK's +-- concatenation of the last response's output_text parts, which remain as +-- text parts) and tool-call capturedAt/providerStartedAt/providerCompletedAt +-- (parse-time stamps and always-null guesses); tool-call id/type/status +-- columns duplicated their output item's and are dropped. + +-- ── 1a. Preconditions ──────────────────────────────────────────────────────── + +DO $$ +DECLARE + offending_investigation_ids TEXT; + offending_attempt_ids TEXT; +BEGIN + SELECT string_agg(i."id", ', ' ORDER BY i."id") + INTO offending_investigation_ids + FROM "Investigation" i + WHERE i."status" = 'COMPLETE' + AND ( + SELECT COUNT(*) + FROM "InvestigationAttempt" a + WHERE a."investigationId" = i."id" + AND a."outcome" = 'SUCCEEDED' + ) <> 1; + + IF offending_investigation_ids IS NOT NULL THEN + RAISE EXCEPTION + 'Cannot record Investigation.model: COMPLETE investigations without exactly one SUCCEEDED InvestigationAttempt have no recorded request model: %', + offending_investigation_ids; + END IF; + + SELECT string_agg(a."id", ', ' ORDER BY a."id") + INTO offending_attempt_ids + FROM "InvestigationAttempt" a + WHERE ( + a."responseId" IS NULL + AND ( + a."responseStatus" IS NOT NULL + OR a."responseModelVersion" IS NOT NULL + OR a."responseOutputText" IS NOT NULL + OR EXISTS (SELECT 1 FROM "InvestigationAttemptOutputItem" o WHERE o."attemptId" = a."id") + OR EXISTS (SELECT 1 FROM "InvestigationAttemptUsage" u WHERE u."attemptId" = a."id") + ) + ) + OR (a."responseId" IS NOT NULL AND a."responseModelVersion" IS NULL); + + IF offending_attempt_ids IS NOT NULL THEN + RAISE EXCEPTION + 'Cannot migrate InvestigationAttempt audits: attempts with a partially recorded response (response data without responseId, or responseId without responseModelVersion): %', + offending_attempt_ids; + END IF; +END +$$; + +-- ── 1b. Investigation.model: recorded at completion ───────────────────────── + +ALTER TABLE "Investigation" ALTER COLUMN "model" DROP NOT NULL; +ALTER TABLE "Investigation" ALTER COLUMN "model" TYPE TEXT USING NULL; + +UPDATE "Investigation" i +SET "model" = a."requestModel" +FROM "InvestigationAttempt" a +WHERE a."investigationId" = i."id" + AND a."outcome" = 'SUCCEEDED' + AND i."status" = 'COMPLETE'; + +ALTER TABLE "Investigation" + ADD CONSTRAINT "Investigation_model_consistency_check" + CHECK (("status" = 'COMPLETE') = ("model" IS NOT NULL)); + +DROP TYPE "InvestigationModel"; + +-- ── 2a. Per-request audit tables ──────────────────────────────────────────── + +CREATE TYPE "InvestigationAttemptRequestKind" AS ENUM ('FACT_CHECK_ROUND', 'CLAIM_VALIDATION', 'LEGACY_COMBINED'); + +CREATE TABLE "InvestigationAttemptRequest" ( + "id" TEXT NOT NULL, + "attemptId" TEXT NOT NULL, + "kind" "InvestigationAttemptRequestKind" NOT NULL, + "factCheckRound" INTEGER, + "claimIndex" INTEGER, + "model" TEXT NOT NULL, + "instructions" TEXT NOT NULL, + "input" JSONB NOT NULL, + "previousResponseId" TEXT, + "reasoningEffort" TEXT, + "reasoningSummary" TEXT, + "include" TEXT[], + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + + CONSTRAINT "InvestigationAttemptRequest_pkey" PRIMARY KEY ("id"), + -- INV-ATTEMPT-REQUEST-SUBJECT + CONSTRAINT "InvestigationAttemptRequest_subject_check" CHECK ( + ("factCheckRound" IS NOT NULL) = ("kind" = 'FACT_CHECK_ROUND') + AND ("claimIndex" IS NOT NULL) = ("kind" = 'CLAIM_VALIDATION') + AND COALESCE("factCheckRound", 0) >= 0 + AND COALESCE("claimIndex", 0) >= 0 + ) +); + +CREATE TABLE "InvestigationAttemptResponse" ( + "id" TEXT NOT NULL, + "requestId" TEXT NOT NULL, + "providerResponseId" TEXT NOT NULL, + "status" TEXT, + "modelVersion" TEXT NOT NULL, + "receivedAt" TIMESTAMP(3), + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + + CONSTRAINT "InvestigationAttemptResponse_pkey" PRIMARY KEY ("id") +); + +-- ── 2b. Legacy attempts → one LEGACY_COMBINED request each ───────────────── +-- The legacy request and response reuse their attempt's id (distinct tables), +-- which lets the children below be re-parented without a mapping table. + +INSERT INTO "InvestigationAttemptRequest" ( + "id", "attemptId", "kind", "factCheckRound", "claimIndex", "model", "instructions", "input", + "previousResponseId", "reasoningEffort", "reasoningSummary", "include", "createdAt" +) +SELECT + a."id", a."id", 'LEGACY_COMBINED', NULL, NULL, a."requestModel", a."requestInstructions", + to_jsonb(a."requestInput"), NULL, a."requestReasoningEffort", a."requestReasoningSummary", + -- Legacy requests sent no `include` parameter. + ARRAY[]::TEXT[], a."createdAt" +FROM "InvestigationAttempt" a; + +INSERT INTO "InvestigationAttemptResponse" ( + "id", "requestId", "providerResponseId", "status", "modelVersion", "receivedAt", "createdAt" +) +SELECT a."id", a."id", a."responseId", a."responseStatus", a."responseModelVersion", NULL, a."createdAt" +FROM "InvestigationAttempt" a +WHERE a."responseId" IS NOT NULL; + +ALTER TABLE "InvestigationAttemptRequestedTool" DROP CONSTRAINT "InvestigationAttemptRequestedTool_attemptId_fkey"; +DROP INDEX "InvestigationAttemptRequestedTool_attemptId_idx"; +DROP INDEX "InvestigationAttemptRequestedTool_attemptId_requestOrder_key"; +ALTER TABLE "InvestigationAttemptRequestedTool" RENAME COLUMN "attemptId" TO "requestId"; + +ALTER TABLE "InvestigationAttemptOutputItem" DROP CONSTRAINT "InvestigationAttemptOutputItem_attemptId_fkey"; +DROP INDEX "InvestigationAttemptOutputItem_attemptId_idx"; +DROP INDEX "InvestigationAttemptOutputItem_attemptId_outputIndex_key"; +ALTER TABLE "InvestigationAttemptOutputItem" RENAME COLUMN "attemptId" TO "responseId"; + +ALTER TABLE "InvestigationAttemptUsage" DROP CONSTRAINT "InvestigationAttemptUsage_attemptId_fkey"; +DROP INDEX "InvestigationAttemptUsage_attemptId_key"; +ALTER TABLE "InvestigationAttemptUsage" RENAME COLUMN "attemptId" TO "responseId"; + +ALTER TABLE "InvestigationAttemptToolCall" DROP CONSTRAINT "InvestigationAttemptToolCall_attemptId_fkey"; +DROP INDEX "InvestigationAttemptToolCall_attemptId_idx"; +DROP INDEX "InvestigationAttemptToolCall_attemptId_outputIndex_key"; +ALTER TABLE "InvestigationAttemptToolCall" + DROP COLUMN "attemptId", + DROP COLUMN "outputIndex", + DROP COLUMN "providerToolCallId", + DROP COLUMN "toolType", + DROP COLUMN "status", + DROP COLUMN "capturedAt", + DROP COLUMN "providerStartedAt", + DROP COLUMN "providerCompletedAt"; + +ALTER TABLE "InvestigationAttempt" + DROP COLUMN "requestModel", + DROP COLUMN "requestInstructions", + DROP COLUMN "requestInput", + DROP COLUMN "requestReasoningEffort", + DROP COLUMN "requestReasoningSummary", + DROP COLUMN "responseId", + DROP COLUMN "responseStatus", + DROP COLUMN "responseModelVersion", + DROP COLUMN "responseOutputText"; + +-- ── 2c. Indexes and foreign keys ──────────────────────────────────────────── + +CREATE INDEX "InvestigationAttemptRequest_attemptId_idx" ON "InvestigationAttemptRequest"("attemptId"); +CREATE UNIQUE INDEX "InvestigationAttemptRequest_attemptId_factCheckRound_key" ON "InvestigationAttemptRequest"("attemptId", "factCheckRound"); +CREATE UNIQUE INDEX "InvestigationAttemptRequest_attemptId_claimIndex_key" ON "InvestigationAttemptRequest"("attemptId", "claimIndex"); +CREATE UNIQUE INDEX "InvestigationAttemptResponse_requestId_key" ON "InvestigationAttemptResponse"("requestId"); +CREATE INDEX "InvestigationAttemptOutputItem_responseId_idx" ON "InvestigationAttemptOutputItem"("responseId"); +CREATE UNIQUE INDEX "InvestigationAttemptOutputItem_responseId_outputIndex_key" ON "InvestigationAttemptOutputItem"("responseId", "outputIndex"); +CREATE INDEX "InvestigationAttemptRequestedTool_requestId_idx" ON "InvestigationAttemptRequestedTool"("requestId"); +CREATE UNIQUE INDEX "InvestigationAttemptRequestedTool_requestId_requestOrder_key" ON "InvestigationAttemptRequestedTool"("requestId", "requestOrder"); +CREATE UNIQUE INDEX "InvestigationAttemptUsage_responseId_key" ON "InvestigationAttemptUsage"("responseId"); + +ALTER TABLE "InvestigationAttemptRequest" ADD CONSTRAINT "InvestigationAttemptRequest_attemptId_fkey" FOREIGN KEY ("attemptId") REFERENCES "InvestigationAttempt"("id") ON DELETE CASCADE ON UPDATE CASCADE; +ALTER TABLE "InvestigationAttemptRequestedTool" ADD CONSTRAINT "InvestigationAttemptRequestedTool_requestId_fkey" FOREIGN KEY ("requestId") REFERENCES "InvestigationAttemptRequest"("id") ON DELETE CASCADE ON UPDATE CASCADE; +ALTER TABLE "InvestigationAttemptResponse" ADD CONSTRAINT "InvestigationAttemptResponse_requestId_fkey" FOREIGN KEY ("requestId") REFERENCES "InvestigationAttemptRequest"("id") ON DELETE CASCADE ON UPDATE CASCADE; +ALTER TABLE "InvestigationAttemptOutputItem" ADD CONSTRAINT "InvestigationAttemptOutputItem_responseId_fkey" FOREIGN KEY ("responseId") REFERENCES "InvestigationAttemptResponse"("id") ON DELETE CASCADE ON UPDATE CASCADE; +ALTER TABLE "InvestigationAttemptUsage" ADD CONSTRAINT "InvestigationAttemptUsage_responseId_fkey" FOREIGN KEY ("responseId") REFERENCES "InvestigationAttemptResponse"("id") ON DELETE CASCADE ON UPDATE CASCADE; diff --git a/src/typescript/api/prisma/migrations/0025_investigation_admission_and_lease_invariant/migration.sql b/src/typescript/api/prisma/migrations/0025_investigation_admission_and_lease_invariant/migration.sql new file mode 100644 index 0000000..c3e9274 --- /dev/null +++ b/src/typescript/api/prisma/migrations/0025_investigation_admission_and_lease_invariant/migration.sql @@ -0,0 +1,351 @@ +-- ============================================================================ +-- Migration 0025: investigation admission, lease invariant, input snapshot +-- ============================================================================ +-- +-- 1. Investigation.origin + admittedAt record who admitted (and therefore pays +-- for) each investigation, so the selector budget can count its own +-- admissions per UTC day. +-- 2. "InvestigationLease row exists iff status = PROCESSING" moves from code +-- comments into deferred constraint triggers checked at commit. +-- 3. Post.identityVerifiedAt latches once url/author come from a server fetch; +-- stored non-HTTPS post URLs are rebuilt from platform identity. +-- 4. InvestigationInput snapshots the prompt context (post URL, author, +-- publication time, video flag) and the source URL behind every [IMAGE:N] +-- markdown placeholder at queue time. +-- 5. Source.snapshotText/snapshotHash/retrievedAt are dropped: they only ever +-- held a copy of the snippet and the save time. +-- 6. SubstackVersionMeta.serverHtmlBlobId is dropped: Substack has no +-- server-side fetch, so it could never be populated. +-- 7. The LessWrong/Substack/Wikipedia version-meta triggers are renamed: they +-- enforce an update policy, they no longer reject updates. + +-- ── 1. Investigation origin / admission ───────────────────────────────────── + +CREATE TYPE "InvestigationOrigin" AS ENUM ('SELECTOR', 'INSTANCE_REQUEST', 'USER_KEY_REQUEST'); + +ALTER TABLE "Investigation" + ADD COLUMN "origin" "InvestigationOrigin", + ADD COLUMN "admittedAt" TIMESTAMP(3); + +-- Historical rows: an attached user-key source identifies a user-key request. +-- Selector and instance-key requests cannot be told apart after the fact, so +-- they are recorded as SELECTOR. That is the conservative choice for the daily +-- budget: rows created earlier today count against today's selector budget. +UPDATE "Investigation" i +SET + "origin" = CASE + WHEN EXISTS ( + SELECT 1 FROM "InvestigationOpenAiKeySource" ks WHERE ks."investigationId" = i."id" + ) THEN 'USER_KEY_REQUEST'::"InvestigationOrigin" + ELSE 'SELECTOR'::"InvestigationOrigin" + END, + "admittedAt" = i."createdAt"; + +ALTER TABLE "Investigation" + ALTER COLUMN "origin" SET NOT NULL, + ALTER COLUMN "admittedAt" SET NOT NULL; + +CREATE INDEX "Investigation_origin_admittedAt_idx" ON "Investigation"("origin", "admittedAt"); + +-- ── 2. Lease row exists iff status = PROCESSING ───────────────────────────── + +-- Repair rows that violate the invariant before enforcing it. +DELETE FROM "InvestigationLease" l +USING "Investigation" i +WHERE i."id" = l."investigationId" + AND i."status" <> 'PROCESSING'; + +UPDATE "Investigation" i +SET "status" = 'PENDING', "queuedAt" = CURRENT_TIMESTAMP +WHERE i."status" = 'PROCESSING' + AND NOT EXISTS ( + SELECT 1 FROM "InvestigationLease" l WHERE l."investigationId" = i."id" + ); + +CREATE FUNCTION "assert_investigation_lease_matches_status"(target_investigation_id TEXT) +RETURNS void +LANGUAGE plpgsql +AS $$ +DECLARE + current_status "CheckStatus"; + has_lease BOOLEAN; +BEGIN + SELECT i."status" + INTO current_status + FROM "Investigation" i + WHERE i."id" = target_investigation_id; + + IF NOT FOUND THEN + -- The investigation was deleted; its lease row cascades with it. + RETURN; + END IF; + + has_lease := EXISTS ( + SELECT 1 FROM "InvestigationLease" l WHERE l."investigationId" = target_investigation_id + ); + + IF (current_status = 'PROCESSING') IS DISTINCT FROM has_lease THEN + RAISE EXCEPTION + 'Investigation % has status % but its lease row %; a lease row must exist iff status = PROCESSING', + target_investigation_id, + current_status, + CASE WHEN has_lease THEN 'exists' ELSE 'is missing' END + USING ERRCODE = 'check_violation'; + END IF; +END; +$$; + +CREATE FUNCTION "enforce_lease_status_on_investigation"() +RETURNS TRIGGER +LANGUAGE plpgsql +AS $$ +BEGIN + PERFORM "assert_investigation_lease_matches_status"(NEW."id"); + RETURN NULL; +END; +$$; + +CREATE FUNCTION "enforce_lease_status_on_lease"() +RETURNS TRIGGER +LANGUAGE plpgsql +AS $$ +BEGIN + IF TG_OP <> 'INSERT' THEN + PERFORM "assert_investigation_lease_matches_status"(OLD."investigationId"); + END IF; + IF TG_OP <> 'DELETE' THEN + PERFORM "assert_investigation_lease_matches_status"(NEW."investigationId"); + END IF; + RETURN NULL; +END; +$$; + +-- Deferred so status and lease changes made in one transaction are checked +-- together at commit. +CREATE CONSTRAINT TRIGGER "enforce_lease_status_on_investigation_trigger" +AFTER INSERT OR UPDATE OF "status" +ON "Investigation" +DEFERRABLE INITIALLY DEFERRED +FOR EACH ROW +EXECUTE FUNCTION "enforce_lease_status_on_investigation"(); + +CREATE CONSTRAINT TRIGGER "enforce_lease_status_on_lease_trigger" +AFTER INSERT OR UPDATE OR DELETE +ON "InvestigationLease" +DEFERRABLE INITIALLY DEFERRED +FOR EACH ROW +EXECUTE FUNCTION "enforce_lease_status_on_lease"(); + +-- ── 3. Post identity ─────────────────────────────────────────────────────── + +ALTER TABLE "Post" ADD COLUMN "identityVerifiedAt" TIMESTAMP(3); + +CREATE FUNCTION "enforce_post_identity_verified_at_latch"() +RETURNS TRIGGER +LANGUAGE plpgsql +AS $$ +BEGIN + IF OLD."identityVerifiedAt" IS NOT NULL AND NEW."identityVerifiedAt" IS NULL THEN + RAISE EXCEPTION + 'Post.identityVerifiedAt cannot be cleared once a server fetch verified the post identity (postId=%)', + NEW."id"; + END IF; + RETURN NEW; +END; +$$; + +CREATE TRIGGER "enforce_post_identity_verified_at_latch_trigger" +BEFORE UPDATE OF "identityVerifiedAt" +ON "Post" +FOR EACH ROW +EXECUTE FUNCTION "enforce_post_identity_verified_at_latch"(); + +-- Post.url used to be overwritten by whatever an unauthenticated client sent, +-- including non-HTTP(S) schemes. Rebuild any such URL from platform identity. +UPDATE "Post" p +SET "url" = CASE p."platform" + WHEN 'LESSWRONG' THEN 'https://www.lesswrong.com/posts/' || p."externalId" + WHEN 'X' THEN 'https://x.com/i/status/' || p."externalId" + WHEN 'WIKIPEDIA' THEN + 'https://' || split_part(p."externalId", ':', 1) || '.wikipedia.org/?curid=' || split_part(p."externalId", ':', 2) + WHEN 'SUBSTACK' THEN ( + SELECT 'https://' || svm."publicationSubdomain" || '.substack.com/p/' || svm."slug" + FROM "SubstackVersionMeta" svm + JOIN "PostVersion" pv ON pv."id" = svm."postVersionId" + WHERE pv."postId" = p."id" + ORDER BY pv."lastSeenAt" DESC, pv."id" DESC + LIMIT 1 + ) +END +WHERE p."url" !~* '^https://'; + +-- ── 4. InvestigationInput prompt-context snapshot ─────────────────────────── + +-- Inputs whose investigation was deleted have nothing to snapshot and are +-- never read again. +DELETE FROM "InvestigationInput" ii +WHERE NOT EXISTS ( + SELECT 1 FROM "Investigation" i WHERE i."inputId" = ii."investigationId" +); + +ALTER TABLE "InvestigationInput" + ADD COLUMN "imagePlaceholderSourceUrls" TEXT[], + ADD COLUMN "postUrl" TEXT, + ADD COLUMN "authorName" TEXT, + ADD COLUMN "postPublishedAt" TIMESTAMP(3), + ADD COLUMN "hasVideo" BOOLEAN; + +ALTER TABLE "InvestigationInput" DISABLE TRIGGER "reject_investigation_input_updates_trigger"; + +-- Backfill from the live rows the worker used to read at run time. Placeholder +-- source URLs cannot be recovered from stored markdown, so historical rows get +-- none (see the in-flight downgrade below). +UPDATE "InvestigationInput" ii +SET + "postUrl" = p."url", + "authorName" = a."displayName", + "postPublishedAt" = COALESCE( + lwm."publishedAt", + xvm."postedAt", + svm."publishedAt", + wvm."lastModifiedAt" + ), + "hasVideo" = COALESCE( + ( + SELECT bool_or( + lower(split_part(split_part(media_url, '?', 1), '#', 1)) ~ '\.(mp4|webm|m3u8|mov|m4v)$' + ) + FROM unnest(xvm."mediaUrls") AS media(media_url) + ), + false + ), + "imagePlaceholderSourceUrls" = ARRAY[]::TEXT[] +FROM "Investigation" i +JOIN "PostVersion" pv ON pv."id" = i."postVersionId" +JOIN "Post" p ON p."id" = pv."postId" +LEFT JOIN "Author" a ON a."id" = p."authorId" +LEFT JOIN "LesswrongVersionMeta" lwm ON lwm."postVersionId" = pv."id" +LEFT JOIN "XVersionMeta" xvm ON xvm."postVersionId" = pv."id" +LEFT JOIN "SubstackVersionMeta" svm ON svm."postVersionId" = pv."id" +LEFT JOIN "WikipediaVersionMeta" wvm ON wvm."postVersionId" = pv."id" +WHERE i."inputId" = ii."investigationId"; + +-- Investigations still waiting to run whose markdown contains image +-- placeholders would otherwise run with placeholders that match no image +-- (their source URLs were never stored). Run them on the flat-text input +-- instead, where images are placed by normalized text offset. +UPDATE "InvestigationInput" ii +SET "markdownSource" = 'NONE', "markdown" = NULL, "markdownRendererVersion" = NULL +FROM "Investigation" i +WHERE i."inputId" = ii."investigationId" + AND i."status" IN ('PENDING', 'PROCESSING') + AND ii."markdown" LIKE '%[IMAGE:%'; + +ALTER TABLE "InvestigationInput" ENABLE TRIGGER "reject_investigation_input_updates_trigger"; + +ALTER TABLE "InvestigationInput" + ALTER COLUMN "postUrl" SET NOT NULL, + ALTER COLUMN "hasVideo" SET NOT NULL; + +ALTER TABLE "InvestigationInput" + ADD CONSTRAINT "InvestigationInput_imagePlaceholderSourceUrls_not_null_chk" + CHECK ("imagePlaceholderSourceUrls" IS NOT NULL); + +ALTER TABLE "InvestigationInput" + ADD CONSTRAINT "InvestigationInput_image_placeholders_require_markdown_chk" + CHECK ( + "markdownSource" <> 'NONE'::"MarkdownSource" + OR cardinality("imagePlaceholderSourceUrls") = 0 + ); + +-- ── 5. Source snapshot placeholders ───────────────────────────────────────── + +ALTER TABLE "Source" + DROP COLUMN "snapshotText", + DROP COLUMN "snapshotHash", + DROP COLUMN "retrievedAt"; + +-- ── 6. SubstackVersionMeta.serverHtmlBlobId ───────────────────────────────── + +-- Substack versions can never be server-verified, so the server-HTML snapshot +-- check rejects serverVerifiedAt on them instead of looking for a column that +-- no longer exists. +CREATE OR REPLACE FUNCTION "enforce_server_verified_html_snapshot"() +RETURNS TRIGGER +LANGUAGE plpgsql +AS $$ +DECLARE + post_platform "Platform"; + has_server_html BOOLEAN; +BEGIN + IF NEW."serverVerifiedAt" IS NULL THEN + RETURN NEW; + END IF; + + SELECT p."platform" + INTO post_platform + FROM "Post" p + WHERE p."id" = NEW."postId"; + + IF NOT FOUND THEN + RAISE EXCEPTION + 'PostVersion references missing Post (postVersionId=%, postId=%)', + NEW."id", + NEW."postId"; + END IF; + + IF post_platform = 'X' THEN + RETURN NEW; + END IF; + + IF post_platform = 'LESSWRONG' THEN + SELECT (lwm."serverHtmlBlobId" IS NOT NULL) + INTO has_server_html + FROM "LesswrongVersionMeta" lwm + WHERE lwm."postVersionId" = NEW."id"; + ELSIF post_platform = 'WIKIPEDIA' THEN + SELECT (wvm."serverHtmlBlobId" IS NOT NULL) + INTO has_server_html + FROM "WikipediaVersionMeta" wvm + WHERE wvm."postVersionId" = NEW."id"; + ELSIF post_platform = 'SUBSTACK' THEN + RAISE EXCEPTION + 'Substack posts have no server-side verification; serverVerifiedAt must stay null (postVersionId=%)', + NEW."id"; + ELSE + RAISE EXCEPTION + 'Unsupported platform on PostVersion (postVersionId=%, platform=%)', + NEW."id", + post_platform; + END IF; + + IF has_server_html IS DISTINCT FROM TRUE THEN + RAISE EXCEPTION + 'serverVerifiedAt requires a server HTML snapshot (postVersionId=%, platform=%)', + NEW."id", + post_platform; + END IF; + + RETURN NEW; +END; +$$; + +ALTER TABLE "SubstackVersionMeta" DROP CONSTRAINT "SubstackVersionMeta_serverHtmlBlobId_fkey"; +ALTER TABLE "SubstackVersionMeta" DROP COLUMN "serverHtmlBlobId"; + +-- ── 7. Version-meta update policy trigger names ───────────────────────────── + +ALTER FUNCTION "reject_lesswrong_version_meta_updates"() + RENAME TO "enforce_lesswrong_version_meta_update_policy"; +ALTER TRIGGER "reject_lesswrong_version_meta_updates_trigger" ON "LesswrongVersionMeta" + RENAME TO "enforce_lesswrong_version_meta_update_policy_trigger"; + +ALTER FUNCTION "reject_substack_version_meta_updates"() + RENAME TO "enforce_substack_version_meta_update_policy"; +ALTER TRIGGER "reject_substack_version_meta_updates_trigger" ON "SubstackVersionMeta" + RENAME TO "enforce_substack_version_meta_update_policy_trigger"; + +ALTER FUNCTION "reject_wikipedia_version_meta_updates"() + RENAME TO "enforce_wikipedia_version_meta_update_policy"; +ALTER TRIGGER "reject_wikipedia_version_meta_updates_trigger" ON "WikipediaVersionMeta" + RENAME TO "enforce_wikipedia_version_meta_update_policy_trigger"; diff --git a/src/typescript/api/prisma/schema.prisma b/src/typescript/api/prisma/schema.prisma index b3195e5..09d3869 100644 --- a/src/typescript/api/prisma/schema.prisma +++ b/src/typescript/api/prisma/schema.prisma @@ -30,13 +30,6 @@ enum InvestigationProvider { ANTHROPIC } -enum InvestigationModel { - OPENAI_GPT_5 - OPENAI_GPT_5_MINI - ANTHROPIC_CLAUDE_SONNET - ANTHROPIC_CLAUDE_OPUS -} - enum ContentProvenance { SERVER_VERIFIED CLIENT_FALLBACK @@ -53,22 +46,53 @@ enum InvestigationAttemptOutcome { FAILED } +enum InvestigationAttemptRequestKind { + // One round of the stage-1 fact-check tool loop. + FACT_CHECK_ROUND + // One stage-2 per-claim validation call. + CLAIM_VALIDATION + // Pre-2026-10 merged audit: attempts recorded before per-request auditing + // squashed every provider request of the attempt into one record (request + // fields of stage 1 with stage-2 text appended, output items of all + // responses, summed usage). Migrated as-is; never written by current code. + LEGACY_COMBINED +} + +// Who admitted an investigation for execution, which also fixes who pays for +// its runs (SPEC §3.7): +// SELECTOR — background selection; server key; counts against the +// selector's per-UTC-day budget (SELECTOR_DAILY_BUDGET). +// INSTANCE_REQUEST — investigateNow from an instance-API-key client; server key. +// USER_KEY_REQUEST — investigateNow funded by the requester's OpenAI key +// (InvestigationOpenAiKeySource). If that key is dropped +// (unusable or rejected by OpenAI) the investigation is +// unfunded until the selector or a new request admits it. +enum InvestigationOrigin { + SELECTOR + INSTANCE_REQUEST + USER_KEY_REQUEST +} + // ─── Models ────────────────────────────────────────────────────────── model Post { - id String @id @default(cuid()) - platform Platform - externalId String - url String - authorId String? - author Author? @relation(fields: [authorId], references: [id]) - viewCount Int @default(0) - uniqueViewScore Int @default(0) - lastViewedAt DateTime? - versions PostVersion[] - viewCredits PostViewCredit[] - createdAt DateTime @default(now()) - updatedAt DateTime @updatedAt + id String @id @default(cuid()) + platform Platform + externalId String + // url and author are identity-bound (SPEC §2.9): when a server fetch verifies + // the post they come from the platform response and identityVerifiedAt is + // set; once set, unverified client data never overwrites them. + url String + authorId String? + author Author? @relation(fields: [authorId], references: [id]) + identityVerifiedAt DateTime? + viewCount Int @default(0) + uniqueViewScore Int @default(0) + lastViewedAt DateTime? + versions PostVersion[] + viewCredits PostViewCredit[] + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt @@unique([platform, externalId]) @@index([viewCount]) @@ -132,7 +156,6 @@ model HtmlBlob { lesswrongServerVersionMetas LesswrongVersionMeta[] @relation("LesswrongServerHtml") lesswrongClientVersionMetas LesswrongVersionMeta[] @relation("LesswrongClientHtml") - substackServerVersionMetas SubstackVersionMeta[] @relation("SubstackServerHtml") substackClientVersionMetas SubstackVersionMeta[] @relation("SubstackClientHtml") wikipediaServerVersionMetas WikipediaVersionMeta[] @relation("WikipediaServerHtml") wikipediaClientVersionMetas WikipediaVersionMeta[] @relation("WikipediaClientHtml") @@ -232,8 +255,7 @@ model SubstackVersionMeta { slug String title String subtitle String? - serverHtmlBlobId String? - serverHtmlBlob HtmlBlob? @relation("SubstackServerHtml", fields: [serverHtmlBlobId], references: [id], onDelete: Restrict) + // Substack has no server-side canonical fetch, so only client HTML exists. clientHtmlBlobId String? clientHtmlBlob HtmlBlob? @relation("SubstackClientHtml", fields: [clientHtmlBlobId], references: [id], onDelete: Restrict) imageUrls String[] @@ -300,23 +322,32 @@ model Investigation { promptId String prompt Prompt @relation(fields: [promptId], references: [id]) provider InvestigationProvider - model InvestigationModel + // Provider model id the stage-1 fact-check requests were sent to (e.g. + // "gpt-6.1-sol"), recorded when the investigation completes. + // INV-INV-MODEL-AT-COMPLETION: model is set iff status = COMPLETE. + // Enforced by CHECK constraint "Investigation_model_consistency_check". + model String? + // Provider-reported model revision of the final stage-1 fact-check response. modelVersion String? // INV-INV-CHECKED-AT-CONSISTENCY: checkedAt is set iff status = COMPLETE. // Enforced by CHECK constraint "Investigation_checked_at_consistency_check". checkedAt DateTime? queuedAt DateTime @default(now()) - // Monotonically increasing attempt counter. Incremented atomically when a - // worker claims the lease. Gives each retry a distinct attemptNumber for - // the InvestigationAttempt audit trail. + origin InvestigationOrigin + // When the current origin admitted (agreed to fund) the investigation. The + // selector's daily budget counts SELECTOR admissions per UTC day. + admittedAt DateTime + // Monotonically increasing attempt counter, never reset. Incremented + // atomically when a worker claims the lease, so every attempt gets a distinct + // attemptNumber in the InvestigationAttempt audit trail. attemptCount Int @default(0) // Set on transient-failure reclaim with exponential backoff. The selector // skips PENDING investigations where retryAfter > now, preventing immediate // re-enqueue from defeating the backoff delay. Cleared on lease claim. retryAfter DateTime? - // INV-LEASE: The InvestigationLease row exists iff the investigation is - // PROCESSING and has an active lease holder. Structurally prevents - // leaseOwner/leaseExpiresAt without PROCESSING, and vice versa. + // INV-LEASE: an InvestigationLease row exists iff status = PROCESSING. + // Enforced at commit by the deferred constraint triggers + // "enforce_lease_status_on_*_trigger". lease InvestigationLease? openAiKeySource InvestigationOpenAiKeySource? attempts InvestigationAttempt[] @@ -330,14 +361,14 @@ model Investigation { @@unique([postVersionId]) @@index([parentInvestigationId]) @@index([status]) + @@index([origin, admittedAt]) } -// INV-LEASE: The existence of an InvestigationLease row means "this -// investigation is PROCESSING and has an active lease holder". All fields +// INV-LEASE: an InvestigationLease row exists iff its investigation is +// PROCESSING (deferred constraint triggers check this at commit). All fields // are NOT NULL — structurally prevents partial lease states. The row is -// deleted on every terminal transition (COMPLETE, FAILED) and on lease -// release (transient failure → PENDING), so progressClaims is automatically -// cleaned up without needing CLEARED_PROGRESS_CLAIMS sentinel values. +// deleted on every transition out of PROCESSING (COMPLETE, FAILED, release to +// PENDING, expired-lease recovery), so progressClaims is cleaned up with it. model InvestigationLease { investigationId String @id investigation Investigation @relation(fields: [investigationId], references: [id], onDelete: Cascade) @@ -351,16 +382,26 @@ model InvestigationLease { @@index([leaseExpiresAt]) } +// Everything the worker feeds the investigator that could change after queue +// time, captured once when the investigation is created so every attempt sees +// the same input (SPEC §2.4.4, §2.12). model InvestigationInput { - investigationId String @id - investigation Investigation? @relation("InvestigationInputOwner") + investigationId String @id + investigation Investigation? @relation("InvestigationInputOwner") // Immutable after insert; enforced by trigger "reject_investigation_input_updates_trigger". - provenance ContentProvenance - contentHash String - markdownSource MarkdownSource - markdown String? // null iff markdownSource = NONE - markdownRendererVersion String? // null iff markdownSource = NONE - createdAt DateTime @default(now()) + provenance ContentProvenance + contentHash String + markdownSource MarkdownSource + markdown String? // null iff markdownSource = NONE + markdownRendererVersion String? // null iff markdownSource = NONE + // Source URL of the image behind each `[IMAGE:N]` placeholder in markdown, + // indexed by N. Empty when markdownSource = NONE. + imagePlaceholderSourceUrls String[] + postUrl String + authorName String? + postPublishedAt DateTime? + hasVideo Boolean + createdAt DateTime @default(now()) } model InvestigationOpenAiKeySource { @@ -402,63 +443,103 @@ model InvestigationImage { @@index([imageBlobId]) } +// One row per worker execution of an investigation (SPEC §2.12). Rows are +// insert-only: attemptNumber strictly increases per investigation, so each +// attempt's audit is written exactly once, at its terminal transition. model InvestigationAttempt { - id String @id @default(cuid()) - investigationId String - investigation Investigation @relation(fields: [investigationId], references: [id], onDelete: Cascade) - attemptNumber Int - outcome InvestigationAttemptOutcome - requestModel String - requestInstructions String - requestInput String - requestReasoningEffort String? - requestReasoningSummary String? - responseId String? - responseStatus String? - responseModelVersion String? - responseOutputText String? - startedAt DateTime - completedAt DateTime? - requestedTools InvestigationAttemptRequestedTool[] - outputItems InvestigationAttemptOutputItem[] - toolCalls InvestigationAttemptToolCall[] - usage InvestigationAttemptUsage? - error InvestigationAttemptError? - createdAt DateTime @default(now()) - updatedAt DateTime @updatedAt + id String @id @default(cuid()) + investigationId String + investigation Investigation @relation(fields: [investigationId], references: [id], onDelete: Cascade) + attemptNumber Int + outcome InvestigationAttemptOutcome + startedAt DateTime + completedAt DateTime? + requests InvestigationAttemptRequest[] + error InvestigationAttemptError? + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt @@unique([investigationId, attemptNumber]) @@index([investigationId, startedAt]) } +// One provider request made during an attempt, exactly as sent. +model InvestigationAttemptRequest { + id String @id @default(cuid()) + attemptId String + attempt InvestigationAttempt @relation(fields: [attemptId], references: [id], onDelete: Cascade) + kind InvestigationAttemptRequestKind + // INV-ATTEMPT-REQUEST-SUBJECT: factCheckRound is set iff kind = FACT_CHECK_ROUND; + // claimIndex is set iff kind = CLAIM_VALIDATION. + // Enforced by CHECK constraint "InvestigationAttemptRequest_subject_check". + factCheckRound Int? + claimIndex Int? + model String + instructions String + // The request's `input` parameter. Image parts carry `imageContentHash` + // (ImageBlob.contentHash) in place of the inline data URI that was sent. + input Json + previousResponseId String? + reasoningEffort String? + reasoningSummary String? + include String[] + requestedTools InvestigationAttemptRequestedTool[] + // Absent when the request failed before the provider returned a response. + response InvestigationAttemptResponse? + createdAt DateTime @default(now()) + + @@unique([attemptId, factCheckRound]) + @@unique([attemptId, claimIndex]) + @@index([attemptId]) +} + model InvestigationAttemptRequestedTool { - id String @id @default(cuid()) - attemptId String - attempt InvestigationAttempt @relation(fields: [attemptId], references: [id], onDelete: Cascade) + id String @id @default(cuid()) + requestId String + request InvestigationAttemptRequest @relation(fields: [requestId], references: [id], onDelete: Cascade) requestOrder Int toolType String rawDefinition Json - createdAt DateTime @default(now()) + createdAt DateTime @default(now()) - @@unique([attemptId, requestOrder]) - @@index([attemptId]) + @@unique([requestId, requestOrder]) + @@index([requestId]) +} + +model InvestigationAttemptResponse { + id String @id @default(cuid()) + requestId String @unique + request InvestigationAttemptRequest @relation(fields: [requestId], references: [id], onDelete: Cascade) + providerResponseId String + // Provider-reported response status; null when the provider omitted it + // (the attempt then fails). + status String? + modelVersion String + // When the response was received. Null only on LEGACY_COMBINED requests, + // which merged responses received at different times. + receivedAt DateTime? + outputItems InvestigationAttemptOutputItem[] + usage InvestigationAttemptUsage? + createdAt DateTime @default(now()) } model InvestigationAttemptOutputItem { id String @id @default(cuid()) - attemptId String - attempt InvestigationAttempt @relation(fields: [attemptId], references: [id], onDelete: Cascade) + responseId String + response InvestigationAttemptResponse @relation(fields: [responseId], references: [id], onDelete: Cascade) outputIndex Int providerItemId String? itemType String itemStatus String? + // Exactly one of the following is populated, by itemType: textParts for + // "message", reasoningSummaries for "reasoning", toolCall for anything else. textParts InvestigationAttemptOutputTextPart[] reasoningSummaries InvestigationAttemptReasoningSummary[] toolCall InvestigationAttemptToolCall? createdAt DateTime @default(now()) - @@unique([attemptId, outputIndex]) - @@index([attemptId]) + @@unique([responseId, outputIndex]) + @@index([responseId]) } model InvestigationAttemptOutputTextPart { @@ -505,36 +586,24 @@ model InvestigationAttemptReasoningSummary { } model InvestigationAttemptToolCall { - id String @id @default(cuid()) - attemptId String - attempt InvestigationAttempt @relation(fields: [attemptId], references: [id], onDelete: Cascade) - outputItemId String - outputItem InvestigationAttemptOutputItem @relation(fields: [outputItemId], references: [id], onDelete: Cascade) - outputIndex Int - providerToolCallId String? - toolType String - status String? - rawPayload Json - capturedAt DateTime - providerStartedAt DateTime? - providerCompletedAt DateTime? - createdAt DateTime @default(now()) - - @@unique([attemptId, outputIndex]) - @@unique([outputItemId]) - @@index([attemptId]) + id String @id @default(cuid()) + outputItemId String @unique + outputItem InvestigationAttemptOutputItem @relation(fields: [outputItemId], references: [id], onDelete: Cascade) + // Full provider output item, as received. + rawPayload Json + createdAt DateTime @default(now()) } model InvestigationAttemptUsage { - id String @id @default(cuid()) - attemptId String @unique - attempt InvestigationAttempt @relation(fields: [attemptId], references: [id], onDelete: Cascade) + id String @id @default(cuid()) + responseId String @unique + response InvestigationAttemptResponse @relation(fields: [responseId], references: [id], onDelete: Cascade) inputTokens Int outputTokens Int totalTokens Int cachedInputTokens Int? reasoningOutputTokens Int? - createdAt DateTime @default(now()) + createdAt DateTime @default(now()) } model InvestigationAttemptError { @@ -574,15 +643,12 @@ model Claim { } model Source { - id String @id @default(cuid()) - claimId String - claim Claim @relation(fields: [claimId], references: [id], onDelete: Cascade) - url String - title String - snippet String - snapshotText String? - snapshotHash String? - retrievedAt DateTime + id String @id @default(cuid()) + claimId String + claim Claim @relation(fields: [claimId], references: [id], onDelete: Cascade) + url String + title String + snippet String @@index([claimId]) } diff --git a/src/typescript/api/scripts/openai-live-smoke.ts b/src/typescript/api/scripts/openai-live-smoke.ts new file mode 100644 index 0000000..d23ced8 --- /dev/null +++ b/src/typescript/api/scripts/openai-live-smoke.ts @@ -0,0 +1,249 @@ +/** + * Live smoke test of the investigation pipeline against the real OpenAI API. + * Spends real tokens; needs no database. + * + * OPENAI_API_KEY=sk-... pnpm --filter @openerrata/api smoke:openai + * + * SMOKE_REASONING_SUMMARY=none|auto|concise|detailed overrides the reasoning + * summary setting for this run only, to check which values the model accepts + * before changing INVESTIGATION_REASONING_SUMMARY. + * + * Runs (a) the request probe the worker runs at startup and (b) one full + * investigation through OpenAIInvestigator of a short synthetic post with one + * false and one true checkable claim, then prints what came back. + */ +import "dotenv/config"; +import process from "node:process"; +import OpenAI, { APIError } from "openai"; +import type { Reasoning } from "openai/resources/shared"; +import { InvestigatorExecutionError } from "../src/lib/investigators/errors.js"; +import type { + InvestigatorInput, + InvestigatorRequestAudit, +} from "../src/lib/investigators/interface.js"; +import { OpenAIInvestigator } from "../src/lib/investigators/openai.js"; +import { probeInvestigationRequest } from "../src/lib/investigators/openai-probe.js"; +import { + INVESTIGATION_REQUEST_CONFIG, + type InvestigationRequestConfig, +} from "../src/lib/investigators/openai-request-config.js"; + +// Plenty for a two-claim post; production uses OPENAI_MAX_RESPONSE_TOOL_ROUNDS. +const SMOKE_MAX_TOOL_ROUNDS = 40; + +const SYNTHETIC_POST: InvestigatorInput = { + platform: "LESSWRONG", + url: "https://www.lesswrong.com/posts/smoke0test/notes-on-engineering-timelines", + authorName: "Smoke Test", + postPublishedAt: "2026-09-30T12:00:00.000Z", + contentText: [ + "Notes on engineering timelines", + "People routinely underestimate how long large projects take, and the planning fallacy shows up even in famous megaprojects.", + "The Eiffel Tower, for example, was completed in 1925 after more than a decade of construction delays.", + "By contrast, the Empire State Building went up remarkably fast: it opened in 1931, barely over a year after construction began.", + "The lesson I take from this is to pad estimates generously when the work is novel.", + ].join("\n\n"), +}; + +type ReasoningSummarySetting = NonNullable | null; + +function parseReasoningSummaryOverride(value: string | undefined): ReasoningSummarySetting { + switch (value) { + case undefined: + return INVESTIGATION_REQUEST_CONFIG.reasoningSummary; + case "none": + return null; + case "auto": + case "concise": + case "detailed": + return value; + default: + throw new Error( + `SMOKE_REASONING_SUMMARY must be one of none|auto|concise|detailed (got "${value}")`, + ); + } +} + +function isOpenAiApiError(error: unknown): error is APIError { + return error instanceof APIError; +} + +function describeError(error: unknown): string { + if (isOpenAiApiError(error)) { + return [ + `${error.constructor.name}: HTTP ${error.status?.toString() ?? "(no status)"}`, + `request id: ${error.requestID ?? "(none)"}`, + `body: ${JSON.stringify(error.error, null, 2)}`, + ].join("\n"); + } + if (error instanceof Error) { + return `${error.name}: ${error.message}${error.stack === undefined ? "" : `\n${error.stack}`}`; + } + return String(error); +} + +function readValidationVerdict(request: InvestigatorRequestAudit): string { + if (request.response === null) return "no response (request failed)"; + const text = request.response.outputItems + .flatMap((item) => (item.content.kind === "MESSAGE" ? item.content.textParts : [])) + .map((part) => part.text) + .join(""); + return text.length > 0 ? text : `no verdict text (status=${String(request.response.status)})`; +} + +function summarizeRequests(requests: InvestigatorRequestAudit[]): void { + const responses = requests.flatMap((request) => + request.response === null ? [] : [request.response], + ); + console.log( + `\nProvider requests: ${requests.length.toString()} sent, ${responses.length.toString()} answered`, + ); + for (const request of requests) { + const label = + request.subject.kind === "FACT_CHECK_ROUND" + ? `fact-check round ${request.subject.round.toString()}` + : `validation of claim ${request.subject.claimIndex.toString()}`; + const response = request.response; + console.log( + ` - ${label}: ${ + response === null + ? "no response" + : `${response.providerResponseId} status=${String(response.status)} model=${response.modelVersion}` + }`, + ); + } + + const webSearches = responses.flatMap((response) => + response.outputItems.flatMap((item) => + item.itemType === "web_search_call" && item.content.kind === "TOOL_CALL" + ? [item.content.rawPayload] + : [], + ), + ); + const sourceCount = webSearches + .map((payload) => { + const action = payload["action"]; + const sources = + typeof action === "object" && action !== null && !Array.isArray(action) + ? action["sources"] + : undefined; + return Array.isArray(sources) ? sources.length : 0; + }) + .reduce((total, count) => total + count, 0); + console.log( + `\nWeb search calls: ${webSearches.length.toString()}, sources returned: ${sourceCount.toString()}`, + ); + + const summaries = responses.flatMap((response) => + response.outputItems.flatMap((item) => + item.content.kind === "REASONING" ? item.content.summaries : [], + ), + ); + const reasoningItemCount = responses + .flatMap((response) => response.outputItems) + .filter((item) => item.content.kind === "REASONING").length; + console.log( + `Reasoning summaries: ${summaries.length > 0 ? "yes" : "no"} (${summaries.length.toString()} summary parts across ${reasoningItemCount.toString()} reasoning items)`, + ); + const [firstSummary] = summaries; + if (firstSummary !== undefined) { + console.log(` first summary: ${firstSummary.slice(0, 300)}`); + } + + console.log("\nClaim validations:"); + const validations = requests.filter((request) => request.subject.kind === "CLAIM_VALIDATION"); + if (validations.length === 0) console.log(" (none — no claims were submitted)"); + for (const validation of validations) { + const claimIndex = + validation.subject.kind === "CLAIM_VALIDATION" ? validation.subject.claimIndex : -1; + console.log(` - claim ${claimIndex.toString()}: ${readValidationVerdict(validation)}`); + } + + const usages = responses.map((response) => response.usage); + const totals = usages.every((usage) => usage !== null) + ? usages.reduce( + (sum, usage) => ({ + input: sum.input + usage.inputTokens, + cached: sum.cached + usage.cachedInputTokens, + output: sum.output + usage.outputTokens, + reasoning: sum.reasoning + usage.reasoningOutputTokens, + total: sum.total + usage.totalTokens, + }), + { input: 0, cached: 0, output: 0, reasoning: 0, total: 0 }, + ) + : null; + console.log( + totals === null + ? "\nToken usage: unknown (a response reported no usage)" + : `\nToken usage: input=${totals.input.toString()} (cached ${totals.cached.toString()}), output=${totals.output.toString()} (reasoning ${totals.reasoning.toString()}), total=${totals.total.toString()}`, + ); +} + +async function main(): Promise { + const requestConfig: InvestigationRequestConfig = { + ...INVESTIGATION_REQUEST_CONFIG, + reasoningSummary: parseReasoningSummaryOverride(process.env["SMOKE_REASONING_SUMMARY"]), + }; + const apiKey = process.env["OPENAI_API_KEY"]?.trim(); + if (apiKey === undefined || apiKey.length === 0) { + console.error("OPENAI_API_KEY is required (set it in the environment or api/.env)."); + return false; + } + console.log( + `Model ${requestConfig.model}, reasoning effort=${requestConfig.reasoningEffort}, summary=${String(requestConfig.reasoningSummary)}, include=${requestConfig.include.join(",")}`, + ); + const client = new OpenAI({ apiKey }); + + console.log("\n(a) Request probe"); + try { + await probeInvestigationRequest(client, requestConfig); + console.log(" accepted"); + } catch (error) { + console.log(` REJECTED\n${describeError(error)}`); + return false; + } + + console.log("\n(b) Full investigation of a synthetic post"); + const investigator = new OpenAIInvestigator({ + client, + requestConfig, + maxToolRounds: SMOKE_MAX_TOOL_ROUNDS, + }); + const startedAt = Date.now(); + try { + const output = await investigator.investigate(SYNTHETIC_POST, { + signal: new AbortController().signal, + }); + console.log( + ` completed in ${((Date.now() - startedAt) / 1000).toFixed(1)}s; model=${output.model}, modelVersion=${output.modelVersion}`, + ); + summarizeRequests(output.attemptAudit.requests); + console.log(`\nConfirmed claims: ${output.result.claims.length.toString()}`); + for (const claim of output.result.claims) { + console.log(` - "${claim.text}"\n ${claim.summary}`); + for (const source of claim.sources) { + console.log(` source: ${source.url} (${source.title})`); + } + } + return true; + } catch (error) { + console.log(` FAILED after ${((Date.now() - startedAt) / 1000).toFixed(1)}s`); + if (error instanceof InvestigatorExecutionError) { + console.log(` ${error.message}\n${describeError(error.cause)}`); + summarizeRequests(error.attemptAudit.requests); + } else { + console.log(describeError(error)); + } + return false; + } +} + +main().then( + (passed) => { + process.exit(passed ? 0 : 1); + }, + (error: unknown) => { + console.error(describeError(error)); + process.exit(1); + }, +); diff --git a/src/typescript/api/src/lib/config/env.ts b/src/typescript/api/src/lib/config/env.ts index 5f4c521..7ceab71 100644 --- a/src/typescript/api/src/lib/config/env.ts +++ b/src/typescript/api/src/lib/config/env.ts @@ -7,7 +7,7 @@ import { z } from "zod"; * the API changes in a way that breaks older extensions, bump this constant * alongside that change. */ -export const MINIMUM_SUPPORTED_EXTENSION_VERSION = "0.2.0"; +export const MINIMUM_SUPPORTED_EXTENSION_VERSION = "0.4.0"; const positiveIntegerFromEnv = z.preprocess((value) => { if (value === undefined || value === null || value === "") return undefined; @@ -42,16 +42,14 @@ const baseEnvironmentSchema = z.object({ "DATABASE_URL must use postgres:// or postgresql://", ), OPENAI_API_KEY: z.string().trim().min(1).optional(), - OPENAI_MODEL_ID: z.string().trim().min(1).default("gpt-5.4"), - OPENAI_MAX_RESPONSE_TOOL_ROUNDS: positiveIntegerFromEnv.default(150), - WORKER_CONCURRENCY: positiveIntegerFromEnv.default(250), - HMAC_SECRET: z.string().trim().min(1, "HMAC_SECRET is required"), - SELECTOR_BUDGET: positiveIntegerFromEnv.default(100), - IP_RANGE_CREDIT_CAP: positiveIntegerFromEnv.default(10), + OPENAI_MAX_RESPONSE_TOOL_ROUNDS: positiveIntegerFromEnv, + WORKER_CONCURRENCY: positiveIntegerFromEnv, + /** Maximum investigations the selector admits per UTC day. */ + SELECTOR_DAILY_BUDGET: positiveIntegerFromEnv, + IP_RANGE_CREDIT_CAP: positiveIntegerFromEnv, BLOB_STORAGE_BUCKET: requiredNonEmptyStringFromEnv, BLOB_STORAGE_ACCESS_KEY_ID: requiredNonEmptyStringFromEnv, BLOB_STORAGE_SECRET_ACCESS_KEY: requiredNonEmptyStringFromEnv, - BLOB_STORAGE_PUBLIC_URL_PREFIX: requiredNonEmptyStringFromEnv, DATABASE_ENCRYPTION_KEY: requiredNonEmptyStringFromEnv, DATABASE_ENCRYPTION_KEY_ID: optionalNonEmptyStringFromEnv.default("primary"), }); diff --git a/src/typescript/api/src/lib/config/runtime.ts b/src/typescript/api/src/lib/config/runtime.ts index bc51d69..eef0847 100644 --- a/src/typescript/api/src/lib/config/runtime.ts +++ b/src/typescript/api/src/lib/config/runtime.ts @@ -1,7 +1,8 @@ import { getEnv } from "./env.js"; -export function getSelectorBudget(): number { - return getEnv().SELECTOR_BUDGET; +/** Maximum SELECTOR admissions per UTC day (SPEC §2.10). */ +export function getSelectorDailyBudget(): number { + return getEnv().SELECTOR_DAILY_BUDGET; } export function getIpRangeCreditCap(): number { diff --git a/src/typescript/api/src/lib/config/startup.ts b/src/typescript/api/src/lib/config/startup.ts index ed9b9fd..f0e80bd 100644 --- a/src/typescript/api/src/lib/config/startup.ts +++ b/src/typescript/api/src/lib/config/startup.ts @@ -1,5 +1,7 @@ import OpenAI from "openai"; import { getPrisma } from "$lib/db/client"; +import { INVESTIGATION_REQUEST_CONFIG } from "$lib/investigators/openai-request-config.js"; +import { probeInvestigationRequest } from "$lib/investigators/openai-probe.js"; import { getEnv, requireOpenAiApiKey } from "./env.js"; type StartupComponent = "api" | "worker" | "selector"; @@ -7,12 +9,13 @@ type StartupComponent = "api" | "worker" | "selector"; interface StartupCheckPolicy { checkDatabase: boolean; checkOpenAiCredentials: boolean; + checkClientAddressSource: boolean; } const startupCheckPolicyByComponent: Record = { - api: { checkDatabase: true, checkOpenAiCredentials: false }, - selector: { checkDatabase: true, checkOpenAiCredentials: false }, - worker: { checkDatabase: true, checkOpenAiCredentials: true }, + api: { checkDatabase: true, checkOpenAiCredentials: false, checkClientAddressSource: true }, + selector: { checkDatabase: true, checkOpenAiCredentials: false, checkClientAddressSource: false }, + worker: { checkDatabase: true, checkOpenAiCredentials: true, checkClientAddressSource: false }, }; const startupCheckPromises = new Map>(); @@ -22,9 +25,35 @@ function startupCheckKey(component: StartupComponent, policy: StartupCheckPolicy component, policy.checkDatabase ? "db:1" : "db:0", policy.checkOpenAiCredentials ? "openai:1" : "openai:0", + policy.checkClientAddressSource ? "client-address:1" : "client-address:0", ].join("|"); } +/** + * In production the API runs behind the chart's ingress proxy, so the socket + * peer is the proxy, not the viewer. adapter-node must be told which header + * carries the client address (ADDRESS_HEADER, plus XFF_DEPTH for + * X-Forwarded-For); without it every anonymous viewer shares the proxy's IP + * range and the per-range view-credit cap (SPEC §2.10) collapses selector + * ranking. Exported for unit tests. + */ +export function assertClientAddressSourceConfigured(env: NodeJS.ProcessEnv): void { + const addressHeader = env["ADDRESS_HEADER"]?.trim().toLowerCase() ?? ""; + if (addressHeader.length === 0) { + throw new Error( + "ADDRESS_HEADER must name the proxy header carrying the client IP (e.g. x-forwarded-for) when NODE_ENV=production", + ); + } + if (addressHeader === "x-forwarded-for") { + const xffDepth = env["XFF_DEPTH"]?.trim() ?? ""; + if (!/^[1-9]\d*$/.test(xffDepth)) { + throw new Error( + "XFF_DEPTH must be set to the number of trusted proxies in front of the API when ADDRESS_HEADER=x-forwarded-for", + ); + } + } +} + async function assertDatabaseCredentials(component: StartupComponent): Promise { try { await getPrisma().$queryRaw`SELECT 1`; @@ -35,12 +64,12 @@ async function assertDatabaseCredentials(component: StartupComponent): Promise { try { - const client = new OpenAI({ apiKey: requireOpenAiApiKey() }); - await client.responses.create({ - model: getEnv().OPENAI_MODEL_ID, - input: "Reply with the single word pong.", - max_output_tokens: 16, - }); + // Probes the investigation request shape, not just the key, so a model or + // request-parameter mismatch stops the worker before it fails every job. + await probeInvestigationRequest( + new OpenAI({ apiKey: requireOpenAiApiKey() }), + INVESTIGATION_REQUEST_CONFIG, + ); } catch (error) { throw new Error(`[startup:${component}] OpenAI credential check failed`, { cause: error }); } @@ -52,6 +81,9 @@ export async function runStartupChecks(component: StartupComponent): Promise { + if (policy.checkClientAddressSource && getEnv().NODE_ENV === "production") { + assertClientAddressSourceConfigured(process.env); + } if (policy.checkDatabase) { await assertDatabaseCredentials(component); } diff --git a/src/typescript/api/src/lib/date.ts b/src/typescript/api/src/lib/date.ts index 6ecb2e3..cad77b6 100644 --- a/src/typescript/api/src/lib/date.ts +++ b/src/typescript/api/src/lib/date.ts @@ -18,3 +18,10 @@ export function toOptionalDate( } return parsed; } + +/** Midnight UTC at the start of the day containing `date`. */ +export function startOfUtcDay(date: Date): Date { + const dayStart = new Date(date); + dayStart.setUTCHours(0, 0, 0, 0); + return dayStart; +} diff --git a/src/typescript/api/src/lib/db/client.ts b/src/typescript/api/src/lib/db/client.ts index 7c92018..5925bab 100644 --- a/src/typescript/api/src/lib/db/client.ts +++ b/src/typescript/api/src/lib/db/client.ts @@ -1,12 +1,15 @@ import "./prisma-enum-compat.js"; import { PrismaPg } from "@prisma/adapter-pg"; import { normalizePgConnectionStringForNode } from "$lib/db/connection-string.js"; -import { PrismaClient } from "$lib/db/prisma-client"; +import { PrismaClient, type Prisma } from "$lib/db/prisma-client"; import { getEnv } from "$lib/config/env.js"; import { Pool } from "pg"; export type { PrismaClient } from "$lib/db/prisma-client"; +/** Either the root client or an interactive-transaction client. */ +export type DbClient = PrismaClient | Prisma.TransactionClient; + declare global { // Reused across HMR reloads in development. var __openerrataPrisma: PrismaClient | undefined; diff --git a/src/typescript/api/src/lib/db/prisma-enum-compat.ts b/src/typescript/api/src/lib/db/prisma-enum-compat.ts index 1cc65fd..f4055e5 100644 --- a/src/typescript/api/src/lib/db/prisma-enum-compat.ts +++ b/src/typescript/api/src/lib/db/prisma-enum-compat.ts @@ -1,7 +1,6 @@ import type { CheckStatus as PrismaCheckStatus, ContentProvenance as PrismaContentProvenance, - InvestigationModel as PrismaInvestigationModel, InvestigationProvider as PrismaInvestigationProvider, MarkdownSource as PrismaMarkdownSource, Platform as PrismaPlatform, @@ -9,7 +8,6 @@ import type { import type { CheckStatus as SharedCheckStatus, ContentProvenance as SharedContentProvenance, - InvestigationModel as SharedInvestigationModel, InvestigationProvider as SharedInvestigationProvider, MarkdownSource as SharedMarkdownSource, Platform as SharedPlatform, @@ -21,13 +19,11 @@ const platformTypesMatch: IsExactly = true; const statusTypesMatch: IsExactly = true; const providerTypesMatch: IsExactly = true; -const modelTypesMatch: IsExactly = true; const provenanceTypesMatch: IsExactly = true; const markdownSourceTypesMatch: IsExactly = true; void platformTypesMatch; void statusTypesMatch; void providerTypesMatch; -void modelTypesMatch; void provenanceTypesMatch; void markdownSourceTypesMatch; diff --git a/src/typescript/api/src/lib/graphql/public-schema.ts b/src/typescript/api/src/lib/graphql/public-schema.ts index 3f20ff8..3b98e10 100644 --- a/src/typescript/api/src/lib/graphql/public-schema.ts +++ b/src/typescript/api/src/lib/graphql/public-schema.ts @@ -83,7 +83,7 @@ const typeDefs = /* GraphQL */ ` """ provider: String! """ - LLM model used (e.g. "GPT4O"). + Provider model id the investigation ran on (e.g. "gpt-6.1-sol"). """ model: String! } @@ -282,9 +282,9 @@ const typeDefs = /* GraphQL */ ` """ investigatedPostsWithFlags: Int! """ - Ratio of posts with flags to total investigated posts (investigatedPostsWithFlags / totalInvestigatedPosts). + Ratio of posts with flags to total investigated posts (investigatedPostsWithFlags / totalInvestigatedPosts). Null when no posts match the filter. """ - factCheckIncidence: Float! + factCheckIncidence: Float } type Query { diff --git a/src/typescript/api/src/lib/investigators/errors.ts b/src/typescript/api/src/lib/investigators/errors.ts new file mode 100644 index 0000000..a181488 --- /dev/null +++ b/src/typescript/api/src/lib/investigators/errors.ts @@ -0,0 +1,66 @@ +import type { InvestigatorFailedAttemptAudit } from "./interface.js"; + +/** + * An investigation attempt failed after making provider requests. Carries the + * attempt's audit so it can be persisted (SPEC §2.12); `cause` is the failure + * the orchestrator classifies for retry (SPEC §3.7). + */ +export class InvestigatorExecutionError extends Error { + readonly attemptAudit: InvestigatorFailedAttemptAudit; + + constructor(message: string, attemptAudit: InvestigatorFailedAttemptAudit, cause: unknown) { + super(message, { cause }); + this.name = "InvestigatorExecutionError"; + this.attemptAudit = attemptAudit; + } +} + +/** + * The provider returned output that is well-formed but unusable by the + * pipeline (e.g. unparseable validation verdict, tool-round limit exceeded). + * Deterministic for a given input, so non-retryable. + */ +export class InvestigatorStructuredOutputError extends Error { + constructor(message: string) { + super(message); + this.name = "InvestigatorStructuredOutputError"; + } +} + +/** + * A provider response ended with a status other than "completed" (or with no + * status at all). SPEC §3.7 classes truncated/incomplete output as PARTIAL: + * the investigation is marked FAILED, not retried. + */ +export class InvestigatorIncompleteResponseError extends Error { + readonly responseStatus: string | null; + readonly responseId: string; + readonly incompleteReason: string | null; + + constructor(input: { + responseStatus: string | null; + responseId: string; + incompleteReason: string | null; + }) { + super( + "OpenAI response did not complete " + + `(status=${input.responseStatus ?? "missing"}, reason=${input.incompleteReason ?? "none"}, responseId=${input.responseId})`, + ); + this.name = "InvestigatorIncompleteResponseError"; + this.responseStatus = input.responseStatus; + this.responseId = input.responseId; + this.incompleteReason = input.incompleteReason; + } +} + +/** + * The investigator was called with input that violates its contract (a caller + * bug, e.g. inconsistent image occurrences). Non-retryable: the same input + * fails the same way. + */ +export class InvestigatorInputError extends Error { + constructor(message: string) { + super(message); + this.name = "InvestigatorInputError"; + } +} diff --git a/src/typescript/api/src/lib/investigators/fetch-url-tool.ts b/src/typescript/api/src/lib/investigators/fetch-url-tool.ts index c0c5ff5..bbb5308 100644 --- a/src/typescript/api/src/lib/investigators/fetch-url-tool.ts +++ b/src/typescript/api/src/lib/investigators/fetch-url-tool.ts @@ -1,21 +1,14 @@ -import { normalizeContent } from "@openerrata/shared"; +import { httpUrlSchema, normalizeContent } from "@openerrata/shared"; import { decodeHTML } from "entities"; import { z } from "zod"; -import { isBlockedHost } from "$lib/network/host-safety.js"; -import { isRedirectStatus } from "$lib/network/http-status.js"; +import { fetchPublicHttp, readBodyPrefix } from "$lib/network/public-http-fetch.js"; const MAX_FETCH_URL_BYTES = 1_000_000; const MAX_FETCH_URL_TEXT_LENGTH = 20_000; const FETCH_URL_TIMEOUT_MS = 15_000; -const MAX_REDIRECT_HOPS = 5; const fetchUrlToolArgumentsSchema = z.object({ - url: z.preprocess( - (value) => (typeof value === "string" ? value.trim() : value), - z - .url("url must be a valid URL") - .refine((value) => /^https?:\/\//i.test(value), "url must use http:// or https://"), - ), + url: z.preprocess((value) => (typeof value === "string" ? value.trim() : value), httpUrlSchema), }); interface FetchUrlToolSuccess { @@ -103,10 +96,6 @@ function parseContentType(contentTypeHeader: string | null): string { return contentTypeHeader.split(";")[0]?.trim().toLowerCase() ?? ""; } -function hasEmbeddedCredentials(url: URL): boolean { - return url.username.length > 0 || url.password.length > 0; -} - function extractContentText( contentType: string, rawBody: string, @@ -142,7 +131,16 @@ function extractContentText( }; } -export async function executeFetchUrlTool(rawArguments: string): Promise { +/** + * Run the `fetch_url` tool: GET a public URL chosen by the model and return + * its normalized text. Untrusted URLs go through the SSRF-safe public fetcher; + * bodies are read up to MAX_FETCH_URL_BYTES. Aborting `signal` (e.g. the run + * lost its lease) aborts the request. + */ +export async function executeFetchUrlTool( + rawArguments: string, + signal: AbortSignal, +): Promise { let parsedArguments: z.infer; try { parsedArguments = fetchUrlToolArgumentsSchema.parse(JSON.parse(rawArguments)); @@ -157,140 +155,37 @@ export async function executeFetchUrlTool(rawArguments: string): Promise 0) { - const contentLength = Number.parseInt(contentLengthHeader, 10); - if (Number.isFinite(contentLength) && contentLength > MAX_FETCH_URL_BYTES) { - return { - ok: false, - errorKind: "FETCH_FAILED", - requestedUrl, - error: `Response too large (${contentLength.toString()} bytes)`, - }; - } - } + const { finalUrl, response } = await fetchPublicHttp({ + url: new URL(requestedUrl), + headers: { + "User-Agent": "OpenErrataInvestigator/1.0 (+https://openerrata.com)", + Accept: "text/html,application/json,text/plain;q=0.9,*/*;q=0.5", + }, + signal: AbortSignal.any([signal, AbortSignal.timeout(FETCH_URL_TIMEOUT_MS)]), + }); - const rawBody = await response.text(); - const byteTruncation = truncateUtf8(rawBody, MAX_FETCH_URL_BYTES); + const body = await readBodyPrefix(response, MAX_FETCH_URL_BYTES); + const rawBody = new TextDecoder().decode(body.bytes); const normalizedContentType = parseContentType(response.headers.get("content-type")); - const extracted = extractContentText(normalizedContentType, byteTruncation.value); + const extracted = extractContentText(normalizedContentType, rawBody); const textTruncation = truncateUtf8(extracted.contentText, MAX_FETCH_URL_TEXT_LENGTH); return { ok: true, requestedUrl, - finalUrl: currentUrl.toString(), + finalUrl: finalUrl.toString(), status: response.status, contentType: normalizedContentType.length > 0 ? normalizedContentType : null, title: extracted.title, contentText: textTruncation.value, - truncated: byteTruncation.truncated || textTruncation.truncated, + truncated: body.truncated || textTruncation.truncated, retrievedAt: new Date().toISOString(), }; } catch (error) { + if (signal.aborted) { + throw error; + } return { ok: false, errorKind: "FETCH_FAILED", diff --git a/src/typescript/api/src/lib/investigators/interface.ts b/src/typescript/api/src/lib/investigators/interface.ts index d6458de..3633098 100644 --- a/src/typescript/api/src/lib/investigators/interface.ts +++ b/src/typescript/api/src/lib/investigators/interface.ts @@ -1,13 +1,4 @@ -import { z } from "zod"; -import type { - InvestigationModel, - InvestigationProvider, - InvestigationResult, - InvestigationClaim, - Platform, -} from "@openerrata/shared"; - -const isoDateTimeSchema = z.iso.datetime(); +import type { InvestigationClaim, InvestigationResult, Platform } from "@openerrata/shared"; export type InvestigatorJsonValue = | string @@ -17,20 +8,7 @@ export type InvestigatorJsonValue = | InvestigatorJsonValue[] | { [key: string]: InvestigatorJsonValue }; -const investigatorJsonValueSchema: z.ZodType = z.lazy(() => - z.union([ - z.string(), - z.number(), - z.boolean(), - z.null(), - z.array(investigatorJsonValueSchema), - z.record(z.string(), investigatorJsonValueSchema), - ]), -); -const investigatorJsonRecordSchema: z.ZodType> = z.record( - z.string(), - investigatorJsonValueSchema, -); +export type InvestigatorJsonRecord = Record; export type InvestigatorImageOccurrence = | { @@ -86,154 +64,116 @@ export type InvestigatorInput = oldClaims: InvestigationClaim[]; }); -export const investigatorRequestedToolAuditSchema = z.object({ - requestOrder: z.number().int().nonnegative(), - toolType: z.string().min(1), - rawDefinition: investigatorJsonRecordSchema, -}); +// ── Attempt audit (SPEC §2.12) ────────────────────────────────────────────── +// Mirrors the persisted tree: InvestigationAttempt → InvestigationAttemptRequest +// (one per provider request) → InvestigationAttemptResponse → output items. +// Every positional index (request order of tools, output index, part index, +// annotation index, summary index) is the element's position in its array. -export const investigatorOutputItemAuditSchema = z - .object({ - outputIndex: z.number().int().nonnegative(), - providerItemId: z.string().nullable(), - itemType: z.string().min(1), - itemStatus: z.string().nullable(), - }) - .superRefine((audit, context) => { - const providerItemIdMissing = audit.providerItemId === null; - const itemStatusMissing = audit.itemStatus === null; - if (providerItemIdMissing !== itemStatusMissing) { - context.addIssue({ - code: "custom", - path: ["providerItemId"], - message: "providerItemId and itemStatus must be either both present or both null", - }); - } - }); +/** Which provider request of the attempt this was. */ +export type InvestigatorRequestSubject = + | { kind: "FACT_CHECK_ROUND"; round: number } + | { kind: "CLAIM_VALIDATION"; claimIndex: number }; -export const investigatorOutputTextPartAuditSchema = z.object({ - outputIndex: z.number().int().nonnegative(), - partIndex: z.number().int().nonnegative(), - partType: z.string().min(1), - text: z.string(), -}); +export interface InvestigatorRequestedToolAudit { + toolType: string; + rawDefinition: InvestigatorJsonRecord; +} -export const investigatorOutputTextAnnotationAuditSchema = z.object({ - outputIndex: z.number().int().nonnegative(), - partIndex: z.number().int().nonnegative(), - annotationIndex: z.number().int().nonnegative(), - annotationType: z.string().min(1), - characterPosition: z - .object({ - start: z.number().int(), - end: z.number().int(), - }) - .optional(), - url: z.string().nullable(), - title: z.string().nullable(), - fileId: z.string().nullable(), -}); +export interface InvestigatorOutputTextAnnotationAudit { + annotationType: string; + startIndex: number | null; + endIndex: number | null; + url: string | null; + title: string | null; + fileId: string | null; +} -export const investigatorReasoningSummaryAuditSchema = z.object({ - outputIndex: z.number().int().nonnegative(), - summaryIndex: z.number().int().nonnegative(), - text: z.string(), -}); +export interface InvestigatorOutputTextPartAudit { + partType: "output_text" | "refusal"; + text: string; + annotations: InvestigatorOutputTextAnnotationAudit[]; +} -export const investigatorToolCallAuditSchema = z - .object({ - outputIndex: z.number().int().nonnegative(), - providerToolCallId: z.string().nullable(), - toolType: z.string().min(1), - status: z.string().nullable(), - rawPayload: investigatorJsonRecordSchema, - capturedAt: isoDateTimeSchema, - providerStartedAt: isoDateTimeSchema.nullable(), - providerCompletedAt: isoDateTimeSchema.nullable(), - }) - .superRefine((toolCall, context) => { - const providerToolCallIdMissing = toolCall.providerToolCallId === null; - const statusMissing = toolCall.status === null; - if (providerToolCallIdMissing !== statusMissing) { - context.addIssue({ - code: "custom", - path: ["providerToolCallId"], - message: "providerToolCallId and status must be either both present or both null", - }); - } - }); +export type InvestigatorOutputItemContentAudit = + | { kind: "MESSAGE"; textParts: InvestigatorOutputTextPartAudit[] } + | { kind: "REASONING"; summaries: string[] } + | { + kind: "TOOL_CALL"; + /** Full provider output item, as received. */ + rawPayload: InvestigatorJsonRecord; + }; + +export interface InvestigatorOutputItemAudit { + providerItemId: string | null; + itemType: string; + itemStatus: string | null; + content: InvestigatorOutputItemContentAudit; +} -export const investigatorUsageAuditSchema = z.object({ - inputTokens: z.number().int().nonnegative(), - outputTokens: z.number().int().nonnegative(), - totalTokens: z.number().int().nonnegative(), - cachedInputTokens: z.number().int().nonnegative().nullable(), - reasoningOutputTokens: z.number().int().nonnegative().nullable(), -}); +export interface InvestigatorUsageAudit { + inputTokens: number; + outputTokens: number; + totalTokens: number; + cachedInputTokens: number; + reasoningOutputTokens: number; +} -export const investigatorResponseAuditSchema = z.object({ - responseId: z.string().nullable(), - responseStatus: z.string().nullable(), - responseModelVersion: z.string().nullable(), - responseOutputText: z.string().nullable(), - outputItems: z.array(investigatorOutputItemAuditSchema), - outputTextParts: z.array(investigatorOutputTextPartAuditSchema), - outputTextAnnotations: z.array(investigatorOutputTextAnnotationAuditSchema), - reasoningSummaries: z.array(investigatorReasoningSummaryAuditSchema), - toolCalls: z.array(investigatorToolCallAuditSchema), - usage: investigatorUsageAuditSchema.nullable(), -}); +export interface InvestigatorResponseAudit { + providerResponseId: string; + /** Provider-reported status; null when the provider omitted it. */ + status: string | null; + modelVersion: string; + receivedAt: Date; + outputItems: InvestigatorOutputItemAudit[]; + usage: InvestigatorUsageAudit | null; +} -export const investigatorErrorAuditSchema = z.object({ - errorName: z.string().min(1), - errorMessage: z.string(), - statusCode: z.number().int().nullable(), -}); +export interface InvestigatorRequestAudit { + subject: InvestigatorRequestSubject; + model: string; + instructions: string; + /** + * The request's `input` parameter as sent, except that image parts carry + * `imageContentHash` (the stored ImageBlob's content hash) instead of the + * inline data URI. + */ + input: string | InvestigatorJsonRecord[]; + previousResponseId: string | null; + reasoningEffort: string | null; + reasoningSummary: string | null; + include: string[]; + tools: InvestigatorRequestedToolAudit[]; + /** Null when the request failed before the provider returned a response. */ + response: InvestigatorResponseAudit | null; +} -const investigatorAttemptAuditBaseSchema = z.object({ - startedAt: isoDateTimeSchema, - completedAt: isoDateTimeSchema.nullable(), - requestModel: z.string().min(1), - requestInstructions: z.string(), - requestInput: z.string(), - requestReasoningEffort: z.string().nullable(), - requestReasoningSummary: z.string().nullable(), - requestedTools: z.array(investigatorRequestedToolAuditSchema), -}); +export interface InvestigatorErrorAudit { + errorName: string; + errorMessage: string; + statusCode: number | null; +} -const investigatorAttemptSucceededAuditSchema = investigatorAttemptAuditBaseSchema.extend({ - response: investigatorResponseAuditSchema, - error: z.null(), -}); +interface InvestigatorAttemptAuditBase { + startedAt: Date; + completedAt: Date; + requests: InvestigatorRequestAudit[]; +} -const investigatorAttemptFailedAuditSchema = investigatorAttemptAuditBaseSchema.extend({ - response: investigatorResponseAuditSchema.nullable(), - error: investigatorErrorAuditSchema, -}); +export type InvestigatorSucceededAttemptAudit = InvestigatorAttemptAuditBase & { + outcome: "SUCCEEDED"; +}; -export const investigatorAttemptAuditSchema = z.union([ - investigatorAttemptSucceededAuditSchema, - investigatorAttemptFailedAuditSchema, -]); +export type InvestigatorFailedAttemptAudit = InvestigatorAttemptAuditBase & { + outcome: "FAILED"; + error: InvestigatorErrorAudit; +}; -export type InvestigatorRequestedToolAudit = z.infer; -export type InvestigatorOutputItemAudit = z.infer; -export type InvestigatorOutputTextPartAudit = z.infer; -export type InvestigatorOutputTextAnnotationAudit = z.infer< - typeof investigatorOutputTextAnnotationAuditSchema ->; -export type InvestigatorReasoningSummaryAudit = z.infer< - typeof investigatorReasoningSummaryAuditSchema ->; -export type InvestigatorToolCallAudit = z.infer; -export type InvestigatorUsageAudit = z.infer; -export type InvestigatorResponseAudit = z.infer; -export type InvestigatorErrorAudit = z.infer; -export type InvestigatorAttemptAudit = z.infer; +export type InvestigatorAttemptAudit = + | InvestigatorSucceededAttemptAudit + | InvestigatorFailedAttemptAudit; -export function parseInvestigatorAttemptAudit(value: unknown): InvestigatorAttemptAudit { - return investigatorAttemptAuditSchema.parse(value); -} +// ── Investigator contract ─────────────────────────────────────────────────── export interface InvestigationProgressCallbacks { onProgressUpdate: ( @@ -242,17 +182,30 @@ export interface InvestigationProgressCallbacks { ) => void; } +export interface InvestigateOptions { + /** Aborts every provider request and tool fetch when the run must stop. */ + signal: AbortSignal; + callbacks?: InvestigationProgressCallbacks; +} + export interface InvestigatorOutput { result: InvestigationResult; - attemptAudit: InvestigatorAttemptAudit; - modelVersion?: string; + attemptAudit: InvestigatorSucceededAttemptAudit; + /** Provider model id the stage-1 fact-check requests were sent to. */ + model: string; + /** Provider-reported model revision of the final stage-1 fact-check response. */ + modelVersion: string; } +/** + * Runs one investigation attempt. Failures reject with + * `InvestigatorExecutionError` (carrying the failed attempt's audit) once a + * provider request has been made, or with `InvestigatorInputError` when the + * input itself violates this contract. + */ export interface Investigator { - investigate( - input: InvestigatorInput, - callbacks?: InvestigationProgressCallbacks, - ): Promise; - readonly provider: InvestigationProvider; - readonly model: InvestigationModel; + investigate(input: InvestigatorInput, options: InvestigateOptions): Promise; } + +/** Builds an investigator that authenticates to the provider with `apiKey`. */ +export type InvestigatorFactory = (apiKey: string) => Investigator; diff --git a/src/typescript/api/src/lib/investigators/openai-attempt-audit-builder.ts b/src/typescript/api/src/lib/investigators/openai-attempt-audit-builder.ts deleted file mode 100644 index 235726c..0000000 --- a/src/typescript/api/src/lib/investigators/openai-attempt-audit-builder.ts +++ /dev/null @@ -1,97 +0,0 @@ -import type { InvestigatorAttemptAudit, InvestigatorResponseAudit } from "./interface.js"; -import { parseInvestigatorAttemptAudit } from "./interface.js"; -import { buildTwoStepRequestInputAudit } from "./openai-input-builder.js"; -import { - buildErrorAudit, - extractRequestedTools, - mergeResponseAudits, - offsetResponseAuditIndices, -} from "./openai-response-audit.js"; -import { INVESTIGATION_VALIDATION_SYSTEM_PROMPT } from "./prompt.js"; - -type AttemptAuditBase = Omit; - -interface RequestReasoning { - effort: "low" | "medium" | "high"; - summary: "auto" | "concise" | "detailed"; -} - -export function createStageOneAttemptAuditBase(input: { - startedAt: string; - openAiModelId: string; - systemPrompt: string; - userPrompt: string; - requestReasoning: RequestReasoning; - requestedTools: unknown; -}): AttemptAuditBase { - return { - startedAt: input.startedAt, - completedAt: null, - requestModel: input.openAiModelId, - requestInstructions: input.systemPrompt, - requestInput: input.userPrompt, - requestReasoningEffort: input.requestReasoning.effort, - requestReasoningSummary: input.requestReasoning.summary, - requestedTools: extractRequestedTools(input.requestedTools), - }; -} - -export function createStageTwoAttemptAuditBase(input: { - stageOneBase: AttemptAuditBase; - userPrompt: string; - validationInputSummary: string; -}): AttemptAuditBase { - return { - ...input.stageOneBase, - requestInstructions: - `=== Stage 1: Fact-check instructions ===\n${input.stageOneBase.requestInstructions}` + - `\n\n=== Stage 2: Validation instructions ===\n${INVESTIGATION_VALIDATION_SYSTEM_PROMPT}`, - requestInput: buildTwoStepRequestInputAudit(input.userPrompt, input.validationInputSummary), - }; -} - -export function buildFailedAttemptAudit(input: { - base: AttemptAuditBase; - response: InvestigatorResponseAudit | null; - error: unknown; - completedAt?: string; -}): InvestigatorAttemptAudit { - return parseInvestigatorAttemptAudit({ - ...input.base, - completedAt: input.completedAt ?? new Date().toISOString(), - response: input.response, - error: buildErrorAudit(input.error), - }); -} - -export function buildSuccessfulAttemptAudit(input: { - base: AttemptAuditBase; - response: InvestigatorResponseAudit; - completedAt?: string; -}): InvestigatorAttemptAudit { - return parseInvestigatorAttemptAudit({ - ...input.base, - completedAt: input.completedAt ?? new Date().toISOString(), - response: input.response, - error: null, - }); -} - -export function buildFullAttemptResponseAudit(input: { - factCheckResponseAudit: InvestigatorResponseAudit; - successfulValidationResponseAudits: readonly InvestigatorResponseAudit[]; - failedValidationResponseAudits: readonly InvestigatorResponseAudit[]; -}): InvestigatorResponseAudit { - let validationOutputOffset = input.factCheckResponseAudit.outputItems.length; - - const orderedValidationResponseAudits = [ - ...input.successfulValidationResponseAudits, - ...input.failedValidationResponseAudits, - ].map((responseAudit) => { - const offsetAudit = offsetResponseAuditIndices(responseAudit, validationOutputOffset); - validationOutputOffset += responseAudit.outputItems.length; - return offsetAudit; - }); - - return mergeResponseAudits([input.factCheckResponseAudit, ...orderedValidationResponseAudits]); -} diff --git a/src/typescript/api/src/lib/investigators/openai-claim-tools.ts b/src/typescript/api/src/lib/investigators/openai-claim-tools.ts new file mode 100644 index 0000000..034840c --- /dev/null +++ b/src/typescript/api/src/lib/investigators/openai-claim-tools.ts @@ -0,0 +1,139 @@ +import type { FunctionTool } from "openai/resources/responses/responses"; +import { z } from "zod"; +import { + investigationClaimPayloadSchema, + type InvestigationClaimPayload, +} from "@openerrata/shared"; + +export const SUBMIT_CORRECTION_TOOL_NAME = "submit_correction"; +export const RETAIN_CORRECTION_TOOL_NAME = "retain_correction"; + +/** + * A claim tool call's arguments, checked against the tool's schema. Invalid + * arguments are reported back to the model (as the call's output) so it can + * correct and resubmit within the same run. + */ +type ClaimToolArguments = { kind: "valid"; value: T } | { kind: "invalid"; error: string }; + +/** + * JSON Schema for a function tool's parameters under OpenAI strict mode, which + * accepts only a subset of JSON Schema. It rejects `minLength` and + * `format: "uri"`, which Zod emits for the shared claim schema's non-empty + * strings and URLs, so those keywords are left out of the provider-facing + * schema; parsing the arguments with the full Zod schema enforces them. + */ +function toStrictModeParameters(schema: z.ZodObject): Record { + // Spread to a plain object: Zod attaches non-enumerable Standard Schema hooks. + return { + ...z.toJSONSchema(schema, { + target: "draft-07", + override: ({ jsonSchema }) => { + delete jsonSchema.minLength; + if (jsonSchema.format === "uri") { + delete jsonSchema.format; + } + }, + }), + }; +} + +function parseToolArguments(schema: z.ZodType, argumentsJson: string): ClaimToolArguments { + let decoded: unknown; + try { + decoded = JSON.parse(argumentsJson); + } catch { + return { kind: "invalid", error: "Arguments are not valid JSON" }; + } + const parsed = schema.safeParse(decoded); + return parsed.success + ? { kind: "valid", value: parsed.data } + : { kind: "invalid", error: z.prettifyError(parsed.error) }; +} + +// ── submit_correction ─────────────────────────────────────────────────────── + +const claimShape = investigationClaimPayloadSchema.shape; +const claimSourceSchema = claimShape.sources.element; + +// The shared claim payload schema with model-facing field descriptions. +// `.describe()` only attaches metadata: the tool advertises exactly the shape +// that `parseSubmitCorrectionArguments` validates with the shared schema. +const submitCorrectionParametersSchema = investigationClaimPayloadSchema.extend({ + text: claimShape.text.describe("The exact text of the incorrect claim."), + context: claimShape.context.describe( + "Surrounding context that disambiguates the claim location.", + ), + summary: claimShape.summary.describe("A one-sentence summary of what is incorrect and why."), + reasoning: claimShape.reasoning.describe( + "Detailed reasoning with evidence for why the claim is incorrect.", + ), + sources: z + .array( + claimSourceSchema.extend({ + url: claimSourceSchema.shape.url.describe("Source URL (absolute http/https)."), + title: claimSourceSchema.shape.title.describe("Title of the source."), + snippet: claimSourceSchema.shape.snippet.describe("Relevant snippet from the source."), + }), + ) + .min(1) + .describe("At least one supporting source."), +}); + +/** Submits one correction; called as the model finds each incorrect claim. */ +export const submitCorrectionToolDefinition: FunctionTool = { + type: "function", + name: SUBMIT_CORRECTION_TOOL_NAME, + description: + "Submit a single factual correction you have found and verified. " + + "Call this tool for each incorrect claim you discover — do not wait " + + "until you have found all claims.", + strict: true, + parameters: toStrictModeParameters(submitCorrectionParametersSchema), +}; + +export function parseSubmitCorrectionArguments( + argumentsJson: string, +): ClaimToolArguments { + return parseToolArguments(investigationClaimPayloadSchema, argumentsJson); +} + +// ── retain_correction (update investigations only) ────────────────────────── + +function retainCorrectionParametersSchema(retainableClaimIds: readonly [string, ...string[]]) { + return z + .object({ + id: z.enum(retainableClaimIds).describe("The ID of the existing claim to retain."), + }) + .strict(); +} + +/** + * Carries a previously validated claim forward unchanged. The `id` enum is + * exactly the prior investigation's claim ids. + */ +export function buildRetainCorrectionToolDefinition( + retainableClaimIds: readonly [string, ...string[]], +): FunctionTool { + return { + type: "function", + name: RETAIN_CORRECTION_TOOL_NAME, + description: + "Retain an existing claim from the previous investigation that is " + + "still correct and relevant. Use this instead of re-submitting the " + + "same claim via submit_correction.", + strict: true, + parameters: toStrictModeParameters(retainCorrectionParametersSchema(retainableClaimIds)), + }; +} + +/** Returns the claim id to retain. */ +export function parseRetainCorrectionArguments( + argumentsJson: string, + retainableClaimIds: readonly [string, ...string[]], +): ClaimToolArguments { + const parsed = parseToolArguments( + retainCorrectionParametersSchema(retainableClaimIds), + argumentsJson, + ); + return parsed.kind === "valid" ? { kind: "valid", value: parsed.value.id } : parsed; +} diff --git a/src/typescript/api/src/lib/investigators/openai-claim-validation-scheduler.ts b/src/typescript/api/src/lib/investigators/openai-claim-validation-scheduler.ts index 23050e7..460ca23 100644 --- a/src/typescript/api/src/lib/investigators/openai-claim-validation-scheduler.ts +++ b/src/typescript/api/src/lib/investigators/openai-claim-validation-scheduler.ts @@ -9,18 +9,18 @@ import { settlePendingValidation, type InvestigationRunState, } from "./openai-investigation-run-state.js"; -import type { PerClaimValidationResult } from "./openai-claim-validator.js"; +import type { ClaimValidationResult } from "./openai-claim-validator.js"; type StageOneClaim = InvestigationResult["claims"][number]; type ValidationLimiter = ( - task: () => Promise, -) => Promise; + task: () => Promise, +) => Promise; type ValidationRunner = ( claimIndex: number, claim: StageOneClaim, -) => Promise; +) => Promise; type RetainClaimResult = | { @@ -31,21 +31,12 @@ type RetainClaimResult = errorMessage: string; }; -export interface ClaimValidationScheduler { +interface ClaimValidationScheduler { getState: () => InvestigationRunState; scheduleClaimValidation: (claim: StageOneClaim) => void; retainClaimById: (claimId: string) => RetainClaimResult; - awaitAllValidations: () => Promise; - settleAllValidations: () => Promise; -} - -function toValidationErrorResult(claimIndex: number, error: unknown): PerClaimValidationResult { - return { - claimIndex, - approved: false, - responseAudit: null, - error: error instanceof Error ? error : new Error(String(error)), - }; + /** Every scheduled validation's result, in scheduling order, once all have settled. */ + awaitAllValidations: () => Promise; } export function createClaimValidationScheduler(input: { @@ -60,7 +51,7 @@ export function createClaimValidationScheduler(input: { input.callbacks?.onProgressUpdate(getPendingClaims(state), getConfirmedClaims(state)); }; - const settleValidation = (pendingIndex: number, result: PerClaimValidationResult): void => { + const settleValidation = (pendingIndex: number, result: ClaimValidationResult): void => { state = settlePendingValidation(state, { pendingIndex, result, @@ -70,9 +61,8 @@ export function createClaimValidationScheduler(input: { const scheduleClaimValidation = (claim: StageOneClaim): void => { const claimIndex = state.nextClaimIndex; - const promise = input - .validationLimiter(() => input.runValidation(claimIndex, claim)) - .catch((error: unknown) => toValidationErrorResult(claimIndex, error)); + // runValidation reports failures as results, never as rejections. + const promise = input.validationLimiter(() => input.runValidation(claimIndex, claim)); const queued = enqueuePendingValidation(state, { claim, @@ -109,8 +99,5 @@ export function createClaimValidationScheduler(input: { scheduleClaimValidation, retainClaimById, awaitAllValidations: async () => Promise.all(getPendingValidationPromises(state)), - settleAllValidations: async () => { - await Promise.allSettled(getPendingValidationPromises(state)); - }, }; } diff --git a/src/typescript/api/src/lib/investigators/openai-claim-validator.ts b/src/typescript/api/src/lib/investigators/openai-claim-validator.ts index 8fac6cb..68d9123 100644 --- a/src/typescript/api/src/lib/investigators/openai-claim-validator.ts +++ b/src/typescript/api/src/lib/investigators/openai-claim-validator.ts @@ -1,129 +1,106 @@ import type OpenAI from "openai"; -import { zodTextFormat } from "openai/helpers/zod"; -import { isNonNullObject, type InvestigationResult } from "@openerrata/shared"; -import type { InvestigatorResponseAudit } from "./interface.js"; -import { claimValidationResultSchema } from "./openai-schemas.js"; +import type { Response } from "openai/resources/responses/responses"; +import type { InvestigationClaimPayload } from "@openerrata/shared"; import { - extractResponseAudit, - readIncompleteReason, - requireCompletedOutputText, - requireJsonObject, -} from "./openai-response-audit.js"; -import { INVESTIGATION_VALIDATION_SYSTEM_PROMPT, buildValidationPrompt } from "./prompt.js"; - -const isRecord = isNonNullObject; + InvestigatorIncompleteResponseError, + InvestigatorStructuredOutputError, +} from "./errors.js"; +import type { InvestigatorRequestAudit } from "./interface.js"; +import { + buildClaimValidationRequestParams, + claimValidationVerdictSchema, + type InvestigationRequestConfig, +} from "./openai-request-config.js"; +import { auditRequest, auditResponse } from "./openai-response-audit.js"; +import { buildValidationPrompt } from "./prompt.js"; export const MAX_PER_CLAIM_VALIDATION_CONCURRENCY = 4; -export class InvestigatorIncompleteResponseError extends Error { - readonly responseStatus: string | null; - readonly responseId: string | null; - readonly incompleteReason: string | null; - readonly outputTextLength: number; +/** Outcome of one stage-2 validation call (SPEC §2.4.3.2). Never a rejection. */ +export type ClaimValidationResult = + | { kind: "approved"; claimIndex: number; request: InvestigatorRequestAudit } + | { kind: "rejected"; claimIndex: number; request: InvestigatorRequestAudit } + | { kind: "failed"; claimIndex: number; request: InvestigatorRequestAudit; error: Error }; + +function toError(caught: unknown): Error { + return caught instanceof Error ? caught : new Error(String(caught)); +} - constructor(input: { - responseStatus: string | null; - responseId: string | null; - incompleteReason: string | null; - outputTextLength: number; - }) { - const statusPart = input.responseStatus ?? "unknown"; - const reasonPart = input.incompleteReason ?? "unknown"; - const responseIdPart = input.responseId ?? "unknown"; - super( - "OpenAI response did not complete " + - `(status=${statusPart}, reason=${reasonPart}, responseId=${responseIdPart}, outputTextLength=${input.outputTextLength.toString()})`, +/** The verdict in a completed validation response; throws when there is none. */ +function readVerdict(response: Response): boolean { + if (response.status !== "completed") { + throw new InvestigatorIncompleteResponseError({ + responseStatus: response.status ?? null, + responseId: response.id, + incompleteReason: response.incomplete_details?.reason ?? null, + }); + } + + const outputText = response.output + .flatMap((item) => (item.type === "message" ? item.content : [])) + .flatMap((part) => (part.type === "output_text" ? [part.text] : [])) + .join(""); + + let decoded: unknown; + try { + decoded = JSON.parse(outputText); + } catch { + throw new InvestigatorStructuredOutputError( + `Claim validation response ${response.id} did not return a JSON verdict`, + ); + } + const verdict = claimValidationVerdictSchema.safeParse(decoded); + if (!verdict.success) { + throw new InvestigatorStructuredOutputError( + `Claim validation response ${response.id} returned an invalid verdict: ${verdict.error.message}`, ); - this.name = "InvestigatorIncompleteResponseError"; - this.responseStatus = input.responseStatus; - this.responseId = input.responseId; - this.incompleteReason = input.incompleteReason; - this.outputTextLength = input.outputTextLength; } + return verdict.data.approved; } -export type PerClaimValidationResult = - | { - claimIndex: number; - approved: boolean; - responseAudit: InvestigatorResponseAudit; - error: null; - } - | { - claimIndex: number; - approved: false; - responseAudit: InvestigatorResponseAudit | null; - error: Error; - }; - -export async function validateClaim( - client: OpenAI, - modelId: string, - claimIndex: number, - claim: InvestigationResult["claims"][number], - contentText: string, - imageContextNotes: string | undefined, - requestReasoning: { - effort: "low" | "medium" | "high"; - summary: "auto" | "concise" | "detailed"; - }, -): Promise { +export async function validateClaim(input: { + client: OpenAI; + requestConfig: InvestigationRequestConfig; + claimIndex: number; + claim: InvestigationClaimPayload; + contentText: string; + imageContextNotes: string | undefined; + signal: AbortSignal; +}): Promise { + const { claimIndex } = input; const validationPrompt = buildValidationPrompt({ - currentPostText: contentText, - candidateClaim: claim, - ...(imageContextNotes === undefined ? {} : { imageContextNotes }), + currentPostText: input.contentText, + candidateClaim: input.claim, + ...(input.imageContextNotes === undefined + ? {} + : { imageContextNotes: input.imageContextNotes }), }); + const params = buildClaimValidationRequestParams(input.requestConfig, validationPrompt); + const subject = { kind: "CLAIM_VALIDATION", claimIndex } as const; - let response: unknown; + let response: Response; try { - response = await client.responses.create({ - model: modelId, - stream: false, - instructions: INVESTIGATION_VALIDATION_SYSTEM_PROMPT, - input: validationPrompt, - reasoning: requestReasoning, - text: { - format: zodTextFormat(claimValidationResultSchema, "claim_validation_result"), - }, - }); + response = await input.client.responses.create(params, { signal: input.signal }); } catch (caught) { return { + kind: "failed", claimIndex, - approved: false, - responseAudit: null, - error: caught instanceof Error ? caught : new Error(String(caught)), + request: auditRequest({ subject, params, auditInput: validationPrompt, response: null }), + error: toError(caught), }; } - const responseRecord = isRecord(response) ? response : {}; - const responseAudit = extractResponseAudit(responseRecord); + const request = auditRequest({ + subject, + params, + auditInput: validationPrompt, + response: auditResponse(response, new Date()), + }); try { - if (responseAudit.responseStatus !== "completed") { - throw new InvestigatorIncompleteResponseError({ - responseStatus: responseAudit.responseStatus, - responseId: responseAudit.responseId, - incompleteReason: readIncompleteReason(responseRecord), - outputTextLength: responseAudit.responseOutputText?.length ?? 0, - }); - } - - const outputText = requireCompletedOutputText({ - responseAudit, - responseRecord, - context: "Claim validation response", - }); - - const parsed: unknown = JSON.parse(outputText); - const { approved } = claimValidationResultSchema.parse( - requireJsonObject(parsed, "Claim validation structured output"), - ); - return { claimIndex, approved, responseAudit, error: null }; + return readVerdict(response) + ? { kind: "approved", claimIndex, request } + : { kind: "rejected", claimIndex, request }; } catch (caught) { - return { - claimIndex, - approved: false, - responseAudit, - error: caught instanceof Error ? caught : new Error(String(caught)), - }; + return { kind: "failed", claimIndex, request, error: toError(caught) }; } } diff --git a/src/typescript/api/src/lib/investigators/openai-errors.ts b/src/typescript/api/src/lib/investigators/openai-errors.ts deleted file mode 100644 index 94789c3..0000000 --- a/src/typescript/api/src/lib/investigators/openai-errors.ts +++ /dev/null @@ -1,11 +0,0 @@ -/** - * Thrown when the LLM returns structured output that is syntactically valid - * but semantically invalid for our investigation pipeline. This is a - * non-retryable error — the model's output cannot be fixed by retrying. - */ -export class InvestigatorStructuredOutputError extends Error { - constructor(message: string) { - super(message); - this.name = "InvestigatorStructuredOutputError"; - } -} diff --git a/src/typescript/api/src/lib/investigators/openai-input-builder.ts b/src/typescript/api/src/lib/investigators/openai-input-builder.ts index 2123dcc..f8e82f2 100644 --- a/src/typescript/api/src/lib/investigators/openai-input-builder.ts +++ b/src/typescript/api/src/lib/investigators/openai-input-builder.ts @@ -1,11 +1,25 @@ import type { ResponseInput } from "openai/resources/responses/responses"; import { validateAndSortImageOccurrences } from "@openerrata/shared"; -import { InvestigatorStructuredOutputError } from "./openai-errors.js"; -import type { InvestigatorImageOccurrence, ImagePlaceholder } from "./interface.js"; +import { InvestigatorInputError } from "./errors.js"; +import type { + InvestigatorImageOccurrence, + ImagePlaceholder, + InvestigatorJsonRecord, +} from "./interface.js"; + +/** + * The stage-1 request's `input`, alongside the form recorded in the attempt + * audit, where each image part carries the image's content hash (its + * ImageBlob) instead of the inline data URI that was sent. + */ +export interface AuditedRequestInput { + request: string | ResponseInput; + audit: string | InvestigatorJsonRecord[]; +} type ContentInputPart = | { type: "input_text"; text: string } - | { type: "input_image"; detail: "auto"; image_url: string }; + | { type: "input_image"; dataUri: string; contentHash: string }; function appendTextInputPart(contentParts: ContentInputPart[], text: string): void { if (text.length === 0) return; @@ -15,6 +29,31 @@ function appendTextInputPart(contentParts: ContentInputPart[], text: string): vo }); } +function toAuditedUserMessage(contentParts: ContentInputPart[]): AuditedRequestInput { + return { + request: [ + { + role: "user", + content: contentParts.map((part) => + part.type === "input_text" + ? part + : { type: "input_image" as const, detail: "auto" as const, image_url: part.dataUri }, + ), + }, + ], + audit: [ + { + role: "user", + content: contentParts.map((part) => + part.type === "input_text" + ? { type: part.type, text: part.text } + : { type: part.type, detail: "auto", imageContentHash: part.contentHash }, + ), + }, + ], + }; +} + function requirePromptContentBounds( userPrompt: string, contentString: string, @@ -27,7 +66,7 @@ function requirePromptContentBounds( contentEnd > userPrompt.length || userPrompt.slice(contentStart, contentEnd) !== contentString ) { - throw new InvestigatorStructuredOutputError( + throw new InvestigatorInputError( "contentOffset does not point to contentString within the stage-1 user prompt", ); } @@ -43,15 +82,13 @@ function normalizeImageOccurrences( onValidationIssue: (issue): never => { switch (issue.code) { case "NON_CONTIGUOUS_ORIGINAL_INDEX": - throw new InvestigatorStructuredOutputError( + throw new InvestigatorInputError( "Image occurrences must use contiguous originalIndex values starting at 0", ); case "OFFSET_EXCEEDS_CONTENT_LENGTH": - throw new InvestigatorStructuredOutputError( - "Image occurrence offset exceeds contentText length", - ); + throw new InvestigatorInputError("Image occurrence offset exceeds contentText length"); case "DECREASING_NORMALIZED_TEXT_OFFSET": - throw new InvestigatorStructuredOutputError( + throw new InvestigatorInputError( "Image occurrences must be non-decreasing by normalizedTextOffset", ); } @@ -93,7 +130,7 @@ function buildInputUsingTextOffsets(input: { contentString: string; contentOffset: number; normalizedOccurrences: InvestigatorImageOccurrence[]; -}): ResponseInput { +}): AuditedRequestInput { const { contentStart, contentEnd } = requirePromptContentBounds( input.userPrompt, input.contentString, @@ -126,8 +163,8 @@ function buildInputUsingTextOffsets(input: { seenResolvedContentHashes.add(occurrence.contentHash); contentParts.push({ type: "input_image", - detail: "auto", - image_url: occurrence.imageDataUri, + dataUri: occurrence.imageDataUri, + contentHash: occurrence.contentHash, }); continue; } @@ -156,12 +193,7 @@ function buildInputUsingTextOffsets(input: { } appendTextInputPart(contentParts, input.userPrompt.slice(contentEnd)); - return [ - { - role: "user", - content: contentParts, - }, - ]; + return toAuditedUserMessage(contentParts); } /** @@ -204,7 +236,7 @@ export function buildInitialInput( contentOffset: number, imageOccurrences: InvestigatorImageOccurrence[] | undefined, imagePlaceholders: ImagePlaceholder[] | undefined, -): string | ResponseInput { +): AuditedRequestInput { const shouldUsePlaceholderInterleaving = imagePlaceholders !== undefined && imagePlaceholders.length > 0; const normalizedOccurrences = normalizeImageOccurrences( @@ -212,7 +244,7 @@ export function buildInitialInput( shouldUsePlaceholderInterleaving ? undefined : contentString, ); if (normalizedOccurrences.length === 0) { - return userPrompt; + return { request: userPrompt, audit: userPrompt }; } // No markdown placeholders available (e.g. markdownSource=NONE): interleave @@ -238,7 +270,7 @@ export function buildInitialInput( const seenResolvedContentHashes = new Set(); // Split content at [IMAGE:N] patterns - const placeholderPattern = /\[IMAGE:(\d+)\]/g; + const placeholderPattern = /\[IMAGE:\d+\]/g; const contentParts: ContentInputPart[] = []; // Text before the content section @@ -249,7 +281,7 @@ export function buildInitialInput( let match: RegExpExecArray | null; while ((match = placeholderPattern.exec(contentString)) !== null) { - const placeholderIndex = parseInt(match[1] ?? "0", 10); + const placeholderIndex = Number.parseInt(match[0].slice("[IMAGE:".length, -"]".length), 10); const placeholder = imagePlaceholders.find((p) => p.index === placeholderIndex); // Text between last position and this placeholder @@ -274,8 +306,8 @@ export function buildInitialInput( consumedUrls.add(occurrence.sourceUrl); contentParts.push({ type: "input_image", - detail: "auto", - image_url: occurrence.imageDataUri, + dataUri: occurrence.imageDataUri, + contentHash: occurrence.contentHash, }); continue; } @@ -314,21 +346,5 @@ export function buildInitialInput( // Text after the content section appendTextInputPart(contentParts, userPrompt.slice(contentEnd)); - return [ - { - role: "user", - content: contentParts, - }, - ]; -} - -export function buildTwoStepRequestInputAudit( - userPrompt: string, - validationPrompt: string, -): string { - return `=== Stage 1: Fact-check input === -${userPrompt} - -=== Stage 2: Validation input === -${validationPrompt}`; + return toAuditedUserMessage(contentParts); } diff --git a/src/typescript/api/src/lib/investigators/openai-investigation-run-state.ts b/src/typescript/api/src/lib/investigators/openai-investigation-run-state.ts index 9fd6e87..3dce5b2 100644 --- a/src/typescript/api/src/lib/investigators/openai-investigation-run-state.ts +++ b/src/typescript/api/src/lib/investigators/openai-investigation-run-state.ts @@ -1,6 +1,6 @@ import type { InvestigationResult } from "@openerrata/shared"; import type { InvestigatorInput } from "./interface.js"; -import type { PerClaimValidationResult } from "./openai-claim-validator.js"; +import type { ClaimValidationResult } from "./openai-claim-validator.js"; type StageOneClaim = InvestigationResult["claims"][number]; type OldClaim = Extract["oldClaims"][number]; @@ -9,7 +9,7 @@ export interface PendingValidationEntry { claim: StageOneClaim; claimIndex: number; submissionOrder: number; - promise: Promise; + promise: Promise; settled: boolean; } @@ -60,7 +60,7 @@ export function enqueuePendingValidation( state: InvestigationRunState, input: { claim: StageOneClaim; - promise: Promise; + promise: Promise; }, ): { nextState: InvestigationRunState; @@ -94,7 +94,7 @@ export function settlePendingValidation( state: InvestigationRunState, input: { pendingIndex: number; - result: PerClaimValidationResult; + result: ClaimValidationResult; }, ): InvestigationRunState { const pending = state.pendingValidations[input.pendingIndex]; @@ -103,7 +103,7 @@ export function settlePendingValidation( } if (pending.settled) { - return state; + throw new Error(`Pending validation already settled: ${input.pendingIndex.toString()}`); } const pendingValidations = state.pendingValidations.map((entry, index) => @@ -111,7 +111,7 @@ export function settlePendingValidation( ); const confirmedClaims = - input.result.error === null && input.result.approved + input.result.kind === "approved" ? [ ...state.confirmedClaims, { @@ -182,6 +182,6 @@ export function getConfirmedClaims(state: InvestigationRunState): StageOneClaim[ export function getPendingValidationPromises( state: InvestigationRunState, -): Promise[] { +): Promise[] { return state.pendingValidations.map((entry) => entry.promise); } diff --git a/src/typescript/api/src/lib/investigators/openai-probe.ts b/src/typescript/api/src/lib/investigators/openai-probe.ts new file mode 100644 index 0000000..13e15d7 --- /dev/null +++ b/src/typescript/api/src/lib/investigators/openai-probe.ts @@ -0,0 +1,26 @@ +import type OpenAI from "openai"; +import { + buildProbeRequestParams, + type InvestigationRequestConfig, +} from "./openai-request-config.js"; + +/** + * Checks that the provider accepts investigation requests from this client: + * the key authenticates, has access to the model, and the request shape + * (tools, include, reasoning options) is valid for it. Rejects with the + * provider's error otherwise. + * + * An "incomplete" response counts as accepted: the probe's tiny output cap + * routinely cuts reasoning short, which says nothing about the request shape. + */ +export async function probeInvestigationRequest( + client: OpenAI, + requestConfig: InvestigationRequestConfig, +): Promise { + const response = await client.responses.create(buildProbeRequestParams(requestConfig)); + if (response.status !== "completed" && response.status !== "incomplete") { + throw new Error( + `OpenAI probe response ${response.id} ended with status ${response.status ?? "missing"}`, + ); + } +} diff --git a/src/typescript/api/src/lib/investigators/openai-request-config.ts b/src/typescript/api/src/lib/investigators/openai-request-config.ts new file mode 100644 index 0000000..e4a6abb --- /dev/null +++ b/src/typescript/api/src/lib/investigators/openai-request-config.ts @@ -0,0 +1,153 @@ +import { zodTextFormat } from "openai/helpers/zod"; +import type { + ResponseCreateParamsNonStreaming, + ResponseIncludable, + ResponseInput, + Tool, +} from "openai/resources/responses/responses"; +import type { Reasoning, ReasoningEffort } from "openai/resources/shared"; +import { z } from "zod"; +import { fetchUrlToolDefinition } from "./fetch-url-tool.js"; +import { + buildRetainCorrectionToolDefinition, + submitCorrectionToolDefinition, +} from "./openai-claim-tools.js"; +import { INVESTIGATION_VALIDATION_SYSTEM_PROMPT } from "./prompt.js"; + +/** + * The one model OpenErrata investigates with. Model choice is a code change, + * not configuration: every request shape below (tool types, reasoning options, + * `include` values) is what this model accepts. + */ +export const INVESTIGATION_MODEL_ID = "gpt-6.1-sol"; + +type ReasoningSummary = NonNullable; +// gpt-6.1-sol accepts low | medium | high | xhigh | max; it rejects "none" and "minimal". +type InvestigationReasoningEffort = Exclude, "none" | "minimal">; + +/** + * Reasoning summaries requested on every investigation request, persisted in + * the attempt audit (SPEC §2.12); `null` requests none. gpt-6.1-sol's model docs + * do not mention reasoning summaries; "detailed" was verified live on + * 2026-10-02. A rejected request parameter fails every investigation + * non-retryably (SPEC §3.7), so verify any new value live with + * `pnpm --filter @openerrata/api smoke:openai` before deploying it. + */ +const INVESTIGATION_REASONING_SUMMARY: ReasoningSummary | null = "detailed"; + +export interface InvestigationRequestConfig { + readonly model: typeof INVESTIGATION_MODEL_ID; + readonly reasoningEffort: InvestigationReasoningEffort; + readonly reasoningSummary: ReasoningSummary | null; + /** Extra response fields the fact-check requests ask the provider to return. */ + readonly include: readonly ResponseIncludable[]; +} + +export const INVESTIGATION_REQUEST_CONFIG: InvestigationRequestConfig = { + model: INVESTIGATION_MODEL_ID, + // gpt-6.1-sol's default, stated explicitly so the audit records it. + reasoningEffort: "medium", + reasoningSummary: INVESTIGATION_REASONING_SUMMARY, + // Without this, web_search_call items carry no sources, and the audit would + // miss the URLs the model consulted. + include: ["web_search_call.action.sources"], +}; + +/** Request parameters for every provider request an investigation makes. */ +export type InvestigationRequestParams = ResponseCreateParamsNonStreaming & { + model: typeof INVESTIGATION_MODEL_ID; + instructions: string; + reasoning: Reasoning; +}; + +function toRequestReasoning(config: InvestigationRequestConfig): Reasoning { + return config.reasoningSummary === null + ? { effort: config.reasoningEffort } + : { effort: config.reasoningEffort, summary: config.reasoningSummary }; +} + +/** + * Tools offered to the stage-1 fact-check. retain_correction is offered only + * to update investigations whose parent has claims to retain. + */ +export function buildFactCheckTools( + retainableClaimIds: readonly [string, ...string[]] | null, +): Tool[] { + return [ + { type: "web_search" }, + fetchUrlToolDefinition, + submitCorrectionToolDefinition, + ...(retainableClaimIds === null + ? [] + : [buildRetainCorrectionToolDefinition(retainableClaimIds)]), + ]; +} + +/** One round of the stage-1 fact-check tool loop. */ +export function buildFactCheckRequestParams( + config: InvestigationRequestConfig, + request: { + instructions: string; + tools: Tool[]; + input: string | ResponseInput; + /** The previous round's response, which this round's input continues. */ + previousResponseId: string | null; + }, +): InvestigationRequestParams { + return { + model: config.model, + stream: false, + instructions: request.instructions, + input: request.input, + tools: request.tools, + include: [...config.include], + reasoning: toRequestReasoning(config), + ...(request.previousResponseId === null + ? {} + : { previous_response_id: request.previousResponseId }), + }; +} + +export const claimValidationVerdictSchema = z + .object({ + approved: z.boolean(), + }) + .strict(); + +/** A stage-2 per-claim validation call: no tools, structured yes/no verdict. */ +export function buildClaimValidationRequestParams( + config: InvestigationRequestConfig, + validationPrompt: string, +): InvestigationRequestParams { + return { + model: config.model, + stream: false, + instructions: INVESTIGATION_VALIDATION_SYSTEM_PROMPT, + input: validationPrompt, + reasoning: toRequestReasoning(config), + text: { + format: zodTextFormat(claimValidationVerdictSchema, "claim_validation_result"), + }, + }; +} + +/** + * A minimal request with the fact-check request's shape (model, tools, + * include, reasoning) that forbids tool use and caps output, so the provider + * validates the shape without running an investigation. + */ +export function buildProbeRequestParams( + config: InvestigationRequestConfig, +): InvestigationRequestParams { + return { + ...buildFactCheckRequestParams(config, { + instructions: "Reply with the single word pong.", + tools: buildFactCheckTools(null), + input: "ping", + previousResponseId: null, + }), + tool_choice: "none", + // The smallest cap the provider has accepted for reasoning models. + max_output_tokens: 16, + }; +} diff --git a/src/typescript/api/src/lib/investigators/openai-response-audit.ts b/src/typescript/api/src/lib/investigators/openai-response-audit.ts index c37dc42..244554c 100644 --- a/src/typescript/api/src/lib/investigators/openai-response-audit.ts +++ b/src/typescript/api/src/lib/investigators/openai-response-audit.ts @@ -1,482 +1,188 @@ -import { isNonNullObject } from "@openerrata/shared"; -import { InvestigatorStructuredOutputError } from "./openai-errors.js"; +import type { + Response, + ResponseOutputItem, + ResponseOutputRefusal, + ResponseOutputText, +} from "openai/resources/responses/responses"; +import { z } from "zod"; +import { readOpenAiStatusCode } from "$lib/openai/errors.js"; import type { InvestigatorErrorAudit, + InvestigatorJsonRecord, InvestigatorJsonValue, InvestigatorOutputItemAudit, InvestigatorOutputTextAnnotationAudit, InvestigatorOutputTextPartAudit, - InvestigatorReasoningSummaryAudit, - InvestigatorRequestedToolAudit, + InvestigatorRequestAudit, + InvestigatorRequestSubject, InvestigatorResponseAudit, - InvestigatorToolCallAudit, - InvestigatorUsageAudit, } from "./interface.js"; -import { readOpenAiStatusCode } from "$lib/openai/errors.js"; - -const isRecord = isNonNullObject; - -export function sanitizeJsonValue(value: unknown, depth = 0): InvestigatorJsonValue { - if (depth > 8) return "[max-depth]"; - if ( - typeof value === "string" || - typeof value === "number" || - typeof value === "boolean" || - value === null - ) { - return value; - } - if (Array.isArray(value)) { - return value.map((entry) => sanitizeJsonValue(entry, depth + 1)); - } - if (isRecord(value)) { - const sanitized: Record = {}; - for (const [key, entry] of Object.entries(value)) { - sanitized[key] = sanitizeJsonValue(entry, depth + 1); - } - return sanitized; - } - if (typeof value === "bigint") return value.toString(); - if (typeof value === "symbol") return value.description ?? "symbol"; - if (typeof value === "function") return "[function]"; - return "[unsupported]"; -} - -export function sanitizeJsonRecord(value: unknown): Record { - const record = requireJsonObject(value, "OpenAI audit payload"); - const sanitized: Record = {}; - for (const [key, entry] of Object.entries(record)) { - sanitized[key] = sanitizeJsonValue(entry, 1); - } - return sanitized; -} - -export function describeJsonValueType(value: unknown): string { - if (value === null) return "null"; - if (Array.isArray(value)) return "array"; - return typeof value; -} - -export function requireJsonObject(value: unknown, context: string): Record { - if (!isRecord(value)) { - throw new InvestigatorStructuredOutputError( - `${context} must be a JSON object (received ${describeJsonValueType(value)})`, - ); - } - return value; -} - -export function requireCompletedOutputText(input: { - responseAudit: InvestigatorResponseAudit; - responseRecord: Record; - context: string; -}): string { - const outputText = input.responseAudit.responseOutputText; - if (outputText === null) { - const rawOutputTextType = describeJsonValueType(input.responseRecord["output_text"]); - throw new InvestigatorStructuredOutputError( - `${input.context} completed without output_text (responseId=${input.responseAudit.responseId ?? "unknown"}, output_text_type=${rawOutputTextType})`, - ); - } - - if (outputText.trim().length === 0) { - throw new InvestigatorStructuredOutputError( - `${input.context} returned empty structured output`, - ); - } - - return outputText; -} - -export function readString(value: unknown): string | null { - return typeof value === "string" ? value : null; -} - -export function readOptionalInteger(value: unknown): number | null { - return typeof value === "number" && Number.isInteger(value) ? value : null; -} - -export function readIncompleteReason(responseRecord: Record): string | null { - const incompleteDetails = responseRecord["incomplete_details"]; - if (!isRecord(incompleteDetails)) return null; - return readString(incompleteDetails["reason"]); -} - -export function buildErrorAudit(error: unknown): InvestigatorErrorAudit { - if (error instanceof Error) { +import type { InvestigationRequestParams } from "./openai-request-config.js"; + +const jsonValueSchema: z.ZodType = z.lazy(() => + z.union([ + z.string(), + z.number(), + z.boolean(), + z.null(), + z.array(jsonValueSchema), + z.record(z.string(), jsonValueSchema), + ]), +); +const jsonRecordSchema = z.record(z.string(), jsonValueSchema); + +/** + * The JSON the SDK puts on (or read off) the wire for an SDK object such as a + * tool definition or output item, checked to be JSON so it can be stored + * verbatim. TypeScript's SDK types can't express JSON-ness themselves. + */ +function toJsonRecord(value: object): InvestigatorJsonRecord { + return jsonRecordSchema.parse(JSON.parse(JSON.stringify(value))); +} + +function auditAnnotation( + annotation: ResponseOutputText["annotations"][number], +): InvestigatorOutputTextAnnotationAudit { + switch (annotation.type) { + case "url_citation": + return { + annotationType: annotation.type, + startIndex: annotation.start_index, + endIndex: annotation.end_index, + url: annotation.url, + title: annotation.title, + fileId: null, + }; + case "container_file_citation": + return { + annotationType: annotation.type, + startIndex: annotation.start_index, + endIndex: annotation.end_index, + url: null, + title: annotation.filename, + fileId: annotation.file_id, + }; + case "file_citation": + return { + annotationType: annotation.type, + startIndex: null, + endIndex: null, + url: null, + title: annotation.filename, + fileId: annotation.file_id, + }; + case "file_path": + return { + annotationType: annotation.type, + startIndex: null, + endIndex: null, + url: null, + title: null, + fileId: annotation.file_id, + }; + } +} + +function auditTextPart( + part: ResponseOutputText | ResponseOutputRefusal, +): InvestigatorOutputTextPartAudit { + switch (part.type) { + case "output_text": + return { + partType: part.type, + text: part.text, + annotations: part.annotations.map(auditAnnotation), + }; + case "refusal": + return { partType: part.type, text: part.refusal, annotations: [] }; + } +} + +function auditOutputItem(item: ResponseOutputItem): InvestigatorOutputItemAudit { + if (item.type === "message") { return { - errorName: error.name, - errorMessage: error.message, - statusCode: readOpenAiStatusCode(error), + providerItemId: item.id, + itemType: item.type, + itemStatus: item.status, + content: { kind: "MESSAGE", textParts: item.content.map(auditTextPart) }, }; } - - return { - errorName: "UnknownError", - errorMessage: typeof error === "string" ? error : "unknown", - statusCode: readOpenAiStatusCode(error), - }; -} - -export function parseTimestamp(value: unknown): string | null { - if (typeof value === "number" && Number.isFinite(value)) { - // Handle seconds and milliseconds unix timestamps. - const normalized = value > 1_000_000_000_000 ? value : value * 1000; - const date = new Date(normalized); - return Number.isNaN(date.valueOf()) ? null : date.toISOString(); - } - - if (typeof value === "string" && value.trim().length > 0) { - const date = new Date(value); - return Number.isNaN(date.valueOf()) ? null : date.toISOString(); - } - - return null; -} - -export function findTimestamp( - value: unknown, - candidateKeys: Set, - depth = 0, -): string | null { - if (depth > 6) return null; - - if (isRecord(value)) { - for (const [key, nested] of Object.entries(value)) { - const normalizedKey = key.toLowerCase(); - if (candidateKeys.has(normalizedKey)) { - const parsed = parseTimestamp(nested); - if (parsed !== null && parsed.length > 0) return parsed; - } - const nestedResult = findTimestamp(nested, candidateKeys, depth + 1); - if (nestedResult !== null && nestedResult.length > 0) return nestedResult; - } - } else if (Array.isArray(value)) { - for (const nested of value) { - const nestedResult = findTimestamp(nested, candidateKeys, depth + 1); - if (nestedResult !== null && nestedResult.length > 0) return nestedResult; - } - } - - return null; -} - -export function extractRequestedTools(tools: unknown): InvestigatorRequestedToolAudit[] { - if (!Array.isArray(tools)) return []; - - const extracted: InvestigatorRequestedToolAudit[] = []; - for (const [index, tool] of tools.entries()) { - if (!isRecord(tool)) continue; - - extracted.push({ - requestOrder: index, - toolType: readString(tool["type"]) ?? "unknown", - rawDefinition: sanitizeJsonRecord(tool), - }); - } - - return extracted; -} - -export function extractOutputItems(outputItems: unknown[]): InvestigatorOutputItemAudit[] { - const extracted: InvestigatorOutputItemAudit[] = []; - - for (const [outputIndex, outputItem] of outputItems.entries()) { - if (!isRecord(outputItem)) { - extracted.push({ - outputIndex, - providerItemId: null, - itemType: "unknown", - itemStatus: null, - }); - continue; - } - - const providerItemId = readString(outputItem["id"]); - const itemStatus = readString(outputItem["status"]); - extracted.push({ - outputIndex, - providerItemId: providerItemId === null || itemStatus === null ? null : providerItemId, - itemType: readString(outputItem["type"]) ?? "unknown", - itemStatus: providerItemId === null || itemStatus === null ? null : itemStatus, - }); - } - - return extracted; -} - -export function extractOutputTextArtifacts(outputItems: unknown[]): { - parts: InvestigatorOutputTextPartAudit[]; - annotations: InvestigatorOutputTextAnnotationAudit[]; -} { - const parts: InvestigatorOutputTextPartAudit[] = []; - const annotations: InvestigatorOutputTextAnnotationAudit[] = []; - - for (const [outputIndex, outputItem] of outputItems.entries()) { - if (!isRecord(outputItem) || outputItem["type"] !== "message") continue; - - const content = outputItem["content"]; - if (!Array.isArray(content)) continue; - - for (const [partIndex, part] of content.entries()) { - if (!isRecord(part)) continue; - - const partType = readString(part["type"]); - if (partType === "output_text") { - const text = readString(part["text"]); - if (text === null || text.length === 0) continue; - - parts.push({ - outputIndex, - partIndex, - partType, - text, - }); - - const partAnnotations = part["annotations"]; - if (!Array.isArray(partAnnotations)) continue; - - for (const [annotationIndex, annotation] of partAnnotations.entries()) { - if (!isRecord(annotation)) continue; - - const startIndex = readOptionalInteger(annotation["start_index"]); - const endIndex = readOptionalInteger(annotation["end_index"]); - - const characterPosition = - startIndex === null || endIndex === null - ? undefined - : { - start: startIndex, - end: endIndex, - }; - - annotations.push({ - outputIndex, - partIndex, - annotationIndex, - annotationType: readString(annotation["type"]) ?? "unknown", - characterPosition, - url: readString(annotation["url"]), - title: readString(annotation["title"]), - fileId: readString(annotation["file_id"]), - }); - } - - continue; - } - - if (partType === "refusal") { - const refusal = readString(part["refusal"]); - if (refusal === null || refusal.length === 0) continue; - - parts.push({ - outputIndex, - partIndex, - partType, - text: refusal, - }); - } - } - } - - return { parts, annotations }; -} - -export function extractReasoningSummaries( - outputItems: unknown[], -): InvestigatorReasoningSummaryAudit[] { - const summaries: InvestigatorReasoningSummaryAudit[] = []; - - for (const [outputIndex, outputItem] of outputItems.entries()) { - if (!isRecord(outputItem) || outputItem["type"] !== "reasoning") continue; - - const summary = outputItem["summary"]; - if (!Array.isArray(summary)) continue; - - for (const [summaryIndex, summaryPart] of summary.entries()) { - if (!isRecord(summaryPart)) continue; - const text = readString(summaryPart["text"]); - if (text === null || text.length === 0) continue; - - summaries.push({ - outputIndex, - summaryIndex, - text, - }); - } - } - - return summaries; -} - -function toToolCallAudit( - outputIndex: number, - outputItem: unknown, -): InvestigatorToolCallAudit | null { - if (!isRecord(outputItem)) return null; - const type = outputItem["type"]; - if (type === "message" || type === "reasoning") { - return null; - } - if (typeof type !== "string" || type.length === 0) return null; - - const providerStartedAt = findTimestamp( - outputItem, - new Set(["started_at", "start_time", "created_at", "createdat", "requested_at", "timestamp"]), - ); - const providerCompletedAt = findTimestamp( - outputItem, - new Set(["completed_at", "finished_at", "ended_at", "updated_at", "completedat", "finishedat"]), - ); - const providerToolCallId = readString(outputItem["id"]); - const status = readString(outputItem["status"]); - - return { - outputIndex, - providerToolCallId: providerToolCallId === null || status === null ? null : providerToolCallId, - toolType: type, - status: providerToolCallId === null || status === null ? null : status, - rawPayload: sanitizeJsonRecord(outputItem), - capturedAt: new Date().toISOString(), - providerStartedAt, - providerCompletedAt, - }; -} - -export function extractToolCalls(outputItems: unknown[]): InvestigatorToolCallAudit[] { - const entries: InvestigatorToolCallAudit[] = []; - for (const [outputIndex, outputItem] of outputItems.entries()) { - const entry = toToolCallAudit(outputIndex, outputItem); - if (entry) entries.push(entry); - } - return entries; -} - -export function extractUsage( - responseRecord: Record, -): InvestigatorUsageAudit | null { - const usageValue = responseRecord["usage"]; - if (!isRecord(usageValue)) return null; - - const inputTokens = readOptionalInteger(usageValue["input_tokens"]); - const outputTokens = readOptionalInteger(usageValue["output_tokens"]); - const totalTokens = readOptionalInteger(usageValue["total_tokens"]); - if (inputTokens === null || outputTokens === null || totalTokens === null) { - return null; + if (item.type === "reasoning") { + return { + providerItemId: item.id, + itemType: item.type, + itemStatus: item.status ?? null, + content: { kind: "REASONING", summaries: item.summary.map((summary) => summary.text) }, + }; } - - const inputDetails = isRecord(usageValue["input_tokens_details"]) - ? usageValue["input_tokens_details"] - : null; - const outputDetails = isRecord(usageValue["output_tokens_details"]) - ? usageValue["output_tokens_details"] - : null; - + // Every other item is a tool call (web_search_call, function_call, …). Their + // shapes vary by tool and some carry no status, so the item is kept verbatim. return { - inputTokens, - outputTokens, - totalTokens, - cachedInputTokens: readOptionalInteger(inputDetails?.["cached_tokens"]), - reasoningOutputTokens: readOptionalInteger(outputDetails?.["reasoning_tokens"]), + providerItemId: item.id ?? null, + itemType: item.type, + itemStatus: "status" in item ? (item.status ?? null) : null, + content: { kind: "TOOL_CALL", rawPayload: toJsonRecord(item) }, }; } -export function extractResponseAudit( - responseRecord: Record, -): InvestigatorResponseAudit { - const outputItems = Array.isArray(responseRecord["output"]) ? responseRecord["output"] : []; - - const outputTextArtifacts = extractOutputTextArtifacts(outputItems); - +export function auditResponse(response: Response, receivedAt: Date): InvestigatorResponseAudit { return { - responseId: readString(responseRecord["id"]), - responseStatus: readString(responseRecord["status"]), - responseModelVersion: readString(responseRecord["model"]), - responseOutputText: readString(responseRecord["output_text"]), - outputItems: extractOutputItems(outputItems), - outputTextParts: outputTextArtifacts.parts, - outputTextAnnotations: outputTextArtifacts.annotations, - reasoningSummaries: extractReasoningSummaries(outputItems), - toolCalls: extractToolCalls(outputItems), - usage: extractUsage(responseRecord), + providerResponseId: response.id, + status: response.status ?? null, + modelVersion: response.model, + receivedAt, + outputItems: response.output.map(auditOutputItem), + usage: + response.usage === undefined + ? null + : { + inputTokens: response.usage.input_tokens, + outputTokens: response.usage.output_tokens, + totalTokens: response.usage.total_tokens, + cachedInputTokens: response.usage.input_tokens_details.cached_tokens, + reasoningOutputTokens: response.usage.output_tokens_details.reasoning_tokens, + }, }; } -export function offsetResponseAuditIndices( - audit: InvestigatorResponseAudit, - outputIndexOffset: number, -): InvestigatorResponseAudit { - if (outputIndexOffset === 0) return audit; - +/** Audits a request exactly as sent (see InvestigatorRequestAudit.input for images). */ +export function auditRequest(input: { + subject: InvestigatorRequestSubject; + params: InvestigationRequestParams; + auditInput: InvestigatorRequestAudit["input"]; + response: InvestigatorResponseAudit | null; +}): InvestigatorRequestAudit { + const { params } = input; return { - ...audit, - outputItems: audit.outputItems.map((item) => ({ - ...item, - outputIndex: item.outputIndex + outputIndexOffset, - })), - outputTextParts: audit.outputTextParts.map((part) => ({ - ...part, - outputIndex: part.outputIndex + outputIndexOffset, - })), - outputTextAnnotations: audit.outputTextAnnotations.map((annotation) => ({ - ...annotation, - outputIndex: annotation.outputIndex + outputIndexOffset, - })), - reasoningSummaries: audit.reasoningSummaries.map((summary) => ({ - ...summary, - outputIndex: summary.outputIndex + outputIndexOffset, - })), - toolCalls: audit.toolCalls.map((toolCall) => ({ - ...toolCall, - outputIndex: toolCall.outputIndex + outputIndexOffset, + subject: input.subject, + model: params.model, + instructions: params.instructions, + input: input.auditInput, + previousResponseId: params.previous_response_id ?? null, + reasoningEffort: params.reasoning.effort ?? null, + reasoningSummary: params.reasoning.summary ?? null, + include: [...(params.include ?? [])], + tools: (params.tools ?? []).map((tool) => ({ + toolType: tool.type, + rawDefinition: toJsonRecord(tool), })), + response: input.response, }; } -export function aggregateUsage( - usages: (InvestigatorUsageAudit | null)[], -): InvestigatorUsageAudit | null { - const presentUsages = usages.filter((usage): usage is InvestigatorUsageAudit => usage !== null); - if (presentUsages.length === 0) return null; - - return presentUsages.reduce( - (accumulator, usage) => ({ - inputTokens: accumulator.inputTokens + usage.inputTokens, - outputTokens: accumulator.outputTokens + usage.outputTokens, - totalTokens: accumulator.totalTokens + usage.totalTokens, - cachedInputTokens: (accumulator.cachedInputTokens ?? 0) + (usage.cachedInputTokens ?? 0), - reasoningOutputTokens: - (accumulator.reasoningOutputTokens ?? 0) + (usage.reasoningOutputTokens ?? 0), - }), - { - inputTokens: 0, - outputTokens: 0, - totalTokens: 0, - cachedInputTokens: 0, - reasoningOutputTokens: 0, - }, - ); -} - -export function mergeResponseAudits( - responseAudits: InvestigatorResponseAudit[], -): InvestigatorResponseAudit { - if (responseAudits.length === 0) { - throw new Error("Cannot merge empty response audits"); +export function buildErrorAudit(error: unknown): InvestigatorErrorAudit { + if (error instanceof Error) { + return { + // OpenAI SDK errors keep the generic name "Error"; their class names the failure. + errorName: error.name === "Error" ? error.constructor.name : error.name, + errorMessage: error.message, + statusCode: readOpenAiStatusCode(error), + }; } - const finalAudit = responseAudits[responseAudits.length - 1]; - if (!finalAudit) throw new Error("Cannot merge empty response audits"); return { - responseId: finalAudit.responseId, - responseStatus: finalAudit.responseStatus, - responseModelVersion: finalAudit.responseModelVersion, - responseOutputText: finalAudit.responseOutputText, - outputItems: responseAudits.flatMap((audit) => audit.outputItems), - outputTextParts: responseAudits.flatMap((audit) => audit.outputTextParts), - outputTextAnnotations: responseAudits.flatMap((audit) => audit.outputTextAnnotations), - reasoningSummaries: responseAudits.flatMap((audit) => audit.reasoningSummaries), - toolCalls: responseAudits.flatMap((audit) => audit.toolCalls), - usage: aggregateUsage(responseAudits.map((audit) => audit.usage)), + errorName: "UnknownError", + errorMessage: typeof error === "string" ? error : "unknown", + statusCode: null, }; } diff --git a/src/typescript/api/src/lib/investigators/openai-schemas.ts b/src/typescript/api/src/lib/investigators/openai-schemas.ts deleted file mode 100644 index 02efbc6..0000000 --- a/src/typescript/api/src/lib/investigators/openai-schemas.ts +++ /dev/null @@ -1,35 +0,0 @@ -import { z } from "zod"; - -export const claimValidationResultSchema = z - .object({ - approved: z.boolean(), - }) - .strict(); - -// OpenAI structured outputs currently reject JSON Schema `format: "uri"`. -// Keep provider-facing schema to plain strings/patterns, then enforce the -// full shared schema (`investigationResultSchema`) before returning. -export const providerStructuredSourceUrlSchema = z - .string() - .min(1) - .regex(/^https?:\/\/\S+$/i, "Source URL must be an absolute http(s) URL"); - -export const providerStructuredInvestigationClaimPayloadSchema = z - .object({ - text: z.string().min(1), - context: z.string().min(1), - summary: z.string().min(1), - reasoning: z.string().min(1), - sources: z - .array( - z - .object({ - url: providerStructuredSourceUrlSchema, - title: z.string().min(1), - snippet: z.string().min(1), - }) - .strict(), - ) - .min(1), - }) - .strict(); diff --git a/src/typescript/api/src/lib/investigators/openai-tool-dispatch.ts b/src/typescript/api/src/lib/investigators/openai-tool-dispatch.ts index 8cb0941..f5e3f14 100644 --- a/src/typescript/api/src/lib/investigators/openai-tool-dispatch.ts +++ b/src/typescript/api/src/lib/investigators/openai-tool-dispatch.ts @@ -1,97 +1,6 @@ -import { isNonNullObject } from "@openerrata/shared"; +import type { Response } from "openai/resources/responses/responses"; import { FETCH_URL_TOOL_NAME, executeFetchUrlTool } from "./fetch-url-tool.js"; -import { readString } from "./openai-response-audit.js"; - -const isRecord = isNonNullObject; - -export const SUBMIT_CORRECTION_TOOL_NAME = "submit_correction"; -export const RETAIN_CORRECTION_TOOL_NAME = "retain_correction"; - -/** - * OpenAI function tool definition for submitting an individual claim. - * The model calls this as it discovers each correction during investigation. - * - * Uses `strict: true` to enable structured outputs for the tool parameters. - * The JSON Schema mirrors `providerStructuredInvestigationClaimPayloadSchema` - * from `openai-schemas.ts` but is expressed as a plain object because the - * OpenAI SDK requires a JSON Schema object, not a Zod schema. - */ -export const submitCorrectionToolDefinition = { - type: "function" as const, - name: SUBMIT_CORRECTION_TOOL_NAME, - description: - "Submit a single factual correction you have found and verified. " + - "Call this tool for each incorrect claim you discover — do not wait " + - "until you have found all claims.", - strict: true as const, - parameters: { - type: "object" as const, - properties: { - text: { type: "string" as const, description: "The exact text of the incorrect claim." }, - context: { - type: "string" as const, - description: "Surrounding context that disambiguates the claim location.", - }, - summary: { - type: "string" as const, - description: "A one-sentence summary of what is incorrect and why.", - }, - reasoning: { - type: "string" as const, - description: "Detailed reasoning with evidence for why the claim is incorrect.", - }, - sources: { - type: "array" as const, - items: { - type: "object" as const, - properties: { - url: { type: "string" as const, description: "Source URL (absolute http/https)." }, - title: { type: "string" as const, description: "Title of the source." }, - snippet: { - type: "string" as const, - description: "Relevant snippet from the source.", - }, - }, - required: ["url", "title", "snippet"] as const, - additionalProperties: false as const, - }, - description: "At least one supporting source.", - }, - }, - required: ["text", "context", "summary", "reasoning", "sources"] as const, - additionalProperties: false as const, - }, -}; - -/** - * OpenAI function tool definition for retaining an existing claim during - * update investigations. The model calls this to carry forward a previously - * validated claim unchanged. - */ -// eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types -- return type is intentionally inferred; the `as const` assertions on each field provide narrow literal types -export function buildRetainCorrectionToolDefinition(oldClaimIds: [string, ...string[]]) { - return { - type: "function" as const, - name: RETAIN_CORRECTION_TOOL_NAME, - description: - "Retain an existing claim from the previous investigation that is " + - "still correct and relevant. Use this instead of re-submitting the " + - "same claim via submit_correction.", - strict: true as const, - parameters: { - type: "object" as const, - properties: { - id: { - type: "string" as const, - enum: oldClaimIds, - description: "The ID of the existing claim to retain.", - }, - }, - required: ["id"] as const, - additionalProperties: false as const, - }, - }; -} +import { RETAIN_CORRECTION_TOOL_NAME, SUBMIT_CORRECTION_TOOL_NAME } from "./openai-claim-tools.js"; export interface PendingFunctionToolCall { callId: string; @@ -109,57 +18,54 @@ export function buildFunctionCallOutput(callId: string, output: string): Functio return { type: "function_call_output", call_id: callId, output }; } -export function extractPendingFunctionToolCalls( - responseRecord: Record, -): PendingFunctionToolCall[] { - const outputItems = Array.isArray(responseRecord["output"]) ? responseRecord["output"] : []; - - const calls: PendingFunctionToolCall[] = []; - for (const outputItem of outputItems) { - if (!isRecord(outputItem) || outputItem["type"] !== "function_call") continue; - - const callId = readString(outputItem["call_id"]); - const name = readString(outputItem["name"]); - const argumentsJson = readString(outputItem["arguments"]); - if ( - callId === null || - callId.length === 0 || - name === null || - name.length === 0 || - argumentsJson === null - ) { - continue; - } - - calls.push({ callId, name, argumentsJson }); - } - - return calls; +/** Function calls the model is waiting on, in the order it emitted them. */ +export function extractFunctionToolCalls(response: Response): PendingFunctionToolCall[] { + return response.output.flatMap((item) => + item.type === "function_call" + ? [{ callId: item.call_id, name: item.name, argumentsJson: item.arguments }] + : [], + ); } -export function deduplicateFunctionToolCalls( - calls: PendingFunctionToolCall[], -): PendingFunctionToolCall[] { - const deduplicated: PendingFunctionToolCall[] = []; - const seen = new Set(); - for (const call of calls) { - if (seen.has(call.callId)) continue; - seen.add(call.callId); - deduplicated.push(call); - } - return deduplicated; +interface FunctionCallHandlers { + submitCorrection: (call: PendingFunctionToolCall) => FunctionCallOutput; + retainCorrection: (call: PendingFunctionToolCall) => FunctionCallOutput; + /** Any other function tool (research tools such as fetch_url). */ + research: (call: PendingFunctionToolCall) => Promise; } -/** Returns true if the tool call is a claim submission/retain tool handled by the investigator loop. */ -export function isClaimToolCall(call: PendingFunctionToolCall): boolean { - return call.name === SUBMIT_CORRECTION_TOOL_NAME || call.name === RETAIN_CORRECTION_TOOL_NAME; +/** + * Answers every call of a round. Claim tool calls are handled synchronously in + * emission order (which fixes claim submission order); research calls run + * concurrently. + */ +export async function dispatchFunctionToolCalls( + calls: readonly PendingFunctionToolCall[], + handlers: FunctionCallHandlers, +): Promise { + return Promise.all( + // The async callback runs synchronously up to its first await, so the + // claim handlers still run one after another in emission order. + calls.map(async (call) => { + switch (call.name) { + case SUBMIT_CORRECTION_TOOL_NAME: + return handlers.submitCorrection(call); + case RETAIN_CORRECTION_TOOL_NAME: + return handlers.retainCorrection(call); + default: + return handlers.research(call); + } + }), + ); } export async function executeFunctionToolCall( call: PendingFunctionToolCall, + signal: AbortSignal, ): Promise { + signal.throwIfAborted(); if (call.name === FETCH_URL_TOOL_NAME) { - const toolOutput = await executeFetchUrlTool(call.argumentsJson); + const toolOutput = await executeFetchUrlTool(call.argumentsJson, signal); return buildFunctionCallOutput(call.callId, JSON.stringify(toolOutput)); } diff --git a/src/typescript/api/src/lib/investigators/openai-tool-loop.ts b/src/typescript/api/src/lib/investigators/openai-tool-loop.ts index c65aeb6..c200700 100644 --- a/src/typescript/api/src/lib/investigators/openai-tool-loop.ts +++ b/src/typescript/api/src/lib/investigators/openai-tool-loop.ts @@ -1,147 +1,108 @@ import type OpenAI from "openai"; -import type { ResponseCreateParamsNonStreaming } from "openai/resources/responses/responses"; -import { isNonNullObject } from "@openerrata/shared"; -import type { InvestigatorResponseAudit } from "./interface.js"; -import { InvestigatorStructuredOutputError } from "./openai-errors.js"; +import type { Response, Tool } from "openai/resources/responses/responses"; +import type { InvestigatorRequestAudit } from "./interface.js"; +import type { AuditedRequestInput } from "./openai-input-builder.js"; import { - extractResponseAudit, - offsetResponseAuditIndices, - readString, -} from "./openai-response-audit.js"; + buildFactCheckRequestParams, + type InvestigationRequestConfig, +} from "./openai-request-config.js"; +import { auditRequest, auditResponse } from "./openai-response-audit.js"; import { - deduplicateFunctionToolCalls, - extractPendingFunctionToolCalls, - isClaimToolCall, - RETAIN_CORRECTION_TOOL_NAME, - SUBMIT_CORRECTION_TOOL_NAME, + extractFunctionToolCalls, type FunctionCallOutput, type PendingFunctionToolCall, } from "./openai-tool-dispatch.js"; -const isRecord = isNonNullObject; - -interface RequestReasoning { - effort: "low" | "medium" | "high"; - summary: "auto" | "concise" | "detailed"; -} - -type RequiredResponseInput = NonNullable; - -interface BaseResponseRequest { - model: NonNullable; - stream: false; - instructions: string; - tools: NonNullable; - reasoning: RequestReasoning; -} - -export class ToolLoopExecutionError extends Error { - readonly responseAudits: readonly InvestigatorResponseAudit[]; - - constructor( - message: string, - responseAudits: readonly InvestigatorResponseAudit[], - cause?: unknown, - ) { - super(message, ...(cause !== undefined ? [{ cause }] : [])); - this.name = "ToolLoopExecutionError"; - this.responseAudits = responseAudits; - } -} - -interface ToolLoopResult { - latestResponseRecord: Record | null; - responseAudits: InvestigatorResponseAudit[]; -} +/** + * How the stage-1 fact-check loop ended. `rounds` audits every request made, + * including a final one that failed without a response. + */ +type ToolLoopResult = + | { + /** The model stopped calling function tools. */ + kind: "completed"; + rounds: InvestigatorRequestAudit[]; + finalResponse: Response; + } + | { + /** The model still had function calls pending when no round was left to answer them. */ + kind: "round_limit"; + rounds: InvestigatorRequestAudit[]; + } + | { + /** A response ended with a status other than "completed". */ + kind: "response_not_completed"; + rounds: InvestigatorRequestAudit[]; + response: Response; + } + | { + /** A provider request or a tool call threw. */ + kind: "failed"; + rounds: InvestigatorRequestAudit[]; + error: unknown; + }; export async function runToolLoop(input: { client: OpenAI; - maxResponseToolRounds: number; - baseResponseRequest: BaseResponseRequest; - initialInput: RequiredResponseInput; - handleSubmittedClaims: ( - calls: PendingFunctionToolCall[], - ) => FunctionCallOutput[] | Promise; - handleRetainedClaims: ( - calls: PendingFunctionToolCall[], - ) => FunctionCallOutput[] | Promise; - handleResearchCalls: ( - calls: PendingFunctionToolCall[], - ) => FunctionCallOutput[] | Promise; + requestConfig: InvestigationRequestConfig; + /** At least 1. */ + maxRounds: number; + instructions: string; + tools: Tool[]; + initialInput: AuditedRequestInput; + signal: AbortSignal; + handleFunctionCalls: (calls: PendingFunctionToolCall[]) => Promise; }): Promise { - let outputIndexOffset = 0; + const rounds: InvestigatorRequestAudit[] = []; + let roundInput = input.initialInput; let previousResponseId: string | null = null; - let latestResponseRecord: Record | null = null; - let nextInput: RequiredResponseInput = input.initialInput; - const responseAudits: InvestigatorResponseAudit[] = []; - let round = 0; - while (round < input.maxResponseToolRounds) { - const responseRequest: ResponseCreateParamsNonStreaming = - round === 0 - ? { - ...input.baseResponseRequest, - input: nextInput, - } - : { - ...input.baseResponseRequest, - previous_response_id: previousResponseId, - input: nextInput, - }; + for (let round = 0; ; round += 1) { + const subject = { kind: "FACT_CHECK_ROUND", round } as const; + const params = buildFactCheckRequestParams(input.requestConfig, { + instructions: input.instructions, + tools: input.tools, + input: roundInput.request, + previousResponseId, + }); - let response: unknown; + let response: Response; try { - response = await input.client.responses.create(responseRequest); + response = await input.client.responses.create(params, { signal: input.signal }); } catch (error) { - throw new ToolLoopExecutionError( - "OpenAI Responses API request failed", - responseAudits, - error, - ); + rounds.push(auditRequest({ subject, params, auditInput: roundInput.audit, response: null })); + return { kind: "failed", rounds, error }; } - - const responseRecord = isRecord(response) ? response : {}; - latestResponseRecord = responseRecord; - previousResponseId = readString(responseRecord["id"]); - - const responseAudit = extractResponseAudit(responseRecord); - responseAudits.push(offsetResponseAuditIndices(responseAudit, outputIndexOffset)); - outputIndexOffset += responseAudit.outputItems.length; - - const pendingFunctionCalls = deduplicateFunctionToolCalls( - extractPendingFunctionToolCalls(responseRecord), + rounds.push( + auditRequest({ + subject, + params, + auditInput: roundInput.audit, + response: auditResponse(response, new Date()), + }), ); - if (pendingFunctionCalls.length === 0) { - break; - } - if (previousResponseId === null || previousResponseId.length === 0) { - throw new ToolLoopExecutionError( - "Tool calls were emitted without a response id", - responseAudits, - new InvestigatorStructuredOutputError("Tool calls were emitted without a response id"), - ); + if (response.status !== "completed") { + return { kind: "response_not_completed", rounds, response }; } - const submittedClaims = pendingFunctionCalls.filter( - (call) => call.name === SUBMIT_CORRECTION_TOOL_NAME, - ); - const retainedClaims = pendingFunctionCalls.filter( - (call) => call.name === RETAIN_CORRECTION_TOOL_NAME, - ); - const researchCalls = pendingFunctionCalls.filter((call) => !isClaimToolCall(call)); - - const outputs: FunctionCallOutput[] = []; - outputs.push(...(await input.handleSubmittedClaims(submittedClaims))); - outputs.push(...(await input.handleRetainedClaims(retainedClaims))); - outputs.push(...(await input.handleResearchCalls(researchCalls))); + const calls = extractFunctionToolCalls(response); + if (calls.length === 0) { + return { kind: "completed", rounds, finalResponse: response }; + } + // Answering these calls needs another round; don't run tools (or schedule + // claim validations) whose outputs could never be sent. + if (round + 1 >= input.maxRounds) { + return { kind: "round_limit", rounds }; + } - nextInput = outputs; - round += 1; + let outputs: FunctionCallOutput[]; + try { + outputs = await input.handleFunctionCalls(calls); + } catch (error) { + return { kind: "failed", rounds, error }; + } + roundInput = { request: outputs, audit: outputs.map((output) => ({ ...output })) }; + previousResponseId = response.id; } - - return { - latestResponseRecord, - responseAudits, - }; } diff --git a/src/typescript/api/src/lib/investigators/openai.ts b/src/typescript/api/src/lib/investigators/openai.ts index bbd349a..01c4682 100644 --- a/src/typescript/api/src/lib/investigators/openai.ts +++ b/src/typescript/api/src/lib/investigators/openai.ts @@ -1,456 +1,255 @@ import OpenAI from "openai"; import pLimit from "p-limit"; -import { - DEFAULT_INVESTIGATION_MODEL, - DEFAULT_INVESTIGATION_PROVIDER, - investigationResultSchema, - isNonNullObject, - type InvestigationResult, -} from "@openerrata/shared"; +import type { InvestigationClaimPayload } from "@openerrata/shared"; import { getEnv } from "$lib/config/env.js"; -import { fetchUrlToolDefinition } from "./fetch-url-tool.js"; +import { + InvestigatorExecutionError, + InvestigatorIncompleteResponseError, + InvestigatorStructuredOutputError, +} from "./errors.js"; import type { - InvestigationProgressCallbacks, + InvestigateOptions, Investigator, - InvestigatorAttemptAudit, InvestigatorInput, InvestigatorOutput, + InvestigatorRequestAudit, } from "./interface.js"; -import { InvestigatorStructuredOutputError } from "./openai-errors.js"; -import { - INVESTIGATION_SYSTEM_PROMPT, - INVESTIGATION_UPDATE_SYSTEM_PROMPT, - buildUserPrompt, -} from "./prompt.js"; -import { providerStructuredInvestigationClaimPayloadSchema } from "./openai-schemas.js"; -import { buildInitialInput, buildValidationImageContextNotes } from "./openai-input-builder.js"; -import { readIncompleteReason, mergeResponseAudits } from "./openai-response-audit.js"; import { - RETAIN_CORRECTION_TOOL_NAME, - SUBMIT_CORRECTION_TOOL_NAME, - buildFunctionCallOutput, - buildRetainCorrectionToolDefinition, - deduplicateFunctionToolCalls, - executeFunctionToolCall, - extractPendingFunctionToolCalls, - type FunctionCallOutput, - type PendingFunctionToolCall, - submitCorrectionToolDefinition, -} from "./openai-tool-dispatch.js"; + parseRetainCorrectionArguments, + parseSubmitCorrectionArguments, +} from "./openai-claim-tools.js"; +import { createClaimValidationScheduler } from "./openai-claim-validation-scheduler.js"; import { - InvestigatorIncompleteResponseError, MAX_PER_CLAIM_VALIDATION_CONCURRENCY, - type PerClaimValidationResult, validateClaim, + type ClaimValidationResult, } from "./openai-claim-validator.js"; -import { - createClaimValidationScheduler, - type ClaimValidationScheduler, -} from "./openai-claim-validation-scheduler.js"; +import { buildInitialInput, buildValidationImageContextNotes } from "./openai-input-builder.js"; import { createInvestigationRunState, getConfirmedClaims, } from "./openai-investigation-run-state.js"; -import { runToolLoop, ToolLoopExecutionError } from "./openai-tool-loop.js"; import { - buildFailedAttemptAudit, - buildFullAttemptResponseAudit, - buildSuccessfulAttemptAudit, - createStageOneAttemptAuditBase, - createStageTwoAttemptAuditBase, -} from "./openai-attempt-audit-builder.js"; - -const isRecord = isNonNullObject; - -const DEFAULT_REASONING_EFFORT = "medium"; -const DEFAULT_REASONING_SUMMARY = "detailed"; -export { InvestigatorStructuredOutputError } from "./openai-errors.js"; - -function getOpenAiModelId(): string { - return getEnv().OPENAI_MODEL_ID; -} + buildFactCheckTools, + INVESTIGATION_REQUEST_CONFIG, + type InvestigationRequestConfig, +} from "./openai-request-config.js"; +import { buildErrorAudit } from "./openai-response-audit.js"; +import { + buildFunctionCallOutput, + dispatchFunctionToolCalls, + executeFunctionToolCall, + type FunctionCallOutput, + type PendingFunctionToolCall, +} from "./openai-tool-dispatch.js"; +import { runToolLoop } from "./openai-tool-loop.js"; +import { + INVESTIGATION_SYSTEM_PROMPT, + INVESTIGATION_UPDATE_SYSTEM_PROMPT, + buildUserPrompt, +} from "./prompt.js"; -function getMaxResponseToolRounds(): number { - return getEnv().OPENAI_MAX_RESPONSE_TOOL_ROUNDS; +interface OpenAIInvestigatorConfig { + client: OpenAI; + requestConfig: InvestigationRequestConfig; + /** Upper bound on stage-1 fact-check rounds (provider requests); at least 1. */ + maxToolRounds: number; } -export class InvestigatorExecutionError extends Error { - readonly attemptAudit: InvestigatorAttemptAudit; - override readonly cause: unknown; +const ACKNOWLEDGED_OUTPUT = JSON.stringify({ acknowledged: true }); - constructor(message: string, attemptAudit: InvestigatorAttemptAudit, cause?: unknown) { - super(message); - this.name = "InvestigatorExecutionError"; - this.attemptAudit = attemptAudit; - this.cause = cause; +function nonEmptyClaimIds(input: InvestigatorInput): readonly [string, ...string[]] | null { + if (input.isUpdate !== true) { + return null; } + const [firstClaim, ...remainingClaims] = input.oldClaims; + return firstClaim === undefined + ? null + : [firstClaim.id, ...remainingClaims.map((claim) => claim.id)]; } -type StageOneClaim = InvestigationResult["claims"][number]; - +/** + * Two-stage OpenAI investigation (SPEC §2.4): a stage-1 fact-check tool loop + * in which the model submits candidate claims, and a stage-2 validation call + * per candidate, started as each claim is submitted. + */ export class OpenAIInvestigator implements Investigator { - readonly provider = DEFAULT_INVESTIGATION_PROVIDER; - readonly model = DEFAULT_INVESTIGATION_MODEL; + private readonly config: OpenAIInvestigatorConfig; - private client: OpenAI; - private readonly overrideModelId: string | undefined; - private readonly overrideMaxToolRounds: number | undefined; - - constructor( - apiKey: string, - overrides?: { client?: OpenAI; modelId?: string; maxToolRounds?: number }, - ) { - this.client = overrides?.client ?? new OpenAI({ apiKey }); - this.overrideModelId = overrides?.modelId; - this.overrideMaxToolRounds = overrides?.maxToolRounds; + constructor(config: OpenAIInvestigatorConfig) { + if (!Number.isInteger(config.maxToolRounds) || config.maxToolRounds < 1) { + throw new Error( + `maxToolRounds must be a positive integer (got ${config.maxToolRounds.toString()})`, + ); + } + this.config = config; } async investigate( input: InvestigatorInput, - callbacks?: InvestigationProgressCallbacks, + options: InvestigateOptions, ): Promise { - const openAiModelId = this.overrideModelId ?? getOpenAiModelId(); - const maxResponseToolRounds = this.overrideMaxToolRounds ?? getMaxResponseToolRounds(); - const systemPrompt = - input.isUpdate === true ? INVESTIGATION_UPDATE_SYSTEM_PROMPT : INVESTIGATION_SYSTEM_PROMPT; - const userPromptResult = buildUserPrompt({ - contentText: input.contentText, - ...(input.contentMarkdown !== undefined && { contentMarkdown: input.contentMarkdown }), - platform: input.platform, - url: input.url, - ...(input.authorName !== undefined && { authorName: input.authorName }), - ...(input.postPublishedAt !== undefined && { postPublishedAt: input.postPublishedAt }), - ...(input.hasVideo !== undefined && { hasVideo: input.hasVideo }), - ...(input.isUpdate - ? { - isUpdate: true as const, - oldClaims: input.oldClaims, - ...(input.contentDiff !== undefined && { contentDiff: input.contentDiff }), - } - : {}), - }); + const { client, requestConfig } = this.config; + const { signal } = options; + const startedAt = new Date(); + + const userPrompt = buildUserPrompt(input); const initialInput = buildInitialInput( - userPromptResult.prompt, - userPromptResult.contentString, - userPromptResult.contentOffset, + userPrompt.prompt, + userPrompt.contentString, + userPrompt.contentOffset, input.imageOccurrences, input.imagePlaceholders, ); const validationImageContextNotes = buildValidationImageContextNotes(input.imageOccurrences); - const client = this.client; - - // ── Build tool set ──────────────────────────────────────────────── - const nonEmptyOldClaimIds: [string, ...string[]] | null = (() => { - if (input.isUpdate !== true) { - return null; - } - const [firstClaim, ...remainingClaims] = input.oldClaims; - if (firstClaim === undefined) { - return null; - } - return [firstClaim.id, ...remainingClaims.map((claim) => claim.id)]; - })(); - - const requestedTools = [ - { type: "web_search_preview" as const }, - fetchUrlToolDefinition, - submitCorrectionToolDefinition, - ...(nonEmptyOldClaimIds !== null - ? [buildRetainCorrectionToolDefinition(nonEmptyOldClaimIds)] - : []), - ]; - - const requestReasoning = { - effort: DEFAULT_REASONING_EFFORT as "low" | "medium" | "high", - summary: DEFAULT_REASONING_SUMMARY as "auto" | "concise" | "detailed", - }; - - const baseResponseRequest = { - model: openAiModelId, - stream: false as const, - instructions: systemPrompt, - tools: requestedTools, - reasoning: requestReasoning, - }; - - const startedAt = new Date().toISOString(); - const stageOneAttemptAuditBase = createStageOneAttemptAuditBase({ - startedAt, - openAiModelId, - systemPrompt, - userPrompt: userPromptResult.prompt, - requestReasoning, - requestedTools, - }); + const retainableClaimIds = nonEmptyClaimIds(input); const validationLimiter = pLimit(MAX_PER_CLAIM_VALIDATION_CONCURRENCY); - const validationScheduler: ClaimValidationScheduler = createClaimValidationScheduler({ + const validations = createClaimValidationScheduler({ initialState: createInvestigationRunState( input.isUpdate === true ? { oldClaims: input.oldClaims } : {}, ), validationLimiter, runValidation: (claimIndex, claim) => - validateClaim( + validateClaim({ client, - openAiModelId, + requestConfig, claimIndex, claim, - input.contentText, - validationImageContextNotes, - requestReasoning, - ), - ...(callbacks === undefined ? {} : { callbacks }), + contentText: input.contentText, + imageContextNotes: validationImageContextNotes, + signal, + }), + ...(options.callbacks === undefined ? {} : { callbacks: options.callbacks }), }); - const handleSubmittedClaims = (calls: PendingFunctionToolCall[]): FunctionCallOutput[] => { - const outputs: FunctionCallOutput[] = []; - for (const call of calls) { - let claim: StageOneClaim; - try { - claim = providerStructuredInvestigationClaimPayloadSchema.parse( - JSON.parse(call.argumentsJson), - ); - } catch (error) { - console.warn( - `Malformed ${SUBMIT_CORRECTION_TOOL_NAME} tool call (call_id=${call.callId}):`, - error instanceof Error ? error.message : error, - ); - outputs.push( - buildFunctionCallOutput( - call.callId, - JSON.stringify({ error: "Invalid claim payload" }), - ), - ); - continue; - } - - validationScheduler.scheduleClaimValidation(claim); - outputs.push(buildFunctionCallOutput(call.callId, '{"acknowledged":true}')); + const submitCorrection = (call: PendingFunctionToolCall): FunctionCallOutput => { + const claim = parseSubmitCorrectionArguments(call.argumentsJson); + if (claim.kind === "invalid") { + return buildFunctionCallOutput( + call.callId, + JSON.stringify({ error: `Invalid claim, not recorded: ${claim.error}` }), + ); } - return outputs; + validations.scheduleClaimValidation(claim.value); + return buildFunctionCallOutput(call.callId, ACKNOWLEDGED_OUTPUT); }; - const handleRetainedClaims = (calls: PendingFunctionToolCall[]): FunctionCallOutput[] => { - const outputs: FunctionCallOutput[] = []; - for (const call of calls) { - let retainId: string; - try { - const raw: unknown = JSON.parse(call.argumentsJson); - if (!isRecord(raw) || typeof raw["id"] !== "string") { - outputs.push( - buildFunctionCallOutput( - call.callId, - JSON.stringify({ error: "Invalid retain arguments: missing id" }), - ), - ); - continue; - } - retainId = raw["id"]; - } catch (error) { - console.warn( - `Malformed ${RETAIN_CORRECTION_TOOL_NAME} tool call (call_id=${call.callId}):`, - error instanceof Error ? error.message : error, - ); - outputs.push( - buildFunctionCallOutput( - call.callId, - JSON.stringify({ error: "Invalid retain arguments" }), - ), - ); - continue; - } - - const retained = validationScheduler.retainClaimById(retainId); - if (retained.kind === "error") { - outputs.push( - buildFunctionCallOutput(call.callId, JSON.stringify({ error: retained.errorMessage })), - ); - continue; - } - - outputs.push(buildFunctionCallOutput(call.callId, '{"acknowledged":true}')); + const retainCorrection = (call: PendingFunctionToolCall): FunctionCallOutput => { + if (retainableClaimIds === null) { + return buildFunctionCallOutput( + call.callId, + JSON.stringify({ error: "There are no prior claims to retain" }), + ); } - return outputs; - }; - - let loopResult: Awaited>; - try { - loopResult = await runToolLoop({ - client, - maxResponseToolRounds, - baseResponseRequest, - initialInput, - handleSubmittedClaims, - handleRetainedClaims, - handleResearchCalls: (calls) => - Promise.all(calls.map((call) => executeFunctionToolCall(call))), - }); - } catch (error) { - await validationScheduler.settleAllValidations(); - - if (error instanceof ToolLoopExecutionError) { - const responseAuditSnapshot = [...error.responseAudits]; - const attemptAudit = buildFailedAttemptAudit({ - base: stageOneAttemptAuditBase, - response: - responseAuditSnapshot.length > 0 ? mergeResponseAudits(responseAuditSnapshot) : null, - error: error.cause ?? error, - }); - throw new InvestigatorExecutionError(error.message, attemptAudit, error.cause ?? error); + const claimId = parseRetainCorrectionArguments(call.argumentsJson, retainableClaimIds); + if (claimId.kind === "invalid") { + return buildFunctionCallOutput( + call.callId, + JSON.stringify({ error: `Invalid retain arguments: ${claimId.error}` }), + ); } - - throw error; - } - - const { latestResponseRecord, responseAudits } = loopResult; - - if (latestResponseRecord === null || responseAudits.length === 0) { - const cause = new InvestigatorStructuredOutputError("Model returned no response payload"); - throw new InvestigatorExecutionError( - cause.message, - buildFailedAttemptAudit({ - base: stageOneAttemptAuditBase, - response: null, - error: cause, - }), - cause, + const retained = validations.retainClaimById(claimId.value); + return buildFunctionCallOutput( + call.callId, + retained.kind === "error" + ? JSON.stringify({ error: retained.errorMessage }) + : ACKNOWLEDGED_OUTPUT, ); - } + }; - const unfinishedToolCalls = deduplicateFunctionToolCalls( - extractPendingFunctionToolCalls(latestResponseRecord), - ); - if (unfinishedToolCalls.length > 0) { - await validationScheduler.settleAllValidations(); - const cause = new InvestigatorStructuredOutputError( - `Model exceeded tool call round limit (${maxResponseToolRounds.toString()})`, - ); - throw new InvestigatorExecutionError( - cause.message, - buildFailedAttemptAudit({ - base: stageOneAttemptAuditBase, - response: mergeResponseAudits(responseAudits), - error: cause, + const loop = await runToolLoop({ + client, + requestConfig, + maxRounds: this.config.maxToolRounds, + instructions: + input.isUpdate === true ? INVESTIGATION_UPDATE_SYSTEM_PROMPT : INVESTIGATION_SYSTEM_PROMPT, + tools: buildFactCheckTools(retainableClaimIds), + initialInput, + signal, + handleFunctionCalls: (calls) => + dispatchFunctionToolCalls(calls, { + submitCorrection, + retainCorrection, + research: (call) => executeFunctionToolCall(call, signal), }), - cause, - ); - } + }); - const factCheckResponseAudit = mergeResponseAudits(responseAudits); - if (factCheckResponseAudit.responseStatus === null) { - console.warn( - `OpenAI response had null status (responseId=${factCheckResponseAudit.responseId ?? "unknown"}); treating as completed`, - ); - } - if ( - factCheckResponseAudit.responseStatus !== "completed" && - factCheckResponseAudit.responseStatus !== null - ) { - await validationScheduler.settleAllValidations(); - const incompleteReason = readIncompleteReason(latestResponseRecord); - const cause = new InvestigatorIncompleteResponseError({ - responseStatus: factCheckResponseAudit.responseStatus, - responseId: factCheckResponseAudit.responseId, - incompleteReason, - outputTextLength: factCheckResponseAudit.responseOutputText?.length ?? 0, - }); - throw new InvestigatorExecutionError( - "OpenAI response was incomplete", - buildFailedAttemptAudit({ - base: stageOneAttemptAuditBase, - response: factCheckResponseAudit, - error: cause, - }), + // Validations already scheduled run to completion on every path, so the + // attempt audit records each request that was made. + const validationResults = await validations.awaitAllValidations(); + const requests: InvestigatorRequestAudit[] = [ + ...loop.rounds, + ...validationResults.map((validation) => validation.request), + ]; + const fail = (message: string, cause: unknown): InvestigatorExecutionError => + new InvestigatorExecutionError( + message, + { + outcome: "FAILED", + startedAt, + completedAt: new Date(), + requests, + error: buildErrorAudit(cause), + }, cause, ); - } - - const validationResults = await validationScheduler.awaitAllValidations(); - const confirmedClaims = getConfirmedClaims(validationScheduler.getState()); - - const validationInputSummary = validationResults - .map( - (result) => - `Claim ${result.claimIndex.toString()}: ${result.approved ? "approved" : "rejected"}`, - ) - .join("\n"); - const stageTwoInputSummary = - validationImageContextNotes === undefined - ? validationInputSummary - : `${validationInputSummary}\n\nImage context notes:\n${validationImageContextNotes}`; - - const stageTwoAttemptAuditBase = createStageTwoAttemptAuditBase({ - stageOneBase: stageOneAttemptAuditBase, - userPrompt: userPromptResult.prompt, - validationInputSummary: stageTwoInputSummary, - }); - type FailedValidation = Extract; - type SuccessfulValidation = Extract; + switch (loop.kind) { + case "failed": + throw fail("OpenAI fact-check round failed", loop.error); + case "round_limit": + throw fail( + "Fact-check exceeded its tool round limit", + new InvestigatorStructuredOutputError( + `Model exceeded tool call round limit (${this.config.maxToolRounds.toString()})`, + ), + ); + case "response_not_completed": + throw fail( + "OpenAI fact-check response was incomplete", + new InvestigatorIncompleteResponseError({ + responseStatus: loop.response.status ?? null, + responseId: loop.response.id, + incompleteReason: loop.response.incomplete_details?.reason ?? null, + }), + ); + case "completed": + break; + } const failedValidations = validationResults.filter( - (result): result is FailedValidation => result.error !== null, - ); - const successfulValidations = validationResults.filter( - (result): result is SuccessfulValidation => result.error === null, + (validation): validation is Extract => + validation.kind === "failed", ); - const validationFailureResponseAudits = failedValidations.flatMap((result) => - result.responseAudit === null ? [] : [result.responseAudit], - ); - - const fullAttemptResponseAudit = buildFullAttemptResponseAudit({ - factCheckResponseAudit, - successfulValidationResponseAudits: successfulValidations.map( - (result) => result.responseAudit, - ), - failedValidationResponseAudits: validationFailureResponseAudits, - }); - - if (failedValidations.length > 0) { - const firstFailure = failedValidations[0]; - if (!firstFailure) { - throw new Error("Invariant violation: failed validations must include at least one item"); - } - - const failedClaimIndicesLabel = failedValidations - .map((failure) => failure.claimIndex.toString()) + const [firstFailedValidation] = failedValidations; + if (firstFailedValidation !== undefined) { + const failedClaimIndices = failedValidations + .map((validation) => validation.claimIndex.toString()) .join(", "); - - throw new InvestigatorExecutionError( - `Per-claim validation failed for claim indices: ${failedClaimIndicesLabel}`, - buildFailedAttemptAudit({ - base: stageTwoAttemptAuditBase, - response: fullAttemptResponseAudit, - error: firstFailure.error, - }), - firstFailure.error, - ); - } - - let result: InvestigationResult; - try { - result = investigationResultSchema.parse({ claims: confirmedClaims }); - } catch (error) { - throw new InvestigatorExecutionError( - "Final investigation result failed schema validation", - buildFailedAttemptAudit({ - base: stageTwoAttemptAuditBase, - response: fullAttemptResponseAudit, - error, - }), - error, + throw fail( + `Per-claim validation failed for claim indices: ${failedClaimIndices}`, + firstFailedValidation.error, ); } + // Submitted claims were validated against the shared claim payload schema + // on submission; retained claims are prior investigations' persisted claims. + const claims: InvestigationClaimPayload[] = getConfirmedClaims(validations.getState()); return { - result, - attemptAudit: buildSuccessfulAttemptAudit({ - base: stageTwoAttemptAuditBase, - response: fullAttemptResponseAudit, - }), - ...(fullAttemptResponseAudit.responseModelVersion != null && { - modelVersion: fullAttemptResponseAudit.responseModelVersion, - }), + result: { claims }, + attemptAudit: { outcome: "SUCCEEDED", startedAt, completedAt: new Date(), requests }, + model: requestConfig.model, + modelVersion: loop.finalResponse.model, }; } } + +/** The production investigator factory: gpt-6.1-sol with the deployment's tool-round budget. */ +export function createOpenAIInvestigator(apiKey: string): Investigator { + return new OpenAIInvestigator({ + client: new OpenAI({ apiKey }), + requestConfig: INVESTIGATION_REQUEST_CONFIG, + maxToolRounds: getEnv().OPENAI_MAX_RESPONSE_TOOL_ROUNDS, + }); +} diff --git a/src/typescript/api/src/lib/network/host-safety.ts b/src/typescript/api/src/lib/network/host-safety.ts index e962693..5add48b 100644 --- a/src/typescript/api/src/lib/network/host-safety.ts +++ b/src/typescript/api/src/lib/network/host-safety.ts @@ -1,41 +1,48 @@ -import { promises as dns } from "node:dns"; import ipaddr from "ipaddr.js"; -type IpFamily = 4 | 6; - -interface ResolvedAddress { - address: string; - family: IpFamily; +type IpAddress = ipaddr.IPv4 | ipaddr.IPv6; + +/** + * IPv6 ranges that ipaddr.js still labels "unicast" but that can reach + * non-public networks. RFC 8215 local-use NAT64 translates into IPv4 space the + * operator chooses, which may be internal. + */ +const NON_PUBLIC_UNICAST_IPV6_RANGES: [ipaddr.IPv6, number][] = [ + [ipaddr.IPv6.parse("64:ff9b:1::"), 48], +]; + +/** + * Whether an address is ordinary public unicast, i.e. safe for the server to + * connect to on behalf of untrusted input. Everything ipaddr.js classifies as + * anything other than "unicast" (private, loopback, link-local, multicast, + * CGNAT, reserved, IPv4-mapped, NAT64, 6to4, Teredo, ...) is rejected; the + * translation ranges are rejected because they can tunnel to internal IPv4. + */ +export function isPublicUnicastAddress(address: IpAddress): boolean { + if (address.range() !== "unicast") { + return false; + } + if (address instanceof ipaddr.IPv4) { + return true; + } + return !NON_PUBLIC_UNICAST_IPV6_RANGES.some(([network, prefixLength]) => + address.match(network, prefixLength), + ); } -const PRIVATE_IPV4_SUBNETS: Record = { - unspecified: [[ipaddr.IPv4.parse("0.0.0.0"), 8]], - private: [ - [ipaddr.IPv4.parse("10.0.0.0"), 8], - [ipaddr.IPv4.parse("172.16.0.0"), 12], - [ipaddr.IPv4.parse("192.168.0.0"), 16], - ], - carrierGradeNat: [[ipaddr.IPv4.parse("100.64.0.0"), 10]], - loopback: [[ipaddr.IPv4.parse("127.0.0.0"), 8]], - linkLocal: [[ipaddr.IPv4.parse("169.254.0.0"), 16]], - ietfProtocol: [[ipaddr.IPv4.parse("192.0.0.0"), 24]], - benchmarking: [[ipaddr.IPv4.parse("198.18.0.0"), 15]], - reserved: [[ipaddr.IPv4.parse("240.0.0.0"), 4]], -}; - -function normalizeIpLiteralCandidate(input: string): string { - const trimmed = input.trim().toLowerCase(); - const withoutBrackets = +/** + * Parse a URL hostname that is an IP literal (`203.0.113.5`, `[2001:db8::1]`, + * `fe80::1%eth0`). Returns null for DNS names. Only strict dotted-quad IPv4 is + * accepted; WHATWG URL parsing already canonicalizes shorthand IPv4 forms. + */ +export function parseIpLiteral(hostname: string): IpAddress | null { + const trimmed = hostname.trim().toLowerCase(); + const unbracketed = trimmed.startsWith("[") && trimmed.endsWith("]") ? trimmed.slice(1, -1) : trimmed; - const zoneSeparatorIndex = withoutBrackets.indexOf("%"); - if (zoneSeparatorIndex === -1) { - return withoutBrackets; - } - return withoutBrackets.slice(0, zoneSeparatorIndex); -} + const zoneSeparatorIndex = unbracketed.indexOf("%"); + const candidate = + zoneSeparatorIndex === -1 ? unbracketed : unbracketed.slice(0, zoneSeparatorIndex); -function parseIpAddress(input: string): ipaddr.IPv4 | ipaddr.IPv6 | null { - const candidate = normalizeIpLiteralCandidate(input); if (ipaddr.IPv4.isValidFourPartDecimal(candidate)) { return ipaddr.IPv4.parse(candidate); } @@ -45,113 +52,13 @@ function parseIpAddress(input: string): ipaddr.IPv4 | ipaddr.IPv6 | null { return null; } -function isIpv4Address(address: ipaddr.IPv4 | ipaddr.IPv6): address is ipaddr.IPv4 { - return address.kind() === "ipv4"; -} - -function isPrivateIPv4(address: ipaddr.IPv4): boolean { - return ipaddr.subnetMatch(address, PRIVATE_IPV4_SUBNETS, "public") !== "public"; -} - -function isPrivateIPv6(address: ipaddr.IPv6): boolean { - if (address.isIPv4MappedAddress()) { - return isPrivateIPv4(address.toIPv4Address()); - } - const range = address.range(); - return ( - range === "unspecified" || - range === "loopback" || - range === "uniqueLocal" || - range === "linkLocal" - ); -} - -export function isPrivateIpAddress(hostnameOrIp: string): boolean { - const parsedAddress = parseIpAddress(hostnameOrIp); - if (!parsedAddress) return false; - if (isIpv4Address(parsedAddress)) { - return isPrivateIPv4(parsedAddress); - } - return isPrivateIPv6(parsedAddress); -} - -function normalizeAddress(address: string): string { - return address.trim().toLowerCase(); -} - -async function resolveHostAddresses(hostname: string): Promise { - const normalizedHost = hostname.trim().toLowerCase(); - const parsedAddress = parseIpAddress(normalizedHost); - if (parsedAddress) { - return [ - { - address: parsedAddress.toNormalizedString(), - family: parsedAddress.kind() === "ipv4" ? 4 : 6, - }, - ]; - } - - const resolvedAddresses = await dns.lookup(normalizedHost, { - all: true, - verbatim: true, - }); - const deduped = new Map(); - for (const resolved of resolvedAddresses) { - if (resolved.family !== 4 && resolved.family !== 6) continue; - const normalizedAddress = normalizeAddress(resolved.address); - deduped.set(normalizedAddress, { - address: normalizedAddress, - family: resolved.family, - }); - } - - return Array.from(deduped.values()); -} - -function isLocallyScopedHostname(normalizedHost: string): boolean { +/** Hostnames that resolve on the local machine or link regardless of DNS. */ +export function isLocallyScopedHostname(hostname: string): boolean { + const normalizedHost = hostname.trim().toLowerCase().replace(/\.$/, ""); return ( + normalizedHost.length === 0 || normalizedHost === "localhost" || normalizedHost.endsWith(".localhost") || normalizedHost.endsWith(".local") ); } - -export async function resolvePublicHostAddresses(hostname: string): Promise { - const normalizedHost = hostname.trim().toLowerCase(); - if (normalizedHost.length === 0 || isLocallyScopedHostname(normalizedHost)) { - return null; - } - - if (isPrivateIpAddress(normalizedHost)) { - return null; - } - - const resolvedAddresses = await resolveHostAddresses(normalizedHost); - if (resolvedAddresses.length === 0) { - return null; - } - if (resolvedAddresses.some((resolved) => isPrivateIpAddress(resolved.address))) { - return null; - } - - return resolvedAddresses.map((resolved) => resolved.address); -} - -export function hasAddressIntersection(left: string[], right: string[]): boolean { - if (left.length === 0 || right.length === 0) return false; - const rightSet = new Set(right.map(normalizeAddress)); - return left.some((address) => rightSet.has(normalizeAddress(address))); -} - -export async function isBlockedHost(hostname: string): Promise { - const normalizedHost = hostname.trim().toLowerCase(); - - if (isLocallyScopedHostname(normalizedHost)) { - return true; - } - - if (isPrivateIpAddress(normalizedHost)) return true; - - const resolvedAddresses = await resolveHostAddresses(normalizedHost); - return resolvedAddresses.some((resolved) => isPrivateIpAddress(resolved.address)); -} diff --git a/src/typescript/api/src/lib/network/ip.ts b/src/typescript/api/src/lib/network/ip.ts index 891c77b..4f00509 100644 --- a/src/typescript/api/src/lib/network/ip.ts +++ b/src/typescript/api/src/lib/network/ip.ts @@ -1,136 +1,37 @@ -import { isIP } from "node:net"; - -type Ipv4Octets = [number, number, number, number]; - -function parseIpv4Octets(value: string): Ipv4Octets | null { - const parts = value.split("."); - if (parts.length !== 4) return null; - - const parsedOctets = parts.map((part) => { - if (!/^\d{1,3}$/.test(part)) return Number.NaN; - const parsed = Number.parseInt(part, 10); - return parsed >= 0 && parsed <= 255 ? parsed : Number.NaN; - }); - - if (parsedOctets.some((octet) => Number.isNaN(octet))) return null; - - const a = parsedOctets[0]; - const b = parsedOctets[1]; - const c = parsedOctets[2]; - const d = parsedOctets[3]; - if (a === undefined || b === undefined || c === undefined || d === undefined) return null; - return [a, b, c, d]; -} - -function ipv4Prefix(octets: number[]): string { - return octets.slice(0, 3).join("."); -} - -function isValidHextet(hextet: string): boolean { - return /^[0-9a-f]{1,4}$/i.test(hextet); -} - -function normalizeHextet(hextet: string): string { - return Number.parseInt(hextet, 16).toString(16); -} - -function expandIpv6(input: string): string[] | null { - let address = input.trim().toLowerCase(); - - const zoneIndex = address.indexOf("%"); - if (zoneIndex >= 0) { - address = address.slice(0, zoneIndex); - } - - if (address.includes(".")) { - const lastColon = address.lastIndexOf(":"); - if (lastColon < 0) return null; - - const ipv4Part = address.slice(lastColon + 1); - const octets = parseIpv4Octets(ipv4Part); - if (!octets) return null; - - const high = ((octets[0] << 8) | octets[1]).toString(16); - const low = ((octets[2] << 8) | octets[3]).toString(16); - address = `${address.slice(0, lastColon)}:${high}:${low}`; - } - - const parts = address.split("::"); - if (parts.length > 2) return null; - - const left = - parts[0] !== undefined && parts[0].length > 0 - ? parts[0].split(":").filter((part) => part.length > 0) - : []; - const right = - parts.length === 2 && parts[1] !== undefined && parts[1].length > 0 - ? parts[1].split(":").filter((part) => part.length > 0) - : []; - - if (!left.every(isValidHextet) || !right.every(isValidHextet)) { - return null; +import ipaddr from "ipaddr.js"; + +function parseClientAddress(clientAddress: string): ipaddr.IPv4 | ipaddr.IPv6 { + const trimmed = clientAddress.trim(); + const zoneSeparatorIndex = trimmed.indexOf("%"); + const withoutZone = zoneSeparatorIndex === -1 ? trimmed : trimmed.slice(0, zoneSeparatorIndex); + if (ipaddr.IPv4.isValidFourPartDecimal(withoutZone)) { + return ipaddr.IPv4.parse(withoutZone); } - - const missing = 8 - (left.length + right.length); - if (parts.length === 1 && missing !== 0) return null; - if (parts.length === 2 && missing < 1) return null; - - const expanded = [ - ...left, - ...(parts.length === 2 ? Array.from({ length: missing }, () => "0") : []), - ...right, - ].map(normalizeHextet); - - return expanded.length === 8 ? expanded : null; -} - -function mappedIpv4FromIpv6(expandedIpv6: string[]): Ipv4Octets | null { - const mappedMarker = expandedIpv6[5]; - const highHextet = expandedIpv6[6]; - const lowHextet = expandedIpv6[7]; - if ( - mappedMarker === undefined || - mappedMarker.length === 0 || - highHextet === undefined || - highHextet.length === 0 || - lowHextet === undefined || - lowHextet.length === 0 - ) { - return null; + if (ipaddr.IPv6.isValid(withoutZone)) { + const ipv6 = ipaddr.IPv6.parse(withoutZone); + return ipv6.isIPv4MappedAddress() ? ipv6.toIPv4Address() : ipv6; } - - const isMappedPrefix = - expandedIpv6.slice(0, 5).every((hextet) => hextet === "0") && mappedMarker === "ffff"; - if (!isMappedPrefix) return null; - - const high = Number.parseInt(highHextet, 16); - const low = Number.parseInt(lowHextet, 16); - return [high >> 8, high & 0xff, low >> 8, low & 0xff]; -} - -function ipv6Prefix(expandedIpv6: string[]): string { - return expandedIpv6.slice(0, 3).join(":"); -} - -export function deriveIpRangePrefix(clientIp: string): string { - const trimmed = clientIp.trim(); - if (trimmed.length === 0) return "unknown"; - - const ipv4 = parseIpv4Octets(trimmed); - if (ipv4) { - return ipv4Prefix(ipv4); + throw new Error(`Client address is not an IP address: ${JSON.stringify(clientAddress)}`); +} + +/** + * Derive the network range a client address belongs to for the per-day + * IP-range view-credit cap (SPEC §2.10): the /24 for IPv4 (first three + * octets) and the /48 for IPv6 (first three hextets). IPv4-mapped IPv6 + * addresses count as their IPv4 address. + * + * The client address comes from the socket peer or the trusted proxy header + * (ADDRESS_HEADER), so anything that is not an IP address means the proxy + * configuration is broken; that throws rather than bucketing such clients + * together. + */ +export function deriveIpRangePrefix(clientAddress: string): string { + const address = parseClientAddress(clientAddress); + if (address instanceof ipaddr.IPv4) { + return address.octets.slice(0, 3).join("."); } - - if (isIP(trimmed) === 6) { - const expanded = expandIpv6(trimmed); - if (!expanded) return `invalid:${trimmed.toLowerCase()}`; - - const mappedIpv4 = mappedIpv4FromIpv6(expanded); - if (mappedIpv4) { - return ipv4Prefix(mappedIpv4); - } - return ipv6Prefix(expanded); - } - - return `invalid:${trimmed.toLowerCase()}`; + return address.parts + .slice(0, 3) + .map((part) => part.toString(16)) + .join(":"); } diff --git a/src/typescript/api/src/lib/network/public-http-fetch.ts b/src/typescript/api/src/lib/network/public-http-fetch.ts new file mode 100644 index 0000000..7675465 --- /dev/null +++ b/src/typescript/api/src/lib/network/public-http-fetch.ts @@ -0,0 +1,186 @@ +/** + * HTTP(S) fetching of URLs chosen by untrusted input (post image URLs from + * anonymous clients, URLs the model asks `fetch_url` to read). + * + * The SSRF check lives on the connection itself: every connection goes through + * an undici Agent whose DNS lookup rejects the whole answer if any resolved + * address is not public unicast, and hands the socket only the addresses it + * validated. A hostname that rebinds between "check" and "connect" therefore + * has nothing to rebind into — there is no separate check. IP-literal hosts + * never reach DNS, so they are validated before dispatch. Redirects are + * followed manually so every hop goes through the same agent. + */ + +import { lookup as dnsLookup, type LookupAddress, type LookupOptions } from "node:dns"; +import type { LookupFunction } from "node:net"; +import ipaddr from "ipaddr.js"; +import { Agent, fetch, type Response } from "undici"; +import { isLocallyScopedHostname, isPublicUnicastAddress, parseIpLiteral } from "./host-safety.js"; +import { isRedirectStatus } from "./http-status.js"; + +const MAX_REDIRECT_HOPS = 5; + +export class BlockedDestinationError extends Error { + constructor(message: string) { + super(message); + this.name = "BlockedDestinationError"; + } +} + +class PublicHttpFetchError extends Error { + constructor(message: string) { + super(message); + this.name = "PublicHttpFetchError"; + } +} + +function nonPublicAddresses(addresses: LookupAddress[]): string[] { + return addresses + .filter((resolved) => !isPublicUnicastAddress(ipaddr.parse(resolved.address))) + .map((resolved) => resolved.address); +} + +/** + * DNS lookup for the public-internet agent: resolves like dns.lookup but fails + * with BlockedDestinationError unless every address is public unicast, and + * returns only addresses it checked. Exported for unit tests. + */ +export const publicOnlyLookup: LookupFunction = (hostname, options: LookupOptions, callback) => { + dnsLookup(hostname, { ...options, all: true, verbatim: true }, (error, addresses) => { + if (error !== null) { + callback(error, "", 0); + return; + } + const [first] = addresses; + if (first === undefined) { + callback(new BlockedDestinationError(`${hostname} did not resolve to any address`), "", 0); + return; + } + const blocked = nonPublicAddresses(addresses); + if (blocked.length > 0) { + callback( + new BlockedDestinationError( + `${hostname} resolves to non-public address(es): ${blocked.join(", ")}`, + ), + "", + 0, + ); + return; + } + if (options.all === true) { + callback(null, addresses); + return; + } + callback(null, first.address, first.family); + }); +}; + +const publicInternetAgent = new Agent({ connect: { lookup: publicOnlyLookup } }); + +function assertFetchableUrl(url: URL): void { + if (url.protocol !== "http:" && url.protocol !== "https:") { + throw new BlockedDestinationError(`Only HTTP(S) URLs are allowed (got ${url.protocol})`); + } + if (url.username.length > 0 || url.password.length > 0) { + throw new BlockedDestinationError("URLs with embedded credentials are not allowed"); + } + if (isLocallyScopedHostname(url.hostname)) { + throw new BlockedDestinationError(`Blocked local hostname ${url.hostname}`); + } + const literal = parseIpLiteral(url.hostname); + if (literal !== null && !isPublicUnicastAddress(literal)) { + throw new BlockedDestinationError(`Blocked non-public address ${url.hostname}`); + } +} + +interface PublicHttpResponse { + /** URL of the final (non-redirect) response. */ + finalUrl: URL; + response: Response; +} + +/** + * GET `url` from the public internet, following up to MAX_REDIRECT_HOPS + * redirects. Throws BlockedDestinationError when any hop targets a non-public + * destination and PublicHttpFetchError for malformed redirect chains; network + * errors and aborts propagate as thrown by undici. + */ +export async function fetchPublicHttp(input: { + url: URL; + headers: Record; + signal: AbortSignal; +}): Promise { + let currentUrl = input.url; + for (let redirectHop = 0; redirectHop <= MAX_REDIRECT_HOPS; redirectHop += 1) { + assertFetchableUrl(currentUrl); + const response = await fetch(currentUrl, { + method: "GET", + redirect: "manual", + headers: input.headers, + signal: input.signal, + dispatcher: publicInternetAgent, + }); + + if (!isRedirectStatus(response.status)) { + return { finalUrl: currentUrl, response }; + } + + await response.body?.cancel(); + const location = response.headers.get("location"); + if (location === null || location.length === 0) { + throw new PublicHttpFetchError("Redirect response missing Location header"); + } + currentUrl = new URL(location, currentUrl); + } + + throw new PublicHttpFetchError(`Too many redirects (more than ${MAX_REDIRECT_HOPS.toString()})`); +} + +/** + * Read at most `maxBytes` of a response body, cancelling the stream as soon as + * the limit is exceeded. `truncated` reports whether any bytes were dropped. + */ +export async function readBodyPrefix( + response: Response, + maxBytes: number, +): Promise<{ bytes: Uint8Array; truncated: boolean }> { + if (response.body === null) { + return { bytes: new Uint8Array(0), truncated: false }; + } + + const reader: ReadableStreamDefaultReader = response.body.getReader(); + const chunks: Uint8Array[] = []; + let totalBytes = 0; + let truncated = false; + try { + while (totalBytes < maxBytes) { + const { done, value } = await reader.read(); + if (done) break; + const remaining = maxBytes - totalBytes; + if (value.byteLength > remaining) { + chunks.push(value.subarray(0, remaining)); + totalBytes += remaining; + truncated = true; + break; + } + chunks.push(value); + totalBytes += value.byteLength; + } + if (!truncated && totalBytes >= maxBytes) { + truncated = !(await reader.read()).done; + } + } finally { + if (truncated) { + await reader.cancel("Response body exceeds byte limit"); + } + reader.releaseLock(); + } + + const bytes = new Uint8Array(totalBytes); + let offset = 0; + for (const chunk of chunks) { + bytes.set(chunk, offset); + offset += chunk.byteLength; + } + return { bytes, truncated }; +} diff --git a/src/typescript/api/src/lib/openai/errors.ts b/src/typescript/api/src/lib/openai/errors.ts index 7c69bad..82c3b21 100644 --- a/src/typescript/api/src/lib/openai/errors.ts +++ b/src/typescript/api/src/lib/openai/errors.ts @@ -1,70 +1,12 @@ -const NON_RETRYABLE_OPENAI_STATUS_CODES = new Set([400, 401, 403, 404, 422]); +import { APIError } from "openai"; -export type OpenAiKeyValidationStatusOutcome = - | { openaiApiKeyStatus: "missing" } - | { openaiApiKeyStatus: "valid" } - | { - openaiApiKeyStatus: "format_invalid"; - openaiApiKeyMessage: string; - } - | { - openaiApiKeyStatus: "authenticated_restricted"; - openaiApiKeyMessage: string; - } - | { - openaiApiKeyStatus: "invalid"; - openaiApiKeyMessage: string; - } - | { - openaiApiKeyStatus: "error"; - openaiApiKeyMessage: string; - }; - -const OPENAI_KEY_VALIDATION_RESULT_BY_STATUS: Partial< - Record -> = { - 401: { - openaiApiKeyStatus: "invalid", - openaiApiKeyMessage: "OpenAI rejected this API key.", - }, - 403: { - openaiApiKeyStatus: "authenticated_restricted", - openaiApiKeyMessage: - "OpenAI authenticated this key, but access is restricted for validation checks.", - }, - 429: { - openaiApiKeyStatus: "error", - openaiApiKeyMessage: "OpenAI rate-limited key validation. Retry in a moment.", - }, -}; - -export function readOpenAiStatusCode(error: unknown): number | null { - if (typeof error !== "object" || error === null) return null; - if (!("status" in error)) return null; - const status = error.status; - return typeof status === "number" ? status : null; +// A guard rather than a bare `instanceof`, which would type the generic +// error's fields as `any`. +function isOpenAiApiError(error: unknown): error is APIError { + return error instanceof APIError; } -export function classifyOpenAiKeyValidationStatus( - statusCode: number | null, -): OpenAiKeyValidationStatusOutcome | null { - if (statusCode === null) return null; - - const knownStatusResult = OPENAI_KEY_VALIDATION_RESULT_BY_STATUS[statusCode]; - if (knownStatusResult !== undefined) { - return knownStatusResult; - } - - if (statusCode >= 400 && statusCode < 500) { - return { - openaiApiKeyStatus: "error", - openaiApiKeyMessage: `OpenAI returned HTTP ${statusCode.toString()} while validating this key.`, - }; - } - - return null; -} - -export function isNonRetryableOpenAiStatusCode(statusCode: number | null): boolean { - return statusCode !== null && NON_RETRYABLE_OPENAI_STATUS_CODES.has(statusCode); +/** HTTP status of an OpenAI API error response; null for any other error (incl. connection errors). */ +export function readOpenAiStatusCode(error: unknown): number | null { + return isOpenAiApiError(error) ? (error.status ?? null) : null; } diff --git a/src/typescript/api/src/lib/services/attempt-audit.ts b/src/typescript/api/src/lib/services/attempt-audit.ts index e6a8af6..ba14dd6 100644 --- a/src/typescript/api/src/lib/services/attempt-audit.ts +++ b/src/typescript/api/src/lib/services/attempt-audit.ts @@ -1,12 +1,99 @@ import { getPrisma } from "$lib/db/client"; -import { - parseInvestigatorAttemptAudit, - type InvestigatorAttemptAudit, +import type { + InvestigatorAttemptAudit, + InvestigatorFailedAttemptAudit, + InvestigatorOutputItemAudit, + InvestigatorRequestAudit, + InvestigatorResponseAudit, } from "$lib/investigators/interface.js"; -import { toDate, toOptionalDate } from "$lib/date.js"; import type { Prisma } from "$lib/db/prisma-client"; import { consumeOpenAiKeySource } from "./user-key-source.js"; +function toOutputItemCreate( + item: InvestigatorOutputItemAudit, + outputIndex: number, +): Prisma.InvestigationAttemptOutputItemCreateWithoutResponseInput { + const base = { + outputIndex, + providerItemId: item.providerItemId, + itemType: item.itemType, + itemStatus: item.itemStatus, + }; + switch (item.content.kind) { + case "MESSAGE": + return { + ...base, + textParts: { + create: item.content.textParts.map((part, partIndex) => ({ + partIndex, + partType: part.partType, + text: part.text, + annotations: { + create: part.annotations.map((annotation, annotationIndex) => ({ + annotationIndex, + ...annotation, + })), + }, + })), + }, + }; + case "REASONING": + return { + ...base, + reasoningSummaries: { + create: item.content.summaries.map((text, summaryIndex) => ({ summaryIndex, text })), + }, + }; + case "TOOL_CALL": + return { ...base, toolCall: { create: { rawPayload: item.content.rawPayload } } }; + } +} + +function toResponseCreate( + response: InvestigatorResponseAudit, +): Prisma.InvestigationAttemptResponseCreateWithoutRequestInput { + return { + providerResponseId: response.providerResponseId, + status: response.status, + modelVersion: response.modelVersion, + receivedAt: response.receivedAt, + outputItems: { create: response.outputItems.map(toOutputItemCreate) }, + ...(response.usage === null ? {} : { usage: { create: response.usage } }), + }; +} + +function toRequestCreate( + request: InvestigatorRequestAudit, +): Prisma.InvestigationAttemptRequestCreateWithoutAttemptInput { + return { + kind: request.subject.kind, + factCheckRound: request.subject.kind === "FACT_CHECK_ROUND" ? request.subject.round : null, + claimIndex: request.subject.kind === "CLAIM_VALIDATION" ? request.subject.claimIndex : null, + model: request.model, + instructions: request.instructions, + input: request.input, + previousResponseId: request.previousResponseId, + reasoningEffort: request.reasoningEffort, + reasoningSummary: request.reasoningSummary, + include: request.include, + requestedTools: { + create: request.tools.map((tool, requestOrder) => ({ + requestOrder, + toolType: tool.toolType, + rawDefinition: tool.rawDefinition, + })), + }, + ...(request.response === null + ? {} + : { response: { create: toResponseCreate(request.response) } }), + }; +} + +/** + * Inserts an attempt's audit (SPEC §2.12). Insert-only: each attemptNumber is + * claimed once per investigation and its audit is written once, at the + * attempt's terminal transition. + */ export async function persistAttemptAudit( tx: Prisma.TransactionClient, input: { @@ -15,194 +102,19 @@ export async function persistAttemptAudit( attemptAudit: InvestigatorAttemptAudit; }, ): Promise { - const attemptAudit = parseInvestigatorAttemptAudit(input.attemptAudit); - // outcome is derived from the audit's discriminated union — error !== null - // means FAILED. No separate parameter needed, no inconsistent state possible. - const outcome = attemptAudit.error !== null ? "FAILED" : "SUCCEEDED"; - - const attempt = await tx.investigationAttempt.upsert({ - where: { - investigationId_attemptNumber: { - investigationId: input.investigationId, - attemptNumber: input.attemptNumber, - }, - }, - create: { + const { attemptAudit } = input; + await tx.investigationAttempt.create({ + data: { investigationId: input.investigationId, attemptNumber: input.attemptNumber, - outcome, - requestModel: attemptAudit.requestModel, - requestInstructions: attemptAudit.requestInstructions, - requestInput: attemptAudit.requestInput, - requestReasoningEffort: attemptAudit.requestReasoningEffort, - requestReasoningSummary: attemptAudit.requestReasoningSummary, - responseId: attemptAudit.response?.responseId ?? null, - responseStatus: attemptAudit.response?.responseStatus ?? null, - responseModelVersion: attemptAudit.response?.responseModelVersion ?? null, - responseOutputText: attemptAudit.response?.responseOutputText ?? null, - startedAt: toDate(attemptAudit.startedAt), - completedAt: toOptionalDate(attemptAudit.completedAt, { strict: true }), + outcome: attemptAudit.outcome, + startedAt: attemptAudit.startedAt, + completedAt: attemptAudit.completedAt, + requests: { create: attemptAudit.requests.map(toRequestCreate) }, + ...(attemptAudit.outcome === "FAILED" ? { error: { create: attemptAudit.error } } : {}), }, - update: { - outcome, - requestModel: attemptAudit.requestModel, - requestInstructions: attemptAudit.requestInstructions, - requestInput: attemptAudit.requestInput, - requestReasoningEffort: attemptAudit.requestReasoningEffort, - requestReasoningSummary: attemptAudit.requestReasoningSummary, - responseId: attemptAudit.response?.responseId ?? null, - responseStatus: attemptAudit.response?.responseStatus ?? null, - responseModelVersion: attemptAudit.response?.responseModelVersion ?? null, - responseOutputText: attemptAudit.response?.responseOutputText ?? null, - startedAt: toDate(attemptAudit.startedAt), - completedAt: toOptionalDate(attemptAudit.completedAt, { strict: true }), - }, - }); - - await tx.investigationAttemptRequestedTool.deleteMany({ - where: { attemptId: attempt.id }, - }); - await tx.investigationAttemptToolCall.deleteMany({ - where: { attemptId: attempt.id }, - }); - await tx.investigationAttemptOutputItem.deleteMany({ - where: { attemptId: attempt.id }, - }); - await tx.investigationAttemptUsage.deleteMany({ - where: { attemptId: attempt.id }, + select: { id: true }, }); - await tx.investigationAttemptError.deleteMany({ - where: { attemptId: attempt.id }, - }); - - for (const requestedTool of attemptAudit.requestedTools) { - await tx.investigationAttemptRequestedTool.create({ - data: { - attemptId: attempt.id, - requestOrder: requestedTool.requestOrder, - toolType: requestedTool.toolType, - rawDefinition: requestedTool.rawDefinition, - }, - }); - } - - const outputItemIdByIndex = new Map(); - for (const outputItem of attemptAudit.response?.outputItems ?? []) { - const createdOutputItem = await tx.investigationAttemptOutputItem.create({ - data: { - attemptId: attempt.id, - outputIndex: outputItem.outputIndex, - providerItemId: outputItem.providerItemId, - itemType: outputItem.itemType, - itemStatus: outputItem.itemStatus, - }, - }); - outputItemIdByIndex.set(outputItem.outputIndex, createdOutputItem.id); - } - - const textPartIdByKey = new Map(); - for (const textPart of attemptAudit.response?.outputTextParts ?? []) { - const outputItemId = outputItemIdByIndex.get(textPart.outputIndex); - if (outputItemId === undefined || outputItemId.length === 0) { - throw new Error(`Missing output item for text part outputIndex=${textPart.outputIndex}`); - } - - const createdTextPart = await tx.investigationAttemptOutputTextPart.create({ - data: { - outputItemId, - partIndex: textPart.partIndex, - partType: textPart.partType, - text: textPart.text, - }, - }); - - textPartIdByKey.set(`${textPart.outputIndex}:${textPart.partIndex}`, createdTextPart.id); - } - - for (const annotation of attemptAudit.response?.outputTextAnnotations ?? []) { - const textPartId = textPartIdByKey.get(`${annotation.outputIndex}:${annotation.partIndex}`); - if (textPartId === undefined || textPartId.length === 0) { - throw new Error( - `Missing text part for annotation outputIndex=${annotation.outputIndex} partIndex=${annotation.partIndex}`, - ); - } - - await tx.investigationAttemptOutputTextAnnotation.create({ - data: { - textPartId, - annotationIndex: annotation.annotationIndex, - annotationType: annotation.annotationType, - startIndex: annotation.characterPosition?.start ?? null, - endIndex: annotation.characterPosition?.end ?? null, - url: annotation.url, - title: annotation.title, - fileId: annotation.fileId, - }, - }); - } - - for (const summary of attemptAudit.response?.reasoningSummaries ?? []) { - const outputItemId = outputItemIdByIndex.get(summary.outputIndex); - if (outputItemId === undefined || outputItemId.length === 0) { - throw new Error( - `Missing output item for reasoning summary outputIndex=${summary.outputIndex}`, - ); - } - - await tx.investigationAttemptReasoningSummary.create({ - data: { - outputItemId, - summaryIndex: summary.summaryIndex, - text: summary.text, - }, - }); - } - - for (const toolCall of attemptAudit.response?.toolCalls ?? []) { - const outputItemId = outputItemIdByIndex.get(toolCall.outputIndex); - if (outputItemId === undefined || outputItemId.length === 0) { - throw new Error(`Missing output item for tool call outputIndex=${toolCall.outputIndex}`); - } - - await tx.investigationAttemptToolCall.create({ - data: { - attemptId: attempt.id, - outputItemId, - outputIndex: toolCall.outputIndex, - providerToolCallId: toolCall.providerToolCallId, - toolType: toolCall.toolType, - status: toolCall.status, - rawPayload: toolCall.rawPayload, - capturedAt: toDate(toolCall.capturedAt), - providerStartedAt: toOptionalDate(toolCall.providerStartedAt, { strict: true }), - providerCompletedAt: toOptionalDate(toolCall.providerCompletedAt, { strict: true }), - }, - }); - } - - if (attemptAudit.response?.usage) { - await tx.investigationAttemptUsage.create({ - data: { - attemptId: attempt.id, - inputTokens: attemptAudit.response.usage.inputTokens, - outputTokens: attemptAudit.response.usage.outputTokens, - totalTokens: attemptAudit.response.usage.totalTokens, - cachedInputTokens: attemptAudit.response.usage.cachedInputTokens, - reasoningOutputTokens: attemptAudit.response.usage.reasoningOutputTokens, - }, - }); - } - - if (attemptAudit.error) { - await tx.investigationAttemptError.create({ - data: { - attemptId: attempt.id, - errorName: attemptAudit.error.errorName, - errorMessage: attemptAudit.error.errorMessage, - statusCode: attemptAudit.error.statusCode, - }, - }); - } } /** @@ -216,7 +128,7 @@ export async function markInvestigationFailedInTx( investigationId: string; workerIdentity: string; attemptNumber: number; - attemptAudit: InvestigatorAttemptAudit | null; + attemptAudit: InvestigatorFailedAttemptAudit | null; }, ): Promise { // Guard: delete the lease row matching our workerIdentity. If it doesn't @@ -260,7 +172,7 @@ export async function persistFailedAttemptAndMarkInvestigationFailed(input: { investigationId: string; workerIdentity: string; attemptNumber: number; - attemptAudit: InvestigatorAttemptAudit | null; + attemptAudit: InvestigatorFailedAttemptAudit | null; }): Promise { return getPrisma().$transaction((tx) => markInvestigationFailedInTx(tx, input)); } @@ -276,7 +188,7 @@ export async function releaseLeaseToRetryInTx( investigationId: string; workerIdentity: string; attemptNumber: number; - attemptAudit: InvestigatorAttemptAudit | null; + attemptAudit: InvestigatorFailedAttemptAudit | null; retryAfter: Date; }, ): Promise { @@ -327,7 +239,7 @@ export async function persistFailedAttemptAndReleaseLease(input: { investigationId: string; workerIdentity: string; attemptNumber: number; - attemptAudit: InvestigatorAttemptAudit | null; + attemptAudit: InvestigatorFailedAttemptAudit | null; retryAfter: Date; }): Promise { return getPrisma().$transaction((tx) => releaseLeaseToRetryInTx(tx, input)); diff --git a/src/typescript/api/src/lib/services/blob-storage.ts b/src/typescript/api/src/lib/services/blob-storage.ts index 62a1cb3..e5349e3 100644 --- a/src/typescript/api/src/lib/services/blob-storage.ts +++ b/src/typescript/api/src/lib/services/blob-storage.ts @@ -7,7 +7,6 @@ interface BlobStorageConfigBase { bucket: string; accessKeyId: string; secretAccessKey: string; - publicUrlPrefix: string; } type AwsBlobStorageConfig = BlobStorageConfigBase & { @@ -24,7 +23,6 @@ type BlobStorageConfig = AwsBlobStorageConfig | S3CompatibleBlobStorageConfig; class BlobStorageService { private readonly client: S3Client; private readonly bucket: string; - private readonly publicUrlPrefix: string; constructor(config: BlobStorageConfig) { this.client = @@ -46,7 +44,6 @@ class BlobStorageService { }, }); this.bucket = config.bucket; - this.publicUrlPrefix = config.publicUrlPrefix.replace(/\/+$/, ""); } async uploadImage(bytes: Uint8Array, contentHash: string, mimeType: string): Promise { @@ -61,10 +58,6 @@ class BlobStorageService { ); return storageKey; } - - getPublicUrl(storageKey: string): string { - return `${this.publicUrlPrefix}/${storageKey}`; - } } let blobStorageService: BlobStorageService | undefined; @@ -79,7 +72,6 @@ function readBlobStorageConfig(): BlobStorageConfig { bucket: env.BLOB_STORAGE_BUCKET, accessKeyId: env.BLOB_STORAGE_ACCESS_KEY_ID, secretAccessKey: env.BLOB_STORAGE_SECRET_ACCESS_KEY, - publicUrlPrefix: env.BLOB_STORAGE_PUBLIC_URL_PREFIX, }; } @@ -90,7 +82,6 @@ function readBlobStorageConfig(): BlobStorageConfig { bucket: env.BLOB_STORAGE_BUCKET, accessKeyId: env.BLOB_STORAGE_ACCESS_KEY_ID, secretAccessKey: env.BLOB_STORAGE_SECRET_ACCESS_KEY, - publicUrlPrefix: env.BLOB_STORAGE_PUBLIC_URL_PREFIX, }; } diff --git a/src/typescript/api/src/lib/services/canonical-resolution.ts b/src/typescript/api/src/lib/services/canonical-resolution.ts index e1eda74..afd3cc0 100644 --- a/src/typescript/api/src/lib/services/canonical-resolution.ts +++ b/src/typescript/api/src/lib/services/canonical-resolution.ts @@ -15,7 +15,7 @@ export type CanonicalContentVersion = provenance: "SERVER_VERIFIED"; /** HTML fetched from the canonical source API (Parse API, LessWrong GraphQL). */ sourceHtml: string; - canonicalIdentity: CanonicalIdentity | null; + canonicalIdentity: CanonicalIdentity; }) | (ObservedContentVersion & { provenance: "CLIENT_FALLBACK"; diff --git a/src/typescript/api/src/lib/services/content-fetcher.ts b/src/typescript/api/src/lib/services/content-fetcher.ts index 4660a71..5cda2de 100644 --- a/src/typescript/api/src/lib/services/content-fetcher.ts +++ b/src/typescript/api/src/lib/services/content-fetcher.ts @@ -1,12 +1,14 @@ import { - CONTENT_BLOCK_SEPARATOR_TAGS, NON_CONTENT_TAGS, + WORD_SEPARATOR_TAGS, hashContent, isNonNullObject, normalizeContent, WIKIPEDIA_LANGUAGE_CODE_REGEX, } from "@openerrata/shared"; +import { setTimeout as sleep } from "node:timers/promises"; import { parseFragment, type DefaultTreeAdapterMap } from "parse5"; +import { z } from "zod"; import { createWikipediaNodeFilter, hasChildren, @@ -21,19 +23,33 @@ type ServerFetchResult = contentText: string; contentHash: string; sourceHtml: string; - canonicalIdentity: CanonicalIdentity | null; + canonicalIdentity: CanonicalIdentity; } | { success: false; failureReason: string; }; -export interface CanonicalIdentity { - platform: "WIKIPEDIA"; - language: string; - pageId: string; - revisionId: string; -} +/** + * Post identity as reported by the platform itself. Identity-bound fields + * (post URL, author, Wikipedia page/revision) come from here whenever the + * server fetch succeeds, never from the client (SPEC §2.9). + */ +export type CanonicalIdentity = + | { + platform: "LESSWRONG"; + url: string; + title: string; + /** Null when LessWrong reports no user for the post (e.g. deleted account). */ + author: { slug: string; displayName: string } | null; + } + | { + platform: "WIKIPEDIA"; + url: string; + language: string; + pageId: string; + revisionId: string; + }; export type CanonicalContentFetchResult = | { @@ -41,7 +57,7 @@ export type CanonicalContentFetchResult = contentText: string; contentHash: string; sourceHtml: string; - canonicalIdentity: CanonicalIdentity | null; + canonicalIdentity: CanonicalIdentity; } | { provenance: "CLIENT_FALLBACK"; @@ -83,6 +99,8 @@ export type CanonicalFetchInput = } | WikipediaCanonicalFetchInput; +const LESSWRONG_GRAPHQL_URL = "https://www.lesswrong.com/graphql"; + function describeFetchError(error: unknown): string { return error instanceof Error ? error.message : String(error); } @@ -97,53 +115,71 @@ function isTransientHttpStatus(status: number): boolean { const TRANSIENT_RETRY_DELAYS_MS = [200, 400, 800] as const; +/** + * Wall-clock budget for one canonical fetch, retries included. The fetch runs + * synchronously inside registerObservedVersion, so a slow or hanging platform + * must degrade to CLIENT_FALLBACK quickly rather than hold the request open. + */ +const CANONICAL_FETCH_DEADLINE_MS = 10_000; + +/** Largest canonical response body we will read (large Wikipedia articles are a few MB). */ +const MAX_CANONICAL_RESPONSE_BYTES = 10 * 1024 * 1024; + /** * Fetch wrapper that retries on transient failures (network errors, HTTP 429, - * HTTP 5xx) with exponential backoff. Non-transient errors (4xx except 429, - * parse failures) propagate immediately. + * HTTP 5xx) with exponential backoff, all within one deadline signal. + * Non-transient errors (4xx except 429) are returned immediately. * - * Returns the successful Response, or throws the last error / returns the - * last non-ok Response if all attempts fail. + * Returns the first non-transient Response or the last transient one once + * retries are exhausted; throws the last network error, or the abort reason + * once the deadline passes. */ async function fetchWithTransientRetry( - input: string | URL | Request, - init?: RequestInit, + input: string | URL, + init: RequestInit & { signal: AbortSignal }, ): Promise { - let lastError: unknown; - for (let attempt = 0; attempt <= TRANSIENT_RETRY_DELAYS_MS.length; attempt += 1) { + for (let attempt = 0; ; attempt += 1) { + const retryDelayMs = TRANSIENT_RETRY_DELAYS_MS[attempt]; try { const response = await fetch(input, init); - if (response.ok || !isTransientHttpStatus(response.status)) { + if (response.ok || !isTransientHttpStatus(response.status) || retryDelayMs === undefined) { return response; } - // Transient HTTP error — retry if attempts remain. - lastError = new Error(`HTTP ${response.status.toString()}`); - if (attempt < TRANSIENT_RETRY_DELAYS_MS.length) { - const delayMs = TRANSIENT_RETRY_DELAYS_MS[attempt]; - if (delayMs !== undefined) { - await new Promise((resolve) => { - setTimeout(resolve, delayMs); - }); - } - continue; - } - return response; + await response.body?.cancel(); } catch (error) { - // Network error — retry if attempts remain. - lastError = error; - if (attempt < TRANSIENT_RETRY_DELAYS_MS.length) { - const delayMs = TRANSIENT_RETRY_DELAYS_MS[attempt]; - if (delayMs !== undefined) { - await new Promise((resolve) => { - setTimeout(resolve, delayMs); - }); - } - continue; + if (init.signal.aborted || retryDelayMs === undefined) { + throw error; } - throw error; } + await sleep(retryDelayMs, undefined, { signal: init.signal }); + } +} + +/** Read and JSON-parse a response body, refusing bodies over MAX_CANONICAL_RESPONSE_BYTES. */ +async function readJsonWithinLimit(response: Response): Promise { + const contentLength = Number.parseInt(response.headers.get("content-length") ?? "", 10); + if (Number.isFinite(contentLength) && contentLength > MAX_CANONICAL_RESPONSE_BYTES) { + await response.body?.cancel(); + throw new Error(`response is ${contentLength.toString()} bytes, over the size limit`); + } + if (response.body === null) { + throw new Error("response has no body"); + } + + const reader = response.body.getReader(); + const chunks: Uint8Array[] = []; + let totalBytes = 0; + for (;;) { + const { done, value } = await reader.read(); + if (done) break; + totalBytes += value.byteLength; + if (totalBytes > MAX_CANONICAL_RESPONSE_BYTES) { + await reader.cancel("Canonical response exceeds size limit"); + throw new Error("response body exceeds the size limit"); + } + chunks.push(value); } - throw lastError; + return JSON.parse(Buffer.concat(chunks).toString("utf8")); } function parseNonNegativeIntegerId(value: unknown): string | null { @@ -157,36 +193,32 @@ function parseNonNegativeIntegerId(value: unknown): string | null { } /** - * Extract the full HTML body from a LessWrong GraphQL response. + * The parts of a LessWrong GraphQL `post` response we rely on. * - * We use the `html` field rather than `plaintextMainText` because the latter - * is truncated to 2000 characters by LessWrong's API, which would cause a + * We use `contents.html` rather than `plaintextMainText` because the latter is + * truncated to 2000 characters by LessWrong's API, which would cause a * canonicalization mismatch for any post longer than that. */ -function extractLesswrongHtml(value: unknown): string | null { - if (!isNonNullObject(value)) return null; - - const data = value["data"]; - if (!isNonNullObject(data)) return null; - - const post = data["post"]; - if (!isNonNullObject(post)) return null; - - const result = post["result"]; - if (!isNonNullObject(result)) return null; - - const contents = result["contents"]; - if (!isNonNullObject(contents)) return null; - - const html = contents["html"]; - return typeof html === "string" ? html : null; -} +const lesswrongPostResponseSchema = z.object({ + data: z.object({ + post: z.object({ + result: z.object({ + _id: z.string().min(1), + slug: z.string().min(1), + title: z.string().min(1), + contents: z.object({ html: z.string().min(1) }), + user: z.object({ slug: z.string().min(1), displayName: z.string().min(1) }).nullable(), + }), + }), + }), +}); /** * Shared parse5 HTML-to-text traversal used by all platform extractors. * * Performs a stack-based DFS over the parse5 fragment tree, collecting text - * node values and injecting word-boundary separators at block element edges. + * node values and injecting word-boundary separators at the edges of + * `WORD_SEPARATOR_TAGS` elements (blocks and line breaks). * * Built-in behavior (unconditional): * - `NON_CONTENT_TAGS` (script, style, noscript) are always excluded. @@ -197,7 +229,7 @@ function extractLesswrongHtml(value: unknown): string | null { */ function parse5HtmlToTextContent(html: string, nodeFilter?: Parse5NodeFilter): string { const fragment = parseFragment(html); - const stack: { node: DefaultTreeAdapterMap["node"]; phase: "enter" | "exit" }[] = []; + const stack: { node: DefaultTreeAdapterMap["childNode"]; phase: "enter" | "exit" }[] = []; for (let index = fragment.childNodes.length - 1; index >= 0; index -= 1) { const child = fragment.childNodes[index]; if (child !== undefined) { @@ -213,7 +245,7 @@ function parse5HtmlToTextContent(html: string, nodeFilter?: Parse5NodeFilter): s const { node, phase } = current; if (phase === "exit") { - if (isElementNode(node) && CONTENT_BLOCK_SEPARATOR_TAGS.has(node.tagName.toLowerCase())) { + if (isElementNode(node) && WORD_SEPARATOR_TAGS.has(node.tagName.toLowerCase())) { chunks.push(" "); } continue; @@ -238,7 +270,7 @@ function parse5HtmlToTextContent(html: string, nodeFilter?: Parse5NodeFilter): s continue; } - if (isElementNode(node) && CONTENT_BLOCK_SEPARATOR_TAGS.has(node.tagName.toLowerCase())) { + if (isElementNode(node) && WORD_SEPARATOR_TAGS.has(node.tagName.toLowerCase())) { chunks.push(" "); } @@ -304,9 +336,10 @@ async function fetchLesswrongContent( input: Extract, ): Promise { const postId = input.externalId; - let response: Response; + const deadline = AbortSignal.timeout(CANONICAL_FETCH_DEADLINE_MS); + let data: unknown; try { - response = await fetchWithTransientRetry("https://www.lesswrong.com/graphql", { + const response = await fetchWithTransientRetry(LESSWRONG_GRAPHQL_URL, { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ @@ -314,6 +347,7 @@ async function fetchLesswrongContent( post(input: { selector: { _id: $id } }) { result { _id + slug title contents { html @@ -327,7 +361,13 @@ async function fetchLesswrongContent( }`, variables: { id: postId }, }), + signal: deadline, }); + if (!response.ok) { + await response.body?.cancel(); + return { success: false, failureReason: `LW API returned ${response.status.toString()}` }; + } + data = await readJsonWithinLimit(response); } catch (error) { return { success: false, @@ -335,30 +375,51 @@ async function fetchLesswrongContent( }; } - if (!response.ok) { - return { success: false, failureReason: `LW API returned ${response.status}` }; - } - - let data: unknown; - try { - data = await response.json(); - } catch (error) { + const parsed = lesswrongPostResponseSchema.safeParse(data); + if (!parsed.success) { return { success: false, - failureReason: `LW API returned invalid JSON: ${describeFetchError(error)}`, + failureReason: "Could not extract post content and identity from LW API response", }; } - const html = extractLesswrongHtml(data); - if (html === null || html.length === 0) { + const post = parsed.data.data.post.result; + if (post._id !== postId) { return { success: false, - failureReason: "Could not extract HTML from LW API response", + failureReason: `LW API returned post ${post._id} for requested post ${postId}`, }; } - const contentText = lesswrongHtmlToNormalizedText(html); + const contentText = lesswrongHtmlToNormalizedText(post.contents.html); const contentHash = await hashContent(contentText); - return { success: true, contentText, contentHash, sourceHtml: html, canonicalIdentity: null }; + return { + success: true, + contentText, + contentHash, + sourceHtml: post.contents.html, + canonicalIdentity: { + platform: "LESSWRONG", + url: lesswrongPostUrl(post._id, post.slug), + title: post.title, + author: post.user, + }, + }; +} + +function lesswrongPostUrl(postId: string, slug: string): string { + return `https://www.lesswrong.com/posts/${encodeURIComponent(postId)}/${encodeURIComponent(slug)}`; +} + +/** + * Article URL for a Wikipedia title as returned by the parse API (spaces, not + * underscores). Slashes and colons stay literal so subpages and namespaces + * read naturally; everything else is percent-encoded. + */ +function wikipediaArticleUrl(language: string, title: string): string { + const encodedTitle = encodeURIComponent(title.replace(/ /g, "_")) + .replace(/%2F/g, "/") + .replace(/%3A/g, ":"); + return `https://${language}.wikipedia.org/wiki/${encodedTitle}`; } function wikipediaHtmlToTextContent(html: string): string { @@ -371,6 +432,7 @@ export function wikipediaHtmlToNormalizedText(html: string): string { function extractWikipediaParsePayload(value: unknown): { html: string; + title: string; pageId: string; revisionId: string; } | null { @@ -379,15 +441,17 @@ function extractWikipediaParsePayload(value: unknown): { if (!isNonNullObject(parse)) return null; const text = parse["text"]; + const title = parse["title"]; const revisionId = parseNonNegativeIntegerId(parse["revid"]); const pageId = parseNonNegativeIntegerId(parse["pageid"]); - if (typeof text !== "string") return null; + if (typeof text !== "string" || typeof title !== "string" || title.length === 0) return null; if (revisionId === null || pageId === null) { return null; } return { html: text, + title, pageId, revisionId, }; @@ -421,29 +485,22 @@ async function fetchWikipediaContent( endpoint.searchParams.set("prop", "text|revid"); endpoint.searchParams.set("oldid", revisionId); - let response: Response; - try { - response = await fetchWithTransientRetry(endpoint); - } catch (error) { - return { - success: false, - failureReason: `Wikipedia parse request failed: ${describeFetchError(error)}`, - }; - } - if (!response.ok) { - return { - success: false, - failureReason: `Wikipedia parse API returned ${response.status}`, - }; - } - + const deadline = AbortSignal.timeout(CANONICAL_FETCH_DEADLINE_MS); let data: unknown; try { - data = await response.json(); + const response = await fetchWithTransientRetry(endpoint, { signal: deadline }); + if (!response.ok) { + await response.body?.cancel(); + return { + success: false, + failureReason: `Wikipedia parse API returned ${response.status.toString()}`, + }; + } + data = await readJsonWithinLimit(response); } catch (error) { return { success: false, - failureReason: `Wikipedia parse API returned invalid JSON: ${describeFetchError(error)}`, + failureReason: `Wikipedia parse request failed: ${describeFetchError(error)}`, }; } const payload = extractWikipediaParsePayload(data); @@ -470,6 +527,7 @@ async function fetchWikipediaContent( sourceHtml: payload.html, canonicalIdentity: { platform: "WIKIPEDIA", + url: wikipediaArticleUrl(language, payload.title), language, pageId: payload.pageId, revisionId: payload.revisionId, diff --git a/src/typescript/api/src/lib/services/hmac.ts b/src/typescript/api/src/lib/services/hmac.ts deleted file mode 100644 index 9d7745b..0000000 --- a/src/typescript/api/src/lib/services/hmac.ts +++ /dev/null @@ -1,28 +0,0 @@ -async function computeHmac(secret: string, body: string): Promise { - const key = await crypto.subtle.importKey( - "raw", - new TextEncoder().encode(secret), - { name: "HMAC", hash: "SHA-256" }, - false, - ["sign"], - ); - const sig = await crypto.subtle.sign("HMAC", key, new TextEncoder().encode(body)); - return Array.from(new Uint8Array(sig)) - .map((b) => b.toString(16).padStart(2, "0")) - .join(""); -} - -export async function verifyHmac( - secret: string, - body: string, - signature: string, -): Promise { - const expected = await computeHmac(secret, body); - // Constant-time comparison - if (expected.length !== signature.length) return false; - let result = 0; - for (let i = 0; i < expected.length; i++) { - result |= expected.charCodeAt(i) ^ signature.charCodeAt(i); - } - return result === 0; -} diff --git a/src/typescript/api/src/lib/services/html-to-markdown.ts b/src/typescript/api/src/lib/services/html-to-markdown.ts index 8722073..7013c1b 100644 --- a/src/typescript/api/src/lib/services/html-to-markdown.ts +++ b/src/typescript/api/src/lib/services/html-to-markdown.ts @@ -17,12 +17,12 @@ import { NON_CONTENT_TAGS } from "@openerrata/shared"; import TurndownService from "turndown"; -import type { ImagePlaceholder } from "$lib/investigators/interface.js"; import { preFilterWikipediaHtml } from "./wikipedia-content-filter.js"; interface HtmlToMarkdownResult { markdown: string; - imagePlaceholders: ImagePlaceholder[]; + /** Absolute source URL of the image behind `[IMAGE:N]`, indexed by N. */ + imageSourceUrls: string[]; } /** @@ -30,7 +30,26 @@ interface HtmlToMarkdownResult { * or image placeholder format changes — ensures InvestigationInput snapshots * record which renderer produced the stored markdown. */ -export const MARKDOWN_RENDERER_VERSION = "1.2.0"; +export const MARKDOWN_RENDERER_VERSION = "1.3.0"; + +/** + * Resolve an `` against the post URL (Wikipedia HTML uses + * protocol-relative URLs) into the absolute form image occurrences use. + * Returns null for sources that can never be fetched (data:, relative junk, + * embedded credentials). + */ +function resolveImageSourceUrl(src: string, baseUrl: string): string | null { + if (src.length === 0) return null; + let resolved: URL; + try { + resolved = new URL(src, baseUrl); + } catch { + return null; + } + if (resolved.protocol !== "http:" && resolved.protocol !== "https:") return null; + if (resolved.username.length > 0 || resolved.password.length > 0) return null; + return resolved.toString(); +} // ── Turndown configuration ──────────────────────────────────────────────── @@ -43,27 +62,27 @@ const TURNDOWN_OPTIONS: TurndownService.Options = { }; /** - * Convert HTML to markdown with `[IMAGE:N]` placeholders for each `` tag. + * Convert HTML to markdown with an `[IMAGE:N]` placeholder for each `` + * whose source is a fetchable URL; images without one are dropped. * - * Returns the markdown string and the ordered list of placeholders so the - * input builder can match placeholders to resolved image occurrences by URL. + * Returns the markdown and the source URL behind each placeholder so the + * investigation input can match placeholders to downloaded images by URL. */ -function htmlToMarkdownWithImages(html: string): HtmlToMarkdownResult { - const placeholders: ImagePlaceholder[] = []; +function htmlToMarkdownWithImages(html: string, baseUrl: string): HtmlToMarkdownResult { + const imageSourceUrls: string[] = []; const service = new TurndownService(TURNDOWN_OPTIONS); service.addRule("imagePlaceholder", { filter: "img", replacement: (_content, node) => { - const src = node.getAttribute("src")?.trim() ?? ""; - const index = placeholders.length; - if (src.length > 0) { - placeholders.push({ index, matchBy: "SOURCE_URL", sourceUrl: src }); - } else { - placeholders.push({ index, matchBy: "ORIGINAL_INDEX" }); + const sourceUrl = resolveImageSourceUrl(node.getAttribute("src")?.trim() ?? "", baseUrl); + if (sourceUrl === null) { + return ""; } - return ` [IMAGE:${index}] `; + const index = imageSourceUrls.length; + imageSourceUrls.push(sourceUrl); + return ` [IMAGE:${index.toString()}] `; }, }); @@ -71,7 +90,7 @@ function htmlToMarkdownWithImages(html: string): HtmlToMarkdownResult { // Substack wrap images in to make them clickable. // Turndown's default link rule would produce "[ [IMAGE:0] ](url)" with extra // blank lines from inner block containers (div, figure, etc.). The anchor URL - // is redundant — the image URL is already captured in imagePlaceholders for + // is redundant — the image URL is already captured in imageSourceUrls for // matching — so we discard it and return just the placeholder(s). service.addRule("imageOnlyLink", { filter: (node) => @@ -103,19 +122,26 @@ function htmlToMarkdownWithImages(html: string): HtmlToMarkdownResult { service.remove((node) => NON_CONTENT_TAGS.has(node.nodeName.toLowerCase())); const markdown = service.turndown(html); - return { markdown, imagePlaceholders: placeholders }; + return { markdown, imageSourceUrls }; } // ── Platform wrappers ──────────────────────────────────────────────────── +// `postUrl` is the base for resolving relative image sources. -export function lesswrongHtmlToContentMarkdown(html: string): HtmlToMarkdownResult { - return htmlToMarkdownWithImages(html); +export function lesswrongHtmlToContentMarkdown( + html: string, + postUrl: string, +): HtmlToMarkdownResult { + return htmlToMarkdownWithImages(html, postUrl); } -export function wikipediaHtmlToContentMarkdown(html: string): HtmlToMarkdownResult { - return htmlToMarkdownWithImages(preFilterWikipediaHtml(html)); +export function wikipediaHtmlToContentMarkdown( + html: string, + postUrl: string, +): HtmlToMarkdownResult { + return htmlToMarkdownWithImages(preFilterWikipediaHtml(html), postUrl); } -export function substackHtmlToContentMarkdown(html: string): HtmlToMarkdownResult { - return htmlToMarkdownWithImages(html); +export function substackHtmlToContentMarkdown(html: string, postUrl: string): HtmlToMarkdownResult { + return htmlToMarkdownWithImages(html, postUrl); } diff --git a/src/typescript/api/src/lib/services/image-downloader.ts b/src/typescript/api/src/lib/services/image-downloader.ts index cbb4449..53faa50 100644 --- a/src/typescript/api/src/lib/services/image-downloader.ts +++ b/src/typescript/api/src/lib/services/image-downloader.ts @@ -7,12 +7,10 @@ import { import { getPrisma } from "$lib/db/client.js"; import { isUniqueConstraintError } from "$lib/db/errors.js"; import type { ImageBlob } from "$lib/db/prisma-client"; -import { hasAddressIntersection, resolvePublicHostAddresses } from "$lib/network/host-safety.js"; -import { isRedirectStatus } from "$lib/network/http-status.js"; +import { fetchPublicHttp, readBodyPrefix } from "$lib/network/public-http-fetch.js"; import { uploadImage } from "./blob-storage.js"; const IMAGE_DOWNLOAD_TIMEOUT_MS = 15_000; -const MAX_REDIRECT_HOPS = 5; const SUPPORTED_MIME_TYPE_SET: ReadonlySet = new Set(SUPPORTED_IMAGE_MIME_TYPE_VALUES); @@ -29,142 +27,39 @@ export function parseImageContentType( return normalized; } -function uniqueImageUrls(urls: string[]): string[] { - const unique = new Set(); - - for (const url of urls) { - const trimmed = url.trim(); - if (trimmed.length === 0) continue; - try { - const parsed = new URL(trimmed); - if (parsed.protocol !== "http:" && parsed.protocol !== "https:") continue; - if (parsed.username.length > 0 || parsed.password.length > 0) continue; - unique.add(parsed.toString()); - } catch { - // Ignore malformed image URLs to keep investigation flow robust. - } - } - - return Array.from(unique); -} - -async function readResponseBytesWithinLimit( - response: Response, - maxBytes: number, -): Promise { - if (!response.body) { - return null; - } - - const reader = response.body.getReader(); - const chunks: Uint8Array[] = []; - let totalBytes = 0; - - try { - while (true) { - const { done, value } = await reader.read(); - if (done) break; - - totalBytes += value.byteLength; - if (totalBytes > maxBytes) { - await reader.cancel("Image exceeds maximum byte limit"); - return null; - } - chunks.push(value); - } - } catch { - await reader.cancel(); - return null; - } finally { - reader.releaseLock(); - } - - const bytes = new Uint8Array(totalBytes); - let offset = 0; - for (const chunk of chunks) { - bytes.set(chunk, offset); - offset += chunk.byteLength; - } - - return bytes; -} - +/** + * Download one image from the public internet (SSRF-safe; see + * public-http-fetch.ts). Returns null when the image is unreachable, blocked, + * not a supported image type, or over MAX_IMAGE_BYTES; a single bad image + * never fails the investigation. Aborting `signal` aborts the download. + */ async function downloadImage( url: string, + signal: AbortSignal, ): Promise<{ bytes: Uint8Array; mimeType: SupportedImageMimeType } | null> { try { - let currentUrl = new URL(url); - - for (let redirectHop = 0; redirectHop <= MAX_REDIRECT_HOPS; redirectHop += 1) { - if (currentUrl.protocol !== "http:" && currentUrl.protocol !== "https:") { - return null; - } - const resolvedBeforeRequest = await resolvePublicHostAddresses(currentUrl.hostname); - if (!resolvedBeforeRequest) { - return null; - } - - const response = await fetch(currentUrl, { - method: "GET", - redirect: "manual", - signal: AbortSignal.timeout(IMAGE_DOWNLOAD_TIMEOUT_MS), - headers: { - "User-Agent": "OpenErrataImageDownloader/1.0 (+https://openerrata.com)", - Accept: "image/*", - }, - }); - - if (isRedirectStatus(response.status)) { - const location = response.headers.get("location"); - if (location === null || location.length === 0) { - return null; - } - - currentUrl = new URL(location, currentUrl); - continue; - } - - if (!response.ok) { - return null; - } - - // Re-resolve and require overlap with pre-request answers. This narrows - // DNS rebinding windows by rejecting responses when hostname resolution - // shifts to a disjoint address set during request handling. - const resolvedAfterRequest = await resolvePublicHostAddresses(currentUrl.hostname); - if ( - !resolvedAfterRequest || - !hasAddressIntersection(resolvedBeforeRequest, resolvedAfterRequest) - ) { - return null; - } - - const contentType = parseImageContentType(response.headers.get("content-type")); - if (contentType === null || contentType.length === 0) { - return null; - } - - const contentLengthHeader = response.headers.get("content-length"); - if (contentLengthHeader !== null && contentLengthHeader.length > 0) { - const contentLength = Number.parseInt(contentLengthHeader, 10); - if (Number.isFinite(contentLength) && contentLength > MAX_IMAGE_BYTES) { - return null; - } - } - - const bytes = await readResponseBytesWithinLimit(response, MAX_IMAGE_BYTES); - if (bytes === null) { - return null; - } + const { response } = await fetchPublicHttp({ + url: new URL(url), + headers: { + "User-Agent": "OpenErrataImageDownloader/1.0 (+https://openerrata.com)", + Accept: "image/*", + }, + signal: AbortSignal.any([signal, AbortSignal.timeout(IMAGE_DOWNLOAD_TIMEOUT_MS)]), + }); - return { - bytes, - mimeType: contentType, - }; + const contentType = parseImageContentType(response.headers.get("content-type")); + if (!response.ok || contentType === null) { + await response.body?.cancel(); + return null; } - return null; - } catch { + const { bytes, truncated } = await readBodyPrefix(response, MAX_IMAGE_BYTES); + if (truncated) { + return null; + } + return { bytes, mimeType: contentType }; + } catch (error) { + if (signal.aborted) throw error; return null; } } @@ -225,20 +120,20 @@ type ImageDownloadResolution = status: "failed"; }; +/** + * Download each URL, dedupe by content hash, and store new images in blob + * storage. Results are reported per input URL, in order. Throws only when + * `signal` aborts (the run lost its lease) or storage fails. + */ export async function downloadAndStoreImages( urls: string[], - maxCount: number, + signal: AbortSignal, ): Promise { - const uniqueUrls = uniqueImageUrls(urls).slice(0, maxCount); - if (uniqueUrls.length === 0) { - return []; - } - const resolutions: ImageDownloadResolution[] = []; const resolvedByContentHash = new Map(); - for (const imageUrl of uniqueUrls) { - const downloaded = await downloadImage(imageUrl); + for (const imageUrl of urls) { + const downloaded = await downloadImage(imageUrl, signal); if (!downloaded) { resolutions.push({ sourceUrl: imageUrl, diff --git a/src/typescript/api/src/lib/services/investigate-now.ts b/src/typescript/api/src/lib/services/investigate-now.ts new file mode 100644 index 0000000..283f5b8 --- /dev/null +++ b/src/typescript/api/src/lib/services/investigate-now.ts @@ -0,0 +1,172 @@ +/** + * investigateNow (SPEC §2.6, §3.3): explicit requests to investigate one + * content version. + * + * - No investigation yet → admit one funded by the requester (their verified + * OpenAI key if they sent one, else the instance key they authenticated + * with), with update lineage, and enqueue it. + * - COMPLETE or FAILED → return it as is. FAILED is terminal (SPEC §3.7). + * - PROCESSING → return it, after recovering it first if its lease expired. + * - PENDING and funded → make sure a queue job exists. The requester's key is + * never attached: whoever admitted it is already paying for it. + * - PENDING and unfunded (its user key was dropped) → fund it as above. + */ + +import { WORD_COUNT_LIMIT } from "@openerrata/shared"; +import type { PrismaClient } from "$lib/db/client"; +import { isUniqueConstraintError } from "$lib/db/errors.js"; +import { + fundUnfundedInvestigation, + insertAdmittedInvestigation, + type InvestigationFunding, +} from "./investigation-admission.js"; +import { buildInvestigationInputSnapshot } from "./investigation-input.js"; +import { recoverExpiredLease } from "./investigation-lease.js"; +import { enqueueInvestigation } from "./queue.js"; +import { resolveUpdateLineage } from "./update-lineage.js"; +import { verifyUserOpenAiApiKey, type UserOpenAiKeyVerification } from "./user-key-source.js"; + +export class InvestigationWordLimitError extends Error { + readonly limit: number; + readonly observedWordCount: number; + + constructor(observedWordCount: number, limit: number) { + super(`Post exceeds word count limit (${limit.toString()} words)`); + this.name = "InvestigationWordLimitError"; + this.observedWordCount = observedWordCount; + this.limit = limit; + } +} + +export class UserOpenAiKeyRejectedError extends Error { + readonly outcome: Extract["outcome"]; + + constructor(outcome: UserOpenAiKeyRejectedError["outcome"]) { + super(`User OpenAI key was not accepted (${outcome.openaiApiKeyStatus})`); + this.name = "UserOpenAiKeyRejectedError"; + this.outcome = outcome; + } +} + +/** Who is asking, and therefore who would pay for a run this request admits. */ +export type InvestigationRequester = + | { kind: "INSTANCE_API_KEY" } + /** The key as sent; it is verified with OpenAI only if it would be attached. */ + | { kind: "USER_OPENAI_KEY"; apiKey: string }; + +async function fundingFor(requester: InvestigationRequester): Promise { + if (requester.kind === "INSTANCE_API_KEY") { + return { origin: "INSTANCE_REQUEST" }; + } + const verification = await verifyUserOpenAiApiKey(requester.apiKey); + if (!verification.verified) { + throw new UserOpenAiKeyRejectedError(verification.outcome); + } + return { origin: "USER_KEY_REQUEST", apiKey: verification.apiKey }; +} + +/** + * Handle one investigateNow request and return the id of the investigation for + * `postVersion`. Throws InvestigationWordLimitError when a new investigation + * would exceed the word limit, and UserOpenAiKeyRejectedError when a user key + * would fund the run but OpenAI does not accept it. + */ +export async function requestInvestigation( + prisma: PrismaClient, + input: { + postVersion: { + id: string; + postId: string; + contentBlob: { contentText: string; wordCount: number }; + }; + promptId: string; + requester: InvestigationRequester; + }, +): Promise<{ investigationId: string }> { + const { postVersion } = input; + // A later pass follows a concurrent creation, an expired-lease recovery, or + // a lost race to fund an unfunded investigation; each re-reads the row. + for (let pass = 0; pass < 3; pass += 1) { + const existing = await prisma.investigation.findUnique({ + where: { postVersionId: postVersion.id }, + select: { + id: true, + status: true, + origin: true, + openAiKeySource: { select: { investigationId: true } }, + lease: { select: { leaseExpiresAt: true } }, + }, + }); + + if (existing === null) { + if (postVersion.contentBlob.wordCount > WORD_COUNT_LIMIT) { + throw new InvestigationWordLimitError(postVersion.contentBlob.wordCount, WORD_COUNT_LIMIT); + } + const lineage = await resolveUpdateLineage(prisma, { + id: postVersion.id, + postId: postVersion.postId, + contentText: postVersion.contentBlob.contentText, + }); + const snapshot = await buildInvestigationInputSnapshot(prisma, postVersion.id); + const funding = await fundingFor(input.requester); + let created: { id: string }; + try { + created = await prisma.$transaction((tx) => + insertAdmittedInvestigation(tx, { + postVersionId: postVersion.id, + promptId: input.promptId, + funding, + lineage, + snapshot, + now: new Date(), + }), + ); + } catch (error) { + if (isUniqueConstraintError(error)) continue; + throw error; + } + await enqueueInvestigation(created.id); + return { investigationId: created.id }; + } + + switch (existing.status) { + case "COMPLETE": + case "FAILED": + return { investigationId: existing.id }; + case "PROCESSING": + if (existing.lease !== null && existing.lease.leaseExpiresAt.getTime() <= Date.now()) { + await recoverExpiredLease(prisma, existing.id); + continue; + } + return { investigationId: existing.id }; + case "PENDING": { + const unfunded = + existing.origin === "USER_KEY_REQUEST" && existing.openAiKeySource === null; + if (unfunded) { + const funding = await fundingFor(input.requester); + let funded: boolean; + try { + funded = await prisma.$transaction((tx) => + fundUnfundedInvestigation(tx, { + investigationId: existing.id, + funding, + now: new Date(), + }), + ); + } catch (error) { + // A concurrent request attached its key first. + if (isUniqueConstraintError(error)) continue; + throw error; + } + if (!funded) continue; + } + await enqueueInvestigation(existing.id); + return { investigationId: existing.id }; + } + } + } + + throw new Error( + `investigateNow could not settle on an investigation for post version ${postVersion.id}: its state kept changing`, + ); +} diff --git a/src/typescript/api/src/lib/services/investigation-admission.ts b/src/typescript/api/src/lib/services/investigation-admission.ts new file mode 100644 index 0000000..fbb378a --- /dev/null +++ b/src/typescript/api/src/lib/services/investigation-admission.ts @@ -0,0 +1,106 @@ +/** + * Admitting investigations for execution (SPEC §3.7). + * + * An investigation is only ever created — or revived after its user key was + * dropped — together with a decision about who pays for it: + * + * - SELECTOR: background selection under the per-UTC-day budget; server key. + * - INSTANCE_REQUEST: investigateNow from an instance-API-key client; server key. + * - USER_KEY_REQUEST: investigateNow funded by the requester's verified OpenAI + * key, attached in the same transaction that admits the investigation. + * + * A user key only ever funds what its own request admits; it never takes over + * an investigation someone else is already paying for. + */ + +import { randomUUID } from "node:crypto"; +import type { Prisma } from "$lib/db/prisma-client"; +import { DEFAULT_INVESTIGATION_PROVIDER } from "@openerrata/shared"; +import { investigationInputRow, type InvestigationInputSnapshot } from "./investigation-input.js"; +import type { UpdateLineage } from "./update-lineage.js"; +import { attachOpenAiKeySource, type VerifiedOpenAiApiKey } from "./user-key-source.js"; + +export type InvestigationFunding = + | { origin: "SELECTOR" } + | { origin: "INSTANCE_REQUEST" } + | { origin: "USER_KEY_REQUEST"; apiKey: VerifiedOpenAiApiKey }; + +/** + * Investigations no one is paying for: user-key admissions whose key was + * dropped. They wait, unqueued, until the selector or a new request funds them. + */ +export const unfundedInvestigationWhere = { + status: "PENDING", + origin: "USER_KEY_REQUEST", + openAiKeySource: { is: null }, +} satisfies Prisma.InvestigationWhereInput; + +async function attachFunding( + tx: Prisma.TransactionClient, + investigationId: string, + funding: InvestigationFunding, + now: Date, +): Promise { + if (funding.origin === "USER_KEY_REQUEST") { + await attachOpenAiKeySource(tx, { investigationId, apiKey: funding.apiKey, now }); + } +} + +/** + * Create the PENDING investigation (and its immutable input snapshot) for a + * post version that has none. Throws a unique-constraint error if another + * request created one first; callers treat that as "already exists". + */ +export async function insertAdmittedInvestigation( + tx: Prisma.TransactionClient, + input: { + postVersionId: string; + promptId: string; + funding: InvestigationFunding; + lineage: UpdateLineage | null; + snapshot: InvestigationInputSnapshot; + now: Date; + }, +): Promise<{ id: string }> { + const investigationId = randomUUID(); + await tx.investigationInput.create({ + data: investigationInputRow(investigationId, input.snapshot), + }); + const investigation = await tx.investigation.create({ + data: { + id: investigationId, + inputId: investigationId, + postVersionId: input.postVersionId, + status: "PENDING", + parentInvestigationId: input.lineage?.parentInvestigationId ?? null, + contentDiff: input.lineage?.contentDiff ?? null, + promptId: input.promptId, + provider: DEFAULT_INVESTIGATION_PROVIDER, + origin: input.funding.origin, + admittedAt: input.now, + queuedAt: input.now, + }, + select: { id: true }, + }); + await attachFunding(tx, investigation.id, input.funding, input.now); + return investigation; +} + +/** + * Fund an unfunded investigation. Returns false when it is no longer + * unfunded (someone else funded it first, or it is no longer PENDING). + */ +export async function fundUnfundedInvestigation( + tx: Prisma.TransactionClient, + input: { investigationId: string; funding: InvestigationFunding; now: Date }, +): Promise { + const admitted = await tx.investigation.updateMany({ + where: { id: input.investigationId, ...unfundedInvestigationWhere }, + data: { origin: input.funding.origin, admittedAt: input.now, queuedAt: input.now }, + }); + if (admitted.count === 0) { + return false; + } + await attachFunding(tx, input.investigationId, input.funding, input.now); + return true; +} diff --git a/src/typescript/api/src/lib/services/investigation-input.ts b/src/typescript/api/src/lib/services/investigation-input.ts new file mode 100644 index 0000000..c1b8d2b --- /dev/null +++ b/src/typescript/api/src/lib/services/investigation-input.ts @@ -0,0 +1,234 @@ +/** + * The immutable InvestigationInput snapshot (SPEC §2.4.4, §2.12). + * + * Everything the worker hands the investigator that could change after an + * investigation is queued — rendered markdown, the source URL behind each + * markdown image placeholder, and the post's URL, author and publication time + * — is captured here once, when the investigation row is created. Every + * attempt then runs on exactly the same input, whatever happens to the live + * Post and version-metadata rows afterwards. + */ + +import type { ContentProvenance, Platform } from "@openerrata/shared"; +import type { DbClient } from "$lib/db/client"; +import type { Prisma } from "$lib/db/prisma-client"; +import { resolveMarkdownForInvestigation, type HtmlSnapshots } from "./markdown-resolution.js"; + +export interface InvestigationInputSnapshot { + provenance: ContentProvenance; + contentHash: string; + markdown: + | { source: "NONE" } + | { + source: "SERVER_HTML" | "CLIENT_HTML"; + markdown: string; + rendererVersion: string; + /** Source URL of the image behind `[IMAGE:N]`, indexed by N. */ + imageSourceUrls: string[]; + }; + postUrl: string; + authorName: string | null; + postPublishedAt: Date | null; + hasVideo: boolean; +} + +const postVersionForInputSnapshotSelect = { + serverVerifiedAt: true, + contentBlob: { select: { contentHash: true } }, + post: { + select: { + platform: true, + url: true, + author: { select: { displayName: true } }, + }, + }, + lesswrongVersionMeta: { + select: { + publishedAt: true, + serverHtmlBlob: { select: { htmlContent: true } }, + clientHtmlBlob: { select: { htmlContent: true } }, + }, + }, + xVersionMeta: { + select: { + postedAt: true, + mediaUrls: true, + }, + }, + substackVersionMeta: { + select: { + publishedAt: true, + clientHtmlBlob: { select: { htmlContent: true } }, + }, + }, + wikipediaVersionMeta: { + select: { + lastModifiedAt: true, + serverHtmlBlob: { select: { htmlContent: true } }, + clientHtmlBlob: { select: { htmlContent: true } }, + }, + }, +} satisfies Prisma.PostVersionSelect; + +type PostVersionForInputSnapshot = Prisma.PostVersionGetPayload<{ + select: typeof postVersionForInputSnapshotSelect; +}>; + +function unreachablePlatform(platform: never): never { + throw new Error(`Unsupported post platform: ${String(platform)}`); +} + +const VIDEO_PATH_SUFFIXES = [".mp4", ".webm", ".m3u8", ".mov", ".m4v"] as const; + +/** Whether a media URL points at a video file, judged by its path extension. */ +export function isLikelyVideoUrl(url: string): boolean { + let pathname: string; + try { + pathname = new URL(url).pathname.toLowerCase(); + } catch { + return false; + } + return VIDEO_PATH_SUFFIXES.some((suffix) => pathname.endsWith(suffix)); +} + +interface PlatformInputFields { + htmlSnapshots: HtmlSnapshots; + postPublishedAt: Date | null; + hasVideo: boolean; +} + +function htmlSnapshots( + serverVerifiedAt: Date | null, + serverHtml: string | null, + clientHtml: string | null, + platform: Platform, +): HtmlSnapshots { + if (serverVerifiedAt === null) { + return { serverVerifiedAt: null, serverHtml, clientHtml }; + } + if (serverHtml === null) { + throw new Error( + `serverVerifiedAt is set but serverHtml is missing for platform ${platform} — violates DB invariant (serverVerifiedAt IS NOT NULL → serverHtmlBlobId IS NOT NULL)`, + ); + } + return { serverVerifiedAt, serverHtml, clientHtml }; +} + +function platformInputFields(postVersion: PostVersionForInputSnapshot): PlatformInputFields { + const platform = postVersion.post.platform; + switch (platform) { + case "LESSWRONG": { + const meta = postVersion.lesswrongVersionMeta; + return { + htmlSnapshots: htmlSnapshots( + postVersion.serverVerifiedAt, + meta?.serverHtmlBlob?.htmlContent ?? null, + meta?.clientHtmlBlob?.htmlContent ?? null, + platform, + ), + postPublishedAt: meta?.publishedAt ?? null, + hasVideo: false, + }; + } + case "X": { + const meta = postVersion.xVersionMeta; + return { + htmlSnapshots: htmlSnapshots(postVersion.serverVerifiedAt, null, null, platform), + postPublishedAt: meta?.postedAt ?? null, + hasVideo: (meta?.mediaUrls ?? []).some(isLikelyVideoUrl), + }; + } + case "SUBSTACK": { + const meta = postVersion.substackVersionMeta; + return { + htmlSnapshots: htmlSnapshots( + postVersion.serverVerifiedAt, + null, + meta?.clientHtmlBlob?.htmlContent ?? null, + platform, + ), + postPublishedAt: meta?.publishedAt ?? null, + hasVideo: false, + }; + } + case "WIKIPEDIA": { + const meta = postVersion.wikipediaVersionMeta; + return { + htmlSnapshots: htmlSnapshots( + postVersion.serverVerifiedAt, + meta?.serverHtmlBlob?.htmlContent ?? null, + meta?.clientHtmlBlob?.htmlContent ?? null, + platform, + ), + postPublishedAt: meta?.lastModifiedAt ?? null, + hasVideo: false, + }; + } + default: + return unreachablePlatform(platform); + } +} + +/** Capture the input snapshot for a new investigation of `postVersionId`. */ +export async function buildInvestigationInputSnapshot( + db: DbClient, + postVersionId: string, +): Promise { + const postVersion = await db.postVersion.findUnique({ + where: { id: postVersionId }, + select: postVersionForInputSnapshotSelect, + }); + if (postVersion === null) { + throw new Error(`PostVersion ${postVersionId} not found`); + } + + const { post } = postVersion; + const fields = platformInputFields(postVersion); + const markdown = resolveMarkdownForInvestigation({ + platform: post.platform, + snapshots: fields.htmlSnapshots, + postUrl: post.url, + }); + + return { + provenance: postVersion.serverVerifiedAt === null ? "CLIENT_FALLBACK" : "SERVER_VERIFIED", + contentHash: postVersion.contentBlob.contentHash, + markdown: + markdown.source === "NONE" + ? { source: "NONE" } + : { + source: markdown.source, + markdown: markdown.markdown, + rendererVersion: markdown.rendererVersion, + imageSourceUrls: markdown.imageSourceUrls, + }, + postUrl: post.url, + authorName: post.author?.displayName ?? null, + postPublishedAt: fields.postPublishedAt, + hasVideo: fields.hasVideo, + }; +} + +/** Row data for persisting `snapshot` as the input of `investigationId`. */ +export function investigationInputRow( + investigationId: string, + snapshot: InvestigationInputSnapshot, +): Prisma.InvestigationInputUncheckedCreateInput { + return { + investigationId, + provenance: snapshot.provenance, + contentHash: snapshot.contentHash, + markdownSource: snapshot.markdown.source, + ...(snapshot.markdown.source === "NONE" + ? { imagePlaceholderSourceUrls: [] } + : { + markdown: snapshot.markdown.markdown, + markdownRendererVersion: snapshot.markdown.rendererVersion, + imagePlaceholderSourceUrls: snapshot.markdown.imageSourceUrls, + }), + postUrl: snapshot.postUrl, + authorName: snapshot.authorName, + postPublishedAt: snapshot.postPublishedAt, + hasVideo: snapshot.hasVideo, + }; +} diff --git a/src/typescript/api/src/lib/services/investigation-lease.ts b/src/typescript/api/src/lib/services/investigation-lease.ts index c25f32b..b248d25 100644 --- a/src/typescript/api/src/lib/services/investigation-lease.ts +++ b/src/typescript/api/src/lib/services/investigation-lease.ts @@ -1,7 +1,21 @@ -import { getPrisma } from "$lib/db/client"; +/** + * Investigation leases (SPEC §3.7). + * + * A worker runs an investigation only while it holds the InvestigationLease + * row; the database guarantees the row exists iff status = PROCESSING. This + * module owns every way into and out of that state that is not a run's own + * outcome: claiming (PENDING → PROCESSING), renewing via heartbeat (and + * noticing when the lease is gone), recovering expired leases, and releasing + * a run whose user key turned out to be unusable. + */ + +import { getPrisma, type PrismaClient } from "$lib/db/client"; import type { Prisma } from "$lib/db/prisma-client"; -import { investigationContextInclude } from "./prompt-context.js"; +import type { InvestigatorAttemptAudit } from "$lib/investigators/interface.js"; +import { persistAttemptAudit } from "./attempt-audit.js"; +import { unfundedInvestigationWhere } from "./investigation-admission.js"; import { formatErrorForLog } from "./orchestrator-errors.js"; +import { consumeOpenAiKeySource } from "./user-key-source.js"; export interface Logger { info(msg: string): void; @@ -13,88 +27,54 @@ const LEASE_TTL_MS = 60_000; const HEARTBEAT_INTERVAL_MS = 15_000; /** - * Maximum number of orchestration attempts before marking FAILED. - * Each attempt is a full orchestration cycle (claim → investigate → persist). - * Transient failures reclaim to PENDING and re-enqueue; this cap prevents - * infinite retry loops. + * Attempt cap. A transient failure on attempt MAX_INVESTIGATION_ATTEMPTS (or + * later), or an expired lease after it, marks the investigation FAILED. */ export const MAX_INVESTIGATION_ATTEMPTS = 4; +/** Exponential backoff base for transient retries. */ +const BASE_BACKOFF_MS = 10_000; + /** - * Exponential backoff base for transient retries, in milliseconds. - * Delay = BASE_BACKOFF_MS * 2^(attemptCount - 1), so: - * attempt 1 → 10s, attempt 2 → 20s, attempt 3 → 40s + * Delay before retrying after transient failure of attempt `attemptNumber` + * (1-indexed): 10s, 20s, 40s, ... */ -export const BASE_BACKOFF_MS = 10_000; - -const investigationWithContextInclude = { - ...investigationContextInclude, - input: true, - parentInvestigation: { - include: { - claims: { - include: { - sources: true, - }, - }, - }, - }, -} satisfies Prisma.InvestigationInclude; - -type InvestigationWithContext = Prisma.InvestigationGetPayload<{ - include: typeof investigationWithContextInclude; -}>; +export function retryBackoffMs(attemptNumber: number): number { + return BASE_BACKOFF_MS * 2 ** (attemptNumber - 1); +} -function nextLeaseExpiry(): Date { - return new Date(Date.now() + LEASE_TTL_MS); +function nextLeaseExpiry(now: Date): Date { + return new Date(now.getTime() + LEASE_TTL_MS); } type LeaseClaimResult = - | { outcome: "CLAIMED"; attemptNumber: number } + | { outcome: "CLAIMED"; attemptNumber: number; leaseExpiresAt: Date } + /** The investigation row is gone (stale job). */ | { outcome: "MISSING" } + /** COMPLETE or FAILED. */ | { outcome: "TERMINAL" } + /** Another worker holds an unexpired lease. */ | { outcome: "LEASE_HELD" } - | { outcome: "ATTEMPTS_EXHAUSTED" }; + /** PENDING, but its user key was dropped and nobody has funded it since. */ + | { outcome: "UNFUNDED" }; /** - * Atomically claim the investigation lease for this worker. - * - * The InvestigationLease table structurally enforces that PROCESSING - * investigations always have a lease holder (the row's existence IS - * the lease). This function handles two paths: - * - * Path 1 (PENDING → PROCESSING): transition investigation status, - * increment attemptCount, and create a new lease row. - * - * Path 2 (stale PROCESSING): delete the expired lease row, increment - * attemptCount, and create a fresh lease for this worker. - * - * Returns the attemptNumber (1-indexed) on CLAIMED so the orchestrator - * can pass it to the audit trail. - * - * Note on retryAfter: this function intentionally does NOT check - * Investigation.retryAfter before claiming. retryAfter is a selector gate - * (prevents the cron from re-enqueueing too early) and a graphile-worker - * scheduling hint (via enqueueInvestigation's runAt). Once a job actually - * arrives at a worker — whether from the scheduled re-enqueue or from a - * user-triggered investigateNow call — the worker may claim immediately. - * This means investigateNow bypasses the backoff window, which is intentional: - * an explicit user request should not be subject to the automatic retry delay. + * Claim a funded PENDING investigation: PENDING → PROCESSING, increment + * attemptCount, create the lease. Returns the new attempt number, or null if + * the investigation was not a funded PENDING investigation. */ -export async function tryClaimLease( +async function claimFundedPending( investigationId: string, workerIdentity: string, -): Promise { +): Promise<{ attemptNumber: number; leaseExpiresAt: Date } | null> { const now = new Date(); - const prisma = getPrisma(); - - // Path 1: PENDING → PROCESSING - const claimedFromPending = await prisma.$transaction(async (tx) => { + const leaseExpiresAt = nextLeaseExpiry(now); + return getPrisma().$transaction(async (tx) => { const transitioned = await tx.investigation.updateMany({ where: { id: investigationId, status: "PENDING", - attemptCount: { lt: MAX_INVESTIGATION_ATTEMPTS }, + NOT: unfundedInvestigationWhere, }, data: { status: "PROCESSING", attemptCount: { increment: 1 }, retryAfter: null }, }); @@ -104,159 +84,280 @@ export async function tryClaimLease( data: { investigationId, leaseOwner: workerIdentity, - leaseExpiresAt: nextLeaseExpiry(), + leaseExpiresAt, startedAt: now, heartbeatAt: now, }, }); - const updated = await tx.investigation.findUnique({ + const { attemptCount } = await tx.investigation.findUniqueOrThrow({ where: { id: investigationId }, select: { attemptCount: true }, }); - return updated?.attemptCount ?? null; + return { attemptNumber: attemptCount, leaseExpiresAt }; }); +} - if (claimedFromPending !== null) { - return { outcome: "CLAIMED", attemptNumber: claimedFromPending }; - } +/** + * Atomically claim the investigation lease for this worker. + * + * Only funded PENDING investigations can be claimed. An expired lease left by + * a dead worker is recovered first (recoverExpiredLease) and the claim retried, + * so stale-lease handling has exactly one path. + * + * retryAfter is intentionally not checked: it is a selector gate and a queue + * scheduling hint (enqueueInvestigation's runAt). Once a job reaches a worker — + * from the scheduled retry or from an explicit investigateNow — it may claim + * immediately, so investigateNow bypasses the automatic retry delay. + */ +export async function tryClaimLease( + investigationId: string, + workerIdentity: string, +): Promise { + const prisma = getPrisma(); + // Two passes: the second follows recovery of an expired lease, or a race in + // which the row changed between the claim attempt and the classification. + for (let pass = 0; pass < 2; pass += 1) { + const claimed = await claimFundedPending(investigationId, workerIdentity); + if (claimed !== null) { + return { outcome: "CLAIMED", ...claimed }; + } - // Check if PENDING but attempts exhausted - const pendingExhausted = await prisma.investigation.findUnique({ - where: { id: investigationId }, - select: { status: true, attemptCount: true }, - }); - if ( - pendingExhausted?.status === "PENDING" && - pendingExhausted.attemptCount >= MAX_INVESTIGATION_ATTEMPTS - ) { - return { outcome: "ATTEMPTS_EXHAUSTED" }; + const state = await prisma.investigation.findUnique({ + where: { id: investigationId }, + select: { + status: true, + origin: true, + openAiKeySource: { select: { investigationId: true } }, + lease: { select: { leaseExpiresAt: true } }, + }, + }); + if (state === null) { + return { outcome: "MISSING" }; + } + + switch (state.status) { + case "COMPLETE": + case "FAILED": + return { outcome: "TERMINAL" }; + case "PENDING": + if (state.origin === "USER_KEY_REQUEST" && state.openAiKeySource === null) { + return { outcome: "UNFUNDED" }; + } + continue; + case "PROCESSING": + if (state.lease === null) { + throw new Error( + `Investigation ${investigationId} is PROCESSING without a lease row, which the database forbids`, + ); + } + if (state.lease.leaseExpiresAt.getTime() > Date.now()) { + return { outcome: "LEASE_HELD" }; + } + await recoverExpiredLease(prisma, investigationId); + continue; + } } - // Path 2: Reclaim stale PROCESSING lease. - // - // IMPORTANT: when the deleteMany succeeds but the updateMany guard fails, - // we must throw (not return) to rollback the transaction. A bare `return null` - // would commit the lease deletion while leaving the investigation PROCESSING - // with no InvestigationLease row — a zombie state the selector perpetually - // re-selects but no worker can ever claim. - class StaleReclaimAborted extends Error {} - const reclaimedStale = await prisma - .$transaction(async (tx) => { - const deleted = await tx.investigationLease.deleteMany({ - where: { investigationId, leaseExpiresAt: { lte: now } }, - }); - if (deleted.count === 0) return null; + throw new Error( + `Could not claim or classify investigation ${investigationId}: its state kept changing`, + ); +} - // Guard: only reclaim if still PROCESSING and under the attempt cap. - // Throws to rollback the lease deletion if the guard fails — prevents - // creating a PROCESSING investigation with no lease row. - const incremented = await tx.investigation.updateMany({ - where: { - id: investigationId, - status: "PROCESSING", - attemptCount: { lt: MAX_INVESTIGATION_ATTEMPTS }, - }, - data: { attemptCount: { increment: 1 } }, - }); - if (incremented.count === 0) { - throw new StaleReclaimAborted("Guard failed; rolling back lease deletion"); - } +/** + * Recover an investigation whose lease expired (its worker died or stalled): + * delete the lease and return it to PENDING, or mark it FAILED when the lost + * attempt was the last one allowed. Returns false if there was no expired + * lease to recover. This is the only stale-lease recovery path; the worker, + * the selector and investigateNow all use it. + */ +export async function recoverExpiredLease( + prisma: PrismaClient, + investigationId: string, +): Promise { + return prisma.$transaction(async (tx) => { + const now = new Date(); + const deleted = await tx.investigationLease.deleteMany({ + where: { investigationId, leaseExpiresAt: { lte: now } }, + }); + if (deleted.count === 0) { + return false; + } - await tx.investigationLease.create({ - data: { - investigationId, - leaseOwner: workerIdentity, - leaseExpiresAt: nextLeaseExpiry(), - startedAt: now, - heartbeatAt: now, - }, + const { attemptCount } = await tx.investigation.findUniqueOrThrow({ + where: { id: investigationId }, + select: { attemptCount: true }, + }); + if (attemptCount >= MAX_INVESTIGATION_ATTEMPTS) { + await tx.investigation.update({ + where: { id: investigationId }, + data: { status: "FAILED" }, }); - - const updated = await tx.investigation.findUnique({ + await consumeOpenAiKeySource(tx, investigationId); + } else { + await tx.investigation.update({ where: { id: investigationId }, - select: { attemptCount: true }, + data: { status: "PENDING", queuedAt: now }, }); - return updated?.attemptCount ?? null; - }) - .catch((error: unknown) => { - if (error instanceof StaleReclaimAborted) return null; - throw error; - }); - - if (reclaimedStale !== null) { - return { outcome: "CLAIMED", attemptNumber: reclaimedStale }; - } - - // Fallback: determine why we couldn't claim - const investigation = await prisma.investigation.findUnique({ - where: { id: investigationId }, - select: { status: true, attemptCount: true }, - }); - const lease = await prisma.investigationLease.findUnique({ - where: { investigationId }, - select: { leaseExpiresAt: true }, + } + return true; }); +} - if (!investigation) return { outcome: "MISSING" }; - if (investigation.status === "COMPLETE" || investigation.status === "FAILED") { - return { outcome: "TERMINAL" }; - } - if (investigation.status === "PROCESSING" && lease !== null && lease.leaseExpiresAt > now) { - return { outcome: "LEASE_HELD" }; +export class LeaseLostError extends Error { + constructor(investigationId: string, reason: string) { + super(`Lost the lease on investigation ${investigationId}: ${reason}`); + this.name = "LeaseLostError"; } - // PROCESSING with exhausted attempts: stale reclaim was rolled back above. - // Signal the orchestrator to mark FAILED only when no active lease exists. - if ( - investigation.status === "PROCESSING" && - investigation.attemptCount >= MAX_INVESTIGATION_ATTEMPTS - ) { - return { outcome: "ATTEMPTS_EXHAUSTED" }; - } - return { outcome: "LEASE_HELD" }; } -export async function loadClaimedInvestigation( - investigationId: string, -): Promise { - return getPrisma().investigation.findUnique({ - where: { id: investigationId }, - include: investigationWithContextInclude, - }); +interface LeaseHeartbeat { + /** + * Aborts (with a LeaseLostError reason) once this worker can no longer show + * it holds the lease: a renewal found no lease row owned by it, or renewals + * kept failing until the last confirmed expiry passed. Every write the run + * makes after this aborts would race the lease's new owner. + */ + readonly leaseLostSignal: AbortSignal; + stop(): void; } -export function startHeartbeat( - investigationId: string, - workerIdentity: string, +/** Renew the lease every HEARTBEAT_INTERVAL_MS until stopped or lost. */ +export function startLeaseHeartbeat( + lease: { investigationId: string; workerIdentity: string; leaseExpiresAt: Date }, logger: Logger, -): { stop(): void } { +): LeaseHeartbeat { const prisma = getPrisma(); - const timer = setInterval(() => { - void prisma.investigationLease - .updateMany({ - where: { - investigationId, - leaseOwner: workerIdentity, - }, - data: { - leaseExpiresAt: nextLeaseExpiry(), - heartbeatAt: new Date(), - }, - }) - .catch((error: unknown) => { - logger.error( - `Investigation ${investigationId} heartbeat update failed: ${formatErrorForLog(error)}`, - ); + const controller = new AbortController(); + let confirmedExpiry = lease.leaseExpiresAt.getTime(); + let timer: ReturnType | null = null; + let stopped = false; + + function loseLease(reason: string): void { + controller.abort(new LeaseLostError(lease.investigationId, reason)); + } + + async function renew(): Promise { + const now = new Date(); + const renewedExpiry = nextLeaseExpiry(now); + try { + const renewed = await prisma.investigationLease.updateMany({ + where: { investigationId: lease.investigationId, leaseOwner: lease.workerIdentity }, + data: { leaseExpiresAt: renewedExpiry, heartbeatAt: now }, }); - }, HEARTBEAT_INTERVAL_MS); + if (renewed.count === 0) { + loseLease("the lease row is gone or owned by another worker"); + return; + } + confirmedExpiry = renewedExpiry.getTime(); + } catch (error) { + logger.error( + `Investigation ${lease.investigationId} heartbeat update failed: ${formatErrorForLog(error)}`, + ); + if (Date.now() >= confirmedExpiry) { + loseLease("renewals failed until the lease expired"); + } + } + } - if (typeof timer.unref === "function") { + function schedule(): void { + timer = setTimeout(() => { + void renew().then(() => { + if (!stopped && !controller.signal.aborted) schedule(); + }); + }, HEARTBEAT_INTERVAL_MS); timer.unref(); } + schedule(); return { + leaseLostSignal: controller.signal, stop() { - clearInterval(timer); + stopped = true; + if (timer !== null) clearTimeout(timer); }, }; } + +/** + * The user key funding this run is unusable (expired, undecryptable, or + * refused by OpenAI): drop it and return the investigation to PENDING without + * re-enqueueing. It is now unfunded; the selector or a new request can fund + * it. The failed attempt is still recorded. Returns false if this worker no + * longer holds the lease. + */ +export async function releaseLeaseDroppingUserKey(input: { + investigationId: string; + workerIdentity: string; + attemptNumber: number; + attemptAudit: InvestigatorAttemptAudit | null; +}): Promise { + return getPrisma().$transaction(async (tx) => { + const released = await tx.investigationLease.deleteMany({ + where: { investigationId: input.investigationId, leaseOwner: input.workerIdentity }, + }); + if (released.count === 0) { + return false; + } + + await tx.investigation.update({ + where: { id: input.investigationId }, + data: { status: "PENDING", queuedAt: new Date(), retryAfter: null }, + }); + await consumeOpenAiKeySource(tx, input.investigationId); + if (input.attemptAudit !== null) { + await persistAttemptAudit(tx, { + investigationId: input.investigationId, + attemptNumber: input.attemptNumber, + attemptAudit: input.attemptAudit, + }); + } + return true; + }); +} + +const investigationForRunInclude = { + input: true, + postVersion: { + select: { + contentBlob: { select: { contentText: true } }, + imageOccurrenceSet: { + select: { + occurrences: { + orderBy: [{ originalIndex: "asc" }], + select: { + originalIndex: true, + normalizedTextOffset: true, + sourceUrl: true, + captionText: true, + }, + }, + }, + }, + post: { select: { platform: true } }, + }, + }, + parentInvestigation: { + include: { + claims: { + include: { + sources: true, + }, + }, + }, + }, +} satisfies Prisma.InvestigationInclude; + +export type InvestigationForRun = Prisma.InvestigationGetPayload<{ + include: typeof investigationForRunInclude; +}>; + +/** Everything a claimed run needs: the input snapshot, version text and images, and parent claims. */ +export async function loadClaimedInvestigation( + investigationId: string, +): Promise { + return getPrisma().investigation.findUnique({ + where: { id: investigationId }, + include: investigationForRunInclude, + }); +} diff --git a/src/typescript/api/src/lib/services/investigation-lifecycle.ts b/src/typescript/api/src/lib/services/investigation-lifecycle.ts deleted file mode 100644 index ea12e53..0000000 --- a/src/typescript/api/src/lib/services/investigation-lifecycle.ts +++ /dev/null @@ -1,420 +0,0 @@ -import { isUniqueConstraintError } from "$lib/db/errors.js"; -import type { Investigation, Prisma, PrismaClient } from "$lib/db/prisma-client"; -import { - DEFAULT_INVESTIGATION_MODEL, - DEFAULT_INVESTIGATION_PROVIDER, - WORD_COUNT_LIMIT, -} from "@openerrata/shared"; -import { resolveMarkdownForInvestigation } from "./markdown-resolution.js"; -import type { HtmlSnapshots } from "./prompt-context.js"; -import { enqueueInvestigation } from "./queue.js"; -import { randomUUID } from "node:crypto"; - -export class InvestigationWordLimitError extends Error { - readonly limit: number; - readonly observedWordCount: number; - - constructor(observedWordCount: number, limit: number) { - super(`Post exceeds word count limit (${limit.toString()} words)`); - this.name = "InvestigationWordLimitError"; - this.observedWordCount = observedWordCount; - this.limit = limit; - } -} - -export function wordCount(text: string): number { - return text.split(/\s+/).filter(Boolean).length; -} - -async function findInvestigation( - prisma: PrismaClient, - postVersionId: string, -): Promise { - return prisma.investigation.findUnique({ - where: { - postVersionId, - }, - }); -} - -async function loadPostVersionWordCount( - prisma: PrismaClient, - postVersionId: string, -): Promise { - const postVersion = await prisma.postVersion.findUnique({ - where: { id: postVersionId }, - select: { - contentBlob: { - select: { - wordCount: true, - }, - }, - }, - }); - - if (postVersion === null) { - throw new Error(`PostVersion ${postVersionId} not found`); - } - - return postVersion.contentBlob.wordCount; -} - -const postVersionForInputSnapshotSelect = { - post: { - select: { - platform: true, - }, - }, - serverVerifiedAt: true, - contentBlob: { - select: { - contentHash: true, - }, - }, - lesswrongVersionMeta: { - select: { - serverHtmlBlob: { select: { htmlContent: true } }, - clientHtmlBlob: { select: { htmlContent: true } }, - }, - }, - substackVersionMeta: { - select: { - serverHtmlBlob: { select: { htmlContent: true } }, - clientHtmlBlob: { select: { htmlContent: true } }, - }, - }, - wikipediaVersionMeta: { - select: { - serverHtmlBlob: { select: { htmlContent: true } }, - clientHtmlBlob: { select: { htmlContent: true } }, - }, - }, -} satisfies Prisma.PostVersionSelect; - -type PostVersionForInputSnapshot = Prisma.PostVersionGetPayload<{ - select: typeof postVersionForInputSnapshotSelect; -}>; - -type InvestigationInputSnapshot = - | { - provenance: "SERVER_VERIFIED" | "CLIENT_FALLBACK"; - contentHash: string; - markdownSource: "NONE"; - } - | { - provenance: "SERVER_VERIFIED" | "CLIENT_FALLBACK"; - contentHash: string; - markdownSource: "SERVER_HTML" | "CLIENT_HTML"; - markdown: string; - markdownRendererVersion: string; - }; - -function unreachablePlatform(platform: never): never { - throw new Error(`Unsupported post platform: ${String(platform)}`); -} - -function resolveHtmlSnapshotsFromPostVersion( - postVersion: PostVersionForInputSnapshot, -): HtmlSnapshots { - const platform = postVersion.post.platform; - let serverHtml: string | null; - let clientHtml: string | null; - switch (platform) { - case "LESSWRONG": - serverHtml = postVersion.lesswrongVersionMeta?.serverHtmlBlob?.htmlContent ?? null; - clientHtml = postVersion.lesswrongVersionMeta?.clientHtmlBlob?.htmlContent ?? null; - break; - case "SUBSTACK": - serverHtml = postVersion.substackVersionMeta?.serverHtmlBlob?.htmlContent ?? null; - clientHtml = postVersion.substackVersionMeta?.clientHtmlBlob?.htmlContent ?? null; - break; - case "WIKIPEDIA": - serverHtml = postVersion.wikipediaVersionMeta?.serverHtmlBlob?.htmlContent ?? null; - clientHtml = postVersion.wikipediaVersionMeta?.clientHtmlBlob?.htmlContent ?? null; - break; - case "X": - serverHtml = null; - clientHtml = null; - break; - default: - return unreachablePlatform(platform); - } - - if (postVersion.serverVerifiedAt !== null) { - if (serverHtml === null) { - throw new Error( - `serverVerifiedAt is set but serverHtml is missing for platform ${platform} — violates DB invariant (serverVerifiedAt IS NOT NULL → serverHtmlBlobId IS NOT NULL)`, - ); - } - return { serverVerifiedAt: postVersion.serverVerifiedAt, serverHtml, clientHtml }; - } - return { serverVerifiedAt: null, serverHtml, clientHtml }; -} - -async function loadInvestigationInputSnapshot( - prisma: PrismaClient, - postVersionId: string, -): Promise { - const postVersion = await prisma.postVersion.findUnique({ - where: { id: postVersionId }, - select: postVersionForInputSnapshotSelect, - }); - if (postVersion === null) { - throw new Error(`PostVersion ${postVersionId} not found`); - } - - const htmlSnapshots = resolveHtmlSnapshotsFromPostVersion(postVersion); - const markdownResolution = resolveMarkdownForInvestigation({ - platform: postVersion.post.platform, - snapshots: htmlSnapshots, - }); - const provenance = - htmlSnapshots.serverVerifiedAt !== null - ? ("SERVER_VERIFIED" as const) - : ("CLIENT_FALLBACK" as const); - - if (markdownResolution.source === "NONE") { - return { - provenance, - contentHash: postVersion.contentBlob.contentHash, - markdownSource: "NONE", - }; - } - - return { - provenance, - contentHash: postVersion.contentBlob.contentHash, - markdownSource: markdownResolution.source, - markdown: markdownResolution.markdown, - markdownRendererVersion: markdownResolution.rendererVersion, - }; -} - -async function createInvestigation( - prisma: PrismaClient, - input: { - postVersionId: string; - promptId: string; - snapshot: InvestigationInputSnapshot; - parentInvestigationId?: string; - contentDiff?: string; - }, -): Promise { - const investigationId = randomUUID(); - const now = new Date(); - return prisma.$transaction(async (tx) => { - await tx.investigationInput.create({ - data: { - investigationId, - provenance: input.snapshot.provenance, - contentHash: input.snapshot.contentHash, - markdownSource: input.snapshot.markdownSource, - ...(input.snapshot.markdownSource === "NONE" - ? {} - : { - markdown: input.snapshot.markdown, - markdownRendererVersion: input.snapshot.markdownRendererVersion, - }), - }, - }); - - return tx.investigation.create({ - data: { - id: investigationId, - inputId: investigationId, - postVersionId: input.postVersionId, - status: "PENDING", - parentInvestigationId: input.parentInvestigationId ?? null, - contentDiff: input.contentDiff ?? null, - promptId: input.promptId, - provider: DEFAULT_INVESTIGATION_PROVIDER, - model: DEFAULT_INVESTIGATION_MODEL, - queuedAt: now, - }, - }); - }); -} - -/** - * Recover a stale PROCESSING investigation whose lease has expired. - * Deletes the expired InvestigationLease row and transitions - * PROCESSING → PENDING. - */ -async function tryRecoverExpiredProcessingInvestigation( - prisma: PrismaClient, - investigationId: string, -): Promise { - const now = new Date(); - - return prisma.$transaction(async (tx) => { - // Try to delete an expired lease row. - const deleted = await tx.investigationLease.deleteMany({ - where: { investigationId, leaseExpiresAt: { lte: now } }, - }); - - if (deleted.count === 0) { - // No expired lease was deleted. Check whether a non-expired lease exists. - const activeLease = await tx.investigationLease.findUnique({ - where: { investigationId }, - select: { investigationId: true }, - }); - if (activeLease) return null; // Active lease, can't recover - - const investigationStatus = await tx.investigation.findUnique({ - where: { id: investigationId }, - select: { status: true }, - }); - if (!investigationStatus) return null; - if (investigationStatus.status !== "PROCESSING") { - // Another concurrent caller likely recovered/transitioned this row - // between our candidate selection and recovery attempt. - return null; - } - - // PROCESSING with no lease row at all is an invariant violation: - // the InvestigationLease row's existence IS the PROCESSING state. - // The migration cleans these up, so hitting this in production - // indicates a bug in lease lifecycle management. - throw new Error( - `Invariant violation: PROCESSING investigation ${investigationId} has no InvestigationLease row. ` + - `This state should not be reachable — lease row existence is required for PROCESSING status.`, - ); - } - - // Expired lease deleted — transition PROCESSING → PENDING - const recovered = await tx.investigation.updateMany({ - where: { id: investigationId, status: "PROCESSING" }, - data: { status: "PENDING", queuedAt: now }, - }); - - if (recovered.count === 0) return null; - - const investigation = await tx.investigation.findUnique({ - where: { id: investigationId }, - }); - - if (!investigation) { - throw new Error( - `Missing investigation during stale-run recovery (investigationId=${investigationId})`, - ); - } - - return investigation; - }); -} - -interface EnsureInvestigationInput { - prisma: PrismaClient; - postVersionId: string; - promptId: string; - parentInvestigationId?: string; - contentDiff?: string; - rejectOverWordLimitOnCreate?: boolean; - allowRequeueFailed?: boolean; - enqueue?: boolean; - onPendingInvestigation?: (input: { - prisma: PrismaClient; - investigation: Investigation; - }) => Promise; -} - -async function ensureInvestigationRecord(input: EnsureInvestigationInput): Promise<{ - investigation: Investigation; - created: boolean; -}> { - const rejectOverWordLimitOnCreate = input.rejectOverWordLimitOnCreate ?? true; - const allowRequeueFailed = input.allowRequeueFailed ?? false; - - let investigation = await findInvestigation(input.prisma, input.postVersionId); - let created = false; - - if (!investigation) { - if (rejectOverWordLimitOnCreate) { - const observedWordCount = await loadPostVersionWordCount(input.prisma, input.postVersionId); - if (observedWordCount > WORD_COUNT_LIMIT) { - throw new InvestigationWordLimitError(observedWordCount, WORD_COUNT_LIMIT); - } - } - - try { - const snapshot = await loadInvestigationInputSnapshot(input.prisma, input.postVersionId); - const createInput: Parameters[1] = { - postVersionId: input.postVersionId, - promptId: input.promptId, - snapshot, - }; - if (input.parentInvestigationId !== undefined) { - createInput.parentInvestigationId = input.parentInvestigationId; - } - if (input.contentDiff !== undefined) { - createInput.contentDiff = input.contentDiff; - } - investigation = await createInvestigation(input.prisma, createInput); - created = true; - } catch (error) { - if (!isUniqueConstraintError(error)) throw error; - investigation = await findInvestigation(input.prisma, input.postVersionId); - if (!investigation) throw error; - } - } - - if (allowRequeueFailed && investigation.status === "FAILED") { - const failedInvestigation = investigation; // capture narrow type for async callback - investigation = await input.prisma.$transaction(async (tx) => { - // Defensive cleanup for any leftover lease row from prior failures. - await tx.investigationLease.deleteMany({ - where: { investigationId: failedInvestigation.id }, - }); - - return tx.investigation.update({ - where: { id: failedInvestigation.id }, - data: { - parentInvestigationId: input.parentInvestigationId ?? null, - contentDiff: input.contentDiff ?? null, - status: "PENDING", - checkedAt: null, - queuedAt: new Date(), - attemptCount: 0, - retryAfter: null, - }, - }); - }); - } - - return { investigation, created }; -} - -export async function ensureInvestigationQueued(input: EnsureInvestigationInput): Promise<{ - investigation: Investigation; - created: boolean; - enqueued: boolean; -}> { - const { investigation: initialInvestigation, created } = await ensureInvestigationRecord(input); - let investigation = initialInvestigation; - - // Recover stale PROCESSING investigations with expired leases - if (investigation.status === "PROCESSING") { - const recovered = await tryRecoverExpiredProcessingInvestigation( - input.prisma, - investigation.id, - ); - if (recovered) { - investigation = recovered; - } - } - - const shouldEnqueue = input.enqueue ?? true; - let enqueued = false; - if (shouldEnqueue && investigation.status === "PENDING") { - if (input.onPendingInvestigation) { - await input.onPendingInvestigation({ - prisma: input.prisma, - investigation, - }); - } - await enqueueInvestigation(investigation.id); - enqueued = true; - } - - return { investigation, created, enqueued }; -} diff --git a/src/typescript/api/src/lib/services/markdown-resolution.ts b/src/typescript/api/src/lib/services/markdown-resolution.ts index 02354da..b07097b 100644 --- a/src/typescript/api/src/lib/services/markdown-resolution.ts +++ b/src/typescript/api/src/lib/services/markdown-resolution.ts @@ -12,8 +12,6 @@ */ import type { Platform } from "@openerrata/shared"; -import type { ImagePlaceholder } from "$lib/investigators/interface.js"; -import type { HtmlSnapshots } from "./prompt-context.js"; import { lesswrongHtmlToContentMarkdown, substackHtmlToContentMarkdown, @@ -21,18 +19,24 @@ import { MARKDOWN_RENDERER_VERSION, } from "./html-to-markdown.js"; +/** + * Source-scoped HTML snapshots with the serverVerifiedAt latch bundled in. + * + * The discriminated union encodes the DB invariant: + * serverVerifiedAt IS NOT NULL → serverHtmlBlobId IS NOT NULL + * When server-verified, serverHtml is guaranteed non-null at the type level. + */ +export type HtmlSnapshots = + | { serverVerifiedAt: Date; serverHtml: string; clientHtml: string | null } + | { serverVerifiedAt: null; serverHtml: string | null; clientHtml: string | null }; + type MarkdownResolution = | { - source: "SERVER_HTML"; + source: "SERVER_HTML" | "CLIENT_HTML"; markdown: string; rendererVersion: string; - imagePlaceholders: ImagePlaceholder[]; - } - | { - source: "CLIENT_HTML"; - markdown: string; - rendererVersion: string; - imagePlaceholders: ImagePlaceholder[]; + /** Source URL of the image behind `[IMAGE:N]`, indexed by N. */ + imageSourceUrls: string[]; } | { source: "NONE" }; @@ -44,34 +48,27 @@ type MarkdownResolution = * - serverVerifiedAt non-null branch → serverHtml: string guaranteed by type * - serverVerifiedAt null + clientHtml non-null → CLIENT_HTML * - otherwise → NONE (X posts, or versions without HTML snapshots) + * + * `postUrl` is the base for resolving relative image sources. */ export function resolveMarkdownForInvestigation(input: { platform: Platform; snapshots: HtmlSnapshots; + postUrl: string; }): MarkdownResolution { if (input.snapshots.serverVerifiedAt !== null) { - const { markdown, imagePlaceholders } = platformMarkdown( - input.platform, - input.snapshots.serverHtml, - ); return { source: "SERVER_HTML", - markdown, rendererVersion: MARKDOWN_RENDERER_VERSION, - imagePlaceholders, + ...platformMarkdown(input.platform, input.snapshots.serverHtml, input.postUrl), }; } if (input.snapshots.clientHtml !== null) { - const { markdown, imagePlaceholders } = platformMarkdown( - input.platform, - input.snapshots.clientHtml, - ); return { source: "CLIENT_HTML", - markdown, rendererVersion: MARKDOWN_RENDERER_VERSION, - imagePlaceholders, + ...platformMarkdown(input.platform, input.snapshots.clientHtml, input.postUrl), }; } @@ -81,40 +78,18 @@ export function resolveMarkdownForInvestigation(input: { function platformMarkdown( platform: Platform, html: string, -): { markdown: string; imagePlaceholders: ImagePlaceholder[] } { + postUrl: string, +): { markdown: string; imageSourceUrls: string[] } { switch (platform) { case "LESSWRONG": - return lesswrongHtmlToContentMarkdown(html); + return lesswrongHtmlToContentMarkdown(html, postUrl); case "SUBSTACK": - return substackHtmlToContentMarkdown(html); + return substackHtmlToContentMarkdown(html, postUrl); case "WIKIPEDIA": - return wikipediaHtmlToContentMarkdown(html); + return wikipediaHtmlToContentMarkdown(html, postUrl); case "X": // X has no HTML; resolveMarkdownForInvestigation returns NONE before // reaching here. If this fires, the caller has a bug. throw new Error("platformMarkdown called for X, which has no HTML content"); } } - -/** - * Extract image placeholders from stored markdown by parsing `[IMAGE:N]` patterns. - * - * Used on retry to reconstruct placeholders from the InvestigationInput snapshot - * without re-resolving from HTML. - */ -export function extractImagePlaceholdersFromMarkdown(markdown: string): ImagePlaceholder[] { - const placeholders: ImagePlaceholder[] = []; - const pattern = /\[IMAGE:(\d+)\]/g; - let match: RegExpExecArray | null; - - while ((match = pattern.exec(markdown)) !== null) { - placeholders.push({ - index: parseInt(match[1] ?? "0", 10), - // sourceUrl is not recoverable from markdown alone; retries must match - // placeholders to image occurrences by originalIndex. - matchBy: "ORIGINAL_INDEX", - }); - } - - return placeholders; -} diff --git a/src/typescript/api/src/lib/services/openai-key-validation-core.ts b/src/typescript/api/src/lib/services/openai-key-validation-core.ts index f7f58a9..096bd8c 100644 --- a/src/typescript/api/src/lib/services/openai-key-validation-core.ts +++ b/src/typescript/api/src/lib/services/openai-key-validation-core.ts @@ -1,25 +1,66 @@ +import OpenAI from "openai"; import { openaiApiKeyFormatSchema } from "@openerrata/shared"; -import { - classifyOpenAiKeyValidationStatus, - readOpenAiStatusCode, - type OpenAiKeyValidationStatusOutcome, -} from "$lib/openai/errors.js"; +import { INVESTIGATION_REQUEST_CONFIG } from "$lib/investigators/openai-request-config.js"; +import { probeInvestigationRequest } from "$lib/investigators/openai-probe.js"; -export type { OpenAiKeyValidationStatusOutcome }; - -type ValidateOpenAiKeyReachability = (openAiApiKey: string) => Promise; - -function isAbortError(error: unknown): boolean { - return error instanceof Error && error.name === "AbortError"; -} +export type OpenAiKeyValidationStatusOutcome = + | { openaiApiKeyStatus: "missing" } + | { openaiApiKeyStatus: "valid" } + | { + openaiApiKeyStatus: "format_invalid"; + openaiApiKeyMessage: string; + } + | { + openaiApiKeyStatus: "authenticated_restricted"; + openaiApiKeyMessage: string; + } + | { + openaiApiKeyStatus: "invalid"; + openaiApiKeyMessage: string; + } + | { + openaiApiKeyStatus: "error"; + openaiApiKeyMessage: string; + }; -function readErrorMessage(error: unknown): string | null { - return error instanceof Error && error.message.trim().length > 0 ? error.message.trim() : null; +function describeProbeFailure(error: unknown): OpenAiKeyValidationStatusOutcome { + if (error instanceof OpenAI.AuthenticationError) { + return { openaiApiKeyStatus: "invalid", openaiApiKeyMessage: "OpenAI rejected this API key." }; + } + if (error instanceof OpenAI.PermissionDeniedError) { + return { + openaiApiKeyStatus: "authenticated_restricted", + openaiApiKeyMessage: + "OpenAI authenticated this key, but access is restricted for validation checks.", + }; + } + if (error instanceof OpenAI.APIConnectionTimeoutError) { + return { + openaiApiKeyStatus: "error", + openaiApiKeyMessage: + "OpenAI key validation timed out. Confirm outbound network access and retry.", + }; + } + // Anything else (no model access, rejected request shape, rate limit, + // network failure) is reported with the provider's own explanation. + const message = error instanceof Error ? error.message.trim() : ""; + return { + openaiApiKeyStatus: "error", + openaiApiKeyMessage: + message.length > 0 + ? message + : "Could not validate this key with OpenAI. Check outbound network access and retry.", + }; } -export async function validateOpenAiApiKeyForSettingsWithReachability( +/** + * Settings-page key check: the key must be able to make the investigation + * request itself (same probe the worker runs at startup), so a key without + * access to the investigation model is not reported as valid. + */ +export async function validateOpenAiApiKeyForSettingsWithClient( openaiApiKey: string | null, - validateOpenAiApiKeyReachability: ValidateOpenAiKeyReachability, + createClient: (apiKey: string) => OpenAI, ): Promise { const normalizedOpenAiApiKey = openaiApiKey?.trim() ?? ""; if (normalizedOpenAiApiKey.length === 0) { @@ -35,34 +76,12 @@ export async function validateOpenAiApiKeyForSettingsWithReachability( } try { - await validateOpenAiApiKeyReachability(normalizedOpenAiApiKey); + await probeInvestigationRequest( + createClient(normalizedOpenAiApiKey), + INVESTIGATION_REQUEST_CONFIG, + ); return { openaiApiKeyStatus: "valid" }; } catch (error) { - if (isAbortError(error)) { - return { - openaiApiKeyStatus: "error", - openaiApiKeyMessage: - "OpenAI key validation timed out. Confirm outbound network access and retry.", - }; - } - - const statusOutcome = classifyOpenAiKeyValidationStatus(readOpenAiStatusCode(error)); - if (statusOutcome && statusOutcome.openaiApiKeyStatus !== "error") { - return statusOutcome; - } - - const specificMessage = readErrorMessage(error); - if (specificMessage !== null) { - return { - openaiApiKeyStatus: "error", - openaiApiKeyMessage: specificMessage, - }; - } - - return { - openaiApiKeyStatus: "error", - openaiApiKeyMessage: - "Could not validate this key with OpenAI. Check outbound network access and retry.", - }; + return describeProbeFailure(error); } } diff --git a/src/typescript/api/src/lib/services/openai-key-validation.ts b/src/typescript/api/src/lib/services/openai-key-validation.ts index c17b3a8..ac56981 100644 --- a/src/typescript/api/src/lib/services/openai-key-validation.ts +++ b/src/typescript/api/src/lib/services/openai-key-validation.ts @@ -1,37 +1,17 @@ import { OPENAI_KEY_VALIDATION_TIMEOUT_MS } from "@openerrata/shared"; import OpenAI from "openai"; -import { getEnv } from "$lib/config/env.js"; import { - validateOpenAiApiKeyForSettingsWithReachability, + validateOpenAiApiKeyForSettingsWithClient, type OpenAiKeyValidationStatusOutcome, } from "./openai-key-validation-core.js"; -async function validateOpenAiApiKeyReachability(openAiApiKey: string): Promise { - const client = new OpenAI({ apiKey: openAiApiKey }); - const abortController = new AbortController(); - const timeoutId = setTimeout(() => { - abortController.abort(); - }, OPENAI_KEY_VALIDATION_TIMEOUT_MS); - - try { - await client.responses.create( - { - model: getEnv().OPENAI_MODEL_ID, - input: "Reply with the single word pong.", - max_output_tokens: 16, - }, - { signal: abortController.signal }, - ); - } finally { - clearTimeout(timeoutId); - } -} - export async function validateOpenAiApiKeyForSettings( openaiApiKey: string | null, ): Promise { - return validateOpenAiApiKeyForSettingsWithReachability( + return validateOpenAiApiKeyForSettingsWithClient( openaiApiKey, - validateOpenAiApiKeyReachability, + // One bounded try: SDK retries would stack further timeouts behind a + // settings request the user is waiting on. + (apiKey) => new OpenAI({ apiKey, timeout: OPENAI_KEY_VALIDATION_TIMEOUT_MS, maxRetries: 0 }), ); } diff --git a/src/typescript/api/src/lib/services/orchestrator-errors.ts b/src/typescript/api/src/lib/services/orchestrator-errors.ts index 3e9c432..953a4f3 100644 --- a/src/typescript/api/src/lib/services/orchestrator-errors.ts +++ b/src/typescript/api/src/lib/services/orchestrator-errors.ts @@ -1,11 +1,17 @@ import { ZodError } from "zod"; -import { isNonRetryableOpenAiStatusCode, readOpenAiStatusCode } from "$lib/openai/errors.js"; +import { readOpenAiStatusCode } from "$lib/openai/errors.js"; import { InvestigatorExecutionError, + InvestigatorIncompleteResponseError, + InvestigatorInputError, InvestigatorStructuredOutputError, -} from "$lib/investigators/openai.js"; +} from "$lib/investigators/errors.js"; import { ExpiredOpenAiKeySourceError, InvalidOpenAiKeySourceError } from "./user-key-source.js"; +// Provider statuses that a retry of the same request cannot fix (SPEC §3.7): +// malformed request, auth, missing model/resource, unprocessable input. +const NON_RETRYABLE_OPENAI_STATUS_CODES = new Set([400, 401, 403, 404, 422]); + type UnwrappedError = Error | Record | string; export function unwrapError(error: unknown): UnwrappedError { @@ -20,9 +26,7 @@ export function unwrapError(error: unknown): UnwrappedError { } export function getErrorStatus(error: unknown): number | null { - const root = unwrapError(error); - if (typeof root === "string") return null; - return readOpenAiStatusCode(root); + return readOpenAiStatusCode(unwrapError(error)); } export function formatErrorForLog(error: unknown): string { @@ -34,9 +38,13 @@ export function formatErrorForLog(error: unknown): string { if (typeof root === "string") { return root; } - return status === null ? "unknown object error" : `status=${status}`; + return "unknown object error"; } +/** + * NON_RETRYABLE and PARTIAL failures of SPEC §3.7: the investigation is marked + * FAILED immediately. Everything else is TRANSIENT and retried with backoff. + */ export function isNonRetryableProviderError(error: unknown): boolean { const root = unwrapError(error); if (root instanceof ExpiredOpenAiKeySourceError) return true; @@ -44,7 +52,9 @@ export function isNonRetryableProviderError(error: unknown): boolean { if (root instanceof SyntaxError) return true; if (root instanceof ZodError) return true; if (root instanceof InvestigatorStructuredOutputError) return true; + if (root instanceof InvestigatorIncompleteResponseError) return true; + if (root instanceof InvestigatorInputError) return true; const status = getErrorStatus(root); - return isNonRetryableOpenAiStatusCode(status); + return status !== null && NON_RETRYABLE_OPENAI_STATUS_CODES.has(status); } diff --git a/src/typescript/api/src/lib/services/orchestrator.ts b/src/typescript/api/src/lib/services/orchestrator.ts index f82d29b..dda2515 100644 --- a/src/typescript/api/src/lib/services/orchestrator.ts +++ b/src/typescript/api/src/lib/services/orchestrator.ts @@ -2,12 +2,21 @@ import { getPrisma } from "$lib/db/client"; import { requireOpenAiApiKey } from "$lib/config/env.js"; import { isRecordNotFoundError } from "$lib/db/errors.js"; import { downloadAndStoreImages, type ResolvedDownloadedImage } from "./image-downloader.js"; -import { consumeOpenAiKeySource, resolveInvestigationKey } from "./user-key-source.js"; -import { InvestigatorExecutionError, OpenAIInvestigator } from "$lib/investigators/openai.js"; +import { + consumeOpenAiKeySource, + ExpiredOpenAiKeySourceError, + InvalidOpenAiKeySourceError, + resolveInvestigationKey, + type InvestigationKeyResolution, +} from "./user-key-source.js"; +import { InvestigatorExecutionError } from "$lib/investigators/errors.js"; import type { + ImagePlaceholder, InvestigationProgressCallbacks, - InvestigatorAttemptAudit, + InvestigatorFactory, InvestigatorImageOccurrence, + InvestigatorInput, + InvestigatorSucceededAttemptAudit, } from "$lib/investigators/interface.js"; import { claimIdSchema, @@ -16,16 +25,21 @@ import { type SupportedImageMimeType, } from "@openerrata/shared"; import type { ImageBlob, Prisma } from "$lib/db/prisma-client"; -import { createHash } from "node:crypto"; -import { formatErrorForLog, isNonRetryableProviderError } from "./orchestrator-errors.js"; -import { toPromptPostContext, type PromptImageOccurrence } from "./prompt-context.js"; +import { + formatErrorForLog, + getErrorStatus, + isNonRetryableProviderError, +} from "./orchestrator-errors.js"; import { tryClaimLease, loadClaimedInvestigation, - startHeartbeat, + startLeaseHeartbeat, + releaseLeaseDroppingUserKey, + retryBackoffMs, + LeaseLostError, MAX_INVESTIGATION_ATTEMPTS, - BASE_BACKOFF_MS, + type InvestigationForRun, type Logger, } from "./investigation-lease.js"; import { @@ -33,31 +47,55 @@ import { persistFailedAttemptAndMarkInvestigationFailed, persistFailedAttemptAndReleaseLease, } from "./attempt-audit.js"; -import { extractImagePlaceholdersFromMarkdown } from "./markdown-resolution.js"; import { enqueueInvestigation } from "./queue.js"; -let serverInvestigator: OpenAIInvestigator | null = null; - -function getServerInvestigator(): OpenAIInvestigator { - if (serverInvestigator) { - return serverInvestigator; +/** + * OpenAI statuses that say "this user key cannot pay for this run" rather than + * anything about the post: rejected (401), not permitted or no model access + * (403, 404), rate-limited or out of quota (429). Retrying on the same key + * cannot help, and letting such failures exhaust attempts or mark the + * investigation FAILED would let a bad key block a post from ever being + * checked — so the key is dropped instead. + */ +const USER_KEY_ATTRIBUTABLE_STATUS_CODES: ReadonlySet = new Set([401, 403, 404, 429]); + +/** Whether `error`, raised while running on `keyType`, means the user key is unusable. */ +function isUserKeyFailure( + error: unknown, + keyType: InvestigationKeyResolution["type"] | null, +): boolean { + if ( + error instanceof ExpiredOpenAiKeySourceError || + error instanceof InvalidOpenAiKeySourceError + ) { + return true; } - - serverInvestigator = new OpenAIInvestigator(requireOpenAiApiKey()); - return serverInvestigator; -} - -function hashSnapshotText(snapshotText: string): string { - return createHash("sha256").update(snapshotText).digest("hex"); + if (keyType !== "USER_OPENAI_KEY") { + return false; + } + const status = getErrorStatus(error); + return status !== null && USER_KEY_ATTRIBUTABLE_STATUS_CODES.has(status); } +/** Replace the investigation's image set, provided this worker still holds the lease. */ async function replaceInvestigationImages( - investigationId: string, + lease: { investigationId: string; workerIdentity: string; leaseLostSignal: AbortSignal }, imageBlobs: ImageBlob[], ): Promise { + const { investigationId } = lease; const uniqueBlobs = [...new Map(imageBlobs.map((b) => [b.id, b])).values()]; + lease.leaseLostSignal.throwIfAborted(); await getPrisma().$transaction(async (tx) => { + // Lock and verify our lease row so a reclaimed run cannot clobber images. + const held = await tx.investigationLease.updateMany({ + where: { investigationId, leaseOwner: lease.workerIdentity }, + data: { heartbeatAt: new Date() }, + }); + if (held.count === 0) { + throw new LeaseLostError(investigationId, "lease not held when writing images"); + } + await tx.investigationImage.deleteMany({ where: { investigationId }, }); @@ -78,35 +116,51 @@ function toDataUri(bytes: Uint8Array, mimeType: SupportedImageMimeType): string return `data:${mimeType};base64,${Buffer.from(bytes).toString("base64")}`; } -function uniqueUrlsInOrder(urls: string[]): string[] { - const seen = new Set(); - const unique: string[] = []; - - for (const url of urls) { - if (seen.has(url)) continue; - seen.add(url); - unique.push(url); +/** + * Canonical form of an image URL, shared by image occurrences, markdown + * placeholders and downloads so they match each other. Null for URLs that can + * never be fetched. + */ +function canonicalImageUrl(url: string): string | null { + let parsed: URL; + try { + parsed = new URL(url); + } catch { + return null; } + return parsed.protocol === "http:" || parsed.protocol === "https:" ? parsed.toString() : null; +} - return unique; +interface StoredImageOccurrence { + originalIndex: number; + normalizedTextOffset: number; + sourceUrl: string; + captionText: string | null; } async function resolvePromptImageOccurrences( - investigationId: string, - imageOccurrences: PromptImageOccurrence[], + lease: { investigationId: string; workerIdentity: string; leaseLostSignal: AbortSignal }, + storedOccurrences: StoredImageOccurrence[], ): Promise { - if (imageOccurrences.length === 0) { - await replaceInvestigationImages(investigationId, []); - return []; - } - - const uniqueSourceUrls = uniqueUrlsInOrder( - imageOccurrences.map((occurrence) => occurrence.sourceUrl), - ); + const imageOccurrences = storedOccurrences.map((occurrence) => ({ + originalIndex: occurrence.originalIndex, + normalizedTextOffset: occurrence.normalizedTextOffset, + sourceUrl: canonicalImageUrl(occurrence.sourceUrl) ?? occurrence.sourceUrl, + fetchable: canonicalImageUrl(occurrence.sourceUrl) !== null, + ...(occurrence.captionText === null ? {} : { captionText: occurrence.captionText }), + })); + + const uniqueSourceUrls = [ + ...new Set( + imageOccurrences + .filter((occurrence) => occurrence.fetchable) + .map((occurrence) => occurrence.sourceUrl), + ), + ]; const urlsWithinBudget = uniqueSourceUrls.slice(0, MAX_IMAGES_PER_INVESTIGATION); const omittedSourceUrls = new Set(uniqueSourceUrls.slice(MAX_IMAGES_PER_INVESTIGATION)); - const resolutions = await downloadAndStoreImages(urlsWithinBudget, MAX_IMAGES_PER_INVESTIGATION); + const resolutions = await downloadAndStoreImages(urlsWithinBudget, lease.leaseLostSignal); const resolvedBySourceUrl = new Map(); const uniqueResolvedBlobs = new Map(); @@ -118,11 +172,11 @@ async function resolvePromptImageOccurrences( } await replaceInvestigationImages( - investigationId, + lease, Array.from(uniqueResolvedBlobs.values()).map((image) => image.blob), ); - return imageOccurrences.map((occurrence) => { + return imageOccurrences.map(({ fetchable: _fetchable, ...occurrence }) => { if (omittedSourceUrls.has(occurrence.sourceUrl)) { return { ...occurrence, @@ -147,6 +201,56 @@ async function resolvePromptImageOccurrences( }); } +/** + * Investigator input for a claimed run, built from the immutable + * InvestigationInput snapshot plus the version's text and resolved images. + */ +function buildInvestigatorInput( + investigation: InvestigationForRun, + imageOccurrences: InvestigatorImageOccurrence[], +): InvestigatorInput { + const { input } = investigation; + const imagePlaceholders: ImagePlaceholder[] = input.imagePlaceholderSourceUrls.map( + (sourceUrl, index) => ({ index, matchBy: "SOURCE_URL", sourceUrl }), + ); + const base = { + contentText: investigation.postVersion.contentBlob.contentText, + ...(input.markdown === null ? {} : { contentMarkdown: input.markdown, imagePlaceholders }), + platform: investigation.postVersion.post.platform, + url: input.postUrl, + ...(input.authorName === null ? {} : { authorName: input.authorName }), + ...(input.postPublishedAt === null + ? {} + : { postPublishedAt: input.postPublishedAt.toISOString() }), + imageOccurrences, + ...(input.hasVideo ? { hasVideo: true } : {}), + }; + + if (investigation.parentInvestigationId === null) { + return base; + } + if (investigation.parentInvestigation === null) { + throw new Error(`Update investigation ${investigation.id} is missing parent investigation`); + } + return { + ...base, + isUpdate: true, + ...(investigation.contentDiff === null ? {} : { contentDiff: investigation.contentDiff }), + oldClaims: investigation.parentInvestigation.claims.map((claim) => ({ + id: claimIdSchema.parse(claim.id), + text: claim.text, + context: claim.context, + summary: claim.summary, + reasoning: claim.reasoning, + sources: claim.sources.map((source) => ({ + url: source.url, + title: source.title, + snippet: source.snippet, + })), + })), + }; +} + /** * Guard-first persist: atomically transition PROCESSING → COMPLETE. * @@ -169,8 +273,10 @@ export async function persistCompletedInvestigation( workerIdentity: string; claims: InvestigationResult["claims"]; attemptNumber: number; - attemptAudit: InvestigatorAttemptAudit; - modelVersion: string | null; + attemptAudit: InvestigatorSucceededAttemptAudit; + /** Provider model id the fact-check ran on (INV-INV-MODEL-AT-COMPLETION). */ + model: string; + modelVersion: string; }, ): Promise { const released = await tx.investigationLease.deleteMany({ @@ -189,6 +295,7 @@ export async function persistCompletedInvestigation( data: { status: "COMPLETE", checkedAt: new Date(), + model: params.model, modelVersion: params.modelVersion, }, }); @@ -218,9 +325,6 @@ export async function persistCompletedInvestigation( url: s.url, title: s.title, snippet: s.snippet, - snapshotText: s.snippet, - snapshotHash: hashSnapshotText(s.snippet), - retrievedAt: new Date(), })), }, }, @@ -236,6 +340,7 @@ export async function orchestrateInvestigation( logger: Logger, options: { workerIdentity: string; + createInvestigator: InvestigatorFactory; }, ): Promise { const inFlightProgressWrites = new Set>(); @@ -256,48 +361,23 @@ export async function orchestrateInvestigation( } const claimResult = await tryClaimLease(investigationId, options.workerIdentity); - if (claimResult.outcome === "MISSING") { - logger.info(`Investigation ${investigationId} no longer exists; skipping stale job`); - return; - } - if (claimResult.outcome === "TERMINAL") { - logger.info(`Investigation ${investigationId} already terminal, skipping`); - return; - } - if (claimResult.outcome === "LEASE_HELD") { - logger.info(`Investigation ${investigationId} already leased, skipping`); - return; - } - if (claimResult.outcome === "ATTEMPTS_EXHAUSTED") { - logger.error( - `Investigation ${investigationId} exhausted ${MAX_INVESTIGATION_ATTEMPTS.toString()} attempts; marking FAILED`, - ); - const prismaForExhausted = getPrisma(); - await prismaForExhausted.$transaction(async (tx) => { - const now = new Date(); - // Defensive cleanup for stale/expired leases. Active leases are preserved. - await tx.investigationLease.deleteMany({ - where: { - investigationId, - leaseExpiresAt: { lte: now }, - }, - }); - // Match both PENDING (normal exhaustion path) and PROCESSING with no - // active lease row (stale reclaim rollback path). Avoid marking FAILED - // while another worker still holds an active lease. - const transitioned = await tx.investigation.updateMany({ - where: { - id: investigationId, - attemptCount: { gte: MAX_INVESTIGATION_ATTEMPTS }, - OR: [{ status: "PENDING" }, { status: "PROCESSING", lease: { is: null } }], - }, - data: { status: "FAILED" }, - }); - if (transitioned.count > 0) { - await consumeOpenAiKeySource(tx, investigationId); - } - }); - return; + switch (claimResult.outcome) { + case "MISSING": + logger.info(`Investigation ${investigationId} no longer exists; skipping stale job`); + return; + case "TERMINAL": + logger.info(`Investigation ${investigationId} already terminal, skipping`); + return; + case "LEASE_HELD": + logger.info(`Investigation ${investigationId} already leased, skipping`); + return; + case "UNFUNDED": + logger.info( + `Investigation ${investigationId} has no funding since its user key was dropped; skipping`, + ); + return; + case "CLAIMED": + break; } const { attemptNumber } = claimResult; @@ -310,39 +390,42 @@ export async function orchestrateInvestigation( const prisma = getPrisma(); - const heartbeat = startHeartbeat(investigationId, options.workerIdentity, logger); - + const heartbeat = startLeaseHeartbeat( + { + investigationId, + workerIdentity: options.workerIdentity, + leaseExpiresAt: claimResult.leaseExpiresAt, + }, + logger, + ); + // Aborts when this worker loses the lease, which stops every in-flight + // provider request, tool fetch and image download of this run. + const { leaseLostSignal } = heartbeat; + const lease = { + investigationId: investigation.id, + workerIdentity: options.workerIdentity, + leaseLostSignal, + }; + + let investigationKeyType: InvestigationKeyResolution["type"] | null = null; try { - const investigationKey = await resolveInvestigationKey(prisma, investigationId); - const investigator = - investigationKey.type === "SERVER_KEY" - ? getServerInvestigator() - : new OpenAIInvestigator(investigationKey.apiKey); - const promptPostContext = toPromptPostContext(investigation.postVersion); - - // ── Resolve or restore InvestigationInput snapshot ── - // All executions (first attempt and retries) must use the immutable - // InvestigationInput snapshot persisted at queue-time. - const contentMarkdown = investigation.input.markdown ?? undefined; - const imagePlaceholders = - contentMarkdown !== undefined - ? extractImagePlaceholdersFromMarkdown(contentMarkdown) - : undefined; + // Resolve the key before touching any attacker-chosen image URL: a + // user-key run whose key is unusable stops here. + const investigationKey = await resolveInvestigationKey(prisma, investigation); + investigationKeyType = investigationKey.type; + const investigator = options.createInvestigator( + investigationKey.type === "SERVER_KEY" ? requireOpenAiApiKey() : investigationKey.apiKey, + ); const resolvedImageOccurrences = await resolvePromptImageOccurrences( - investigation.id, - promptPostContext.imageOccurrences, + lease, + investigation.postVersion.imageOccurrenceSet.occurrences, ); - - if ( - investigation.parentInvestigationId !== null && - investigation.parentInvestigation === null - ) { - throw new Error(`Update investigation ${investigation.id} is missing parent investigation`); - } + const investigatorInput = buildInvestigatorInput(investigation, resolvedImageOccurrences); const progressCallbacks: InvestigationProgressCallbacks = { onProgressUpdate: (pending, confirmed) => { + if (leaseLostSignal.aborted) return; // Guard on leaseOwner to avoid writing progressClaims after a // terminal transition or lease reclaim (the lease row won't exist). const write = prisma.investigationLease @@ -362,37 +445,15 @@ export async function orchestrateInvestigation( }, }; - const output = await investigator.investigate( - { - contentText: investigation.postVersion.contentBlob.contentText, - ...promptPostContext, - ...(contentMarkdown !== undefined && { contentMarkdown }), - ...(imagePlaceholders !== undefined && { imagePlaceholders }), - imageOccurrences: resolvedImageOccurrences, - ...(promptPostContext.hasVideo ? { hasVideo: true } : {}), - ...(investigation.parentInvestigation !== null && { - isUpdate: true, - ...(investigation.contentDiff === null ? {} : { contentDiff: investigation.contentDiff }), - oldClaims: investigation.parentInvestigation.claims.map((claim) => ({ - id: claimIdSchema.parse(claim.id), - text: claim.text, - context: claim.context, - summary: claim.summary, - reasoning: claim.reasoning, - sources: claim.sources.map((source) => ({ - url: source.url, - title: source.title, - snippet: source.snippet, - })), - })), - }), - }, - progressCallbacks, - ); + const output = await investigator.investigate(investigatorInput, { + signal: leaseLostSignal, + callbacks: progressCallbacks, + }); // Ensure all progress writes settle before terminal transition. await flushProgressWrites(); + leaseLostSignal.throwIfAborted(); const completed = await prisma.$transaction((tx) => persistCompletedInvestigation(tx, { investigationId: investigation.id, @@ -400,7 +461,8 @@ export async function orchestrateInvestigation( claims: output.result.claims, attemptNumber, attemptAudit: output.attemptAudit, - modelVersion: output.modelVersion ?? null, + model: output.model, + modelVersion: output.modelVersion, }), ); @@ -417,6 +479,13 @@ export async function orchestrateInvestigation( // Drain callback writes so FAILED/lease-release transition is the final state. await flushProgressWrites(); + if (leaseLostSignal.aborted || error instanceof LeaseLostError) { + logger.warn( + `Investigation ${investigation.id} attempt ${attemptNumber.toString()} abandoned: ${formatErrorForLog(leaseLostSignal.aborted ? leaseLostSignal.reason : error)}`, + ); + return; + } + if (isRecordNotFoundError(error)) { logger.info( `Investigation ${investigation.id} disappeared during processing; skipping stale job`, @@ -426,6 +495,27 @@ export async function orchestrateInvestigation( const attemptAudit = error instanceof InvestigatorExecutionError ? error.attemptAudit : null; + // USER KEY UNUSABLE: drop the key; the investigation waits, unfunded, for + // the selector or a new request instead of failing. + if (isUserKeyFailure(error, investigationKeyType)) { + const released = await releaseLeaseDroppingUserKey({ + investigationId: investigation.id, + workerIdentity: options.workerIdentity, + attemptNumber, + attemptAudit, + }); + if (released) { + logger.warn( + `Investigation ${investigation.id} dropped its user OpenAI key and is unfunded: ${formatErrorForLog(error)}`, + ); + } else { + logger.info( + `Investigation ${investigation.id} no longer PROCESSING; ignoring user key failure`, + ); + } + return; + } + // NON_RETRYABLE: deterministic provider or parsing failures. if (isNonRetryableProviderError(error)) { const marked = await persistFailedAttemptAndMarkInvestigationFailed({ @@ -469,7 +559,7 @@ export async function orchestrateInvestigation( // Not last attempt — reclaim to PENDING and explicitly re-enqueue. // Do NOT rethrow to graphile-worker — we control retry timing ourselves. - const backoffMs = BASE_BACKOFF_MS * Math.pow(2, attemptNumber - 1); + const backoffMs = retryBackoffMs(attemptNumber); const retryAfter = new Date(Date.now() + backoffMs); const released = await persistFailedAttemptAndReleaseLease({ diff --git a/src/typescript/api/src/lib/services/prompt-context.ts b/src/typescript/api/src/lib/services/prompt-context.ts deleted file mode 100644 index ec5c914..0000000 --- a/src/typescript/api/src/lib/services/prompt-context.ts +++ /dev/null @@ -1,235 +0,0 @@ -import type { Platform } from "@openerrata/shared"; -import type { Prisma } from "$lib/db/prisma-client"; - -export interface PromptImageOccurrence { - originalIndex: number; - normalizedTextOffset: number; - sourceUrl: string; - captionText?: string; -} - -interface PromptPostContext { - platform: Platform; - url: string; - authorName?: string; - postPublishedAt?: string; - imageOccurrences: PromptImageOccurrence[]; - hasVideo?: boolean; -} - -/** Prisma include fragment that loads everything needed by `toPromptPostContext`. */ -export const investigationContextInclude = { - postVersion: { - select: { - serverVerifiedAt: true, - contentBlob: { - select: { - contentText: true, - contentHash: true, - }, - }, - imageOccurrenceSet: { - select: { - occurrences: { - orderBy: [{ originalIndex: "asc" }], - select: { - originalIndex: true, - normalizedTextOffset: true, - sourceUrl: true, - captionText: true, - }, - }, - }, - }, - lesswrongVersionMeta: { - select: { - publishedAt: true, - serverHtmlBlob: { select: { htmlContent: true } }, - clientHtmlBlob: { select: { htmlContent: true } }, - }, - }, - xVersionMeta: { - select: { - postedAt: true, - mediaUrls: true, - }, - }, - substackVersionMeta: { - select: { - publishedAt: true, - serverHtmlBlob: { select: { htmlContent: true } }, - clientHtmlBlob: { select: { htmlContent: true } }, - }, - }, - wikipediaVersionMeta: { - select: { - lastModifiedAt: true, - serverHtmlBlob: { select: { htmlContent: true } }, - clientHtmlBlob: { select: { htmlContent: true } }, - }, - }, - post: { - select: { - platform: true, - url: true, - author: { select: { displayName: true } }, - }, - }, - }, - }, -} satisfies Prisma.InvestigationInclude; - -type InvestigationWithContext = Prisma.InvestigationGetPayload<{ - include: typeof investigationContextInclude; -}>; -type InvestigationVersionContext = InvestigationWithContext["postVersion"]; - -function unreachablePlatform(platform: never): never { - throw new Error(`Unsupported post platform: ${String(platform)}`); -} - -export function isLikelyVideoUrl(url: string): boolean { - let pathname = url.toLowerCase(); - try { - pathname = new URL(url).pathname.toLowerCase(); - } catch { - // Keep best-effort behavior for malformed values already stored in metadata. - } - - return ( - pathname.endsWith(".mp4") || - pathname.endsWith(".webm") || - pathname.endsWith(".m3u8") || - pathname.endsWith(".mov") || - pathname.endsWith(".m4v") - ); -} - -export function hasXVideoMedia(mediaUrls: string[]): boolean { - for (const mediaUrl of mediaUrls) { - if (isLikelyVideoUrl(mediaUrl)) { - return true; - } - } - return false; -} - -/** - * Source-scoped HTML snapshots with the serverVerifiedAt latch bundled in. - * - * The discriminated union encodes the DB invariant: - * serverVerifiedAt IS NOT NULL → serverHtmlBlobId IS NOT NULL - * When server-verified, serverHtml is guaranteed non-null at the type level. - */ -export type HtmlSnapshots = - | { serverVerifiedAt: Date; serverHtml: string; clientHtml: string | null } - | { serverVerifiedAt: null; serverHtml: string | null; clientHtml: string | null }; - -/** - * Resolve source-scoped HTML snapshots from version metadata. - * - * Throws if serverVerifiedAt is set but serverHtml is absent — that state - * violates the DB trigger that enforces the invariant, so it represents data - * corruption and should surface immediately rather than silently falling back. - */ -export function resolveHtmlSnapshotsFromVersionMeta( - postVersion: InvestigationVersionContext, -): HtmlSnapshots { - const post = postVersion.post; - let serverHtml: string | null; - let clientHtml: string | null; - switch (post.platform) { - case "LESSWRONG": - serverHtml = postVersion.lesswrongVersionMeta?.serverHtmlBlob?.htmlContent ?? null; - clientHtml = postVersion.lesswrongVersionMeta?.clientHtmlBlob?.htmlContent ?? null; - break; - case "SUBSTACK": - serverHtml = postVersion.substackVersionMeta?.serverHtmlBlob?.htmlContent ?? null; - clientHtml = postVersion.substackVersionMeta?.clientHtmlBlob?.htmlContent ?? null; - break; - case "WIKIPEDIA": - serverHtml = postVersion.wikipediaVersionMeta?.serverHtmlBlob?.htmlContent ?? null; - clientHtml = postVersion.wikipediaVersionMeta?.clientHtmlBlob?.htmlContent ?? null; - break; - case "X": - serverHtml = null; - clientHtml = null; - break; - default: - return unreachablePlatform(post.platform); - } - - if (postVersion.serverVerifiedAt !== null) { - if (serverHtml === null) { - throw new Error( - `serverVerifiedAt is set but serverHtml is missing for platform ${post.platform} — violates DB invariant (serverVerifiedAt IS NOT NULL → serverHtmlBlobId IS NOT NULL)`, - ); - } - return { serverVerifiedAt: postVersion.serverVerifiedAt, serverHtml, clientHtml }; - } - return { serverVerifiedAt: null, serverHtml, clientHtml }; -} - -export function toPromptPostContext(postVersion: InvestigationVersionContext): PromptPostContext { - const post = postVersion.post; - const authorName = post.author?.displayName; - const imageOccurrences = postVersion.imageOccurrenceSet.occurrences.map((occurrence) => ({ - originalIndex: occurrence.originalIndex, - normalizedTextOffset: occurrence.normalizedTextOffset, - sourceUrl: occurrence.sourceUrl, - ...(occurrence.captionText === null ? {} : { captionText: occurrence.captionText }), - })); - - switch (post.platform) { - case "LESSWRONG": { - const publishedAt = postVersion.lesswrongVersionMeta?.publishedAt; - return { - platform: "LESSWRONG", - url: post.url, - ...(authorName != null && { authorName }), - ...(publishedAt != null && { postPublishedAt: publishedAt.toISOString() }), - imageOccurrences, - hasVideo: false, - }; - } - case "X": { - const postedAt = postVersion.xVersionMeta?.postedAt; - const mediaUrls = postVersion.xVersionMeta?.mediaUrls ?? []; - const hasVideo = hasXVideoMedia(mediaUrls); - return { - platform: "X", - url: post.url, - ...(authorName != null && { authorName }), - ...(postedAt != null && { postPublishedAt: postedAt.toISOString() }), - imageOccurrences, - hasVideo, - }; - } - case "SUBSTACK": { - const publishedAt = postVersion.substackVersionMeta?.publishedAt; - return { - platform: "SUBSTACK", - url: post.url, - ...(authorName != null && { authorName }), - ...(publishedAt != null && { postPublishedAt: publishedAt.toISOString() }), - imageOccurrences, - hasVideo: false, - }; - } - case "WIKIPEDIA": { - const lastModifiedAt = postVersion.wikipediaVersionMeta?.lastModifiedAt; - return { - platform: "WIKIPEDIA", - url: post.url, - ...(authorName != null && { authorName }), - ...(lastModifiedAt != null && { - postPublishedAt: lastModifiedAt.toISOString(), - }), - imageOccurrences, - hasVideo: false, - }; - } - default: - return unreachablePlatform(post.platform); - } -} diff --git a/src/typescript/api/src/lib/services/public-read-model.ts b/src/typescript/api/src/lib/services/public-read-model.ts index 6c798f8..ba962bb 100644 --- a/src/typescript/api/src/lib/services/public-read-model.ts +++ b/src/typescript/api/src/lib/services/public-read-model.ts @@ -1,10 +1,12 @@ import { Prisma, type PrismaClient } from "$lib/db/prisma-client"; -import { platformSchema, type Platform } from "@openerrata/shared"; +import type { ContentProvenance, InvestigationProvider, Platform } from "@openerrata/shared"; -interface PublicInvestigationOrigin { - provenance: "SERVER_VERIFIED" | "CLIENT_FALLBACK"; - serverVerifiedAt: Date | null; -} +// A SERVER_VERIFIED investigation ran on verified content, so its post +// version's verification latch is always set; a CLIENT_FALLBACK one's may be +// set later by a subsequent server fetch of the same content. +type PublicInvestigationOrigin = + | { provenance: Extract; serverVerifiedAt: Date } + | { provenance: Extract; serverVerifiedAt: Date | null }; interface PublicTrustSignals { origin: PublicInvestigationOrigin; @@ -30,7 +32,7 @@ type PublicInvestigation = PublicTrustSignals & { id: string; checkedAt: Date; promptVersion: string; - provider: string; + provider: InvestigationProvider; model: string; }; @@ -87,7 +89,8 @@ interface SearchInvestigationPageRow { interface PublicMetricsResult { totalInvestigatedPosts: number; investigatedPostsWithFlags: number; - factCheckIncidence: number; + /** investigatedPostsWithFlags / totalInvestigatedPosts; null when nothing was investigated. */ + factCheckIncidence: number | null; } interface PublicMetricsInput { @@ -105,10 +108,6 @@ interface PublicSearchInvestigationsInput { offset: number; } -function parsePlatform(value: string): Platform { - return platformSchema.parse(value); -} - function escapeLikePattern(query: string): string { return query.replace(/[\\%_]/g, (char) => `\\${char}`); } @@ -124,28 +123,6 @@ function invariantViolation(message: string): never { throw new PublicReadModelInvariantError(`Public read-model invariant violation: ${message}`); } -function parsePublicOrigin(input: { - investigationId: string; - provenance: string | undefined; - serverVerifiedAt: Date | null; -}): PublicInvestigationOrigin { - // provenance lives on InvestigationInput (1:1); older investigations may lack it. - if (input.provenance === undefined) { - invariantViolation( - `Investigation ${input.investigationId} has no InvestigationInput (missing provenance)`, - ); - } - if (input.provenance !== "SERVER_VERIFIED" && input.provenance !== "CLIENT_FALLBACK") { - invariantViolation( - `Investigation ${input.investigationId} has invalid provenance "${input.provenance}"`, - ); - } - return { - provenance: input.provenance, - serverVerifiedAt: input.serverVerifiedAt, - }; -} - function requireCompleteCheckedAt(input: { investigationId: string; checkedAt: Date | null; @@ -156,26 +133,44 @@ function requireCompleteCheckedAt(input: { return input.checkedAt; } -function parsePublicLifecycle(input: { - investigationId: string; - provenance: string | undefined; - serverVerifiedAt: Date | null; - checkedAt: Date | null; -}): { origin: PublicInvestigationOrigin; checkedAt: Date } { - const origin = parsePublicOrigin({ - investigationId: input.investigationId, - provenance: input.provenance, - serverVerifiedAt: input.serverVerifiedAt, - }); +function requireCompleteModel(input: { investigationId: string; model: string | null }): string { + if (input.model === null) { + invariantViolation(`Investigation ${input.investigationId} is COMPLETE with null model`); + } + return input.model; +} - const checkedAt = requireCompleteCheckedAt({ - investigationId: input.investigationId, - checkedAt: input.checkedAt, - }); +function publicOrigin(investigation: { + id: string; + input: { provenance: ContentProvenance }; + postVersion: { serverVerifiedAt: Date | null }; +}): PublicInvestigationOrigin { + const { serverVerifiedAt } = investigation.postVersion; + switch (investigation.input.provenance) { + case "SERVER_VERIFIED": + if (serverVerifiedAt === null) { + invariantViolation( + `Investigation ${investigation.id} is SERVER_VERIFIED but its post version has no serverVerifiedAt`, + ); + } + return { provenance: "SERVER_VERIFIED", serverVerifiedAt }; + case "CLIENT_FALLBACK": + return { provenance: "CLIENT_FALLBACK", serverVerifiedAt }; + } +} +function publicLifecycle(investigation: { + id: string; + input: { provenance: ContentProvenance }; + postVersion: { serverVerifiedAt: Date | null }; + checkedAt: Date | null; +}): { origin: PublicInvestigationOrigin; checkedAt: Date } { return { - origin, - checkedAt, + origin: publicOrigin(investigation), + checkedAt: requireCompleteCheckedAt({ + investigationId: investigation.id, + checkedAt: investigation.checkedAt, + }), }; } @@ -259,12 +254,7 @@ export async function getPublicInvestigationById( return null; } - const lifecycle = parsePublicLifecycle({ - investigationId: investigation.id, - provenance: investigation.input.provenance, - serverVerifiedAt: investigation.postVersion.serverVerifiedAt, - checkedAt: investigation.checkedAt, - }); + const lifecycle = publicLifecycle(investigation); return { investigation: { @@ -274,10 +264,13 @@ export async function getPublicInvestigationById( checkedAt: lifecycle.checkedAt, promptVersion: investigation.prompt.version, provider: investigation.provider, - model: investigation.model, + model: requireCompleteModel({ + investigationId: investigation.id, + model: investigation.model, + }), }, post: { - platform: parsePlatform(investigation.postVersion.post.platform), + platform: investigation.postVersion.post.platform, externalId: investigation.postVersion.post.externalId, url: investigation.postVersion.post.url, }, @@ -361,17 +354,12 @@ export async function getPublicPostInvestigations( return { post: { - platform: parsePlatform(post.platform), + platform: post.platform, externalId: post.externalId, url: post.url, }, investigations: investigations.map((investigation) => { - const lifecycle = parsePublicLifecycle({ - investigationId: investigation.id, - provenance: investigation.input.provenance, - serverVerifiedAt: investigation.postVersion.serverVerifiedAt, - checkedAt: investigation.checkedAt, - }); + const lifecycle = publicLifecycle(investigation); return { id: investigation.id, contentHash: investigation.postVersion.contentBlob.contentHash, @@ -459,17 +447,12 @@ export async function searchPublicInvestigations( ); } - const lifecycle = parsePublicLifecycle({ - investigationId: investigation.id, - provenance: investigation.input.provenance, - serverVerifiedAt: investigation.postVersion.serverVerifiedAt, - checkedAt: investigation.checkedAt, - }); + const lifecycle = publicLifecycle(investigation); return { id: investigation.id, contentHash: investigation.postVersion.contentBlob.contentHash, checkedAt: lifecycle.checkedAt, - platform: parsePlatform(investigation.postVersion.post.platform), + platform: investigation.postVersion.post.platform, externalId: investigation.postVersion.post.externalId, url: investigation.postVersion.post.url, origin: lifecycle.origin, @@ -509,14 +492,15 @@ export async function getPublicMetrics( WHERE ${Prisma.join(conditions, " AND ")} `; - const { total_investigated, with_flags } = result[0] ?? { - total_investigated: 0, - with_flags: 0, - }; + const [counts] = result; + if (counts === undefined) { + invariantViolation("public metrics aggregate query returned no row"); + } + const { total_investigated, with_flags } = counts; return { totalInvestigatedPosts: total_investigated, investigatedPostsWithFlags: with_flags, - factCheckIncidence: total_investigated > 0 ? with_flags / total_investigated : 0, + factCheckIncidence: total_investigated > 0 ? with_flags / total_investigated : null, }; } diff --git a/src/typescript/api/src/lib/services/queue-lifecycle.ts b/src/typescript/api/src/lib/services/queue-lifecycle.ts deleted file mode 100644 index 4c33968..0000000 --- a/src/typescript/api/src/lib/services/queue-lifecycle.ts +++ /dev/null @@ -1,195 +0,0 @@ -// ── Queue lifecycle state machine ──────────────────────────────────────── -// -// Manages a lazily-initialized, closeable resource pool. The state machine -// ensures that initialization, usage, and shutdown are properly serialized -// even under concurrent access. -// -// Transitions: -// idle → initializing acquire starts connection -// idle → closed close requested with nothing to release -// initializing → ready connection succeeds -// initializing → idle connection fails (retry allowed) -// initializing → closing close requested during connection -// ready → closing close requested -// closing → closed release succeeds -// closing → idle release fails (retry allowed) -// closed → (terminal) acquire throws - -export interface Releasable { - release(): void | Promise; -} - -export const QUEUE_ERROR_CODES = { - CLOSED: "QUEUE_CLOSED", - CONNECT_FAILED: "QUEUE_CONNECT_FAILED", - RELEASE_FAILED: "QUEUE_RELEASE_FAILED", -} as const; - -export type QueueErrorCode = (typeof QUEUE_ERROR_CODES)[keyof typeof QUEUE_ERROR_CODES]; - -class QueueLifecycleError extends Error { - constructor( - readonly code: QueueErrorCode, - message: string, - options?: ErrorOptions, - ) { - super(message, options); - this.name = new.target.name; - } -} - -export class QueueClosedError extends QueueLifecycleError { - constructor() { - super(QUEUE_ERROR_CODES.CLOSED, "Queue utilities are closed"); - } -} - -export class QueueConnectError extends QueueLifecycleError { - constructor(cause: unknown) { - super(QUEUE_ERROR_CODES.CONNECT_FAILED, "Queue utilities failed to initialize", { cause }); - } -} - -export class QueueReleaseError extends QueueLifecycleError { - constructor(cause: unknown) { - super(QUEUE_ERROR_CODES.RELEASE_FAILED, "Queue utilities failed to release", { cause }); - } -} - -interface QueueManager { - acquire(): Promise; - close(): Promise; -} - -type QueueState = - | { phase: "idle" } - | { phase: "initializing"; promise: Promise } - | { phase: "ready"; utils: T } - | { phase: "closing"; promise: Promise } - | { phase: "closed" }; - -export function createQueueManager( - connect: () => Promise, -): QueueManager { - let state: QueueState = { phase: "idle" }; - - /** - * Checks whether the given promise is still the active initialization - * attempt. State can change across await boundaries (e.g. close requested - * during init), so callers must re-check after any suspension. This is - * extracted as a function so TypeScript reads `state` without the enclosing - * switch-case narrowing that would make the check look redundant to the - * linter. - */ - function isActiveInit(promise: Promise): boolean { - return state.phase === "initializing" && state.promise === promise; - } - - async function acquire(): Promise { - while (true) { - switch (state.phase) { - case "closed": - throw new QueueClosedError(); - - case "closing": - // Wait for close to finish, then re-check. Swallow close errors — - // an acquire caller should not see release failures. - try { - await state.promise; - } catch { - // Close failed; state is now idle. Loop will retry initialization. - } - continue; - - case "ready": - return state.utils; - - case "idle": { - const promise = connect(); - state = { phase: "initializing", promise }; - continue; - } - - case "initializing": { - const { promise } = state; - let utils: T; - try { - utils = await promise; - } catch (error) { - if (isActiveInit(promise)) { - state = { phase: "idle" }; - throw new QueueConnectError(error); - } - // State changed during init (e.g. close was requested). - // Swallow the connection error and re-check — callers should - // see "closed", not a transient connection failure. - continue; - } - if (isActiveInit(promise)) { - state = { phase: "ready", utils }; - return utils; - } - // State changed during init (e.g. close was requested). Re-check. - continue; - } - } - } - } - - async function releaseAndClose(utils: T): Promise { - try { - await utils.release(); - state = { phase: "closed" }; - } catch (error) { - // Release failed — revert to idle so close can be retried. - state = { phase: "idle" }; - throw new QueueReleaseError(error); - } - } - - async function awaitInitThenClose(initPromise: Promise): Promise { - let utils: T; - try { - utils = await initPromise; - } catch { - // Init failed — nothing to release. Close succeeds. - state = { phase: "closed" }; - return; - } - await releaseAndClose(utils); - } - - async function close(): Promise { - switch (state.phase) { - case "closed": - return; - - case "closing": - await state.promise; - return; - - case "idle": { - state = { phase: "closed" }; - return; - } - - case "ready": { - const { utils } = state; - const promise = releaseAndClose(utils); - state = { phase: "closing", promise }; - await promise; - return; - } - - case "initializing": { - const { promise: initPromise } = state; - const promise = awaitInitThenClose(initPromise); - state = { phase: "closing", promise }; - await promise; - return; - } - } - } - - return { acquire, close }; -} diff --git a/src/typescript/api/src/lib/services/queue.ts b/src/typescript/api/src/lib/services/queue.ts index b668afa..d4bcacf 100644 --- a/src/typescript/api/src/lib/services/queue.ts +++ b/src/typescript/api/src/lib/services/queue.ts @@ -1,27 +1,47 @@ -import { makeWorkerUtils } from "graphile-worker"; +import { makeWorkerUtils, type WorkerUtils } from "graphile-worker"; import { getEnv } from "$lib/config/env.js"; import { normalizePgConnectionStringForNode } from "$lib/db/connection-string.js"; -import { createQueueManager } from "./queue-lifecycle.js"; -const manager = createQueueManager(() => - makeWorkerUtils({ +let workerUtils: Promise | null = null; + +/** Lazily connect once per process; a failed connection is retried on the next call. */ +function getWorkerUtils(): Promise { + workerUtils ??= makeWorkerUtils({ connectionString: normalizePgConnectionStringForNode(getEnv().DATABASE_URL), - }), -); + }).catch((error: unknown) => { + workerUtils = null; + throw error; + }); + return workerUtils; +} +/** + * Enqueue (or replace) the single graphile-worker job for an investigation. + * The per-investigation jobKey collapses concurrent enqueues from + * investigateNow, the selector and retry scheduling into one job; retries are + * application-controlled, so graphile-worker never retries a job itself. + */ export async function enqueueInvestigation( investigationId: string, options?: { runAt?: Date }, ): Promise { - const utils = await manager.acquire(); - const spec = { - maxAttempts: 1, - jobKey: `investigate:${investigationId}`, - ...(options?.runAt !== undefined && { runAt: options.runAt }), - }; - await utils.addJob("investigate", { investigationId }, spec); + const utils = await getWorkerUtils(); + await utils.addJob( + "investigate", + { investigationId }, + { + maxAttempts: 1, + jobKey: `investigate:${investigationId}`, + ...(options?.runAt !== undefined && { runAt: options.runAt }), + }, + ); } +/** Release the queue's database pool (lets short-lived processes such as tests exit). */ export async function closeQueueUtils(): Promise { - await manager.close(); + const pending = workerUtils; + workerUtils = null; + if (pending !== null) { + await (await pending).release(); + } } diff --git a/src/typescript/api/src/lib/services/request-identity.ts b/src/typescript/api/src/lib/services/request-identity.ts index 517af29..f84a624 100644 --- a/src/typescript/api/src/lib/services/request-identity.ts +++ b/src/typescript/api/src/lib/services/request-identity.ts @@ -3,25 +3,23 @@ interface RequestIdentityInput { userAgent: string; instanceApiKey: string | null | undefined; userOpenAiApiKey: string | null | undefined; - attestationSignature: string | null | undefined; - attestationBody: string | null; } interface RequestIdentityDependencies { hashContent: (value: string) => Promise; findActiveInstanceApiKeyHash: (apiKey: string) => Promise; deriveIpRangePrefix: (ipAddress: string) => string; - verifyHmac: (body: string, signature: string) => Promise; } interface RequestIdentity { - authenticatedApiKeyHash: string | null; + /** Stable hashed viewer: the instance API key when authenticated, else address + user agent. */ viewerKey: string; + /** Stable hashed /24 (IPv4) or /48 (IPv6) of the client address. */ ipRangeKey: string; - userOpenAiApiKey: string | null; + /** Whether the request carries an active instance API key. */ isAuthenticated: boolean; - canInvestigate: boolean; - hasValidAttestation: boolean; + /** The request-scoped user OpenAI key, as sent; unverified. */ + userOpenAiApiKey: string | null; } function trimToOptional(value: string | null | undefined): string | null { @@ -29,84 +27,28 @@ function trimToOptional(value: string | null | undefined): string | null { return trimmed !== undefined && trimmed.length > 0 ? trimmed : null; } -async function resolveAuthenticatedApiKeyHash(input: { - instanceApiKey: string | null; - findActiveInstanceApiKeyHash: RequestIdentityDependencies["findActiveInstanceApiKeyHash"]; -}): Promise { - if (input.instanceApiKey === null) { - return null; - } - return input.findActiveInstanceApiKeyHash(input.instanceApiKey); -} - -async function resolveViewerKey(input: { - authenticatedApiKeyHash: string | null; - clientAddress: string; - userAgent: string; - hashContent: RequestIdentityDependencies["hashContent"]; -}): Promise { - if (input.authenticatedApiKeyHash !== null) { - return input.hashContent(`apikey:${input.authenticatedApiKeyHash}`); - } - return input.hashContent(`anon:${input.clientAddress}:${input.userAgent}`); -} - -async function resolveHasValidAttestation(input: { - attestationSignature: string | null; - attestationBody: string | null; - verifyHmac: RequestIdentityDependencies["verifyHmac"]; -}): Promise { - if ( - input.attestationSignature === null || - input.attestationBody === null || - input.attestationBody.length === 0 - ) { - return false; - } - - try { - return await input.verifyHmac(input.attestationBody, input.attestationSignature); - } catch { - return false; - } -} - export async function deriveRequestIdentity( input: RequestIdentityInput, dependencies: RequestIdentityDependencies, ): Promise { const instanceApiKey = trimToOptional(input.instanceApiKey); - const userOpenAiApiKey = trimToOptional(input.userOpenAiApiKey); - const attestationSignature = trimToOptional(input.attestationSignature); - - const authenticatedApiKeyHash = await resolveAuthenticatedApiKeyHash({ - instanceApiKey, - findActiveInstanceApiKeyHash: dependencies.findActiveInstanceApiKeyHash, - }); - const viewerKey = await resolveViewerKey({ - authenticatedApiKeyHash, - clientAddress: input.clientAddress, - userAgent: input.userAgent, - hashContent: dependencies.hashContent, - }); + const authenticatedApiKeyHash = + instanceApiKey === null + ? null + : await dependencies.findActiveInstanceApiKeyHash(instanceApiKey); + + const viewerKey = await dependencies.hashContent( + authenticatedApiKeyHash === null + ? `anon:${input.clientAddress}:${input.userAgent}` + : `apikey:${authenticatedApiKeyHash}`, + ); const ipRangePrefix = dependencies.deriveIpRangePrefix(input.clientAddress); const ipRangeKey = await dependencies.hashContent(`iprange:${ipRangePrefix}`); - const hasValidAttestation = await resolveHasValidAttestation({ - attestationSignature, - attestationBody: input.attestationBody, - verifyHmac: dependencies.verifyHmac, - }); - - const isAuthenticated = authenticatedApiKeyHash !== null; - const canInvestigate = isAuthenticated || userOpenAiApiKey !== null; return { - authenticatedApiKeyHash, viewerKey, ipRangeKey, - userOpenAiApiKey, - isAuthenticated, - canInvestigate, - hasValidAttestation, + isAuthenticated: authenticatedApiKeyHash !== null, + userOpenAiApiKey: trimToOptional(input.userOpenAiApiKey), }; } diff --git a/src/typescript/api/src/lib/services/selector-entrypoint.ts b/src/typescript/api/src/lib/services/selector-entrypoint.ts index d8f30a1..a0568b5 100644 --- a/src/typescript/api/src/lib/services/selector-entrypoint.ts +++ b/src/typescript/api/src/lib/services/selector-entrypoint.ts @@ -1,3 +1,4 @@ +import { getSelectorDailyBudget } from "$lib/config/runtime.js"; import { runStartupChecks } from "$lib/config/startup.js"; import { getPrisma } from "$lib/db/client"; import { runSelector } from "./selector.js"; @@ -6,8 +7,16 @@ async function runOnce(): Promise { let exitCode = 0; try { await runStartupChecks("selector"); - const count = await runSelector(); - console.log(`Selector: enqueued ${count} investigations`); + const summary = await runSelector({ dailyBudget: getSelectorDailyBudget() }); + console.log( + `Selector: admitted ${summary.admitted.toString()} (budget left today: ${summary.budgetRemaining.toString()}), re-enqueued ${summary.requeued.toString()}, recovered ${summary.recovered.toString()} expired lease(s)`, + ); + for (const failure of summary.failures) { + console.error(`Selector ${failure.stage} failed for ${failure.subjectId}:`, failure.error); + } + if (summary.failures.length > 0) { + exitCode = 1; + } } catch (err) { console.error("Selector error:", err); exitCode = 1; diff --git a/src/typescript/api/src/lib/services/selector.ts b/src/typescript/api/src/lib/services/selector.ts index 396d683..b1a08e6 100644 --- a/src/typescript/api/src/lib/services/selector.ts +++ b/src/typescript/api/src/lib/services/selector.ts @@ -1,71 +1,270 @@ -import { getPrisma } from "$lib/db/client"; -import { getOrCreateCurrentPrompt } from "./prompt.js"; -import { ensureInvestigationQueued } from "./investigation-lifecycle.js"; +/** + * Investigation selector (SPEC §2.10, §3.6). + * + * Each run, in order: + * 1. Recovers every investigation whose lease expired (dead or stalled worker). + * 2. Re-enqueues every funded PENDING investigation that is due, so a lost + * queue job never strands one. This is not new spending and is unbudgeted. + * 3. Admits new work, highest capped unique-view score first: latest post + * versions with no investigation, and unfunded investigations (whose user + * key was dropped). Admissions are SELECTOR-funded and capped at + * SELECTOR_DAILY_BUDGET per UTC day, however often the cron runs. + * + * A failure on one candidate is recorded and the run moves on; the run's + * summary carries the failures so the entrypoint can exit non-zero. + */ + +import { getPrisma, type PrismaClient } from "$lib/db/client"; +import { startOfUtcDay } from "$lib/date.js"; +import { isUniqueConstraintError } from "$lib/db/errors.js"; +import type { Prisma } from "$lib/db/prisma-client"; import { WORD_COUNT_LIMIT } from "@openerrata/shared"; -import { getSelectorBudget } from "$lib/config/runtime.js"; +import { getOrCreateCurrentPrompt } from "./prompt.js"; +import { + fundUnfundedInvestigation, + insertAdmittedInvestigation, + unfundedInvestigationWhere, +} from "./investigation-admission.js"; +import { buildInvestigationInputSnapshot } from "./investigation-input.js"; +import { recoverExpiredLease } from "./investigation-lease.js"; +import { enqueueInvestigation } from "./queue.js"; +import { resolveUpdateLineage } from "./update-lineage.js"; -export async function runSelector(): Promise { - const prisma = getPrisma(); - const budget = getSelectorBudget(); - const prompt = await getOrCreateCurrentPrompt(); - - // Consider the most recently seen version for each post and enqueue - // investigations that are missing or in recoverable pending/processing states. - const candidates = await prisma.$queryRaw< - { - postVersionId: string; - investigationId: string | null; - investigationStatus: "PENDING" | "PROCESSING" | "COMPLETE" | "FAILED" | null; - }[] +interface SelectorFailure { + stage: "RECOVER" | "REQUEUE" | "ADMIT"; + /** Investigation id for RECOVER/REQUEUE, post version id for ADMIT. */ + subjectId: string; + error: unknown; +} + +interface SelectorRunSummary { + recovered: number; + requeued: number; + admitted: number; + /** SELECTOR admissions still allowed today after this run. */ + budgetRemaining: number; + failures: SelectorFailure[]; +} + +type AdmissionCandidate = + | { kind: "NEW"; postVersionId: string } + | { kind: "UNFUNDED"; postVersionId: string; investigationId: string }; + +/** Serializes budget checks across concurrent selector runs. */ +async function lockSelectorBudget(tx: Prisma.TransactionClient): Promise { + await tx.$executeRaw`SELECT pg_advisory_xact_lock(hashtext('openerrata.selector_daily_budget')::bigint)`; +} + +async function countSelectorAdmissionsSince( + db: PrismaClient | Prisma.TransactionClient, + dayStart: Date, +): Promise { + return db.investigation.count({ + where: { origin: "SELECTOR", admittedAt: { gte: dayStart } }, + }); +} + +async function recoverExpiredLeases( + prisma: PrismaClient, + failures: SelectorFailure[], +): Promise { + const expired = await prisma.investigationLease.findMany({ + where: { leaseExpiresAt: { lte: new Date() } }, + select: { investigationId: true }, + }); + + let recovered = 0; + for (const { investigationId } of expired) { + try { + if (await recoverExpiredLease(prisma, investigationId)) { + recovered += 1; + } + } catch (error) { + failures.push({ stage: "RECOVER", subjectId: investigationId, error }); + } + } + return recovered; +} + +async function requeueDueFundedInvestigations( + prisma: PrismaClient, + failures: SelectorFailure[], +): Promise { + const due = await prisma.investigation.findMany({ + where: { + status: "PENDING", + NOT: unfundedInvestigationWhere, + OR: [{ retryAfter: null }, { retryAfter: { lte: new Date() } }], + }, + select: { id: true }, + }); + + let requeued = 0; + for (const { id } of due) { + try { + await enqueueInvestigation(id); + requeued += 1; + } catch (error) { + failures.push({ stage: "REQUEUE", subjectId: id, error }); + } + } + return requeued; +} + +async function loadAdmissionCandidates( + prisma: PrismaClient, + limit: number, +): Promise { + const rows = await prisma.$queryRaw< + { postVersionId: string; unfundedInvestigationId: string | null }[] >` WITH latest_versions AS ( SELECT DISTINCT ON (pv."postId") pv."id" AS "postVersionId", pv."postId", - pv."contentBlobId", - pv."lastSeenAt" + pv."contentBlobId" FROM "PostVersion" pv ORDER BY pv."postId", pv."lastSeenAt" DESC, pv."id" DESC ) SELECT lv."postVersionId", - i."id" AS "investigationId", - i."status" AS "investigationStatus" + i."id" AS "unfundedInvestigationId" FROM latest_versions lv - JOIN "Post" p - ON p."id" = lv."postId" - JOIN "ContentBlob" cb - ON cb."id" = lv."contentBlobId" - LEFT JOIN "Investigation" i - ON i."postVersionId" = lv."postVersionId" - LEFT JOIN "InvestigationLease" il - ON il."investigationId" = i."id" + JOIN "Post" p ON p."id" = lv."postId" + JOIN "ContentBlob" cb ON cb."id" = lv."contentBlobId" + LEFT JOIN "Investigation" i ON i."postVersionId" = lv."postVersionId" WHERE cb."wordCount" <= ${WORD_COUNT_LIMIT} AND ( - i."id" IS NULL - OR (i."status" = 'PENDING' AND (i."retryAfter" IS NULL OR i."retryAfter" <= NOW())) - OR ( - i."status" = 'PROCESSING' - AND (il."investigationId" IS NULL OR il."leaseExpiresAt" <= NOW()) + i."id" IS NULL + OR ( + i."status" = 'PENDING' + AND i."origin" = 'USER_KEY_REQUEST' + AND NOT EXISTS ( + SELECT 1 FROM "InvestigationOpenAiKeySource" ks WHERE ks."investigationId" = i."id" + ) + ) ) - ) - ORDER BY p."uniqueViewScore" DESC - LIMIT ${budget} + ORDER BY p."uniqueViewScore" DESC, lv."postVersionId" + LIMIT ${limit} `; - let enqueued = 0; - for (const candidate of candidates) { - const { enqueued: wasEnqueued } = await ensureInvestigationQueued({ - prisma, - postVersionId: candidate.postVersionId, - promptId: prompt.id, - rejectOverWordLimitOnCreate: false, + return rows.map((row) => + row.unfundedInvestigationId === null + ? { kind: "NEW", postVersionId: row.postVersionId } + : { + kind: "UNFUNDED", + postVersionId: row.postVersionId, + investigationId: row.unfundedInvestigationId, + }, + ); +} + +/** + * Run `admit` in a transaction holding the selector budget lock, but only if + * fewer than `dailyBudget` SELECTOR admissions happened since `dayStart`. + * Returns null when the budget is spent or `admit` declined; a unique + * violation means a request or concurrent run admitted the version first. + */ +async function admitWithinDailyBudget( + prisma: PrismaClient, + budget: { dailyBudget: number; dayStart: Date }, + admit: (tx: Prisma.TransactionClient, now: Date) => Promise, +): Promise { + try { + return await prisma.$transaction(async (tx) => { + await lockSelectorBudget(tx); + if ((await countSelectorAdmissionsSince(tx, budget.dayStart)) >= budget.dailyBudget) { + return null; + } + return admit(tx, new Date()); }); + } catch (error) { + if (isUniqueConstraintError(error)) { + return null; + } + throw error; + } +} + +/** Admit one candidate under the daily budget; returns the admitted investigation id. */ +async function admitCandidate( + prisma: PrismaClient, + input: { candidate: AdmissionCandidate; promptId: string; dailyBudget: number; dayStart: Date }, +): Promise { + const { candidate } = input; + switch (candidate.kind) { + case "UNFUNDED": + return admitWithinDailyBudget(prisma, input, async (tx, now) => + (await fundUnfundedInvestigation(tx, { + investigationId: candidate.investigationId, + funding: { origin: "SELECTOR" }, + now, + })) + ? candidate.investigationId + : null, + ); + case "NEW": { + const postVersion = await prisma.postVersion.findUniqueOrThrow({ + where: { id: candidate.postVersionId }, + select: { id: true, postId: true, contentBlob: { select: { contentText: true } } }, + }); + const lineage = await resolveUpdateLineage(prisma, { + id: postVersion.id, + postId: postVersion.postId, + contentText: postVersion.contentBlob.contentText, + }); + const snapshot = await buildInvestigationInputSnapshot(prisma, postVersion.id); + return admitWithinDailyBudget(prisma, input, async (tx, now) => { + const created = await insertAdmittedInvestigation(tx, { + postVersionId: postVersion.id, + promptId: input.promptId, + funding: { origin: "SELECTOR" }, + lineage, + snapshot, + now, + }); + return created.id; + }); + } + } +} + +/** One selector pass; `dailyBudget` caps SELECTOR admissions per UTC day. */ +export async function runSelector(input: { dailyBudget: number }): Promise { + const prisma = getPrisma(); + const { dailyBudget } = input; + const failures: SelectorFailure[] = []; + + const recovered = await recoverExpiredLeases(prisma, failures); + const requeued = await requeueDueFundedInvestigations(prisma, failures); - if (wasEnqueued) { - enqueued += 1; + const dayStart = startOfUtcDay(new Date()); + const remainingAtStart = dailyBudget - (await countSelectorAdmissionsSince(prisma, dayStart)); + let admitted = 0; + if (remainingAtStart > 0) { + const prompt = await getOrCreateCurrentPrompt(); + const candidates = await loadAdmissionCandidates(prisma, remainingAtStart); + for (const candidate of candidates) { + try { + const investigationId = await admitCandidate(prisma, { + candidate, + promptId: prompt.id, + dailyBudget, + dayStart, + }); + if (investigationId !== null) { + admitted += 1; + await enqueueInvestigation(investigationId); + } + } catch (error) { + failures.push({ stage: "ADMIT", subjectId: candidate.postVersionId, error }); + } } } - return enqueued; + const budgetRemaining = Math.max( + 0, + dailyBudget - (await countSelectorAdmissionsSince(prisma, dayStart)), + ); + return { recovered, requeued, admitted, budgetRemaining, failures }; } diff --git a/src/typescript/api/src/lib/services/update-lineage.ts b/src/typescript/api/src/lib/services/update-lineage.ts new file mode 100644 index 0000000..1678439 --- /dev/null +++ b/src/typescript/api/src/lib/services/update-lineage.ts @@ -0,0 +1,87 @@ +/** + * Update lineage for new investigations (SPEC §2.4.3). + * + * When a post is edited and an earlier version already has a complete + * SERVER_VERIFIED investigation, the new investigation runs in update mode: + * it records that investigation as its parent and carries a line diff from the + * parent's text, so the prompt can keep unchanged claims stable. Every path + * that creates investigations (investigateNow and the selector) goes through + * here, so update-aware prompting does not depend on who queued the run. + */ + +import type { DbClient } from "$lib/db/client"; + +export interface UpdateLineage { + parentInvestigationId: string; + contentDiff: string; +} + +/** + * Lineage for a new investigation of `postVersion`, or null when no other + * version of the post has a complete SERVER_VERIFIED investigation. + */ +export async function resolveUpdateLineage( + db: DbClient, + postVersion: { id: string; postId: string; contentText: string }, +): Promise { + const parent = await db.investigation.findFirst({ + where: { + status: "COMPLETE", + input: { provenance: "SERVER_VERIFIED" }, + postVersion: { postId: postVersion.postId, id: { not: postVersion.id } }, + }, + orderBy: [{ checkedAt: "desc" }, { id: "desc" }], + select: { + id: true, + postVersion: { select: { contentBlob: { select: { contentText: true } } } }, + }, + }); + if (parent === null) { + return null; + } + + return { + parentInvestigationId: parent.id, + contentDiff: buildLineDiff(parent.postVersion.contentBlob.contentText, postVersion.contentText), + }; +} + +/** + * Deterministic line-oriented diff: the common prefix and suffix are trimmed + * and the differing middle is reported as removed and added lines. + */ +export function buildLineDiff(previous: string, current: string): string { + if (previous === current) { + return "No changes detected."; + } + + const previousLines = previous.split("\n"); + const currentLines = current.split("\n"); + const maxStart = Math.min(previousLines.length, currentLines.length); + let start = 0; + while (start < maxStart && previousLines[start] === currentLines[start]) { + start += 1; + } + + let previousEnd = previousLines.length; + let currentEnd = currentLines.length; + while ( + previousEnd > start && + currentEnd > start && + previousLines[previousEnd - 1] === currentLines[currentEnd - 1] + ) { + previousEnd -= 1; + currentEnd -= 1; + } + + const removed = previousLines.slice(start, previousEnd); + const added = currentLines.slice(start, currentEnd); + + return [ + "Diff summary (line context):", + "- Removed lines:", + removed.length > 0 ? removed.join("\n") : "(none)", + "+ Added lines:", + added.length > 0 ? added.join("\n") : "(none)", + ].join("\n"); +} diff --git a/src/typescript/api/src/lib/services/user-key-source.ts b/src/typescript/api/src/lib/services/user-key-source.ts index 55500a9..6b31710 100644 --- a/src/typescript/api/src/lib/services/user-key-source.ts +++ b/src/typescript/api/src/lib/services/user-key-source.ts @@ -1,7 +1,19 @@ +/** + * Request-scoped user OpenAI keys (SPEC §2.2.6, §3.7). + * + * A user key funds only an investigation its own request admits. It is + * verified with OpenAI before it is attached to anything, stored encrypted + * with a short TTL on InvestigationOpenAiKeySource, and deleted on every + * terminal transition — or as soon as it turns out to be unusable, in which + * case the investigation becomes unfunded rather than FAILED. + */ + import { createCipheriv, createDecipheriv, createHash, randomBytes } from "node:crypto"; -import { isUniqueConstraintError } from "$lib/db/errors.js"; -import type { Prisma, PrismaClient } from "$lib/db/prisma-client"; +import type { DbClient } from "$lib/db/client"; +import type { InvestigationOrigin, Prisma } from "$lib/db/prisma-client"; import { getDatabaseEncryptionConfig } from "$lib/config/env.js"; +import type { OpenAiKeyValidationStatusOutcome } from "./openai-key-validation-core.js"; +import { validateOpenAiApiKeyForSettings } from "./openai-key-validation.js"; const OPENAI_KEY_SOURCE_TTL_MS = 30 * 60 * 1000; const AES_GCM_IV_BYTES = 12; @@ -64,56 +76,47 @@ function decryptOpenAiKey(input: { return decrypted.toString("utf8"); } -type AttachOpenAiKeySourceResult = - | "ATTACHED" - | "ALREADY_ATTACHED" - | "NOT_PENDING" - | "MISSING_INVESTIGATION"; - -export async function attachOpenAiKeySourceIfPending( - prisma: PrismaClient, - input: { - investigationId: string; - openAiApiKey: string; - }, -): Promise { - const encrypted = encryptOpenAiKey(input.openAiApiKey); - const expiresAt = new Date(Date.now() + OPENAI_KEY_SOURCE_TTL_MS); +declare const verifiedOpenAiApiKeyBrand: unique symbol; - try { - return await prisma.$transaction(async (tx) => { - const investigation = await tx.investigation.findUnique({ - where: { id: input.investigationId }, - select: { id: true, status: true }, - }); - if (!investigation) return "MISSING_INVESTIGATION"; - if (investigation.status !== "PENDING") return "NOT_PENDING"; - - const existing = await tx.investigationOpenAiKeySource.findUnique({ - where: { investigationId: input.investigationId }, - select: { investigationId: true }, - }); - if (existing) return "ALREADY_ATTACHED"; - - await tx.investigationOpenAiKeySource.create({ - data: { - investigationId: input.investigationId, - ciphertext: encrypted.ciphertext, - iv: encrypted.iv, - authTag: encrypted.authTag, - keyId: encrypted.keyId, - expiresAt, - }, - }); - return "ATTACHED"; - }); - } catch (error) { - if (!isUniqueConstraintError(error)) throw error; - return "ALREADY_ATTACHED"; +/** A user OpenAI key that OpenAI accepted moments ago. Only `verifyUserOpenAiApiKey` makes one. */ +export type VerifiedOpenAiApiKey = string & { readonly [verifiedOpenAiApiKeyBrand]: true }; + +export type UserOpenAiKeyVerification = + | { verified: true; apiKey: VerifiedOpenAiApiKey } + | { + verified: false; + outcome: Exclude; + }; + +/** Ask OpenAI whether `apiKey` works before letting it fund anything. */ +export async function verifyUserOpenAiApiKey(apiKey: string): Promise { + const outcome = await validateOpenAiApiKeyForSettings(apiKey); + if (outcome.openaiApiKeyStatus !== "valid") { + return { verified: false, outcome }; } + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- branded only after OpenAI accepted the key above + return { verified: true, apiKey: apiKey as VerifiedOpenAiApiKey }; } -type InvestigationKeyResolution = +/** Store `apiKey` (encrypted, short-lived) as the funding source of `investigationId`. */ +export async function attachOpenAiKeySource( + tx: Prisma.TransactionClient, + input: { investigationId: string; apiKey: VerifiedOpenAiApiKey; now: Date }, +): Promise { + const encrypted = encryptOpenAiKey(input.apiKey); + await tx.investigationOpenAiKeySource.create({ + data: { + investigationId: input.investigationId, + ciphertext: encrypted.ciphertext, + iv: encrypted.iv, + authTag: encrypted.authTag, + keyId: encrypted.keyId, + expiresAt: new Date(input.now.getTime() + OPENAI_KEY_SOURCE_TTL_MS), + }, + }); +} + +export type InvestigationKeyResolution = | { type: "SERVER_KEY" } | { type: "USER_OPENAI_KEY"; apiKey: string }; @@ -131,12 +134,22 @@ export class InvalidOpenAiKeySourceError extends Error { } } +/** + * The key a worker must use for `investigation`: the server key for SELECTOR + * and INSTANCE_REQUEST admissions, the attached user key for USER_KEY_REQUEST. + * A user-key investigation never falls back to the server key — a missing, + * expired or undecryptable key source throws, and the caller drops the key. + */ export async function resolveInvestigationKey( - prisma: PrismaClient, - investigationId: string, + db: DbClient, + investigation: { id: string; origin: InvestigationOrigin }, ): Promise { - const keySource = await prisma.investigationOpenAiKeySource.findUnique({ - where: { investigationId }, + if (investigation.origin !== "USER_KEY_REQUEST") { + return { type: "SERVER_KEY" }; + } + + const keySource = await db.investigationOpenAiKeySource.findUnique({ + where: { investigationId: investigation.id }, select: { keyId: true, ciphertext: true, @@ -145,22 +158,25 @@ export async function resolveInvestigationKey( expiresAt: true, }, }); - if (!keySource) return { type: "SERVER_KEY" }; + if (keySource === null) { + throw new InvalidOpenAiKeySourceError(investigation.id, "no key source is attached"); + } if (keySource.expiresAt.getTime() <= Date.now()) { - throw new ExpiredOpenAiKeySourceError(investigationId); + throw new ExpiredOpenAiKeySourceError(investigation.id); } + let apiKey: string; try { - const apiKey = decryptOpenAiKey(keySource); - if (apiKey.trim().length === 0) { - throw new Error("decrypted key was empty"); - } - return { type: "USER_OPENAI_KEY", apiKey }; + apiKey = decryptOpenAiKey(keySource); } catch (error) { const reason = error instanceof Error ? error.message : String(error); - throw new InvalidOpenAiKeySourceError(investigationId, reason); + throw new InvalidOpenAiKeySourceError(investigation.id, reason); + } + if (apiKey.trim().length === 0) { + throw new InvalidOpenAiKeySourceError(investigation.id, "decrypted key was empty"); } + return { type: "USER_OPENAI_KEY", apiKey }; } export async function consumeOpenAiKeySource( diff --git a/src/typescript/api/src/lib/services/view-credit.ts b/src/typescript/api/src/lib/services/view-credit.ts index e17e2f3..b03ad39 100644 --- a/src/typescript/api/src/lib/services/view-credit.ts +++ b/src/typescript/api/src/lib/services/view-credit.ts @@ -1,13 +1,8 @@ import { getIpRangeCreditCap } from "$lib/config/runtime.js"; +import { startOfUtcDay } from "$lib/date.js"; import { isUniqueConstraintError } from "$lib/db/errors.js"; import type { PrismaClient } from "$lib/db/prisma-client"; -function startOfUTCDay(date: Date): Date { - const dayStart = new Date(date); - dayStart.setUTCHours(0, 0, 0, 0); - return dayStart; -} - export async function maybeIncrementUniqueViewScore( prisma: PrismaClient, postId: string, @@ -15,7 +10,7 @@ export async function maybeIncrementUniqueViewScore( ipRangeKey: string, ): Promise { const ipRangeCreditCap = getIpRangeCreditCap(); - const bucketDay = startOfUTCDay(new Date()); + const bucketDay = startOfUtcDay(new Date()); return prisma.$transaction(async (tx) => { // Serialize per-post view-credit updates so cap checks stay consistent diff --git a/src/typescript/api/src/lib/services/wikipedia-content-filter.ts b/src/typescript/api/src/lib/services/wikipedia-content-filter.ts index 1541318..41feaa3 100644 --- a/src/typescript/api/src/lib/services/wikipedia-content-filter.ts +++ b/src/typescript/api/src/lib/services/wikipedia-content-filter.ts @@ -3,15 +3,15 @@ import { effectiveHeadingText, headingLevelFromTag, isExcludedWikipediaSectionTitle, - normalizeWikipediaSectionTitle, shouldExcludeWikipediaElement, type WikipediaHeadingLevelDescriptor, type WikipediaNodeDescriptor, } from "@openerrata/shared"; import { parseFragment, serialize, type DefaultTreeAdapterMap } from "parse5"; -export type Parse5Node = DefaultTreeAdapterMap["node"]; -export type Parse5NodeFilter = (node: Parse5Node) => "include" | "skip"; +type Parse5Node = DefaultTreeAdapterMap["node"]; +/** Decides, per node of a fragment visited in document order, whether its subtree is kept. */ +export type Parse5NodeFilter = (node: DefaultTreeAdapterMap["childNode"]) => "include" | "skip"; export function isElementNode(node: Parse5Node): node is DefaultTreeAdapterMap["element"] { return "tagName" in node; @@ -103,57 +103,73 @@ function shouldSkipWikipediaElement(node: DefaultTreeAdapterMap["element"]): boo return shouldExcludeWikipediaElement({ tagName: node.tagName, classTokens: classTokens(node), + role: attrValue(node, "role"), }); } /** - * Creates a stateful node filter for Wikipedia content extraction and rendering. + * Creates a stateful node filter for Wikipedia content extraction and rendering, + * for nodes visited in document order (pre-order, skipped subtrees not entered). * * This handles: - * - section-level exclusion (e.g. "References", "External links") - * - element-level exclusion (e.g. citation superscripts, edit links) - * - text-node suppression while inside an excluded section + * - element-level exclusion (e.g. citation superscripts, edit links, navboxes) + * - section-level exclusion (e.g. "References", "External links"): an excluded + * heading (or its Parsoid `div.mw-heading` wrapper) and its following + * siblings, up to the next sibling heading of the same or a higher level — + * the extension's rule too (`sectionElements` in its Wikipedia adapter). + * Being sibling-based, it holds for flat Parse API output and for read views + * that nest each section in a `

` element alike. */ export function createWikipediaNodeFilter(): Parse5NodeFilter { - let skipSectionLevel: number | null = null; - - return (node: Parse5Node): "include" | "skip" => { - if (isElementNode(node)) { - const classTokenValues = classTokens(node); - const firstChildHeadingNode = firstDirectChildHeadingNode(node); - const nodeHeadingLevel = effectiveHeadingLevel( - toHeadingLevelDescriptor(node, classTokenValues, firstChildHeadingNode), - ); - if (nodeHeadingLevel !== null) { - if (skipSectionLevel !== null && nodeHeadingLevel <= skipSectionLevel) { - skipSectionLevel = null; - } - - const headingText = normalizeWikipediaSectionTitle( - effectiveHeadingText(toNodeDescriptor(node, classTokenValues, firstChildHeadingNode)), - ); - if (isExcludedWikipediaSectionTitle(headingText)) { - skipSectionLevel = nodeHeadingLevel; - return "skip"; - } - } + let excludedSection: { + parent: DefaultTreeAdapterMap["parentNode"] | null; + level: number; + } | null = null; + + return (node) => { + // Pre-order traversal reaches a node outside the section's parent only + // once it has left that parent for good. + if (excludedSection !== null && node.parentNode !== excludedSection.parent) { + excludedSection = null; + } - if (skipSectionLevel !== null || shouldSkipWikipediaElement(node)) { + const headingLevel = isElementNode(node) ? headingLevelOf(node) : null; + if (excludedSection !== null) { + if (headingLevel === null || headingLevel > excludedSection.level) { return "skip"; } + excludedSection = null; + } + if (!isElementNode(node)) { return "include"; } - - // Text nodes: suppress when inside an excluded section. - if (skipSectionLevel !== null) { + if (shouldSkipWikipediaElement(node)) { + return "skip"; + } + if (headingLevel !== null && hasExcludedSectionTitle(node)) { + excludedSection = { parent: node.parentNode, level: headingLevel }; return "skip"; } - return "include"; }; } +/** The level of the section `node` opens (a heading or Parsoid heading wrapper), or null. */ +function headingLevelOf(node: DefaultTreeAdapterMap["element"]): number | null { + return effectiveHeadingLevel( + toHeadingLevelDescriptor(node, classTokens(node), firstDirectChildHeadingNode(node)), + ); +} + +/** Whether the heading (or heading wrapper) `node` titles an excluded section. */ +function hasExcludedSectionTitle(node: DefaultTreeAdapterMap["element"]): boolean { + const headingText = effectiveHeadingText( + toNodeDescriptor(node, classTokens(node), firstDirectChildHeadingNode(node)), + ); + return isExcludedWikipediaSectionTitle(headingText); +} + /** * Strip excluded Wikipedia sections/elements from a parse5 tree in place. */ diff --git a/src/typescript/api/src/lib/services/word-count.ts b/src/typescript/api/src/lib/services/word-count.ts new file mode 100644 index 0000000..09a4476 --- /dev/null +++ b/src/typescript/api/src/lib/services/word-count.ts @@ -0,0 +1,7 @@ +/** + * Whitespace-delimited word count stored on ContentBlob and compared against + * WORD_COUNT_LIMIT by investigateNow and the selector (SPEC §2.4). + */ +export function wordCount(text: string): number { + return text.split(/\s+/).filter(Boolean).length; +} diff --git a/src/typescript/api/src/lib/services/worker-runner.ts b/src/typescript/api/src/lib/services/worker-runner.ts index ef083fa..d0b42e3 100644 --- a/src/typescript/api/src/lib/services/worker-runner.ts +++ b/src/typescript/api/src/lib/services/worker-runner.ts @@ -1,6 +1,7 @@ import { run } from "graphile-worker"; import { getEnv } from "$lib/config/env.js"; import { normalizePgConnectionStringForNode } from "$lib/db/connection-string.js"; +import { createOpenAIInvestigator } from "$lib/investigators/openai.js"; import { orchestrateInvestigation } from "./orchestrator.js"; function isInvestigatePayload(payload: unknown): payload is { investigationId: string } { @@ -23,6 +24,7 @@ export async function startWorker(): Promise { await orchestrateInvestigation(payload.investigationId, helpers.logger, { workerIdentity: `worker-job-${helpers.job.id}`, + createInvestigator: createOpenAIInvestigator, }); }, }, diff --git a/src/typescript/api/src/lib/trpc/context.ts b/src/typescript/api/src/lib/trpc/context.ts index 31aff29..e3d4575 100644 --- a/src/typescript/api/src/lib/trpc/context.ts +++ b/src/typescript/api/src/lib/trpc/context.ts @@ -1,13 +1,16 @@ import { getPrisma, type PrismaClient } from "$lib/db/client"; -import { getEnv, MINIMUM_SUPPORTED_EXTENSION_VERSION } from "$lib/config/env.js"; +import { MINIMUM_SUPPORTED_EXTENSION_VERSION } from "$lib/config/env.js"; import { hashContent, trimToOptionalNonEmpty } from "@openerrata/shared"; -import { verifyHmac } from "$lib/services/hmac.js"; import { deriveIpRangePrefix } from "$lib/network/ip.js"; import { findActiveInstanceApiKeyHash } from "$lib/services/instance-api-key.js"; import { deriveRequestIdentity } from "$lib/services/request-identity.js"; export interface RequestEventLike { request: Request; + /** + * The client's IP address. Behind the chart's ingress this comes from the + * proxy header named by ADDRESS_HEADER (see startup checks), not the socket. + */ getClientAddress: () => string; } @@ -17,9 +20,8 @@ export interface Context { viewerKey: string; ipRangeKey: string; isAuthenticated: boolean; - canInvestigate: boolean; + /** Request-scoped user OpenAI key as sent (`x-openai-api-key`); not yet verified. */ userOpenAiApiKey: string | null; - hasValidAttestation: boolean; extensionVersion: string | null; minimumSupportedExtensionVersion: string; } @@ -32,13 +34,10 @@ export async function createContext(event: RequestEventLike): Promise { userAgent: event.request.headers.get("user-agent") ?? "", instanceApiKey: event.request.headers.get("x-api-key"), userOpenAiApiKey: event.request.headers.get("x-openai-api-key"), - attestationSignature: event.request.headers.get("x-openerrata-signature"), - attestationBody: await readRequestBody(event), }, { hashContent, deriveIpRangePrefix, - verifyHmac: (body, signature) => verifyHmac(getEnv().HMAC_SECRET, body, signature), findActiveInstanceApiKeyHash: async (apiKey) => findActiveInstanceApiKeyHash(prisma, apiKey), }, ); @@ -49,19 +48,9 @@ export async function createContext(event: RequestEventLike): Promise { viewerKey: identity.viewerKey, ipRangeKey: identity.ipRangeKey, isAuthenticated: identity.isAuthenticated, - canInvestigate: identity.canInvestigate, userOpenAiApiKey: identity.userOpenAiApiKey, - hasValidAttestation: identity.hasValidAttestation, extensionVersion: trimToOptionalNonEmpty(event.request.headers.get("x-openerrata-extension-version")) ?? null, minimumSupportedExtensionVersion: MINIMUM_SUPPORTED_EXTENSION_VERSION, }; } - -async function readRequestBody(event: RequestEventLike): Promise { - try { - return await event.request.clone().text(); - } catch { - return null; - } -} diff --git a/src/typescript/api/src/lib/trpc/router.ts b/src/typescript/api/src/lib/trpc/router.ts index de35d88..5cde4d0 100644 --- a/src/typescript/api/src/lib/trpc/router.ts +++ b/src/typescript/api/src/lib/trpc/router.ts @@ -1,12 +1,11 @@ import { router } from "./init.js"; import { postRouter } from "./routes/post.js"; -import { publicRouter } from "./routes/public.js"; import type { EXTENSION_TRPC_PATH, ExtensionApiProcedureContract } from "@openerrata/shared"; import type { inferRouterInputs, inferRouterOutputs } from "@trpc/server"; +/** Extension-facing RPC. Public read access is GraphQL only (SPEC §3.4). */ export const appRouter = router({ post: postRouter, - public: publicRouter, }); type IsExactly = [A] extends [B] ? ([B] extends [A] ? true : never) : never; diff --git a/src/typescript/api/src/lib/trpc/routes/post.ts b/src/typescript/api/src/lib/trpc/routes/post.ts index 1e484b1..4d4b2e9 100644 --- a/src/typescript/api/src/lib/trpc/routes/post.ts +++ b/src/typescript/api/src/lib/trpc/routes/post.ts @@ -20,46 +20,31 @@ import { batchStatusInputSchema, batchStatusOutputSchema, settingsValidationOutputSchema, - contentProvenanceSchema, isExtensionVersionAtLeast, - type ContentProvenance, type ExtensionRuntimeErrorCode, type Platform, } from "@openerrata/shared"; import { getOrCreateCurrentPrompt } from "$lib/services/prompt.js"; import { TRPCError } from "@trpc/server"; - -/** Parse provenance from required InvestigationInput snapshots. */ -function parseProvenance(input: { - investigationId: string; - snapshot: { provenance: string } | null; -}): ContentProvenance { - if (input.snapshot === null) { - throw new TRPCError({ - code: "INTERNAL_SERVER_ERROR", - message: `Investigation ${input.investigationId} has no InvestigationInput snapshot`, - }); - } - return contentProvenanceSchema.parse(input.snapshot.provenance); -} -import { InvestigationWordLimitError } from "$lib/services/investigation-lifecycle.js"; +import { + InvestigationWordLimitError, + requestInvestigation, + UserOpenAiKeyRejectedError, + type InvestigationRequester, +} from "$lib/services/investigate-now.js"; import { maybeIncrementUniqueViewScore } from "$lib/services/view-credit.js"; -import { attachOpenAiKeySourceIfPending } from "$lib/services/user-key-source.js"; import { validateOpenAiApiKeyForSettings } from "$lib/services/openai-key-validation.js"; import { registerObservedVersion, findPostVersionById } from "./post/content-storage.js"; import { loadInvestigationWithClaims, - findCompletedInvestigationByPostVersionId, - findLatestServerVerifiedCompleteInvestigationForPost, - selectSourceInvestigationForUpdate, - toPriorInvestigationResult, + findCarriedForwardClaims, formatClaims, - ensureInvestigationsWithUpdateMetadata, maybeRecordCorroboration, unreachableInvestigationStatus, requireCompleteCheckedAtIso, prismaInvestigationRepository, parseProgressClaims, + type InvestigationRepository, } from "./post/investigation-queries.js"; // --------------------------------------------------------------------------- @@ -123,6 +108,107 @@ function assertSupportedExtensionVersion(input: { }); } +// --------------------------------------------------------------------------- +// investigateNow funding +// --------------------------------------------------------------------------- + +/** + * Who would pay for a run this investigateNow admits: the request's own + * OpenAI key when it sends one, otherwise the instance key it authenticated + * with. Requests with neither cannot ask for investigations. + */ +function investigationRequester(ctx: { + isAuthenticated: boolean; + userOpenAiApiKey: string | null; +}): InvestigationRequester { + if (ctx.userOpenAiApiKey !== null) { + return { kind: "USER_OPENAI_KEY", apiKey: ctx.userOpenAiApiKey }; + } + if (ctx.isAuthenticated) { + return { kind: "INSTANCE_API_KEY" }; + } + throw new TRPCError({ + code: "UNAUTHORIZED", + message: "Valid API key or x-openai-api-key required for investigateNow", + }); +} + +function userOpenAiKeyRejectedError(error: UserOpenAiKeyRejectedError): TRPCError { + const { outcome } = error; + switch (outcome.openaiApiKeyStatus) { + case "missing": + return new TRPCError({ code: "UNAUTHORIZED", message: "x-openai-api-key is empty" }); + case "format_invalid": + case "invalid": + return new TRPCError({ + code: "UNAUTHORIZED", + message: `x-openai-api-key was rejected: ${outcome.openaiApiKeyMessage}`, + }); + case "authenticated_restricted": + return new TRPCError({ + code: "FORBIDDEN", + message: `x-openai-api-key cannot run investigations: ${outcome.openaiApiKeyMessage}`, + }); + case "error": + return new TRPCError({ + code: "BAD_GATEWAY", + message: `Could not verify x-openai-api-key with OpenAI: ${outcome.openaiApiKeyMessage}`, + }); + } +} + +// --------------------------------------------------------------------------- +// Investigation status projection +// --------------------------------------------------------------------------- + +type LoadedInvestigation = NonNullable>>; + +/** + * Client-facing status of one investigation. Shared by `recordViewAndGetStatus` + * (lookup by post version) and `getInvestigation` (lookup by id) so a viewer + * sees the same projection — including live progress claims and the interim + * claims carried forward to the version (spec §2.8) — however the + * investigation was found. + */ +async function projectInvestigationStatus( + repo: InvestigationRepository, + investigation: LoadedInvestigation, +) { + const provenance = investigation.input.provenance; + + switch (investigation.status) { + case "COMPLETE": + return { + investigationState: "INVESTIGATED" as const, + provenance, + claims: formatClaims(investigation.claims), + }; + case "PENDING": + case "PROCESSING": { + const progress = parseProgressClaims(investigation.lease?.progressClaims ?? null); + return { + investigationState: "INVESTIGATING" as const, + status: investigation.status, + provenance, + pendingClaims: progress.pendingClaims, + confirmedClaims: progress.confirmedClaims, + priorInvestigationResult: await findCarriedForwardClaims(repo, { + id: investigation.postVersion.id, + postId: investigation.postVersion.postId, + contentText: investigation.postVersion.contentBlob.contentText, + }), + }; + } + case "FAILED": + return { + investigationState: "FAILED" as const, + provenance, + }; + default: + return unreachableInvestigationStatus(investigation.status); + } +} + // --------------------------------------------------------------------------- // Router // --------------------------------------------------------------------------- @@ -154,10 +240,10 @@ export const postRouter = router({ .mutation(async ({ input, ctx }) => { const postVersion = await findPostVersionById(ctx.prisma, input.postVersionId); if (postVersion === null) { - return { - investigationState: "NOT_INVESTIGATED" as const, - priorInvestigationResult: null, - }; + throw new TRPCError({ + code: "BAD_REQUEST", + message: "Unknown post version", + }); } await ctx.prisma.post.update({ @@ -175,42 +261,32 @@ export const postRouter = router({ ctx.ipRangeKey, ); - await maybeRecordCorroboration( - prismaInvestigationRepository(ctx.prisma), - postVersion.id, - ctx.viewerKey, - ctx.isAuthenticated, - ); + const repo = prismaInvestigationRepository(ctx.prisma); + await maybeRecordCorroboration(repo, postVersion.id, ctx.viewerKey, ctx.isAuthenticated); - const complete = await findCompletedInvestigationByPostVersionId( - prismaInvestigationRepository(ctx.prisma), - postVersion.id, - ); - - if (complete) { + // At most one investigation exists per post version (spec §3.5). If it + // exists in any state, report that state with its id: a viewer of a post + // someone else (or the selector) queued must be able to poll it. + const existing = await ctx.prisma.investigation.findUnique({ + where: { postVersionId: postVersion.id }, + select: { id: true }, + }); + const investigation = + existing === null ? null : await loadInvestigationWithClaims(repo, existing.id); + if (investigation !== null) { return { - investigationState: "INVESTIGATED" as const, - provenance: parseProvenance({ - investigationId: complete.id, - snapshot: complete.input, - }), - claims: formatClaims(complete.claims), + investigationId: investigation.id, + ...(await projectInvestigationStatus(repo, investigation)), }; } - const latestServerVerifiedSource = await findLatestServerVerifiedCompleteInvestigationForPost( - prismaInvestigationRepository(ctx.prisma), - postVersion.post.id, - ); - - const sourceInvestigation = selectSourceInvestigationForUpdate( - latestServerVerifiedSource, - postVersion.id, - ); - return { investigationState: "NOT_INVESTIGATED" as const, - priorInvestigationResult: toPriorInvestigationResult(sourceInvestigation), + priorInvestigationResult: await findCarriedForwardClaims(repo, { + id: postVersion.id, + postId: postVersion.post.id, + contentText: postVersion.contentBlob.contentText, + }), }; }), @@ -218,10 +294,8 @@ export const postRouter = router({ .input(getInvestigationInputSchema) .output(getInvestigationOutputSchema) .query(async ({ input, ctx }) => { - const investigation = await loadInvestigationWithClaims( - prismaInvestigationRepository(ctx.prisma), - input.investigationId, - ); + const repo = prismaInvestigationRepository(ctx.prisma); + const investigation = await loadInvestigationWithClaims(repo, input.investigationId); if (!investigation) { return { @@ -230,60 +304,21 @@ export const postRouter = router({ }; } - const provenance = parseProvenance({ - investigationId: investigation.id, - snapshot: investigation.input, - }); - - switch (investigation.status) { - case "COMPLETE": - return { - investigationState: "INVESTIGATED" as const, - provenance, - claims: formatClaims(investigation.claims), - checkedAt: requireCompleteCheckedAtIso(investigation.id, investigation.checkedAt), - }; - case "PENDING": - case "PROCESSING": { - const progress = parseProgressClaims(investigation.lease?.progressClaims ?? null); - return { - investigationState: "INVESTIGATING" as const, - status: investigation.status, - provenance, - pendingClaims: progress.pendingClaims, - confirmedClaims: progress.confirmedClaims, - priorInvestigationResult: - investigation.parentInvestigation !== null && - investigation.parentInvestigation.status === "COMPLETE" - ? { - oldClaims: formatClaims(investigation.parentInvestigation.claims), - sourceInvestigationId: investigation.parentInvestigation.id, - } - : null, - checkedAt: investigation.checkedAt?.toISOString(), - }; - } - case "FAILED": - return { - investigationState: "FAILED" as const, - provenance, - checkedAt: investigation.checkedAt?.toISOString(), - }; - default: - return unreachableInvestigationStatus(investigation.status); + const status = await projectInvestigationStatus(repo, investigation); + if (status.investigationState === "INVESTIGATED") { + return { + ...status, + checkedAt: requireCompleteCheckedAtIso(investigation.id, investigation.checkedAt), + }; } + return status; }), investigateNow: extensionProcedure .input(investigateNowInputSchema) .output(investigateNowOutputSchema) .mutation(async ({ input, ctx }) => { - if (!ctx.canInvestigate) { - throw new TRPCError({ - code: "UNAUTHORIZED", - message: "Valid API key or x-openai-api-key required for investigateNow", - }); - } + const requester = investigationRequester(ctx); const postVersion = await findPostVersionById(ctx.prisma, input.postVersionId); if (postVersion === null) { @@ -293,84 +328,14 @@ export const postRouter = router({ }); } - const complete = await findCompletedInvestigationByPostVersionId( - prismaInvestigationRepository(ctx.prisma), - postVersion.id, - ); - if (complete) { - return { - investigationId: complete.id, - status: complete.status, - provenance: parseProvenance({ - investigationId: complete.id, - snapshot: complete.input, - }), - claims: formatClaims(complete.claims), - }; - } - - const latestServerVerifiedSource = await findLatestServerVerifiedCompleteInvestigationForPost( - prismaInvestigationRepository(ctx.prisma), - postVersion.post.id, - ); - - const sourceInvestigation = selectSourceInvestigationForUpdate( - latestServerVerifiedSource, - postVersion.id, - ); - const prompt = await getOrCreateCurrentPrompt(); + let investigationId: string; try { - const { investigation } = await ensureInvestigationsWithUpdateMetadata({ - prisma: ctx.prisma, + ({ investigationId } = await requestInvestigation(ctx.prisma, { postVersion, promptId: prompt.id, - sourceInvestigation, - onPendingInvestigation: async ({ prisma, investigation: pendingInvestigation }) => { - if (ctx.userOpenAiApiKey === null) return; - await attachOpenAiKeySourceIfPending(prisma, { - investigationId: pendingInvestigation.id, - openAiApiKey: ctx.userOpenAiApiKey, - }); - }, - }); - - const loadedInvestigation = await loadInvestigationWithClaims( - prismaInvestigationRepository(ctx.prisma), - investigation.id, - ); - if (!loadedInvestigation) { - throw new TRPCError({ - code: "INTERNAL_SERVER_ERROR", - message: `Investigation ${investigation.id} disappeared after ensureInvestigationsWithUpdateMetadata`, - }); - } - - const provenance = parseProvenance({ - investigationId: loadedInvestigation.id, - snapshot: loadedInvestigation.input, - }); - - switch (loadedInvestigation.status) { - case "COMPLETE": { - return { - investigationId: loadedInvestigation.id, - status: loadedInvestigation.status, - provenance, - claims: formatClaims(loadedInvestigation.claims), - }; - } - case "PENDING": - case "PROCESSING": - case "FAILED": - return { - investigationId: loadedInvestigation.id, - status: loadedInvestigation.status, - provenance, - }; - default: - return unreachableInvestigationStatus(loadedInvestigation.status); - } + requester, + })); } catch (error) { if (error instanceof InvestigationWordLimitError) { throw new TRPCError({ @@ -378,8 +343,43 @@ export const postRouter = router({ message: error.message, }); } + if (error instanceof UserOpenAiKeyRejectedError) { + throw userOpenAiKeyRejectedError(error); + } throw error; } + + const investigation = await loadInvestigationWithClaims( + prismaInvestigationRepository(ctx.prisma), + investigationId, + ); + if (!investigation) { + throw new TRPCError({ + code: "INTERNAL_SERVER_ERROR", + message: `Investigation ${investigationId} disappeared after investigateNow`, + }); + } + + const provenance = investigation.input.provenance; + switch (investigation.status) { + case "COMPLETE": + return { + investigationId: investigation.id, + status: investigation.status, + provenance, + claims: formatClaims(investigation.claims), + }; + case "PENDING": + case "PROCESSING": + case "FAILED": + return { + investigationId: investigation.id, + status: investigation.status, + provenance, + }; + default: + return unreachableInvestigationStatus(investigation.status); + } }), validateSettings: extensionProcedure @@ -400,39 +400,37 @@ export const postRouter = router({ const lookupKey = (platform: Platform, externalId: string, versionHash: string): string => `${platform}:${externalId}:${versionHash}`; - const versions = - input.posts.length === 0 - ? [] - : await ctx.prisma.postVersion.findMany({ - where: { - OR: input.posts.map((post) => ({ - versionHash: post.versionHash, - post: { - platform: post.platform, - externalId: post.externalId, - }, - })), - }, - select: { - versionHash: true, - post: { - select: { - platform: true, - externalId: true, - }, - }, - investigation: { - select: { - status: true, - _count: { - select: { - claims: true, - }, - }, - }, + // batchStatusInputSchema requires at least one post. + const versions = await ctx.prisma.postVersion.findMany({ + where: { + OR: input.posts.map((post) => ({ + versionHash: post.versionHash, + post: { + platform: post.platform, + externalId: post.externalId, + }, + })), + }, + select: { + versionHash: true, + post: { + select: { + platform: true, + externalId: true, + }, + }, + investigation: { + select: { + status: true, + _count: { + select: { + claims: true, }, }, - }); + }, + }, + }, + }); const byLookupKey = new Map(); for (const version of versions) { diff --git a/src/typescript/api/src/lib/trpc/routes/post/content-storage.ts b/src/typescript/api/src/lib/trpc/routes/post/content-storage.ts index 0f6931c..e574bbe 100644 --- a/src/typescript/api/src/lib/trpc/routes/post/content-storage.ts +++ b/src/typescript/api/src/lib/trpc/routes/post/content-storage.ts @@ -5,6 +5,5 @@ * `post/content-storage/` to keep each file maintainable. */ -export type { ResolvedPostVersion } from "./content-storage/shared.js"; export { findPostVersionById } from "./content-storage/post-version.js"; export { registerObservedVersion } from "./content-storage/register-observed-version.js"; diff --git a/src/typescript/api/src/lib/trpc/routes/post/content-storage/blobs.ts b/src/typescript/api/src/lib/trpc/routes/post/content-storage/blobs.ts index dce1729..7f7bbd4 100644 --- a/src/typescript/api/src/lib/trpc/routes/post/content-storage/blobs.ts +++ b/src/typescript/api/src/lib/trpc/routes/post/content-storage/blobs.ts @@ -1,5 +1,5 @@ import { TRPCError } from "@trpc/server"; -import { wordCount } from "$lib/services/investigation-lifecycle.js"; +import { wordCount } from "$lib/services/word-count.js"; import type { ContentBlob } from "$lib/db/prisma-client"; import { createOrFindByUniqueConstraint, type DbClient } from "./shared.js"; import { sha256 } from "./hashing.js"; diff --git a/src/typescript/api/src/lib/trpc/routes/post/content-storage/content-preparation.ts b/src/typescript/api/src/lib/trpc/routes/post/content-storage/content-preparation.ts index 70b65aa..f4aa45c 100644 --- a/src/typescript/api/src/lib/trpc/routes/post/content-storage/content-preparation.ts +++ b/src/typescript/api/src/lib/trpc/routes/post/content-storage/content-preparation.ts @@ -26,7 +26,7 @@ export function applyServerVerifiedWikipediaIdentity(input: { if ( input.preparedInput.platform !== "WIKIPEDIA" || input.canonical.provenance !== "SERVER_VERIFIED" || - input.canonical.canonicalIdentity?.platform !== "WIKIPEDIA" + input.canonical.canonicalIdentity.platform !== "WIKIPEDIA" ) { return input.preparedInput; } diff --git a/src/typescript/api/src/lib/trpc/routes/post/content-storage/metadata.ts b/src/typescript/api/src/lib/trpc/routes/post/content-storage/metadata.ts index 738dd18..14484bf 100644 --- a/src/typescript/api/src/lib/trpc/routes/post/content-storage/metadata.ts +++ b/src/typescript/api/src/lib/trpc/routes/post/content-storage/metadata.ts @@ -1,5 +1,5 @@ import type { ViewPostInput } from "@openerrata/shared"; -import { trimToOptionalNonEmpty } from "@openerrata/shared"; +import { observedImageUrlsFromOccurrences, trimToOptionalNonEmpty } from "@openerrata/shared"; import { TRPCError } from "@trpc/server"; import { toOptionalDate } from "$lib/date.js"; import type { Prisma } from "$lib/db/prisma-client"; @@ -8,10 +8,37 @@ import { createOrFindByUniqueConstraint, type DbClient } from "./shared.js"; import { getOrCreateHtmlBlob } from "./blobs.js"; import type { PreparedViewPostInput } from "../wikipedia.js"; -export async function createPlatformVersionMetadataIfMissing( +/** + * LessWrong title/author for version metadata: the server's when it verified + * the post (identity binding, SPEC §2.9), otherwise what the client reported. + */ +function lesswrongDisplayMetadata( + metadata: Extract["metadata"], + canonical: CanonicalContentVersion, +): { title: string | undefined; authorName: string | undefined; authorSlug: string | undefined } { + if ( + canonical.provenance === "SERVER_VERIFIED" && + canonical.canonicalIdentity.platform === "LESSWRONG" + ) { + const { title, author } = canonical.canonicalIdentity; + return { title, authorName: author?.displayName, authorSlug: author?.slug }; + } + return { + title: trimToOptionalNonEmpty(metadata.title), + authorName: trimToOptionalNonEmpty(metadata.authorName), + authorSlug: trimToOptionalNonEmpty(metadata.authorSlug), + }; +} + +/** + * Create the version's platform metadata row, or update its mutable fields + * (latest wins) when it already exists; client HTML is first-write-wins. + */ +export async function upsertPlatformVersionMetadata( prisma: DbClient, input: { preparedInput: PreparedViewPostInput; + canonical: CanonicalContentVersion; postVersionId: string; htmlBlobIds: { serverHtmlBlobId: string | null; @@ -19,6 +46,7 @@ export async function createPlatformVersionMetadataIfMissing( }; }, ): Promise { + const imageUrls = observedImageUrlsFromOccurrences(input.preparedInput.observedImageOccurrences); switch (input.preparedInput.platform) { case "LESSWRONG": { const metadata = input.preparedInput.metadata; @@ -29,9 +57,7 @@ export async function createPlatformVersionMetadataIfMissing( message: `LessWrong version requires at least one HTML snapshot (postVersionId=${input.postVersionId})`, }); } - const title = trimToOptionalNonEmpty(metadata.title); - const authorName = trimToOptionalNonEmpty(metadata.authorName); - const authorSlug = trimToOptionalNonEmpty(metadata.authorSlug); + const { title, authorName, authorSlug } = lesswrongDisplayMetadata(metadata, input.canonical); const existingOrCreated = await createOrFindByUniqueConstraint({ findExisting: () => @@ -49,7 +75,7 @@ export async function createPlatformVersionMetadataIfMissing( ...(title !== undefined && { title }), ...(serverHtmlBlobId !== null && { serverHtmlBlobId }), ...(clientHtmlBlobId !== null && { clientHtmlBlobId }), - imageUrls: input.preparedInput.observedImageUrls ?? [], + imageUrls, ...(authorName !== undefined && { authorName }), ...(authorSlug !== undefined && { authorSlug }), tags: metadata.tags, @@ -73,7 +99,7 @@ export async function createPlatformVersionMetadataIfMissing( slug: metadata.slug, title: title ?? null, ...(serverHtmlBlobId !== null && { serverHtmlBlobId }), - imageUrls: input.preparedInput.observedImageUrls ?? [], + imageUrls, authorName: authorName ?? null, authorSlug: authorSlug ?? null, tags: metadata.tags, @@ -136,6 +162,12 @@ export async function createPlatformVersionMetadataIfMissing( case "SUBSTACK": { const metadata = input.preparedInput.metadata; const { serverHtmlBlobId, clientHtmlBlobId } = input.htmlBlobIds; + if (serverHtmlBlobId !== null) { + throw new TRPCError({ + code: "INTERNAL_SERVER_ERROR", + message: `Substack has no server-side fetch, yet server HTML was stored (postVersionId=${input.postVersionId})`, + }); + } const authorSubstackHandle = trimToOptionalNonEmpty(metadata.authorSubstackHandle); const existingOrCreated = await createOrFindByUniqueConstraint({ @@ -156,9 +188,8 @@ export async function createPlatformVersionMetadataIfMissing( slug: metadata.slug, title: metadata.title, subtitle: metadata.subtitle ?? null, - ...(serverHtmlBlobId !== null && { serverHtmlBlobId }), ...(clientHtmlBlobId !== null && { clientHtmlBlobId }), - imageUrls: input.preparedInput.observedImageUrls ?? [], + imageUrls, authorName: metadata.authorName, authorSubstackHandle: authorSubstackHandle ?? null, publishedAt: toOptionalDate(metadata.publishedAt), @@ -184,8 +215,7 @@ export async function createPlatformVersionMetadataIfMissing( slug: metadata.slug, title: metadata.title, subtitle: metadata.subtitle ?? null, - ...(serverHtmlBlobId !== null && { serverHtmlBlobId }), - imageUrls: input.preparedInput.observedImageUrls ?? [], + imageUrls, authorName: metadata.authorName, authorSubstackHandle: authorSubstackHandle ?? null, publishedAt: toOptionalDate(metadata.publishedAt), @@ -233,7 +263,7 @@ export async function createPlatformVersionMetadataIfMissing( ...(clientHtmlBlobId !== null && { clientHtmlBlobId }), revisionId: metadata.revisionId, lastModifiedAt: toOptionalDate(metadata.lastModifiedAt), - imageUrls: input.preparedInput.observedImageUrls ?? [], + imageUrls, }, select: { pageId: true, @@ -256,7 +286,7 @@ export async function createPlatformVersionMetadataIfMissing( ...(serverHtmlBlobId !== null && { serverHtmlBlobId }), revisionId: metadata.revisionId, lastModifiedAt: toOptionalDate(metadata.lastModifiedAt), - imageUrls: input.preparedInput.observedImageUrls ?? [], + imageUrls, } satisfies Prisma.WikipediaVersionMetaUpdateInput; await Promise.all([ diff --git a/src/typescript/api/src/lib/trpc/routes/post/content-storage/observed-url.ts b/src/typescript/api/src/lib/trpc/routes/post/content-storage/observed-url.ts new file mode 100644 index 0000000..9302772 --- /dev/null +++ b/src/typescript/api/src/lib/trpc/routes/post/content-storage/observed-url.ts @@ -0,0 +1,82 @@ +/** + * Validation of the post URL a client reports to registerObservedVersion. + * + * Until a server fetch supplies the canonical URL (SPEC §2.9), the client's + * URL is what we store and link to publicly, so it must be an https URL on the + * platform's own host that names the same post as `externalId`. Substack + * publications may use custom domains, so only the post path is checked there. + */ + +import { TRPCError } from "@trpc/server"; +import type { PreparedViewPostInput } from "../wikipedia.js"; + +const LESSWRONG_HOSTS: ReadonlySet = new Set(["lesswrong.com", "www.lesswrong.com"]); +const X_HOSTS = ["x.com", "twitter.com"] as const; + +function isXHost(hostname: string): boolean { + return X_HOSTS.some((host) => hostname === host || hostname.endsWith(`.${host}`)); +} + +/** Status id named by an X status path (`//status/`, `/i/status/`, `/i/web/status/`). */ +function xStatusIdFromPath(pathname: string): string | null { + const segments = pathname.split("/").filter((segment) => segment.length > 0); + const statusIndex = segments.indexOf("status"); + if (statusIndex < 1) return null; + return segments[statusIndex + 1] ?? null; +} + +function lesswrongPathNamesPost(pathname: string, externalId: string): boolean { + const prefix = `/posts/${externalId}`; + return pathname === prefix || pathname.startsWith(`${prefix}/`); +} + +function rejectUrl(platform: string, reason: string): never { + throw new TRPCError({ + code: "BAD_REQUEST", + message: `Invalid ${platform} post URL: ${reason}`, + }); +} + +export function assertObservedPostUrlMatchesPlatform(input: PreparedViewPostInput): void { + let url: URL; + try { + url = new URL(input.url); + } catch { + return rejectUrl(input.platform, "not a URL"); + } + if (url.protocol !== "https:") { + rejectUrl(input.platform, "must use https"); + } + if (url.username.length > 0 || url.password.length > 0) { + rejectUrl(input.platform, "must not embed credentials"); + } + const hostname = url.hostname.toLowerCase(); + + switch (input.platform) { + case "LESSWRONG": + if (!LESSWRONG_HOSTS.has(hostname)) { + rejectUrl(input.platform, `unexpected host ${hostname}`); + } + if (!lesswrongPathNamesPost(url.pathname, input.externalId)) { + rejectUrl(input.platform, "path does not name the post's externalId"); + } + return; + case "X": + if (!isXHost(hostname)) { + rejectUrl(input.platform, `unexpected host ${hostname}`); + } + if (xStatusIdFromPath(url.pathname) !== input.externalId) { + rejectUrl(input.platform, "path does not name the post's externalId"); + } + return; + case "SUBSTACK": + if (!/^\/p\/[^/]+/.test(url.pathname)) { + rejectUrl(input.platform, "path is not a /p/ post path"); + } + return; + case "WIKIPEDIA": + // Host and article identity are checked against metadata when the + // external ID is derived (prepareViewPostInput). + return; + } +} diff --git a/src/typescript/api/src/lib/trpc/routes/post/content-storage/post-upsert.ts b/src/typescript/api/src/lib/trpc/routes/post/content-storage/post-upsert.ts index 3b7574f..dc18671 100644 --- a/src/typescript/api/src/lib/trpc/routes/post/content-storage/post-upsert.ts +++ b/src/typescript/api/src/lib/trpc/routes/post/content-storage/post-upsert.ts @@ -1,157 +1,173 @@ -import { trimToOptionalNonEmpty } from "@openerrata/shared"; -import type { DbClient, UpsertPostInput } from "./shared.js"; -import type { Post } from "$lib/db/prisma-client"; +/** + * Post identity storage (SPEC §2.9 identity binding). + * + * A post's URL and author are identity-bound. When the server fetch verified + * the post, they come from the platform's response and the post is latched as + * identity-verified (Post.identityVerifiedAt). Otherwise the client's + * validated URL and reported author are used — but never over a post whose + * identity a server fetch has already verified. + */ + +import { trimToOptionalNonEmpty, type Platform } from "@openerrata/shared"; +import type { DbClient } from "$lib/db/client"; +import type { CanonicalContentVersion } from "$lib/services/canonical-resolution.js"; import type { PreparedViewPostInput } from "../wikipedia.js"; -async function upsertAuthorAndAttachToPost( - prisma: DbClient, - input: { - postId: string; - platform: UpsertPostInput["platform"]; - platformUserId: string; - displayName: string; - }, -): Promise { - const author = await prisma.author.upsert({ - where: { - platform_platformUserId: { - platform: input.platform, - platformUserId: input.platformUserId, - }, - }, - create: { - platform: input.platform, - platformUserId: input.platformUserId, - displayName: input.displayName, - }, - update: { - displayName: input.displayName, - }, - select: { id: true }, - }); +interface AuthorIdentity { + platformUserId: string; + displayName: string; +} - await prisma.post.update({ - where: { id: input.postId }, - data: { authorId: author.id }, - }); +interface PostIdentity { + source: "SERVER_VERIFIED" | "CLIENT_OBSERVED"; + url: string; + /** Null when the source names no author; the stored author link is then left as is. */ + author: AuthorIdentity | null; } -/** - * Upsert the Author record for a post and link it via Post.authorId. - * Extracted from the deleted linkAuthorAndMetadata; handles only author - * identity — platform-specific metadata lives in version meta tables. - */ -async function upsertAuthorForPost( - prisma: DbClient, - input: { postId: string } & UpsertPostInput, -): Promise { +function clientReportedAuthor(input: PreparedViewPostInput): AuthorIdentity | null { switch (input.platform) { case "LESSWRONG": { const authorName = trimToOptionalNonEmpty(input.metadata.authorName); const authorSlug = trimToOptionalNonEmpty(input.metadata.authorSlug); - const authorDisplayName = authorName ?? authorSlug; - if (authorDisplayName !== undefined && authorDisplayName.length > 0) { - const platformUserId = authorSlug ?? `name:${authorDisplayName.toLowerCase()}`; - await upsertAuthorAndAttachToPost(prisma, { - postId: input.postId, - platform: "LESSWRONG", - platformUserId, - displayName: authorDisplayName, - }); - } - return; + const displayName = authorName ?? authorSlug; + if (displayName === undefined) return null; + return { + platformUserId: authorSlug ?? `name:${displayName.toLowerCase()}`, + displayName, + }; } case "X": { const authorHandle = input.metadata.authorHandle; - const authorDisplayName = trimToOptionalNonEmpty(input.metadata.authorDisplayName); - await upsertAuthorAndAttachToPost(prisma, { - postId: input.postId, - platform: "X", + return { platformUserId: authorHandle, - displayName: authorDisplayName ?? authorHandle, - }); - return; + displayName: trimToOptionalNonEmpty(input.metadata.authorDisplayName) ?? authorHandle, + }; } case "SUBSTACK": { const authorName = input.metadata.authorName.trim(); - const authorSubstackHandle = trimToOptionalNonEmpty(input.metadata.authorSubstackHandle); - const platformUserId = - authorSubstackHandle ?? - `publication:${input.metadata.publicationSubdomain}:name:${authorName.toLowerCase()}`; - await upsertAuthorAndAttachToPost(prisma, { - postId: input.postId, - platform: "SUBSTACK", - platformUserId, + return { + platformUserId: + trimToOptionalNonEmpty(input.metadata.authorSubstackHandle) ?? + `publication:${input.metadata.publicationSubdomain}:name:${authorName.toLowerCase()}`, displayName: authorName, - }); - return; + }; } case "WIKIPEDIA": - return; + return null; + } +} + +function resolvePostIdentity( + input: PreparedViewPostInput, + canonical: CanonicalContentVersion, +): PostIdentity { + if (canonical.provenance === "SERVER_VERIFIED") { + const identity = canonical.canonicalIdentity; + switch (identity.platform) { + case "LESSWRONG": + return { + source: "SERVER_VERIFIED", + url: identity.url, + author: + identity.author === null + ? null + : { platformUserId: identity.author.slug, displayName: identity.author.displayName }, + }; + case "WIKIPEDIA": + return { source: "SERVER_VERIFIED", url: identity.url, author: null }; + } } + return { source: "CLIENT_OBSERVED", url: input.url, author: clientReportedAuthor(input) }; } -async function upsertPost(prisma: DbClient, input: UpsertPostInput) { - const post = await prisma.post.upsert({ +function externalIdOf(input: PreparedViewPostInput): string { + return input.platform === "WIKIPEDIA" ? input.derivedExternalId : input.externalId; +} + +async function linkAuthor( + db: DbClient, + input: { + postId: string; + platform: Platform; + author: AuthorIdentity; + /** Only link when the post's identity is not server-verified. */ + onlyIfUnverified: boolean; + }, +): Promise { + const author = await db.author.upsert({ where: { - platform_externalId: { + platform_platformUserId: { platform: input.platform, - externalId: input.externalId, + platformUserId: input.author.platformUserId, }, }, create: { platform: input.platform, - externalId: input.externalId, - url: input.url, + platformUserId: input.author.platformUserId, + displayName: input.author.displayName, }, update: { - url: input.url, + displayName: input.author.displayName, }, + select: { id: true }, }); - await upsertAuthorForPost(prisma, { - postId: post.id, - ...input, + await db.post.updateMany({ + where: { + id: input.postId, + ...(input.onlyIfUnverified ? { identityVerifiedAt: null } : {}), + }, + data: { authorId: author.id }, }); - - return post; } +/** Upsert the Post row for an observed version and apply its identity. Returns the post id. */ export async function upsertPostFromViewInput( - prisma: DbClient, + db: DbClient, input: PreparedViewPostInput, -): Promise { - if (input.platform === "LESSWRONG") { - return upsertPost(prisma, { - url: input.url, - platform: "LESSWRONG", - externalId: input.externalId, - metadata: input.metadata, - }); + canonical: CanonicalContentVersion, +): Promise<{ id: string }> { + const identity = resolvePostIdentity(input, canonical); + const verified = identity.source === "SERVER_VERIFIED"; + const now = new Date(); + + const post = await db.post.upsert({ + where: { + platform_externalId: { + platform: input.platform, + externalId: externalIdOf(input), + }, + }, + create: { + platform: input.platform, + externalId: externalIdOf(input), + url: identity.url, + identityVerifiedAt: verified ? now : null, + }, + update: verified ? { url: identity.url, identityVerifiedAt: now } : {}, + select: { id: true, identityVerifiedAt: true }, + }); + + if (!verified && post.identityVerifiedAt !== null) { + // A server fetch verified this post earlier; unverified client data + // never overwrites that. + return { id: post.id }; } - if (input.platform === "X") { - return upsertPost(prisma, { - url: input.url, - platform: "X", - externalId: input.externalId, - metadata: input.metadata, + if (!verified) { + await db.post.updateMany({ + where: { id: post.id, identityVerifiedAt: null }, + data: { url: identity.url }, }); } - - if (input.platform === "WIKIPEDIA") { - return upsertPost(prisma, { - url: input.url, - platform: "WIKIPEDIA", - externalId: input.derivedExternalId, - metadata: input.metadata, + if (identity.author !== null) { + await linkAuthor(db, { + postId: post.id, + platform: input.platform, + author: identity.author, + onlyIfUnverified: !verified, }); } - - return upsertPost(prisma, { - url: input.url, - platform: "SUBSTACK", - externalId: input.externalId, - metadata: input.metadata, - }); + return { id: post.id }; } diff --git a/src/typescript/api/src/lib/trpc/routes/post/content-storage/register-observed-version.ts b/src/typescript/api/src/lib/trpc/routes/post/content-storage/register-observed-version.ts index 889e97c..2cebcf3 100644 --- a/src/typescript/api/src/lib/trpc/routes/post/content-storage/register-observed-version.ts +++ b/src/typescript/api/src/lib/trpc/routes/post/content-storage/register-observed-version.ts @@ -2,7 +2,6 @@ import type { PrismaClient } from "$lib/db/prisma-client"; import type { ViewPostInput } from "@openerrata/shared"; import { fetchCanonicalContent } from "$lib/services/content-fetcher.js"; import { resolveCanonicalContentVersion } from "$lib/services/canonical-resolution.js"; -import { TRPCError } from "@trpc/server"; import { isUniqueConstraintError } from "$lib/db/errors.js"; import { prepareViewPostInput } from "../wikipedia.js"; import { @@ -11,10 +10,11 @@ import { delay, type ResolvedPostVersion, } from "./shared.js"; +import { assertObservedPostUrlMatchesPlatform } from "./observed-url.js"; import { upsertPostFromViewInput } from "./post-upsert.js"; import { upsertPostVersion } from "./post-version.js"; import { - createPlatformVersionMetadataIfMissing, + upsertPlatformVersionMetadata, resolveHtmlBlobIdsForStorage, resolveHtmlSnapshotsForStorage, } from "./metadata.js"; @@ -35,6 +35,7 @@ export async function registerObservedVersion( input: ViewPostInput, ): Promise { const initiallyPreparedInput = prepareViewPostInput(input); + assertObservedPostUrlMatchesPlatform(initiallyPreparedInput); const observed = await toObservedContentVersion(initiallyPreparedInput); const canonical = await resolveCanonicalContentVersion({ @@ -55,10 +56,12 @@ export async function registerObservedVersion( preparedInput: initiallyPreparedInput, canonical, }); - for (let attempt = 0; attempt < UNIQUE_CONSTRAINT_RACE_RETRY_ATTEMPTS; attempt += 1) { + // Concurrent registrations of the same post race on unique constraints; + // retry the whole transaction, re-throwing once the retries are spent. + for (let attempt = 1; ; attempt += 1) { try { return await prisma.$transaction(async (tx) => { - const post = await upsertPostFromViewInput(tx, preparedInput); + const post = await upsertPostFromViewInput(tx, preparedInput, canonical); const postVersion = await upsertPostVersion(tx, { postId: post.id, canonical, @@ -67,26 +70,19 @@ export async function registerObservedVersion( : { observedImageOccurrences: preparedInput.observedImageOccurrences }), }); const htmlBlobIds = await resolveHtmlBlobIdsForStorage(tx, htmlSnapshotsForStorage); - await createPlatformVersionMetadataIfMissing(tx, { + await upsertPlatformVersionMetadata(tx, { preparedInput, + canonical, postVersionId: postVersion.id, htmlBlobIds, }); return postVersion; }); } catch (error) { - if ( - !isUniqueConstraintError(error) || - attempt === UNIQUE_CONSTRAINT_RACE_RETRY_ATTEMPTS - 1 - ) { + if (!isUniqueConstraintError(error) || attempt >= UNIQUE_CONSTRAINT_RACE_RETRY_ATTEMPTS) { throw error; } await delay(UNIQUE_CONSTRAINT_RACE_RETRY_DELAY_MS); } } - - throw new TRPCError({ - code: "INTERNAL_SERVER_ERROR", - message: "Failed to register observed version due to repeated unique-constraint races", - }); } diff --git a/src/typescript/api/src/lib/trpc/routes/post/content-storage/shared.ts b/src/typescript/api/src/lib/trpc/routes/post/content-storage/shared.ts index 75676c4..929f7cf 100644 --- a/src/typescript/api/src/lib/trpc/routes/post/content-storage/shared.ts +++ b/src/typescript/api/src/lib/trpc/routes/post/content-storage/shared.ts @@ -1,15 +1,6 @@ -import type { Platform, PlatformMetadataByPlatform } from "@openerrata/shared"; -import type { PrismaClient, Prisma } from "$lib/db/prisma-client"; -import { isUniqueConstraintError } from "$lib/db/errors.js"; +import type { Platform } from "@openerrata/shared"; -export type UpsertPostInput = { - [P in Platform]: { - platform: P; - externalId: string; - url: string; - metadata: PlatformMetadataByPlatform[P]; - }; -}[Platform]; +export type { DbClient } from "$lib/db/client"; export interface ResolvedPostVersion { id: string; @@ -32,49 +23,26 @@ export interface ResolvedPostVersion { export const UNIQUE_CONSTRAINT_RACE_RETRY_ATTEMPTS = 30; export const UNIQUE_CONSTRAINT_RACE_RETRY_DELAY_MS = 20; -export type DbClient = PrismaClient | Prisma.TransactionClient; - export function delay(ms: number): Promise { return new Promise((resolve) => { setTimeout(resolve, ms); }); } +/** + * Find-or-create inside an interactive transaction. A unique violation from a + * concurrent insert aborts the transaction, so it propagates for the caller's + * whole-transaction retry (registerObservedVersion) rather than being handled here. + */ export async function createOrFindByUniqueConstraint(input: { findExisting: () => Promise; create: () => Promise; assertEquivalent: (existing: T) => void; - /** - * Safe only outside of interactive transactions. - * Inside a transaction, unique violations abort the transaction until rollback. - */ - retryReadAfterUniqueConflict?: boolean; }): Promise { const existing = await input.findExisting(); if (existing !== null) { input.assertEquivalent(existing); return existing; } - - try { - return await input.create(); - } catch (error) { - if (!isUniqueConstraintError(error)) { - throw error; - } - if (input.retryReadAfterUniqueConflict !== true) { - throw error; - } - // For non-transactional callers, the winning insert can briefly be - // invisible right after a unique-constraint conflict. - for (let attempt = 0; attempt < UNIQUE_CONSTRAINT_RACE_RETRY_ATTEMPTS; attempt += 1) { - const raced = await input.findExisting(); - if (raced !== null) { - input.assertEquivalent(raced); - return raced; - } - await delay(UNIQUE_CONSTRAINT_RACE_RETRY_DELAY_MS); - } - throw error; - } + return input.create(); } diff --git a/src/typescript/api/src/lib/trpc/routes/post/investigation-queries.ts b/src/typescript/api/src/lib/trpc/routes/post/investigation-queries.ts index 3504716..d00f513 100644 --- a/src/typescript/api/src/lib/trpc/routes/post/investigation-queries.ts +++ b/src/typescript/api/src/lib/trpc/routes/post/investigation-queries.ts @@ -1,23 +1,19 @@ /** - * Investigation query helpers for the post router. - * - * Provides loading, formatting, and lifecycle helpers for investigations: - * claim formatting, diff computation for update investigations, - * corroboration recording, and investigation queueing with update metadata. + * Investigation query helpers for the post router: loading, claim formatting, + * interim claim carry-forward, and corroboration recording. */ import { claimIdSchema, investigationClaimPayloadSchema, + normalizeContent, type InvestigationClaim, type InvestigationClaimPayload, } from "@openerrata/shared"; import { z } from "zod"; -import { ensureInvestigationQueued } from "$lib/services/investigation-lifecycle.js"; import { isUniqueConstraintError } from "$lib/db/errors.js"; import type { Prisma, PrismaClient } from "$lib/db/prisma-client"; import { TRPCError } from "@trpc/server"; -import type { ResolvedPostVersion } from "./content-storage.js"; // --------------------------------------------------------------------------- // Prisma include shapes and derived payload types @@ -28,6 +24,8 @@ const investigationWithClaimsInclude = { lease: true, postVersion: { select: { + id: true, + postId: true, contentBlob: { select: { contentText: true, @@ -41,72 +39,12 @@ const investigationWithClaimsInclude = { sources: true, }, }, - parentInvestigation: { - include: { - claims: { - include: { - sources: true, - }, - }, - postVersion: { - select: { - contentBlob: { - select: { - contentText: true, - }, - }, - }, - }, - }, - }, } satisfies Prisma.InvestigationInclude; type InvestigationWithClaims = Prisma.InvestigationGetPayload<{ include: typeof investigationWithClaimsInclude; }>; -const completedInvestigationInclude = { - input: true, - postVersion: { - select: { - id: true, - contentBlob: { - select: { - contentText: true, - contentHash: true, - }, - }, - }, - }, - claims: { - include: { - sources: true, - }, - }, -} satisfies Prisma.InvestigationInclude; - -type CompletedInvestigation = Prisma.InvestigationGetPayload<{ - include: typeof completedInvestigationInclude; -}>; - -const serverVerifiedSourceInclude = { - postVersion: { - select: { - id: true, - contentBlob: { - select: { - contentText: true, - }, - }, - }, - }, - claims: { - include: { - sources: true, - }, - }, -} satisfies Prisma.InvestigationInclude; - interface ClaimSourceSummary { url: string; title: string; @@ -122,71 +60,12 @@ interface ClaimSummary { sources: ClaimSourceSummary[]; } -interface SourceInvestigationForUpdate { +/** A complete investigation whose claims may be carried forward to another version. */ +interface CompleteInvestigationClaims { id: string; - postVersion: { - id: string; - contentBlob: { - contentText: string; - }; - }; claims: ClaimSummary[]; } -type LatestServerVerifiedCompleteInvestigation = SourceInvestigationForUpdate | null; - -type EnsuredInvestigationStatus = Awaited< - ReturnType ->["investigation"]["status"]; - -interface EnsureInvestigationResult { - investigation: { - id: string; - status: EnsuredInvestigationStatus; - }; -} - -interface EnsureQueuedInput { - prisma: TPrisma; - postVersionId: string; - promptId: string; - parentInvestigationId?: string; - contentDiff?: string; - rejectOverWordLimitOnCreate: true; - allowRequeueFailed: true; - onPendingInvestigation?: (input: { - prisma: TPrisma; - investigation: { - id: string; - status: EnsuredInvestigationStatus; - }; - }) => Promise; -} - -type EnsureQueued = ( - input: EnsureQueuedInput, -) => Promise; - -interface EnsureWithDefaultInput { - prisma: PrismaClient; - promptId: string; - postVersion: ResolvedPostVersion; - sourceInvestigation: LatestServerVerifiedCompleteInvestigation; - onPendingInvestigation?: Parameters< - typeof ensureInvestigationQueued - >[0]["onPendingInvestigation"]; - ensureQueued?: undefined; -} - -interface EnsureWithCustomInput { - prisma: TPrisma; - promptId: string; - postVersion: ResolvedPostVersion; - sourceInvestigation: LatestServerVerifiedCompleteInvestigation; - onPendingInvestigation?: EnsureQueuedInput["onPendingInvestigation"]; - ensureQueued: EnsureQueued; -} - /** * Semantic repository interface for investigation queries. Both PrismaClient * (production) and test stubs implement this — decoupled from Prisma's exact @@ -194,8 +73,14 @@ interface EnsureWithCustomInput { */ export interface InvestigationRepository { findInvestigationWithClaims(id: string): Promise; - findCompletedByPostVersionId(postVersionId: string): Promise; - findLatestServerVerifiedComplete(postId: string): Promise; + /** + * The post's most recently completed investigation of any provenance on a + * version other than `excludedPostVersionId`, or null when there is none. + */ + findLatestCompleteOnOtherVersion( + postId: string, + excludedPostVersionId: string, + ): Promise; findClientFallbackInvestigationId(postVersionId: string): Promise; recordCorroborationCredit(investigationId: string, reporterKey: string): Promise; } @@ -209,21 +94,17 @@ export function prismaInvestigationRepository(prisma: PrismaClient): Investigati include: investigationWithClaimsInclude, }); }, - async findCompletedByPostVersionId(postVersionId) { - return prisma.investigation.findFirst({ - where: { postVersionId, status: "COMPLETE" }, - include: completedInvestigationInclude, - }); - }, - async findLatestServerVerifiedComplete(postId) { + async findLatestCompleteOnOtherVersion(postId, excludedPostVersionId) { return prisma.investigation.findFirst({ where: { status: "COMPLETE", - postVersion: { postId }, - input: { provenance: "SERVER_VERIFIED" }, + postVersion: { postId, id: { not: excludedPostVersionId } }, + }, + orderBy: [{ checkedAt: "desc" }, { id: "desc" }], + select: { + id: true, + claims: { include: { sources: true } }, }, - orderBy: { checkedAt: "desc" }, - include: serverVerifiedSourceInclude, }); }, async findClientFallbackInvestigationId(postVersionId) { @@ -331,87 +212,61 @@ export async function loadInvestigationWithClaims( return repo.findInvestigationWithClaims(investigationId); } -export async function findCompletedInvestigationByPostVersionId( - repo: InvestigationRepository, - postVersionId: string, -): Promise { - return repo.findCompletedByPostVersionId(postVersionId); -} - -export async function findLatestServerVerifiedCompleteInvestigationForPost( - repo: InvestigationRepository, - postId: string, -): Promise { - return repo.findLatestServerVerifiedComplete(postId); -} - // --------------------------------------------------------------------------- -// Update investigation helpers +// Interim claim carry-forward (spec §2.8 "Interim carry-forward") // --------------------------------------------------------------------------- -export function selectSourceInvestigationForUpdate( - latestServerVerifiedSource: LatestServerVerifiedCompleteInvestigation, - currentPostVersionId: string, -): LatestServerVerifiedCompleteInvestigation { - if (latestServerVerifiedSource === null) { - return null; - } +/** Claims shown on a post version while it has no finished investigation of its own. */ +interface CarriedForwardClaims { + oldClaims: InvestigationClaim[]; + sourceInvestigationId: string; +} - return latestServerVerifiedSource.postVersion.id === currentPostVersionId - ? null - : latestServerVerifiedSource; +/** + * Whether a claim's quoted text occurs verbatim in `contentText`, compared + * after the normalization all content text gets (spec §3.8) — the text the + * extension locates claims in. Empty text quotes nothing. + */ +function claimTextOccursIn(claimText: string, contentText: string): boolean { + const normalizedClaimText = normalizeContent(claimText); + return normalizedClaimText.length > 0 && contentText.includes(normalizedClaimText); } -export function toPriorInvestigationResult( - sourceInvestigation: LatestServerVerifiedCompleteInvestigation, -): { - oldClaims: InvestigationClaim[]; - sourceInvestigationId: string; -} | null { - if (sourceInvestigation === null) { +/** + * The claims of `source` that are still about `contentText`: those whose + * quoted text still occurs in it. A correction is only shown while the exact + * text it corrects is still on the page. Null when no claim survives. + */ +export function carryForwardClaims( + source: CompleteInvestigationClaims, + contentText: string, +): CarriedForwardClaims | null { + const survivingClaims = source.claims.filter((claim) => + claimTextOccursIn(claim.text, contentText), + ); + if (survivingClaims.length === 0) { return null; } - return { - oldClaims: formatClaims(sourceInvestigation.claims), - sourceInvestigationId: sourceInvestigation.id, + oldClaims: formatClaims(survivingClaims), + sourceInvestigationId: source.id, }; } -function buildLineDiff(previous: string, current: string): string { - if (previous === current) { - return "No changes detected."; - } - - const previousLines = previous.split("\n"); - const currentLines = current.split("\n"); - const maxStart = Math.min(previousLines.length, currentLines.length); - let start = 0; - while (start < maxStart && previousLines[start] === currentLines[start]) { - start += 1; - } - - let previousEnd = previousLines.length; - let currentEnd = currentLines.length; - while ( - previousEnd > start && - currentEnd > start && - previousLines[previousEnd - 1] === currentLines[currentEnd - 1] - ) { - previousEnd -= 1; - currentEnd -= 1; - } - - const removed = previousLines.slice(start, previousEnd); - const added = currentLines.slice(start, currentEnd); - - return [ - "Diff summary (line context):", - "- Removed lines:", - removed.length > 0 ? removed.join("\n") : "(none)", - "+ Added lines:", - added.length > 0 ? added.join("\n") : "(none)", - ].join("\n"); +/** + * Interim claims for a post version with no finished investigation (spec §2.8 + * "Interim carry-forward"): the claims of the post's latest complete + * investigation of another version — of either provenance — that still occur + * in this version's content text. Null when there is no such investigation or + * none of its claims survive. Every status that reports interim claims + * (not investigated, investigating) takes them from here. + */ +export async function findCarriedForwardClaims( + repo: InvestigationRepository, + postVersion: { id: string; postId: string; contentText: string }, +): Promise { + const source = await repo.findLatestCompleteOnOtherVersion(postVersion.postId, postVersion.id); + return source === null ? null : carryForwardClaims(source, postVersion.contentText); } // --------------------------------------------------------------------------- @@ -431,101 +286,3 @@ export async function maybeRecordCorroboration( await repo.recordCorroborationCredit(investigationId, viewerKey); } - -// --------------------------------------------------------------------------- -// Investigation queueing with update metadata -// --------------------------------------------------------------------------- - -function buildEnsureQueuedInputWithoutPending(input: { - prisma: TPrisma; - promptId: string; - postVersion: ResolvedPostVersion; - sourceInvestigation: LatestServerVerifiedCompleteInvestigation; -}): Omit, "onPendingInvestigation"> { - const baseInput: Omit, "onPendingInvestigation"> = { - prisma: input.prisma, - postVersionId: input.postVersion.id, - promptId: input.promptId, - rejectOverWordLimitOnCreate: true, - allowRequeueFailed: true, - }; - - if (input.sourceInvestigation === null) { - return baseInput; - } - - return { - ...baseInput, - parentInvestigationId: input.sourceInvestigation.id, - contentDiff: buildLineDiff( - input.sourceInvestigation.postVersion.contentBlob.contentText, - input.postVersion.contentBlob.contentText, - ), - }; -} - -export async function ensureInvestigationsWithUpdateMetadata( - input: EnsureWithDefaultInput, -): Promise; - -export async function ensureInvestigationsWithUpdateMetadata(input: { - prisma: TPrisma; - promptId: string; - postVersion: ResolvedPostVersion; - sourceInvestigation: LatestServerVerifiedCompleteInvestigation; - onPendingInvestigation?: EnsureQueuedInput["onPendingInvestigation"]; - ensureQueued: EnsureQueued; -}): Promise; - -export async function ensureInvestigationsWithUpdateMetadata( - input: - | { - prisma: PrismaClient; - promptId: string; - postVersion: ResolvedPostVersion; - sourceInvestigation: LatestServerVerifiedCompleteInvestigation; - onPendingInvestigation?: Parameters< - typeof ensureInvestigationQueued - >[0]["onPendingInvestigation"]; - ensureQueued?: undefined; - } - | EnsureWithCustomInput, -): Promise { - if (input.ensureQueued !== undefined) { - const queuedInput: EnsureQueuedInput = { - ...buildEnsureQueuedInputWithoutPending({ - prisma: input.prisma, - promptId: input.promptId, - postVersion: input.postVersion, - sourceInvestigation: input.sourceInvestigation, - }), - ...(input.onPendingInvestigation === undefined - ? {} - : { onPendingInvestigation: input.onPendingInvestigation }), - }; - return input.ensureQueued(queuedInput); - } - - const queuedInput: Parameters[0] = { - ...buildEnsureQueuedInputWithoutPending({ - prisma: input.prisma, - promptId: input.promptId, - postVersion: input.postVersion, - sourceInvestigation: input.sourceInvestigation, - }), - ...(input.onPendingInvestigation === undefined - ? {} - : { onPendingInvestigation: input.onPendingInvestigation }), - }; - const ensured = await ensureInvestigationQueued(queuedInput); - return { - investigation: { - id: ensured.investigation.id, - status: ensured.investigation.status, - }, - }; -} - -export const investigationQueriesInternals = { - buildLineDiff, -}; diff --git a/src/typescript/api/src/lib/trpc/routes/post/wikipedia.ts b/src/typescript/api/src/lib/trpc/routes/post/wikipedia.ts index b79b3f2..6d25c7f 100644 --- a/src/typescript/api/src/lib/trpc/routes/post/wikipedia.ts +++ b/src/typescript/api/src/lib/trpc/routes/post/wikipedia.ts @@ -8,7 +8,7 @@ import { normalizeWikipediaTitleToken, - parseWikipediaIdentity, + parseWikipediaUrlIdentity, wikipediaExternalIdFromPageId, type ViewPostInput, } from "@openerrata/shared"; @@ -49,10 +49,15 @@ function canonicalizeWikipediaMetadata( }; } +/** + * The stored external ID is always `{language}:{pageId}` from metadata. The + * submitted URL must be consistent with that metadata: same language, and the + * same page — by page ID when the URL carries one, otherwise by title. + */ function deriveWikipediaExternalId( input: Pick & { metadata: WikipediaViewInput["metadata"] }, ): string { - const urlIdentity = parseWikipediaIdentity(input.url); + const urlIdentity = parseWikipediaUrlIdentity(input.url); if (urlIdentity === null) { throw new TRPCError({ code: "BAD_REQUEST", @@ -67,22 +72,23 @@ function deriveWikipediaExternalId( }); } - if (urlIdentity.pageId !== null && urlIdentity.pageId !== input.metadata.pageId) { - throw new TRPCError({ - code: "BAD_REQUEST", - message: "Wikipedia metadata.pageId does not match URL page ID", - }); - } - - if ( - urlIdentity.pageId === null && - urlIdentity.title !== null && - urlIdentity.title !== input.metadata.title - ) { - throw new TRPCError({ - code: "BAD_REQUEST", - message: "Wikipedia metadata.title does not match URL article title", - }); + switch (urlIdentity.kind) { + case "PAGE_ID": + if (urlIdentity.pageId !== input.metadata.pageId) { + throw new TRPCError({ + code: "BAD_REQUEST", + message: "Wikipedia metadata.pageId does not match URL page ID", + }); + } + break; + case "TITLE": + if (urlIdentity.title !== input.metadata.title) { + throw new TRPCError({ + code: "BAD_REQUEST", + message: "Wikipedia metadata.title does not match URL article title", + }); + } + break; } return wikipediaExternalIdFromPageId(input.metadata.language, input.metadata.pageId); diff --git a/src/typescript/api/src/lib/trpc/routes/public.ts b/src/typescript/api/src/lib/trpc/routes/public.ts deleted file mode 100644 index e64eb84..0000000 --- a/src/typescript/api/src/lib/trpc/routes/public.ts +++ /dev/null @@ -1,104 +0,0 @@ -import { router, publicProcedure } from "../init.js"; -import { - getPublicInvestigationInputSchema, - getPostInvestigationsInputSchema, - searchInvestigationsInputSchema, - getMetricsInputSchema, - publicGetInvestigationOutputSchema, - publicGetPostInvestigationsOutputSchema, - publicSearchInvestigationsOutputSchema, - publicGetMetricsOutputSchema, -} from "@openerrata/shared"; -import { - getPublicInvestigationById, - getPublicPostInvestigations, - searchPublicInvestigations, - getPublicMetrics, -} from "$lib/services/public-read-model.js"; - -interface PublicOrigin { - provenance: "SERVER_VERIFIED" | "CLIENT_FALLBACK"; - serverVerifiedAt: Date | null; -} - -function toOriginOutput(origin: PublicOrigin) { - return { - provenance: origin.provenance, - ...(origin.serverVerifiedAt !== null && { - serverVerifiedAt: origin.serverVerifiedAt.toISOString(), - }), - }; -} - -export const publicRouter = router({ - getInvestigation: publicProcedure - .input(getPublicInvestigationInputSchema) - .output(publicGetInvestigationOutputSchema) - .query(async ({ input, ctx }) => { - const result = await getPublicInvestigationById(ctx.prisma, input.investigationId); - if (!result) { - return null; - } - - return { - investigation: { - id: result.investigation.id, - corroborationCount: result.investigation.corroborationCount, - checkedAt: result.investigation.checkedAt.toISOString(), - origin: toOriginOutput(result.investigation.origin), - promptVersion: result.investigation.promptVersion, - provider: result.investigation.provider, - model: result.investigation.model, - }, - post: result.post, - claims: result.claims, - }; - }), - - getPostInvestigations: publicProcedure - .input(getPostInvestigationsInputSchema) - .output(publicGetPostInvestigationsOutputSchema) - .query(async ({ input, ctx }) => { - const result = await getPublicPostInvestigations(ctx.prisma, input); - return { - post: result.post, - investigations: result.investigations.map((investigation) => ({ - id: investigation.id, - contentHash: investigation.contentHash, - corroborationCount: investigation.corroborationCount, - checkedAt: investigation.checkedAt.toISOString(), - claimCount: investigation.claimCount, - claimSummaries: investigation.claimSummaries, - origin: toOriginOutput(investigation.origin), - })), - }; - }), - - searchInvestigations: publicProcedure - .input(searchInvestigationsInputSchema) - .output(publicSearchInvestigationsOutputSchema) - .query(async ({ input, ctx }) => { - const result = await searchPublicInvestigations(ctx.prisma, input); - - return { - investigations: result.investigations.map((investigation) => ({ - id: investigation.id, - contentHash: investigation.contentHash, - checkedAt: investigation.checkedAt.toISOString(), - platform: investigation.platform, - externalId: investigation.externalId, - url: investigation.url, - corroborationCount: investigation.corroborationCount, - claimCount: investigation.claimCount, - claimSummaries: investigation.claimSummaries, - origin: toOriginOutput(investigation.origin), - })), - hasMore: result.hasMore, - }; - }), - - getMetrics: publicProcedure - .input(getMetricsInputSchema) - .output(publicGetMetricsOutputSchema) - .query(async ({ input, ctx }) => getPublicMetrics(ctx.prisma, input)), -}); diff --git a/src/typescript/api/test/helpers/fake-openai.ts b/src/typescript/api/test/helpers/fake-openai.ts new file mode 100644 index 0000000..1e3701f --- /dev/null +++ b/src/typescript/api/test/helpers/fake-openai.ts @@ -0,0 +1,202 @@ +/* eslint-disable @typescript-eslint/explicit-module-boundary-types */ +import assert from "node:assert/strict"; +import OpenAI from "openai"; +import type { + Response as OpenAiResponse, + ResponseCreateParamsNonStreaming, + ResponseFunctionToolCall, + ResponseFunctionWebSearch, + ResponseOutputItem, + ResponseOutputMessage, + ResponseReasoningItem, +} from "openai/resources/responses/responses"; +import type { InvestigatorRequestAudit } from "../../src/lib/investigators/interface.js"; + +/** + * A real OpenAI SDK client whose HTTP transport is a test double: tests see + * exactly the request bodies the SDK puts on the wire, and replies go through + * the SDK's own response handling and error classes. + */ + +export interface RecordedOpenAiRequest { + path: string; + /** The JSON body as sent. */ + body: ResponseCreateParamsNonStreaming; +} + +export type FakeOpenAiReply = + | { kind: "response"; response: OpenAiResponse } + | { kind: "http_error"; status: number; message: string } + /** Never answers; the request ends only when the SDK aborts it. */ + | { kind: "hang" }; + +export function createFakeOpenAiClient( + reply: ( + request: RecordedOpenAiRequest, + index: number, + ) => FakeOpenAiReply | Promise, + clientOptions: { timeoutMs?: number } = {}, +) { + const requests: RecordedOpenAiRequest[] = []; + const client = new OpenAI({ + apiKey: "sk-test-fake-key", + maxRetries: 0, + ...(clientOptions.timeoutMs === undefined ? {} : { timeout: clientOptions.timeoutMs }), + fetch: async (url, init) => { + const requestBody = init?.body; + if (typeof requestBody !== "string") { + assert.fail("the SDK sends JSON string bodies"); + } + const request: RecordedOpenAiRequest = { + path: new URL(url instanceof Request ? url.url : url).pathname, + body: JSON.parse(requestBody), + }; + requests.push(request); + const result = await reply(request, requests.length - 1); + if (result.kind === "hang") { + return new Promise((_resolve, reject) => { + const signal = init?.signal; + signal?.addEventListener("abort", () => { + reject(new DOMException("The operation was aborted.", "AbortError")); + }); + }); + } + const status = result.kind === "response" ? 200 : result.status; + const responseBody = + result.kind === "response" + ? result.response + : { error: { message: result.message, type: "invalid_request_error" } }; + return new globalThis.Response(JSON.stringify(responseBody), { + status, + headers: { "content-type": "application/json" }, + }); + }, + }); + return { client, requests }; +} + +export function makeResponse(input: { + id: string; + output: ResponseOutputItem[]; + status?: OpenAiResponse["status"]; + model?: string; + incompleteReason?: "max_output_tokens" | "content_filter"; +}): OpenAiResponse { + return { + id: input.id, + object: "response", + created_at: 1_760_000_000, + model: input.model ?? "gpt-6.1-sol-2026-09-01", + status: input.status ?? "completed", + output: input.output, + output_text: "", + error: null, + incomplete_details: + input.incompleteReason === undefined ? null : { reason: input.incompleteReason }, + instructions: null, + metadata: null, + parallel_tool_calls: true, + temperature: null, + tool_choice: "auto", + tools: [], + top_p: null, + usage: { + input_tokens: 100, + input_tokens_details: { cached_tokens: 10 }, + output_tokens: 50, + output_tokens_details: { reasoning_tokens: 20 }, + total_tokens: 150, + }, + }; +} + +export function makeFunctionCall( + callId: string, + name: string, + args: unknown, +): ResponseFunctionToolCall { + return { + type: "function_call", + id: `fc_${callId}`, + call_id: callId, + name, + arguments: JSON.stringify(args), + status: "completed", + }; +} + +export function makeMessage(id: string, text: string): ResponseOutputMessage { + return { + type: "message", + id, + role: "assistant", + status: "completed", + content: [{ type: "output_text", text, annotations: [] }], + }; +} + +export function makeReasoning(id: string, summaries: string[]): ResponseReasoningItem { + return { + type: "reasoning", + id, + summary: summaries.map((text) => ({ type: "summary_text", text })), + }; +} + +export function makeWebSearch( + id: string, + query: string, + sourceUrls: string[], +): ResponseFunctionWebSearch { + return { + type: "web_search_call", + id, + status: "completed", + action: { + type: "search", + query, + sources: sourceUrls.map((url) => ({ type: "url", url })), + }, + }; +} + +export function makeVerdictResponse(id: string, approved: boolean): OpenAiResponse { + return makeResponse({ id, output: [makeMessage(`msg_${id}`, JSON.stringify({ approved }))] }); +} + +export function makeClaim(label: string) { + return { + text: `Incorrect claim: ${label}`, + context: `The article states ${label}`, + summary: `${label} is wrong because of evidence`, + reasoning: `Detailed reasoning for ${label}`, + sources: [ + { + url: `https://example.com/${label.toLowerCase()}`, + title: `Source ${label}`, + snippet: `Evidence for ${label}`, + }, + ], + }; +} + +/** Whether a recorded request is a stage-2 validation call (structured verdict, no tools). */ +export function isValidationRequest(request: RecordedOpenAiRequest): boolean { + return request.body.text !== undefined; +} + +/** A validation request's audit, for tests that only need some request to carry. */ +export function makeValidationRequestAudit(claimIndex: number): InvestigatorRequestAudit { + return { + subject: { kind: "CLAIM_VALIDATION", claimIndex }, + model: "gpt-6.1-sol", + instructions: "validation instructions", + input: "validation prompt", + previousResponseId: null, + reasoningEffort: "medium", + reasoningSummary: "detailed", + include: [], + tools: [], + response: null, + }; +} diff --git a/src/typescript/api/test/integration/api-endpoints.attempt-audit.integration.test.ts b/src/typescript/api/test/integration/api-endpoints.attempt-audit.integration.test.ts new file mode 100644 index 0000000..1ef114e --- /dev/null +++ b/src/typescript/api/test/integration/api-endpoints.attempt-audit.integration.test.ts @@ -0,0 +1,329 @@ +import type { + InvestigatorOutput, + InvestigatorRequestAudit, +} from "../../src/lib/investigators/interface.js"; +import { + assert, + orchestrateInvestigation, + prisma, + seedInvestigation, + seedPost, + test, + withIntegrationPrefix, +} from "./api-endpoints.integration.shared.js"; + +const RECEIVED_AT = new Date("2026-10-02T12:00:00.000Z"); + +function factCheckRound(round: number): InvestigatorRequestAudit { + return { + subject: { kind: "FACT_CHECK_ROUND", round }, + model: "gpt-6.1-sol", + instructions: "fact-check instructions", + input: + round === 0 + ? [ + { + role: "user", + content: [ + { type: "input_text", text: "Post text" }, + { type: "input_image", detail: "auto", imageContentHash: "hash-image" }, + ], + }, + ] + : [{ type: "function_call_output", call_id: "call-1", output: '{"acknowledged":true}' }], + previousResponseId: round === 0 ? null : "resp_round_0", + reasoningEffort: "medium", + reasoningSummary: "detailed", + include: ["web_search_call.action.sources"], + tools: [ + { toolType: "web_search", rawDefinition: { type: "web_search" } }, + { toolType: "function", rawDefinition: { type: "function", name: "submit_correction" } }, + ], + response: { + providerResponseId: `resp_round_${round.toString()}`, + status: "completed", + modelVersion: "gpt-6.1-sol-2026-09-01", + receivedAt: RECEIVED_AT, + outputItems: + round === 0 + ? [ + { + providerItemId: "rs_1", + itemType: "reasoning", + itemStatus: null, + content: { kind: "REASONING", summaries: ["Check the date.", "Search."] }, + }, + { + providerItemId: "ws_1", + itemType: "web_search_call", + itemStatus: "completed", + content: { + kind: "TOOL_CALL", + rawPayload: { + type: "web_search_call", + id: "ws_1", + status: "completed", + action: { + type: "search", + query: "q", + sources: [{ type: "url", url: "https://a.example" }], + }, + }, + }, + }, + ] + : [ + { + providerItemId: "msg_1", + itemType: "message", + itemStatus: "completed", + content: { + kind: "MESSAGE", + textParts: [ + { + partType: "output_text", + text: "Done, see source.", + annotations: [ + { + annotationType: "url_citation", + startIndex: 0, + endIndex: 4, + url: "https://a.example", + title: "A", + fileId: null, + }, + ], + }, + ], + }, + }, + ], + usage: { + inputTokens: 100, + outputTokens: 20, + totalTokens: 120, + cachedInputTokens: 0, + reasoningOutputTokens: 10, + }, + }, + }; +} + +function claimValidation(claimIndex: number): InvestigatorRequestAudit { + return { + subject: { kind: "CLAIM_VALIDATION", claimIndex }, + model: "gpt-6.1-sol", + instructions: "validation instructions", + input: `validation prompt ${claimIndex.toString()}`, + previousResponseId: null, + reasoningEffort: "medium", + reasoningSummary: "detailed", + include: [], + tools: [], + response: null, + }; +} + +void test("a completed investigation records its model and one audit row per provider request", async () => { + const post = await seedPost({ + platform: "X", + externalId: "attempt-audit-requests-1", + url: "https://x.com/openerrata/status/attempt-audit-requests-1", + contentText: "Every provider request gets its own audit row.", + }); + const investigation = await seedInvestigation({ + postId: post.id, + contentHash: post.contentHash, + contentText: post.contentText, + provenance: "CLIENT_FALLBACK", + status: "PENDING", + promptLabel: "attempt-audit-requests", + }); + const output: InvestigatorOutput = { + result: { claims: [] }, + attemptAudit: { + outcome: "SUCCEEDED", + startedAt: RECEIVED_AT, + completedAt: RECEIVED_AT, + requests: [factCheckRound(0), factCheckRound(1), claimValidation(0)], + }, + model: "gpt-6.1-sol", + modelVersion: "gpt-6.1-sol-2026-09-01", + }; + + await orchestrateInvestigation( + investigation.id, + { info() {}, warn() {}, error() {} }, + { + workerIdentity: withIntegrationPrefix("worker-attempt-audit"), + createInvestigator: () => ({ investigate: async () => output }), + }, + ); + + const stored = await prisma.investigation.findUniqueOrThrow({ + where: { id: investigation.id }, + select: { status: true, model: true, modelVersion: true }, + }); + assert.deepEqual(stored, { + status: "COMPLETE", + model: "gpt-6.1-sol", + modelVersion: "gpt-6.1-sol-2026-09-01", + }); + + const attempt = await prisma.investigationAttempt.findFirstOrThrow({ + where: { investigationId: investigation.id }, + include: { + error: true, + requests: { + // Enum order: FACT_CHECK_ROUND before CLAIM_VALIDATION. + orderBy: [{ kind: "asc" }, { factCheckRound: "asc" }, { claimIndex: "asc" }], + include: { + requestedTools: { orderBy: { requestOrder: "asc" } }, + response: { + include: { + usage: true, + outputItems: { + orderBy: { outputIndex: "asc" }, + include: { + textParts: { include: { annotations: true } }, + reasoningSummaries: { orderBy: { summaryIndex: "asc" } }, + toolCall: true, + }, + }, + }, + }, + }, + }, + }, + }); + assert.equal(attempt.outcome, "SUCCEEDED"); + assert.equal(attempt.error, null); + + assert.deepEqual( + attempt.requests.map((request) => [request.kind, request.factCheckRound, request.claimIndex]), + [ + ["FACT_CHECK_ROUND", 0, null], + ["FACT_CHECK_ROUND", 1, null], + ["CLAIM_VALIDATION", null, 0], + ], + ); + const [round0, round1, validation] = attempt.requests; + assert.ok(validation && round0 && round1); + assert.equal(validation.response, null); + assert.equal(validation.input, "validation prompt 0"); + assert.deepEqual(validation.include, []); + + assert.deepEqual(round0.input, factCheckRound(0).input); + assert.equal(round0.previousResponseId, null); + assert.equal(round1.previousResponseId, "resp_round_0"); + assert.deepEqual(round0.include, ["web_search_call.action.sources"]); + assert.deepEqual( + round0.requestedTools.map((tool) => [tool.requestOrder, tool.toolType]), + [ + [0, "web_search"], + [1, "function"], + ], + ); + + assert.ok(round0.response); + assert.equal(round0.response.providerResponseId, "resp_round_0"); + assert.equal(round0.response.receivedAt?.toISOString(), RECEIVED_AT.toISOString()); + assert.equal(round0.response.usage?.totalTokens, 120); + const [reasoning, webSearch] = round0.response.outputItems; + assert.ok(reasoning && webSearch); + assert.equal(reasoning.providerItemId, "rs_1"); + assert.equal(reasoning.itemStatus, null); + assert.deepEqual( + reasoning.reasoningSummaries.map((summary) => summary.text), + ["Check the date.", "Search."], + ); + assert.ok(webSearch.toolCall); + assert.deepEqual(webSearch.toolCall.rawPayload, { + type: "web_search_call", + id: "ws_1", + status: "completed", + action: { type: "search", query: "q", sources: [{ type: "url", url: "https://a.example" }] }, + }); + + const [message] = round1.response?.outputItems ?? []; + assert.equal(message?.textParts[0]?.text, "Done, see source."); + assert.equal(message.textParts[0].annotations[0]?.url, "https://a.example"); +}); + +void test("the database rejects a COMPLETE investigation without a recorded model", async () => { + const post = await seedPost({ + platform: "X", + externalId: "attempt-audit-model-check-1", + url: "https://x.com/openerrata/status/attempt-audit-model-check-1", + contentText: "COMPLETE requires a model.", + }); + const investigation = await seedInvestigation({ + postId: post.id, + contentHash: post.contentHash, + contentText: post.contentText, + provenance: "CLIENT_FALLBACK", + status: "PENDING", + promptLabel: "attempt-audit-model-check", + }); + + await assert.rejects( + prisma.investigation.update({ + where: { id: investigation.id }, + data: { status: "COMPLETE", checkedAt: new Date() }, + }), + /Investigation_model_consistency_check/, + ); + await assert.rejects( + prisma.investigation.update({ + where: { id: investigation.id }, + data: { model: "gpt-6.1-sol" }, + }), + /Investigation_model_consistency_check/, + ); +}); + +void test("the database rejects an attempt request whose subject does not match its kind", async () => { + const post = await seedPost({ + platform: "X", + externalId: "attempt-audit-subject-check-1", + url: "https://x.com/openerrata/status/attempt-audit-subject-check-1", + contentText: "Request subjects must match their kind.", + }); + const investigation = await seedInvestigation({ + postId: post.id, + contentHash: post.contentHash, + contentText: post.contentText, + provenance: "CLIENT_FALLBACK", + status: "FAILED", + promptLabel: "attempt-audit-subject-check", + }); + const attempt = await prisma.investigationAttempt.create({ + data: { + investigationId: investigation.id, + attemptNumber: 1, + outcome: "FAILED", + startedAt: new Date(), + completedAt: new Date(), + }, + }); + + const request = { + attemptId: attempt.id, + model: "gpt-6.1-sol", + instructions: "instructions", + input: "input", + include: [], + }; + for (const subject of [ + { kind: "FACT_CHECK_ROUND" as const, factCheckRound: null, claimIndex: null }, + { kind: "FACT_CHECK_ROUND" as const, factCheckRound: 0, claimIndex: 0 }, + { kind: "CLAIM_VALIDATION" as const, factCheckRound: 0, claimIndex: null }, + { kind: "LEGACY_COMBINED" as const, factCheckRound: 0, claimIndex: null }, + ]) { + await assert.rejects( + prisma.investigationAttemptRequest.create({ data: { ...request, ...subject } }), + /InvestigationAttemptRequest_subject_check/, + JSON.stringify(subject), + ); + } +}); diff --git a/src/typescript/api/test/integration/api-endpoints.integration.attempt-audit.ts b/src/typescript/api/test/integration/api-endpoints.integration.attempt-audit.ts index 204a6a5..89bf4f3 100644 --- a/src/typescript/api/test/integration/api-endpoints.integration.attempt-audit.ts +++ b/src/typescript/api/test/integration/api-endpoints.integration.attempt-audit.ts @@ -1,43 +1,65 @@ -/* eslint-disable @typescript-eslint/explicit-module-boundary-types */ -export function buildSucceededAttemptAudit(label: string) { - const now = new Date().toISOString(); +import type { + InvestigatorFailedAttemptAudit, + InvestigatorOutput, + InvestigatorRequestAudit, + InvestigatorSucceededAttemptAudit, +} from "../../src/lib/investigators/interface.js"; + +function buildFactCheckRequestAudit(label: string): InvestigatorRequestAudit { return { - startedAt: now, - completedAt: now, - requestModel: `test-model-${label}`, - requestInstructions: `instructions-${label}`, - requestInput: `input-${label}`, - requestReasoningEffort: null, - requestReasoningSummary: null, - requestedTools: [], + subject: { kind: "FACT_CHECK_ROUND", round: 0 }, + model: "gpt-6.1-sol", + instructions: `instructions-${label}`, + input: `input-${label}`, + previousResponseId: null, + reasoningEffort: "medium", + reasoningSummary: "detailed", + include: ["web_search_call.action.sources"], + tools: [{ toolType: "web_search", rawDefinition: { type: "web_search" } }], response: { - responseId: `response-${label}`, - responseStatus: "completed", - responseModelVersion: "test-model-version", - responseOutputText: '{"claims":[]}', - outputItems: [], - outputTextParts: [], - outputTextAnnotations: [], - reasoningSummaries: [], - toolCalls: [], - usage: null, + providerResponseId: `response-${label}`, + status: "completed", + modelVersion: "test-model-version", + receivedAt: new Date(), + outputItems: [ + { + providerItemId: `msg-${label}`, + itemType: "message", + itemStatus: "completed", + content: { + kind: "MESSAGE", + textParts: [{ partType: "output_text", text: "No issues found.", annotations: [] }], + }, + }, + ], + usage: { + inputTokens: 10, + outputTokens: 5, + totalTokens: 15, + cachedInputTokens: 0, + reasoningOutputTokens: 2, + }, }, - error: null, }; } -export function buildFailedAttemptAudit(label: string) { - const now = new Date().toISOString(); +export function buildSucceededAttemptAudit(label: string): InvestigatorSucceededAttemptAudit { + const now = new Date(); return { + outcome: "SUCCEEDED", startedAt: now, completedAt: now, - requestModel: `test-model-${label}`, - requestInstructions: `instructions-${label}`, - requestInput: `input-${label}`, - requestReasoningEffort: null, - requestReasoningSummary: null, - requestedTools: [], - response: null, + requests: [buildFactCheckRequestAudit(label)], + }; +} + +export function buildFailedAttemptAudit(label: string): InvestigatorFailedAttemptAudit { + const now = new Date(); + return { + outcome: "FAILED", + startedAt: now, + completedAt: now, + requests: [{ ...buildFactCheckRequestAudit(label), response: null }], error: { errorName: "TransientTestFailure", errorMessage: `transient-error-${label}`, @@ -45,3 +67,13 @@ export function buildFailedAttemptAudit(label: string) { }, }; } + +/** A successful investigator result with no claims, as the fake investigators return it. */ +export function buildSucceededInvestigatorOutput(label: string): InvestigatorOutput { + return { + result: { claims: [] }, + attemptAudit: buildSucceededAttemptAudit(label), + model: "gpt-6.1-sol", + modelVersion: "test-model-version", + }; +} diff --git a/src/typescript/api/test/integration/api-endpoints.integration.shared.ts b/src/typescript/api/test/integration/api-endpoints.integration.shared.ts index 38528ab..a95b24e 100644 --- a/src/typescript/api/test/integration/api-endpoints.integration.shared.ts +++ b/src/typescript/api/test/integration/api-endpoints.integration.shared.ts @@ -6,6 +6,7 @@ import { hashContent, isNonNullObject, normalizeContent, + observedImageUrlsFromOccurrences, serializeVersionHashSeed, serializeVersionIdentityImageOccurrences, WORD_COUNT_LIMIT, @@ -22,11 +23,17 @@ import { import { buildFailedAttemptAudit, buildSucceededAttemptAudit, + buildSucceededInvestigatorOutput, } from "./api-endpoints.integration.attempt-audit.js"; import { applyIntegrationEnvironmentDefaults } from "./integration-env.js"; +import { + installMockOpenAiApi, + lesswrongPostIdFromGraphqlBody, +} from "./helpers/external-api-mocks.js"; import { INTEGRATION_LESSWRONG_FIXTURE_KEYS, readLesswrongFixture } from "./lesswrong-fixtures.js"; applyIntegrationEnvironmentDefaults(process.env); +installMockOpenAiApi(); const INTEGRATION_TEST_RUN_ID = [ Date.now().toString(36), @@ -48,11 +55,12 @@ const [ { buildHealthResponse }, { handlePublicGraphqlRequest }, { closeQueueUtils }, - { ensureInvestigationQueued }, { runSelector }, + { requestInvestigation, UserOpenAiKeyRejectedError }, + { LeaseLostError }, { lesswrongHtmlToNormalizedText }, { orchestrateInvestigation }, - { OpenAIInvestigator, InvestigatorExecutionError }, + { InvestigatorExecutionError }, ] = await Promise.all([ import("../../src/lib/trpc/router.js"), import("../../src/lib/db/client.js"), @@ -61,11 +69,12 @@ const [ import("../../src/lib/services/health.js"), import("../../src/lib/graphql/handler.js"), import("../../src/lib/services/queue.js"), - import("../../src/lib/services/investigation-lifecycle.js"), import("../../src/lib/services/selector.js"), + import("../../src/lib/services/investigate-now.js"), + import("../../src/lib/services/investigation-lease.js"), import("../../src/lib/services/content-fetcher.js"), import("../../src/lib/services/orchestrator.js"), - import("../../src/lib/investigators/openai.js"), + import("../../src/lib/investigators/errors.js"), ]); const prisma = getPrisma(); @@ -137,15 +146,12 @@ function createCaller(options: CallerOptions = {}): AppCaller { viewerKey: options.viewerKey ?? "integration-viewer", ipRangeKey: options.ipRangeKey ?? "integration-ip-range", isAuthenticated, - canInvestigate: isAuthenticated || userOpenAiApiKey !== null, userOpenAiApiKey, - hasValidAttestation: false, extensionVersion: options.extensionVersion ?? MINIMUM_SUPPORTED_EXTENSION_VERSION, minimumSupportedExtensionVersion: MINIMUM_SUPPORTED_EXTENSION_VERSION, }); return { - public: caller.public, post: { registerObservedVersion: caller.post.registerObservedVersion, getInvestigation: caller.post.getInvestigation, @@ -464,6 +470,8 @@ async function assertSchemaCatalogInvariants(): Promise { WHERE n.nspname = 'public' AND NOT tg.tgisinternal AND tg.tgname IN ( + 'enforce_lease_status_on_investigation_trigger', + 'enforce_lease_status_on_lease_trigger', 'enforce_investigation_parent_semantics_trigger', 'enforce_referenced_parent_investigation_validity_trigger', 'enforce_server_verified_at_latch_trigger', @@ -476,6 +484,22 @@ async function assertSchemaCatalogInvariants(): Promise { string, { tableName: string; functionName: string; requiredFragments: string[] } >([ + [ + "enforce_lease_status_on_investigation_trigger", + { + tableName: "Investigation", + functionName: "enforce_lease_status_on_investigation", + requiredFragments: ["after insert or update of status", "deferrable initially deferred"], + }, + ], + [ + "enforce_lease_status_on_lease_trigger", + { + tableName: "InvestigationLease", + functionName: "enforce_lease_status_on_lease", + requiredFragments: ["after insert or delete or update", "deferrable initially deferred"], + }, + ], [ "enforce_investigation_parent_semantics_trigger", { @@ -565,6 +589,8 @@ async function assertSchemaCatalogInvariants(): Promise { FROM information_schema.triggers WHERE trigger_schema = 'public' AND trigger_name IN ( + 'enforce_lease_status_on_investigation_trigger', + 'enforce_lease_status_on_lease_trigger', 'enforce_investigation_parent_semantics_trigger', 'enforce_referenced_parent_investigation_validity_trigger', 'enforce_server_verified_at_latch_trigger', @@ -575,6 +601,11 @@ async function assertSchemaCatalogInvariants(): Promise { const expectedTriggerEvents = [ "enforce_investigation_parent_semantics_trigger|Investigation|BEFORE|INSERT", "enforce_investigation_parent_semantics_trigger|Investigation|BEFORE|UPDATE", + "enforce_lease_status_on_investigation_trigger|Investigation|AFTER|INSERT", + "enforce_lease_status_on_investigation_trigger|Investigation|AFTER|UPDATE", + "enforce_lease_status_on_lease_trigger|InvestigationLease|AFTER|DELETE", + "enforce_lease_status_on_lease_trigger|InvestigationLease|AFTER|INSERT", + "enforce_lease_status_on_lease_trigger|InvestigationLease|AFTER|UPDATE", "enforce_referenced_parent_investigation_validity_trigger|Investigation|BEFORE|UPDATE", "enforce_server_verified_at_latch_trigger|PostVersion|BEFORE|UPDATE", "enforce_server_verified_html_snapshot_trigger|PostVersion|AFTER|INSERT", @@ -791,34 +822,8 @@ async function ensureServerHtmlSnapshotForSeed(input: { } return; } - case "SUBSTACK": { - const existing = await prisma.substackVersionMeta.findUnique({ - where: { postVersionId: input.postVersionId }, - select: { serverHtmlBlobId: true }, - }); - if (existing === null) { - await prisma.substackVersionMeta.create({ - data: { - postVersionId: input.postVersionId, - substackPostId: `seed-${input.externalId}`, - publicationSubdomain: "seed", - slug: `seed-${input.externalId}`, - title: "Seeded Substack Post", - serverHtmlBlobId: htmlBlob.id, - imageUrls: [], - authorName: "Seed Author", - }, - }); - return; - } - if (existing.serverHtmlBlobId === null) { - await prisma.substackVersionMeta.updateMany({ - where: { postVersionId: input.postVersionId, serverHtmlBlobId: null }, - data: { serverHtmlBlobId: htmlBlob.id }, - }); - } - return; - } + case "SUBSTACK": + throw new Error("Substack has no server-side fetch; seed Substack posts as CLIENT_FALLBACK"); case "WIKIPEDIA": { const existing = await prisma.wikipediaVersionMeta.findUnique({ where: { postVersionId: input.postVersionId }, @@ -1095,6 +1100,10 @@ async function seedInvestigation(input: { contentDiff?: string; leaseOwner?: string | null; leaseExpiresAt?: Date | null; + /** Defaults to INSTANCE_REQUEST so seeds never count against the selector's daily budget. */ + origin?: "SELECTOR" | "INSTANCE_REQUEST" | "USER_KEY_REQUEST"; + admittedAt?: Date; + attemptCount?: number; }): Promise<{ id: string }> { const prompt = await seedPrompt(input.promptLabel); const checkedAt = input.status === "COMPLETE" ? (input.checkedAt ?? new Date()) : null; @@ -1107,12 +1116,19 @@ async function seedInvestigation(input: { const investigationId = randomUUID(); const investigation = await prisma.$transaction(async (tx) => { + const post = await tx.post.findUniqueOrThrow({ + where: { id: input.postId }, + select: { url: true }, + }); await tx.investigationInput.create({ data: { investigationId, provenance: input.provenance, contentHash: input.contentHash, markdownSource: "NONE", + imagePlaceholderSourceUrls: [], + postUrl: post.url, + hasVideo: false, }, }); @@ -1124,7 +1140,10 @@ async function seedInvestigation(input: { status: input.status, promptId: prompt.id, provider: "OPENAI", - model: "OPENAI_GPT_5", + model: input.status === "COMPLETE" ? "gpt-6.1-sol" : null, + origin: input.origin ?? "INSTANCE_REQUEST", + admittedAt: input.admittedAt ?? new Date(), + attemptCount: input.attemptCount ?? 0, checkedAt, parentInvestigationId: input.parentInvestigationId ?? null, contentDiff: input.contentDiff ?? null, @@ -1174,14 +1193,19 @@ async function seedInvestigationWithLeaseFields(input: { }); } +/** + * Seed one claim with one source. `text` is the passage the claim quotes; + * tests that do not depend on where the claim sits get a generic label. + */ async function seedClaimWithSource( investigationId: string, index: number, + options: { text?: string } = {}, ): Promise<{ id: string }> { const claim = await prisma.claim.create({ data: { investigationId, - text: `Claim ${index.toString()}`, + text: options.text ?? `Claim ${index.toString()}`, context: `Context ${index.toString()}`, summary: `Summary ${index.toString()}`, reasoning: `Reasoning ${index.toString()}`, @@ -1195,7 +1219,6 @@ async function seedClaimWithSource( url: `https://example.com/source-${index.toString()}`, title: `Source ${index.toString()}`, snippet: `Snippet ${index.toString()}`, - retrievedAt: new Date("2026-02-19T00:00:00.000Z"), }, }); @@ -1216,7 +1239,6 @@ async function seedCorroborationCredits(investigationId: string, count: number): function buildXViewInput(input: { externalId: string; observedContentText: string; - observedImageUrls?: string[]; observedImageOccurrences?: { originalIndex: number; normalizedTextOffset: number; @@ -1226,14 +1248,12 @@ function buildXViewInput(input: { }) { const externalId = withIntegrationPrefix(input.externalId); const observedContentText = input.observedContentText; - const observedImageUrls = input.observedImageUrls ?? []; return { platform: "X" as const, externalId, url: `https://x.com/openerrata/status/${externalId}`, observedContentText, - ...(observedImageUrls.length === 0 ? {} : { observedImageUrls }), ...(input.observedImageOccurrences === undefined ? {} : { observedImageOccurrences: input.observedImageOccurrences }), @@ -1241,7 +1261,7 @@ function buildXViewInput(input: { authorHandle: withIntegrationPrefix("author"), authorDisplayName: "Integration Author", text: observedContentText, - mediaUrls: observedImageUrls, + mediaUrls: observedImageUrlsFromOccurrences(input.observedImageOccurrences), }, }; } @@ -1270,6 +1290,12 @@ async function withMockLesswrongFetch( return withMockLesswrongCanonicalHtml(fixture.html, run); } +/** Identity the mocked LessWrong GraphQL API reports for every post. */ +const LESSWRONG_MOCK_SERVER_SLUG = "server-slug"; +const LESSWRONG_MOCK_SERVER_TITLE = "Server LW Title"; +const LESSWRONG_MOCK_SERVER_AUTHOR_SLUG = withIntegrationPrefix("server-lw-author"); +const LESSWRONG_MOCK_SERVER_AUTHOR_NAME = "Server LW Author"; + async function withMockLesswrongCanonicalHtml( html: string, run: () => Promise, @@ -1284,15 +1310,23 @@ async function withMockLesswrongCanonicalHtml( return originalFetch(input, init); } sawLesswrongRequest = true; + const postId = lesswrongPostIdFromGraphqlBody(init?.body); return new Response( JSON.stringify({ data: { post: { result: { + _id: postId, + slug: LESSWRONG_MOCK_SERVER_SLUG, + title: LESSWRONG_MOCK_SERVER_TITLE, contents: { html, }, + user: { + slug: LESSWRONG_MOCK_SERVER_AUTHOR_SLUG, + displayName: LESSWRONG_MOCK_SERVER_AUTHOR_NAME, + }, }, }, }, @@ -1413,9 +1447,13 @@ export { EMPTY_IMAGE_OCCURRENCES_HASH, INTEGRATION_DATA_PREFIX, INTEGRATION_LESSWRONG_FIXTURE_KEYS, + LeaseLostError, + LESSWRONG_MOCK_SERVER_AUTHOR_NAME, + LESSWRONG_MOCK_SERVER_AUTHOR_SLUG, + LESSWRONG_MOCK_SERVER_SLUG, + LESSWRONG_MOCK_SERVER_TITLE, InvestigatorExecutionError, MINIMUM_SUPPORTED_EXTENSION_VERSION, - OpenAIInvestigator, WORD_COUNT_LIMIT, appRouter, assert, @@ -1424,13 +1462,13 @@ export { buildHealthResponse, buildLesswrongViewInput, buildSucceededAttemptAudit, + buildSucceededInvestigatorOutput, buildXViewInput, closeQueueUtils, createCaller, createContext, createDeterministicRandom, createMockRequestEvent, - ensureInvestigationQueued, ensurePostVersionForSeed, errorHasOpenErrataCode, getPrisma, @@ -1446,6 +1484,7 @@ export { randomChance, randomInt, readLesswrongFixture, + requestInvestigation, resetDatabase, runConcurrentInvestigateNowScenario, runSelector, @@ -1465,6 +1504,7 @@ export { sha256, sleep, test, + UserOpenAiKeyRejectedError, versionHashFromContentHash, withIntegrationPrefix, withMockLesswrongCanonicalHtml, diff --git a/src/typescript/api/test/integration/api-endpoints.lifecycle.integration.test.ts b/src/typescript/api/test/integration/api-endpoints.lifecycle.integration.test.ts index 3f1311c..62fd989 100644 --- a/src/typescript/api/test/integration/api-endpoints.lifecycle.integration.test.ts +++ b/src/typescript/api/test/integration/api-endpoints.lifecycle.integration.test.ts @@ -1,11 +1,11 @@ -import type { InvestigatorInput } from "../../src/lib/investigators/interface.js"; +import OpenAI from "openai"; +import type { Investigator } from "../../src/lib/investigators/interface.js"; import { EMPTY_IMAGE_OCCURRENCES_HASH, INTEGRATION_DATA_PREFIX, INTEGRATION_LESSWRONG_FIXTURE_KEYS, InvestigatorExecutionError, MINIMUM_SUPPORTED_EXTENSION_VERSION, - OpenAIInvestigator, WORD_COUNT_LIMIT, appRouter, assert, @@ -13,13 +13,13 @@ import { buildFailedAttemptAudit, buildLesswrongViewInput, buildSucceededAttemptAudit, + buildSucceededInvestigatorOutput, buildXViewInput, closeQueueUtils, createCaller, createContext, createDeterministicRandom, createMockRequestEvent, - ensureInvestigationQueued, ensurePostVersionForSeed, errorHasOpenErrataCode, getPrisma, @@ -58,6 +58,11 @@ import { withIntegrationPrefix, withMockLesswrongCanonicalHtml, withMockLesswrongFetch, + LeaseLostError, + LESSWRONG_MOCK_SERVER_AUTHOR_NAME, + LESSWRONG_MOCK_SERVER_SLUG, + requestInvestigation, + UserOpenAiKeyRejectedError, } from "./api-endpoints.integration.shared.js"; void [ @@ -66,7 +71,6 @@ void [ INTEGRATION_LESSWRONG_FIXTURE_KEYS, InvestigatorExecutionError, MINIMUM_SUPPORTED_EXTENSION_VERSION, - OpenAIInvestigator, WORD_COUNT_LIMIT, appRouter, assert, @@ -74,13 +78,13 @@ void [ buildFailedAttemptAudit, buildLesswrongViewInput, buildSucceededAttemptAudit, + buildSucceededInvestigatorOutput, buildXViewInput, closeQueueUtils, createCaller, createContext, createDeterministicRandom, createMockRequestEvent, - ensureInvestigationQueued, ensurePostVersionForSeed, errorHasOpenErrataCode, getPrisma, @@ -141,36 +145,21 @@ void test("orchestrateInvestigation skips work when lease is held by another wor }); let investigateCalled = false; - const originalInvestigateDescriptor = Object.getOwnPropertyDescriptor( - OpenAIInvestigator.prototype, - "investigate", - ); - assert.ok(originalInvestigateDescriptor); - assert.equal(typeof originalInvestigateDescriptor.value, "function"); - OpenAIInvestigator.prototype.investigate = async () => { - investigateCalled = true; - return { - result: { claims: [] }, - attemptAudit: buildSucceededAttemptAudit("lease-held"), - modelVersion: "test-model-version", - }; - }; + const createInvestigator = (): Investigator => ({ + investigate: async () => { + investigateCalled = true; + return buildSucceededInvestigatorOutput("lease-held"); + }, + }); - try { - await orchestrateInvestigation( - investigation.id, - { info() {}, warn() {}, error() {} }, - { - workerIdentity: withIntegrationPrefix("contending-worker"), - }, - ); - } finally { - Object.defineProperty( - OpenAIInvestigator.prototype, - "investigate", - originalInvestigateDescriptor, - ); - } + await orchestrateInvestigation( + investigation.id, + { info() {}, warn() {}, error() {} }, + { + workerIdentity: withIntegrationPrefix("contending-worker"), + createInvestigator, + }, + ); assert.equal(investigateCalled, false); const storedInvestigation = await prisma.investigation.findUnique({ @@ -214,54 +203,39 @@ void test("orchestrateInvestigation passes update context to investigator for up }); let sawExpectedUpdateContext = false; - const originalInvestigateDescriptor = Object.getOwnPropertyDescriptor( - OpenAIInvestigator.prototype, - "investigate", - ); - assert.ok(originalInvestigateDescriptor); - assert.equal(typeof originalInvestigateDescriptor.value, "function"); - OpenAIInvestigator.prototype.investigate = async (input: InvestigatorInput) => { - assert.equal(input.isUpdate, true); - assert.equal(input.contentDiff, contentDiff); - assert.deepStrictEqual(input.oldClaims, [ - { - id: parentClaim.id, - text: "Claim 1", - context: "Context 1", - summary: "Summary 1", - reasoning: "Reasoning 1", - sources: [ - { - url: "https://example.com/source-1", - title: "Source 1", - snippet: "Snippet 1", - }, - ], - }, - ]); - sawExpectedUpdateContext = true; - return { - result: { claims: [] }, - attemptAudit: buildSucceededAttemptAudit("update-context"), - modelVersion: "test-model-version", - }; - }; + const createInvestigator = (): Investigator => ({ + investigate: async (input) => { + assert.equal(input.isUpdate, true); + assert.equal(input.contentDiff, contentDiff); + assert.deepStrictEqual(input.oldClaims, [ + { + id: parentClaim.id, + text: "Claim 1", + context: "Context 1", + summary: "Summary 1", + reasoning: "Reasoning 1", + sources: [ + { + url: "https://example.com/source-1", + title: "Source 1", + snippet: "Snippet 1", + }, + ], + }, + ]); + sawExpectedUpdateContext = true; + return buildSucceededInvestigatorOutput("update-context"); + }, + }); - try { - await orchestrateInvestigation( - updateInvestigation.id, - { info() {}, warn() {}, error() {} }, - { - workerIdentity: withIntegrationPrefix("worker-update-context"), - }, - ); - } finally { - Object.defineProperty( - OpenAIInvestigator.prototype, - "investigate", - originalInvestigateDescriptor, - ); - } + await orchestrateInvestigation( + updateInvestigation.id, + { info() {}, warn() {}, error() {} }, + { + workerIdentity: withIntegrationPrefix("worker-update-context"), + createInvestigator, + }, + ); assert.equal(sawExpectedUpdateContext, true); @@ -294,74 +268,59 @@ void test("orchestrateInvestigation does not persist late progress updates after resolveLateCallbackFired = resolve; }); - const originalInvestigateDescriptor = Object.getOwnPropertyDescriptor( - OpenAIInvestigator.prototype, - "investigate", - ); - assert.ok(originalInvestigateDescriptor); - assert.equal(typeof originalInvestigateDescriptor.value, "function"); - OpenAIInvestigator.prototype.investigate = async (_input: InvestigatorInput, callbacks) => { - const latePending = [ - { - text: "Late pending claim", - context: "Late pending context", - summary: "Late pending summary", - reasoning: "Late pending reasoning", - sources: [ - { - url: "https://example.com/late-pending", - title: "Late Pending Source", - snippet: "Late pending snippet", - }, - ], - }, - ]; - const lateConfirmed = [ - { - text: "Late confirmed claim", - context: "Late confirmed context", - summary: "Late confirmed summary", - reasoning: "Late confirmed reasoning", - sources: [ - { - url: "https://example.com/late-confirmed", - title: "Late Confirmed Source", - snippet: "Late confirmed snippet", - }, - ], - }, - ]; + const createInvestigator = (): Investigator => ({ + investigate: async (_input, options) => { + const latePending = [ + { + text: "Late pending claim", + context: "Late pending context", + summary: "Late pending summary", + reasoning: "Late pending reasoning", + sources: [ + { + url: "https://example.com/late-pending", + title: "Late Pending Source", + snippet: "Late pending snippet", + }, + ], + }, + ]; + const lateConfirmed = [ + { + text: "Late confirmed claim", + context: "Late confirmed context", + summary: "Late confirmed summary", + reasoning: "Late confirmed reasoning", + sources: [ + { + url: "https://example.com/late-confirmed", + title: "Late Confirmed Source", + snippet: "Late confirmed snippet", + }, + ], + }, + ]; - setTimeout(() => { - callbacks?.onProgressUpdate(latePending, lateConfirmed); - resolveLateCallbackFired(); - }, 25); + setTimeout(() => { + options.callbacks?.onProgressUpdate(latePending, lateConfirmed); + resolveLateCallbackFired(); + }, 25); - return { - result: { claims: [] }, - attemptAudit: buildSucceededAttemptAudit("late-progress"), - modelVersion: "test-model-version", - }; - }; + return buildSucceededInvestigatorOutput("late-progress"); + }, + }); - try { - await orchestrateInvestigation( - investigation.id, - { info() {}, warn() {}, error() {} }, - { - workerIdentity: withIntegrationPrefix("worker-late-progress"), - }, - ); - await lateCallbackFired; - // Allow the asynchronous callback write attempt to settle. - await sleep(50); - } finally { - Object.defineProperty( - OpenAIInvestigator.prototype, - "investigate", - originalInvestigateDescriptor, - ); - } + await orchestrateInvestigation( + investigation.id, + { info() {}, warn() {}, error() {} }, + { + workerIdentity: withIntegrationPrefix("worker-late-progress"), + createInvestigator, + }, + ); + await lateCallbackFired; + // Allow the asynchronous callback write attempt to settle. + await sleep(50); const storedInvestigation = await prisma.investigation.findUnique({ where: { id: investigation.id }, @@ -407,29 +366,21 @@ void test("orchestrateInvestigation ignores stale transient failure after anothe }; }); - const originalInvestigateDescriptor = Object.getOwnPropertyDescriptor( - OpenAIInvestigator.prototype, - "investigate", - ); - assert.ok(originalInvestigateDescriptor); - assert.equal(typeof originalInvestigateDescriptor.value, "function"); - OpenAIInvestigator.prototype.investigate = async () => { - callCount += 1; - if (callCount === 1) { - markFirstStarted(); - await firstWorkerContinue; - throw new InvestigatorExecutionError( - "simulated transient failure from stale worker", - buildFailedAttemptAudit("stale"), - new Error("network timeout"), - ); - } - return { - result: { claims: [] }, - attemptAudit: buildSucceededAttemptAudit("winner"), - modelVersion: "test-model-version", - }; - }; + const createInvestigator = (): Investigator => ({ + investigate: async () => { + callCount += 1; + if (callCount === 1) { + markFirstStarted(); + await firstWorkerContinue; + throw new InvestigatorExecutionError( + "simulated transient failure from stale worker", + buildFailedAttemptAudit("stale"), + new Error("network timeout"), + ); + } + return buildSucceededInvestigatorOutput("winner"); + }, + }); try { const firstWorker = orchestrateInvestigation( @@ -437,6 +388,7 @@ void test("orchestrateInvestigation ignores stale transient failure after anothe { info() {}, warn() {}, error() {} }, { workerIdentity: withIntegrationPrefix("worker-a"), + createInvestigator, }, ); await firstWorkerStarted; @@ -455,6 +407,7 @@ void test("orchestrateInvestigation ignores stale transient failure after anothe { info() {}, warn() {}, error() {} }, { workerIdentity: withIntegrationPrefix("worker-b"), + createInvestigator, }, ); @@ -462,11 +415,6 @@ void test("orchestrateInvestigation ignores stale transient failure after anothe await firstWorker; } finally { releaseFirstWorker(); - Object.defineProperty( - OpenAIInvestigator.prototype, - "investigate", - originalInvestigateDescriptor, - ); } assert.equal(callCount, 2); @@ -518,6 +466,7 @@ void test("orchestrateInvestigation marks exhausted stale PROCESSING investigati { info() {}, warn() {}, error() {} }, { workerIdentity: withIntegrationPrefix("worker-exhausted-stale"), + createInvestigator: () => assert.fail("an exhausted investigation must not run"), }, ); @@ -533,12 +482,12 @@ void test("orchestrateInvestigation marks exhausted stale PROCESSING investigati assert.equal(stored.lease, null); }); -void test("ensureInvestigationQueued requeueing FAILED resets retry counters and clears stale lease row", async () => { +void test("investigateNow leaves FAILED investigations terminal and never resets attempt numbering", async () => { const post = await seedPost({ platform: "X", - externalId: withIntegrationPrefix("ensure-queued-requeue-failed-reset-1"), - url: "https://x.com/openerrata/status/ensure-queued-requeue-failed-reset-1", - contentText: "Failed investigations should become runnable when requeued.", + externalId: "investigate-now-failed-terminal-1", + url: "https://x.com/openerrata/status/investigate-now-failed-terminal-1", + contentText: "Failed investigations stay failed for this content version.", }); const investigation = await seedInvestigation({ postId: post.id, @@ -546,58 +495,20 @@ void test("ensureInvestigationQueued requeueing FAILED resets retry counters and contentText: post.contentText, provenance: "CLIENT_FALLBACK", status: "FAILED", - promptLabel: "ensure-queued-requeue-failed-reset", + promptLabel: "investigate-now-failed-terminal", + attemptCount: 4, }); - const prompt = await seedPrompt("ensure-queued-requeue-failed-reset-prompt"); - await prisma.investigation.update({ - where: { id: investigation.id }, - data: { - attemptCount: 4, // MAX_INVESTIGATION_ATTEMPTS - retryAfter: new Date(Date.now() + 10 * 60_000), - }, - }); - await prisma.investigationLease.create({ - data: { - investigationId: investigation.id, - leaseOwner: withIntegrationPrefix("stale-failed-lease"), - leaseExpiresAt: new Date(Date.now() - 5 * 60_000), - startedAt: new Date(Date.now() - 10 * 60_000), - heartbeatAt: new Date(Date.now() - 5 * 60_000), - }, - }); - - const storedBefore = await prisma.investigation.findUnique({ - where: { id: investigation.id }, - select: { postVersionId: true }, - }); - assert.ok(storedBefore); - - const result = await ensureInvestigationQueued({ - prisma, - postVersionId: storedBefore.postVersionId, - promptId: prompt.id, - allowRequeueFailed: true, - enqueue: false, - }); - - assert.equal(result.investigation.id, investigation.id); - assert.equal(result.investigation.status, "PENDING"); + const caller = createCaller({ isAuthenticated: true }); + const result = await caller.post.investigateNow({ postVersionId: post.postVersionId }); - const storedAfter = await prisma.investigation.findUnique({ + assert.equal(result.investigationId, investigation.id); + assert.equal(result.status, "FAILED"); + const stored = await prisma.investigation.findUniqueOrThrow({ where: { id: investigation.id }, - select: { - status: true, - attemptCount: true, - retryAfter: true, - lease: { select: { investigationId: true } }, - }, + select: { status: true, attemptCount: true }, }); - assert.ok(storedAfter); - assert.equal(storedAfter.status, "PENDING"); - assert.equal(storedAfter.attemptCount, 0); - assert.equal(storedAfter.retryAfter, null); - assert.equal(storedAfter.lease, null); + assert.deepEqual(stored, { status: "FAILED", attemptCount: 4 }); }); void test("investigateNow persists InvestigationInput snapshot at queue time", async () => { @@ -618,6 +529,7 @@ void test("investigateNow persists InvestigationInput snapshot at queue time", a select: { id: true, inputId: true, + origin: true, input: { select: { investigationId: true, @@ -625,257 +537,388 @@ void test("investigateNow persists InvestigationInput snapshot at queue time", a markdownSource: true, markdown: true, markdownRendererVersion: true, + postUrl: true, + authorName: true, + hasVideo: true, + imagePlaceholderSourceUrls: true, }, }, }, }); assert.ok(investigation); assert.equal(investigation.inputId, investigation.id); + assert.equal(investigation.origin, "INSTANCE_REQUEST"); assert.equal(investigation.input.investigationId, investigation.id); assert.equal(investigation.input.provenance, "SERVER_VERIFIED"); assert.equal(investigation.input.markdownSource, "SERVER_HTML"); assert.equal(typeof investigation.input.markdown, "string"); assert.equal(typeof investigation.input.markdownRendererVersion, "string"); + // Prompt context comes from the server-verified identity, frozen at queue time. + assert.equal( + investigation.input.postUrl, + `https://www.lesswrong.com/posts/${viewInput.externalId}/${LESSWRONG_MOCK_SERVER_SLUG}`, + ); + assert.equal(investigation.input.authorName, LESSWRONG_MOCK_SERVER_AUTHOR_NAME); + assert.equal(investigation.input.hasVideo, false); + assert.deepEqual(investigation.input.imagePlaceholderSourceUrls, []); + + // Later changes to the live Post row do not reach the snapshot. + await prisma.post.updateMany({ + where: { externalId: viewInput.externalId }, + data: { url: "https://www.lesswrong.com/posts/changed" }, + }); + const snapshotAfterEdit = await prisma.investigationInput.findUniqueOrThrow({ + where: { investigationId: investigation.id }, + select: { postUrl: true }, + }); + assert.equal(snapshotAfterEdit.postUrl, investigation.input.postUrl); }); -void test("ensureInvestigationQueued randomized state model preserves lifecycle invariants", async () => { - const random = createDeterministicRandom(0x94ab73d1); - const rounds = 18; - const seedCases = [ - { name: "new", status: null }, - { name: "failed", status: "FAILED" }, - { name: "pending", status: "PENDING" }, - { name: "processing-stale", status: "PROCESSING", leaseKind: "STALE" }, - { name: "processing-active", status: "PROCESSING", leaseKind: "ACTIVE" }, - ] as const; - - for (let round = 0; round < rounds; round += 1) { - const seedCaseIndex = randomInt(random, 0, seedCases.length - 1); - const seedCase = seedCases[seedCaseIndex]; - assert.ok(seedCase, `seed case index out of bounds: ${seedCaseIndex.toString()}`); - const allowRequeueFailed = randomChance(random, 0.5); - const enqueue = randomChance(random, 0.7); - const includeOnPendingInvestigation = randomChance(random, 0.6); - const canonicalProvenance = randomChance(random, 0.5) ? "SERVER_VERIFIED" : "CLIENT_FALLBACK"; - const roundTag = [ - `round=${round.toString()}`, - `seedCase=${seedCase.name}`, - `allowRequeueFailed=${allowRequeueFailed.toString()}`, - `enqueue=${enqueue.toString()}`, - `canonicalProvenance=${canonicalProvenance}`, - ].join(" "); +type RequestedState = + | "NONE" + | "COMPLETE" + | "FAILED" + | "PENDING_FUNDED" + | "PENDING_UNFUNDED" + | "PROCESSING_ACTIVE" + | "PROCESSING_STALE" + | "PROCESSING_STALE_EXHAUSTED"; + +interface RequestInvestigationCase { + state: RequestedState; + requester: "INSTANCE_API_KEY" | "USER_OPENAI_KEY"; + expected: { + status: "PENDING" | "PROCESSING" | "COMPLETE" | "FAILED"; + origin: "SELECTOR" | "INSTANCE_REQUEST" | "USER_KEY_REQUEST"; + hasKeySource: boolean; + hasLease: boolean; + }; +} + +const REQUEST_INVESTIGATION_CASES: RequestInvestigationCase[] = [ + { + state: "NONE", + requester: "INSTANCE_API_KEY", + expected: { + status: "PENDING", + origin: "INSTANCE_REQUEST", + hasKeySource: false, + hasLease: false, + }, + }, + { + state: "NONE", + requester: "USER_OPENAI_KEY", + expected: { + status: "PENDING", + origin: "USER_KEY_REQUEST", + hasKeySource: true, + hasLease: false, + }, + }, + { + state: "COMPLETE", + requester: "USER_OPENAI_KEY", + expected: { status: "COMPLETE", origin: "SELECTOR", hasKeySource: false, hasLease: false }, + }, + { + state: "FAILED", + requester: "USER_OPENAI_KEY", + expected: { status: "FAILED", origin: "SELECTOR", hasKeySource: false, hasLease: false }, + }, + // A user key never takes over an investigation someone else is paying for. + { + state: "PENDING_FUNDED", + requester: "USER_OPENAI_KEY", + expected: { status: "PENDING", origin: "SELECTOR", hasKeySource: false, hasLease: false }, + }, + { + state: "PENDING_UNFUNDED", + requester: "INSTANCE_API_KEY", + expected: { + status: "PENDING", + origin: "INSTANCE_REQUEST", + hasKeySource: false, + hasLease: false, + }, + }, + { + state: "PENDING_UNFUNDED", + requester: "USER_OPENAI_KEY", + expected: { + status: "PENDING", + origin: "USER_KEY_REQUEST", + hasKeySource: true, + hasLease: false, + }, + }, + { + state: "PROCESSING_ACTIVE", + requester: "USER_OPENAI_KEY", + expected: { status: "PROCESSING", origin: "SELECTOR", hasKeySource: false, hasLease: true }, + }, + { + state: "PROCESSING_STALE", + requester: "INSTANCE_API_KEY", + expected: { status: "PENDING", origin: "SELECTOR", hasKeySource: false, hasLease: false }, + }, + { + state: "PROCESSING_STALE_EXHAUSTED", + requester: "INSTANCE_API_KEY", + expected: { status: "FAILED", origin: "SELECTOR", hasKeySource: false, hasLease: false }, + }, +]; +function seedStatusFor(state: Exclude) { + switch (state) { + case "COMPLETE": + return "COMPLETE" as const; + case "FAILED": + return "FAILED" as const; + case "PENDING_FUNDED": + case "PENDING_UNFUNDED": + return "PENDING" as const; + case "PROCESSING_ACTIVE": + case "PROCESSING_STALE": + case "PROCESSING_STALE_EXHAUSTED": + return "PROCESSING" as const; + } +} + +void test("requestInvestigation follows the investigateNow state table for every starting state", async () => { + const prompt = await seedPrompt("request-investigation-state-table"); + for (const [index, testCase] of REQUEST_INVESTIGATION_CASES.entries()) { + const caseTag = `${testCase.state}/${testCase.requester}`; const post = await seedPost({ platform: "X", - externalId: `ensure-queued-fuzz-${round.toString()}`, - url: `https://x.com/openerrata/status/${withIntegrationPrefix(`ensure-queued-fuzz-${round.toString()}`)}`, - contentText: `ensureInvestigationQueued fuzz payload ${roundTag}`, + externalId: `request-investigation-state-${index.toString()}`, + url: `https://x.com/openerrata/status/${withIntegrationPrefix(`request-investigation-state-${index.toString()}`)}`, + contentText: `requestInvestigation state table ${caseTag}`, }); - const prompt = await seedPrompt(`ensure-queued-fuzz-${round.toString()}`); - - let seededInvestigationId: string | null = null; - let seededExistingProvenance: "SERVER_VERIFIED" | "CLIENT_FALLBACK" | null = null; - let seededActiveLeaseOwner: string | null = null; - let seededActiveLeaseExpiresAt: Date | null = null; - if (seedCase.status !== null) { - seededExistingProvenance = randomChance(random, 0.5) ? "SERVER_VERIFIED" : "CLIENT_FALLBACK"; - const leaseKind = "leaseKind" in seedCase ? seedCase.leaseKind : null; - - const activeLeaseOwner = - leaseKind === "ACTIVE" ? withIntegrationPrefix(`active-worker-${round.toString()}`) : null; - const activeLeaseExpiresAt = - leaseKind === "ACTIVE" ? new Date(Date.now() + 10 * 60_000) : null; - seededActiveLeaseOwner = activeLeaseOwner; - seededActiveLeaseExpiresAt = activeLeaseExpiresAt; - - const investigation = await seedInvestigation({ + let seededId: string | null = null; + if (testCase.state !== "NONE") { + const stale = testCase.state.startsWith("PROCESSING_STALE"); + const seeded = await seedInvestigation({ postId: post.id, contentHash: post.contentHash, contentText: post.contentText, - provenance: seededExistingProvenance, - status: seedCase.status, - promptLabel: `ensure-queued-existing-${round.toString()}`, - ...(leaseKind === "ACTIVE" && activeLeaseOwner !== null && activeLeaseExpiresAt !== null - ? { - leaseOwner: activeLeaseOwner, - leaseExpiresAt: activeLeaseExpiresAt, - } - : leaseKind === "STALE" - ? { - leaseOwner: withIntegrationPrefix(`stale-worker-${round.toString()}`), - leaseExpiresAt: new Date(Date.now() - 5 * 60_000), - } - : {}), + provenance: "CLIENT_FALLBACK", + status: seedStatusFor(testCase.state), + promptLabel: `request-investigation-state-${index.toString()}`, + origin: testCase.state === "PENDING_UNFUNDED" ? "USER_KEY_REQUEST" : "SELECTOR", + attemptCount: testCase.state === "PROCESSING_STALE_EXHAUSTED" ? 4 : 1, + ...(stale ? { leaseExpiresAt: new Date(Date.now() - 60_000) } : {}), }); - seededInvestigationId = investigation.id; - // The lease row (including leaseOwner, leaseExpiresAt) is fully created - // by seedInvestigation above. No follow-up seedInvestigationWithLeaseFields - // call is needed here — startedAt/heartbeatAt are not asserted in this test. + seededId = seeded.id; } - let onPendingInvestigationCalls = 0; - let onPendingInvestigationId: string | null = null; - const canonicalPostVersion = await ensurePostVersionForSeed({ - postId: post.id, - contentHash: post.contentHash, - contentText: post.contentText, - provenance: canonicalProvenance, + const postVersion = await prisma.postVersion.findUniqueOrThrow({ + where: { id: post.postVersionId }, + select: { + id: true, + postId: true, + contentBlob: { select: { contentText: true, wordCount: true } }, + }, }); - const result = await ensureInvestigationQueued({ - prisma, - postVersionId: canonicalPostVersion.id, + const { investigationId } = await requestInvestigation(prisma, { + postVersion, promptId: prompt.id, - allowRequeueFailed, - enqueue, - ...(includeOnPendingInvestigation - ? { - onPendingInvestigation: async ({ investigation }) => { - onPendingInvestigationCalls += 1; - onPendingInvestigationId = investigation.id; + requester: + testCase.requester === "INSTANCE_API_KEY" + ? { kind: "INSTANCE_API_KEY" } + : { + kind: "USER_OPENAI_KEY", + apiKey: `sk-test-state-table-${index.toString()}-0123456789`, }, - } - : {}), }); - const startedWithoutInvestigation = seedCase.status === null; - const expectedCreated = startedWithoutInvestigation; - const statusAfterRecord = startedWithoutInvestigation - ? "PENDING" - : seedCase.status === "FAILED" && allowRequeueFailed - ? "PENDING" - : seedCase.status; - const leaseKind = "leaseKind" in seedCase ? seedCase.leaseKind : null; - const expectedRecoveredFromStaleProcessing = - statusAfterRecord === "PROCESSING" && leaseKind === "STALE"; - const expectedFinalStatus = expectedRecoveredFromStaleProcessing - ? "PENDING" - : statusAfterRecord; - const expectedEnqueued = enqueue && expectedFinalStatus === "PENDING"; - - assert.equal(result.created, expectedCreated, `created mismatch (${roundTag})`); - assert.equal(result.enqueued, expectedEnqueued, `enqueued mismatch (${roundTag})`); - assert.equal( - result.investigation.status, - expectedFinalStatus, - `result status mismatch (${roundTag})`, - ); - if (seededInvestigationId !== null) { - assert.equal( - result.investigation.id, - seededInvestigationId, - `existing investigation identity mismatch (${roundTag})`, - ); - } - - const expectedOnPendingInvestigationCalls = - includeOnPendingInvestigation && expectedEnqueued ? 1 : 0; - assert.equal( - onPendingInvestigationCalls, - expectedOnPendingInvestigationCalls, - `onPendingInvestigation invocation mismatch (${roundTag})`, - ); - if (expectedOnPendingInvestigationCalls === 1) { - assert.equal( - onPendingInvestigationId, - result.investigation.id, - `onPendingInvestigation investigation mismatch (${roundTag})`, - ); + if (seededId !== null) { + assert.equal(investigationId, seededId, `existing investigation reused (${caseTag})`); } - - const storedInvestigations = await prisma.investigation.findMany({ - where: { - postVersion: { - postId: post.id, - }, - }, + const stored = await prisma.investigation.findUniqueOrThrow({ + where: { id: investigationId }, select: { - id: true, status: true, - queuedAt: true, - postVersion: { - select: { - serverVerifiedAt: true, - }, - }, + origin: true, + openAiKeySource: { select: { investigationId: true } }, + lease: { select: { investigationId: true } }, }, }); - assert.equal( - storedInvestigations.length, - 1, - `exactly one investigation row expected (${roundTag})`, - ); - const storedInvestigation = storedInvestigations[0]; - assert.ok(storedInvestigation, `missing stored investigation (${roundTag})`); - assert.equal( - storedInvestigation.id, - result.investigation.id, - `stored investigation id mismatch (${roundTag})`, - ); - assert.equal( - storedInvestigation.status, - expectedFinalStatus, - `stored status mismatch (${roundTag})`, + assert.deepEqual( + { + status: stored.status, + origin: stored.origin, + hasKeySource: stored.openAiKeySource !== null, + hasLease: stored.lease !== null, + }, + testCase.expected, + caseTag, ); + } +}); - const expectedServerVerified = - seedCase.status === null - ? canonicalProvenance === "SERVER_VERIFIED" - : seededExistingProvenance === "SERVER_VERIFIED" || - canonicalProvenance === "SERVER_VERIFIED"; - if (expectedServerVerified) { - assert.notEqual( - storedInvestigation.postVersion.serverVerifiedAt, - null, - `server-verified rows should have serverVerifiedAt (${roundTag})`, - ); - } else { - assert.equal( - storedInvestigation.postVersion.serverVerifiedAt, - null, - `client-fallback rows should not have serverVerifiedAt (${roundTag})`, - ); - } +void test("requestInvestigation verifies a user key before it funds anything", async () => { + const prompt = await seedPrompt("request-investigation-rejected-key"); + const post = await seedPost({ + platform: "X", + externalId: "request-investigation-rejected-key-1", + url: `https://x.com/openerrata/status/${withIntegrationPrefix("request-investigation-rejected-key-1")}`, + contentText: "A key OpenAI rejects must not create or fund an investigation.", + }); + const postVersion = await prisma.postVersion.findUniqueOrThrow({ + where: { id: post.postVersionId }, + select: { + id: true, + postId: true, + contentBlob: { select: { contentText: true, wordCount: true } }, + }, + }); - // queuedAt is now always non-null (@default(now())), so just verify it's set - assert.notEqual( - storedInvestigation.queuedAt, - null, - `queuedAt should always be populated (${roundTag})`, - ); + await assert.rejects( + requestInvestigation(prisma, { + postVersion, + promptId: prompt.id, + requester: { kind: "USER_OPENAI_KEY", apiKey: "sk-test-rejected-key-0123456789" }, + }), + (error: unknown) => + error instanceof UserOpenAiKeyRejectedError && error.outcome.openaiApiKeyStatus === "invalid", + ); + assert.equal( + await prisma.investigation.count({ where: { postVersionId: post.postVersionId } }), + 0, + ); +}); - // Check lease state via InvestigationLease table - const storedLease = await prisma.investigationLease.findUnique({ - where: { investigationId: storedInvestigation.id }, - select: { leaseOwner: true, leaseExpiresAt: true }, - }); +async function requestUserKeyInvestigation(externalId: string): Promise { + const caller = createCaller({ userOpenAiApiKey: `sk-test-${externalId}` }); + const result = await caller.post.investigateNow( + buildXViewInput({ externalId, observedContentText: `User key funding for ${externalId}.` }), + ); + assert.equal(result.status, "PENDING"); + return result.investigationId; +} - if (expectedRecoveredFromStaleProcessing) { - assert.equal( - storedLease, - null, - `recovered stale investigations should have no lease row (${roundTag})`, - ); - } +void test("orchestrateInvestigation drops a user key OpenAI refuses instead of failing the investigation", async () => { + const investigationId = await requestUserKeyInvestigation("user-key-refused-1"); - if (statusAfterRecord === "PROCESSING" && !expectedRecoveredFromStaleProcessing) { - assert.equal( - leaseKind, - "ACTIVE", - `non-recovered processing cases must come from active lease seeds (${roundTag})`, - ); - assert.ok(storedLease, `active processing investigation should have lease row (${roundTag})`); - assert.equal( - storedLease.leaseOwner, - seededActiveLeaseOwner, - `active processing investigation should keep lease owner (${roundTag})`, - ); - assert.ok(seededActiveLeaseExpiresAt, `active lease should have seeded expiry (${roundTag})`); - assert.equal( - storedLease.leaseExpiresAt.getTime(), - seededActiveLeaseExpiresAt.getTime(), - `active processing investigation should keep lease expiry (${roundTag})`, - ); - } - } + await orchestrateInvestigation( + investigationId, + { info() {}, warn() {}, error() {} }, + { + workerIdentity: withIntegrationPrefix("worker-user-key-refused"), + createInvestigator: () => ({ + investigate: () => + Promise.reject( + new InvestigatorExecutionError( + "OpenAI rejected the key", + buildFailedAttemptAudit("user-key-refused"), + OpenAI.APIError.generate( + 401, + { error: { message: "Incorrect API key provided" } }, + undefined, + new Headers(), + ), + ), + ), + }), + }, + ); + + const stored = await prisma.investigation.findUniqueOrThrow({ + where: { id: investigationId }, + select: { + status: true, + origin: true, + attemptCount: true, + openAiKeySource: { select: { investigationId: true } }, + attempts: { select: { attemptNumber: true, outcome: true } }, + }, + }); + assert.equal(stored.status, "PENDING"); + assert.equal(stored.origin, "USER_KEY_REQUEST"); + assert.equal(stored.openAiKeySource, null); + assert.deepEqual(stored.attempts, [{ attemptNumber: 1, outcome: "FAILED" }]); + + // Unfunded: a worker that picks the job up again does nothing, and never + // falls back to the server key. + await orchestrateInvestigation( + investigationId, + { info() {}, warn() {}, error() {} }, + { + workerIdentity: withIntegrationPrefix("worker-user-key-unfunded"), + createInvestigator: () => assert.fail("an unfunded investigation must not run"), + }, + ); + const afterSkip = await prisma.investigation.findUniqueOrThrow({ + where: { id: investigationId }, + select: { status: true, attemptCount: true }, + }); + assert.deepEqual(afterSkip, { status: "PENDING", attemptCount: stored.attemptCount }); +}); + +void test("orchestrateInvestigation drops an expired user key before calling OpenAI", async () => { + const investigationId = await requestUserKeyInvestigation("user-key-expired-1"); + await prisma.investigationOpenAiKeySource.update({ + where: { investigationId }, + data: { expiresAt: new Date(Date.now() - 1_000) }, + }); + + await orchestrateInvestigation( + investigationId, + { info() {}, warn() {}, error() {} }, + { + workerIdentity: withIntegrationPrefix("worker-user-key-expired"), + createInvestigator: () => assert.fail("an expired user key must not reach OpenAI"), + }, + ); + + const stored = await prisma.investigation.findUniqueOrThrow({ + where: { id: investigationId }, + select: { status: true, openAiKeySource: { select: { investigationId: true } } }, + }); + assert.equal(stored.status, "PENDING"); + assert.equal(stored.openAiKeySource, null); +}); + +void test("orchestrateInvestigation abandons an attempt whose lease was lost without writing", async () => { + const post = await seedPost({ + platform: "X", + externalId: "orchestrator-lease-lost-1", + url: "https://x.com/openerrata/status/orchestrator-lease-lost-1", + contentText: "A run that lost its lease must not write results.", + }); + const investigation = await seedPendingInvestigation({ + postId: post.id, + contentHash: post.contentHash, + contentText: post.contentText, + provenance: "CLIENT_FALLBACK", + }); + const workerIdentity = withIntegrationPrefix("worker-lease-lost"); + + await orchestrateInvestigation( + investigation.id, + { info() {}, warn() {}, error() {} }, + { + workerIdentity, + createInvestigator: () => ({ + investigate: () => + Promise.reject(new LeaseLostError(investigation.id, "simulated lost lease")), + }), + }, + ); + + const stored = await prisma.investigation.findUniqueOrThrow({ + where: { id: investigation.id }, + select: { + status: true, + attempts: { select: { id: true } }, + lease: { select: { leaseOwner: true } }, + }, + }); + // Left for expired-lease recovery: still PROCESSING under this worker's + // lease, with no attempt audit or status change written. + assert.equal(stored.status, "PROCESSING"); + assert.deepEqual(stored.attempts, []); + assert.equal(stored.lease?.leaseOwner, workerIdentity); }); diff --git a/src/typescript/api/test/integration/api-endpoints.public-and-auth.integration.test.ts b/src/typescript/api/test/integration/api-endpoints.public-and-auth.integration.test.ts index cee6b54..af133c9 100644 --- a/src/typescript/api/test/integration/api-endpoints.public-and-auth.integration.test.ts +++ b/src/typescript/api/test/integration/api-endpoints.public-and-auth.integration.test.ts @@ -5,7 +5,6 @@ import { INTEGRATION_LESSWRONG_FIXTURE_KEYS, InvestigatorExecutionError, MINIMUM_SUPPORTED_EXTENSION_VERSION, - OpenAIInvestigator, WORD_COUNT_LIMIT, appRouter, assert, @@ -13,13 +12,13 @@ import { buildFailedAttemptAudit, buildLesswrongViewInput, buildSucceededAttemptAudit, + buildSucceededInvestigatorOutput, buildXViewInput, closeQueueUtils, createCaller, createContext, createDeterministicRandom, createMockRequestEvent, - ensureInvestigationQueued, ensurePostVersionForSeed, errorHasOpenErrataCode, getPrisma, @@ -66,7 +65,6 @@ void [ INTEGRATION_LESSWRONG_FIXTURE_KEYS, InvestigatorExecutionError, MINIMUM_SUPPORTED_EXTENSION_VERSION, - OpenAIInvestigator, WORD_COUNT_LIMIT, appRouter, assert, @@ -74,13 +72,13 @@ void [ buildFailedAttemptAudit, buildLesswrongViewInput, buildSucceededAttemptAudit, + buildSucceededInvestigatorOutput, buildXViewInput, closeQueueUtils, createCaller, createContext, createDeterministicRandom, createMockRequestEvent, - ensureInvestigationQueued, ensurePostVersionForSeed, errorHasOpenErrataCode, getPrisma, @@ -145,11 +143,11 @@ void test("post.investigateNow allows user OpenAI key callers and returns inline assert.equal(result.claims.length, 1); }); -void test("post.investigateNow requeues existing failed investigations as PENDING", async () => { +void test("post.investigateNow returns existing FAILED investigations unchanged", async () => { const caller = createCaller({ isAuthenticated: true }); const input = buildXViewInput({ - externalId: "investigate-now-requeue-failed-1", - observedContentText: "Canonical content should be retried after FAILED state.", + externalId: "investigate-now-failed-terminal-1", + observedContentText: "FAILED is terminal for this content version.", }); const seeded = await seedInvestigationForXViewInput({ viewInput: input, @@ -160,67 +158,94 @@ void test("post.investigateNow requeues existing failed investigations as PENDIN const result = await caller.post.investigateNow(input); assert.equal(result.investigationId, seeded.investigationId); - assert.equal(result.status, "PENDING"); + assert.equal(result.status, "FAILED"); assert.equal(result.provenance, "CLIENT_FALLBACK"); - const stored = await prisma.investigation.findUnique({ + const stored = await prisma.investigation.findUniqueOrThrow({ where: { id: seeded.investigationId }, - select: { status: true, checkedAt: true }, + select: { status: true }, }); - assert.ok(stored); - assert.equal(stored.status, "PENDING"); - assert.equal(stored.checkedAt, null); + assert.equal(stored.status, "FAILED"); }); -void test("post.investigateNow attaches first user key source while investigation is PENDING", async () => { +void test("post.investigateNow attaches a user key only to the investigation its request creates", async () => { const firstCaller = createCaller({ - userOpenAiApiKey: "sk-test-user-key-first", + userOpenAiApiKey: "sk-test-user-key-first-0123456789", }); const secondCaller = createCaller({ - userOpenAiApiKey: "sk-test-user-key-second", + userOpenAiApiKey: "sk-test-user-key-second-0123456789", }); const input = buildXViewInput({ externalId: "investigate-now-user-key-first-wins-1", - observedContentText: "Pending investigation should keep the first user key source.", + observedContentText: "The creating request's key funds the investigation.", + }); + + const firstResult = await firstCaller.post.investigateNow(input); + assert.equal(firstResult.status, "PENDING"); + const created = await prisma.investigation.findUniqueOrThrow({ + where: { id: firstResult.investigationId }, + select: { + origin: true, + openAiKeySource: { select: { ciphertext: true, iv: true, authTag: true, keyId: true } }, + }, + }); + assert.equal(created.origin, "USER_KEY_REQUEST"); + assert.ok(created.openAiKeySource); + + const secondResult = await secondCaller.post.investigateNow(input); + assert.equal(secondResult.investigationId, firstResult.investigationId); + const afterSecond = await prisma.investigationOpenAiKeySource.findUniqueOrThrow({ + where: { investigationId: firstResult.investigationId }, + select: { ciphertext: true, iv: true, authTag: true, keyId: true }, + }); + assert.deepEqual(afterSecond, created.openAiKeySource); +}); + +void test("post.investigateNow never attaches a user key to an investigation the server is paying for", async () => { + const caller = createCaller({ userOpenAiApiKey: "sk-test-user-key-late-0123456789" }); + const input = buildXViewInput({ + externalId: "investigate-now-user-key-no-takeover-1", + observedContentText: "A selector-admitted investigation keeps its server funding.", }); const seeded = await seedInvestigationForXViewInput({ viewInput: input, status: "PENDING", provenance: "CLIENT_FALLBACK", }); + await prisma.investigation.update({ + where: { id: seeded.investigationId }, + data: { origin: "SELECTOR" }, + }); - const firstResult = await firstCaller.post.investigateNow(input); - assert.equal(firstResult.investigationId, seeded.investigationId); - assert.equal(firstResult.status, "PENDING"); + const result = await caller.post.investigateNow(input); - const storedAfterFirst = await prisma.investigationOpenAiKeySource.findUnique({ - where: { investigationId: seeded.investigationId }, - select: { - ciphertext: true, - iv: true, - authTag: true, - keyId: true, - expiresAt: true, - }, + assert.equal(result.investigationId, seeded.investigationId); + assert.equal(result.status, "PENDING"); + const stored = await prisma.investigation.findUniqueOrThrow({ + where: { id: seeded.investigationId }, + select: { origin: true, openAiKeySource: { select: { investigationId: true } } }, }); - assert.ok(storedAfterFirst); + assert.equal(stored.origin, "SELECTOR"); + assert.equal(stored.openAiKeySource, null); +}); - const secondResult = await secondCaller.post.investigateNow(input); - assert.equal(secondResult.investigationId, seeded.investigationId); - assert.equal(secondResult.status, "PENDING"); +void test("post.investigateNow rejects a user key OpenAI refuses before creating anything", async () => { + const caller = createCaller({ userOpenAiApiKey: "sk-test-rejected-user-key" }); + const input = buildXViewInput({ + externalId: "investigate-now-user-key-rejected-1", + observedContentText: "A refused key must not create an investigation.", + }); - const storedAfterSecond = await prisma.investigationOpenAiKeySource.findUnique({ - where: { investigationId: seeded.investigationId }, - select: { - ciphertext: true, - iv: true, - authTag: true, - keyId: true, - expiresAt: true, - }, + await assert.rejects(caller.post.investigateNow(input), /x-openai-api-key was rejected/); + + const registered = await prisma.post.findUniqueOrThrow({ + where: { platform_externalId: { platform: "X", externalId: input.externalId } }, + select: { versions: { select: { investigation: { select: { id: true } } } } }, }); - assert.ok(storedAfterSecond); - assert.deepEqual(storedAfterSecond, storedAfterFirst); + assert.deepEqual( + registered.versions.map((version) => version.investigation), + [null], + ); }); void test("post.investigateNow recovers stale PROCESSING investigations to PENDING", async () => { @@ -332,8 +357,9 @@ void test("selector recovers stale PROCESSING investigations using shared lifecy heartbeatAt: new Date(Date.now() - 5 * 60_000), }); - const enqueued = await runSelector(); - assert.ok(enqueued >= 1); + const summary = await runSelector({ dailyBudget: 0 }); + assert.ok(summary.recovered >= 1); + assert.deepEqual(summary.failures, []); const stored = await prisma.investigation.findUnique({ where: { id: processingInvestigation.id }, @@ -353,6 +379,104 @@ void test("selector recovers stale PROCESSING investigations using shared lifecy assert.equal(storedLease, null); }); +async function selectorAdmissionsToday(): Promise { + const dayStart = new Date(); + dayStart.setUTCHours(0, 0, 0, 0); + return prisma.investigation.count({ + where: { origin: "SELECTOR", admittedAt: { gte: dayStart } }, + }); +} + +async function seedTopScoredPost(externalId: string, uniqueViewScore: number) { + const post = await seedPost({ + platform: "X", + externalId, + url: `https://x.com/openerrata/status/${withIntegrationPrefix(externalId)}`, + contentText: `Selector admission candidate ${externalId}.`, + }); + await prisma.post.update({ where: { id: post.id }, data: { uniqueViewScore } }); + return post; +} + +void test("selector admits at most its daily budget of new investigations per UTC day", async () => { + const post = await seedTopScoredPost("selector-daily-budget-1", 2_000_000_000); + const dailyBudget = (await selectorAdmissionsToday()) + 1; + + const first = await runSelector({ dailyBudget }); + assert.equal(first.admitted, 1); + assert.equal(first.budgetRemaining, 0); + assert.deepEqual(first.failures, []); + const admitted = await prisma.investigation.findUniqueOrThrow({ + where: { postVersionId: post.postVersionId }, + select: { + status: true, + origin: true, + input: { select: { postUrl: true } }, + }, + }); + assert.deepEqual(admitted, { + status: "PENDING", + origin: "SELECTOR", + input: { postUrl: post.url }, + }); + + // Running again the same day admits nothing more, however often it runs. + await seedTopScoredPost("selector-daily-budget-2", 2_000_000_001); + const second = await runSelector({ dailyBudget }); + assert.equal(second.admitted, 0); + assert.equal(second.budgetRemaining, 0); +}); + +void test("selector funds an investigation whose user key was dropped, within its budget", async () => { + const post = await seedTopScoredPost("selector-adopts-unfunded-1", 2_000_000_010); + const unfunded = await seedPendingInvestigation({ + postId: post.id, + contentHash: post.contentHash, + contentText: post.contentText, + provenance: "CLIENT_FALLBACK", + }); + await prisma.investigation.update({ + where: { id: unfunded.id }, + data: { origin: "USER_KEY_REQUEST" }, + }); + + const summary = await runSelector({ dailyBudget: (await selectorAdmissionsToday()) + 1 }); + + assert.equal(summary.admitted, 1); + const stored = await prisma.investigation.findUniqueOrThrow({ + where: { id: unfunded.id }, + select: { origin: true, status: true }, + }); + assert.deepEqual(stored, { origin: "SELECTOR", status: "PENDING" }); +}); + +void test("selector-created investigations of edited posts get update lineage", async () => { + const post = await seedTopScoredPost("selector-update-lineage-1", 2_000_000_020); + const parent = await seedCompleteInvestigation({ + postId: post.id, + contentHash: post.contentHash, + contentText: post.contentText, + provenance: "SERVER_VERIFIED", + }); + const editedText = normalizeContent(`${post.contentText} An edited sentence.`); + const editedVersion = await ensurePostVersionForSeed({ + postId: post.id, + contentHash: await hashContent(editedText), + contentText: editedText, + provenance: "CLIENT_FALLBACK", + }); + + const summary = await runSelector({ dailyBudget: (await selectorAdmissionsToday()) + 1 }); + + assert.equal(summary.admitted, 1); + const update = await prisma.investigation.findUniqueOrThrow({ + where: { postVersionId: editedVersion.id }, + select: { parentInvestigationId: true, contentDiff: true }, + }); + assert.equal(update.parentInvestigationId, parent.id); + assert.match(update.contentDiff ?? "", /An edited sentence\./); +}); + void test("post.investigateNow rejects unauthenticated callers", async () => { const caller = createCaller({ isAuthenticated: false }); const input = buildXViewInput({ @@ -457,8 +581,108 @@ void test("post.batchStatus returns investigation state and incorrect claim coun assert.equal(notInvestigated.incorrectClaimCount, 0); }); -void test("public.getInvestigation hides non-COMPLETE investigations", async () => { - const caller = createCaller(); +interface GraphqlOrigin { + provenance: "SERVER_VERIFIED" | "CLIENT_FALLBACK"; + serverVerifiedAt: string | null; +} + +async function queryPublicInvestigation(investigationId: string) { + const result = await queryPublicGraphql<{ + publicInvestigation: { + investigation: { id: string; origin: GraphqlOrigin; corroborationCount: number }; + post: { platform: Platform; externalId: string; url: string }; + claims: { id: string }[]; + } | null; + }>( + ` + query PublicInvestigation($investigationId: ID!) { + publicInvestigation(investigationId: $investigationId) { + investigation { + id + origin { + provenance + serverVerifiedAt + } + corroborationCount + } + post { + platform + externalId + url + } + claims { + id + } + } + } + `, + { investigationId }, + ); + return result.publicInvestigation; +} + +async function querySearchInvestigations(variables: { + query?: string; + platform?: Platform; + minClaimCount?: number; +}): Promise<{ id: string; platform: Platform; origin: GraphqlOrigin }[]> { + const result = await queryPublicGraphql<{ + searchInvestigations: { + investigations: { id: string; platform: Platform; origin: GraphqlOrigin }[]; + }; + }>( + ` + query SearchInvestigations($query: String, $platform: Platform, $minClaimCount: Int) { + searchInvestigations( + query: $query + platform: $platform + minClaimCount: $minClaimCount + limit: 20 + offset: 0 + ) { + investigations { + id + platform + origin { + provenance + serverVerifiedAt + } + } + } + } + `, + variables, + ); + return result.searchInvestigations.investigations; +} + +async function queryPublicMetrics(variables: { + windowStart: string; + windowEnd: string; + platform?: Platform; +}) { + const result = await queryPublicGraphql<{ + publicMetrics: { + totalInvestigatedPosts: number; + investigatedPostsWithFlags: number; + factCheckIncidence: number | null; + }; + }>( + ` + query PublicMetrics($windowStart: DateTime, $windowEnd: DateTime, $platform: Platform) { + publicMetrics(windowStart: $windowStart, windowEnd: $windowEnd, platform: $platform) { + totalInvestigatedPosts + investigatedPostsWithFlags + factCheckIncidence + } + } + `, + variables, + ); + return result.publicMetrics; +} + +void test("publicInvestigation hides non-COMPLETE investigations", async () => { const post = await seedPost({ platform: "LESSWRONG", externalId: "public-investigation-non-complete-hidden-1", @@ -484,35 +708,11 @@ void test("public.getInvestigation hides non-COMPLETE investigations", async () }); for (const investigationId of [pending.id, failed.id]) { - const trpcResult = await caller.public.getInvestigation({ - investigationId, - }); - assert.equal(trpcResult, null); - - const graphqlResult = await queryPublicGraphql<{ - publicInvestigation: { - investigation: { - id: string; - }; - } | null; - }>( - ` - query PublicInvestigation($investigationId: ID!) { - publicInvestigation(investigationId: $investigationId) { - investigation { - id - } - } - } - `, - { investigationId }, - ); - assert.equal(graphqlResult.publicInvestigation, null); + assert.equal(await queryPublicInvestigation(investigationId), null); } }); -void test("public.getInvestigation returns complete investigation and trust signals", async () => { - const caller = createCaller(); +void test("publicInvestigation returns complete investigation and trust signals", async () => { const post = await seedPost({ platform: "LESSWRONG", externalId: "public-investigation-1", @@ -528,70 +728,18 @@ void test("public.getInvestigation returns complete investigation and trust sign }); await seedClaimWithSource(investigation.id, 1); - const result = await caller.public.getInvestigation({ - investigationId: investigation.id, - }); - + const result = await queryPublicInvestigation(investigation.id); assert.ok(result); assert.equal(result.investigation.id, investigation.id); assert.equal(result.investigation.origin.provenance, "SERVER_VERIFIED"); assert.equal(result.investigation.corroborationCount, 0); - assert.equal(typeof result.investigation.origin.serverVerifiedAt, "string"); + assert.notEqual(result.investigation.origin.serverVerifiedAt, null); assert.equal(result.post.platform, post.platform); assert.equal(result.post.externalId, post.externalId); assert.equal(result.claims.length, 1); - - const graphqlResult = await queryPublicGraphql<{ - publicInvestigation: { - investigation: { - id: string; - origin: { - provenance: "SERVER_VERIFIED" | "CLIENT_FALLBACK"; - serverVerifiedAt: string | null; - }; - corroborationCount: number; - }; - post: { - platform: Platform; - externalId: string; - }; - claims: { id: string }[]; - } | null; - }>( - ` - query PublicInvestigation($investigationId: ID!) { - publicInvestigation(investigationId: $investigationId) { - investigation { - id - origin { - provenance - serverVerifiedAt - } - corroborationCount - } - post { - platform - externalId - } - claims { - id - } - } - } - `, - { investigationId: investigation.id }, - ); - - assert.ok(graphqlResult.publicInvestigation); - const graphqlInvestigation = graphqlResult.publicInvestigation.investigation; - assert.equal(graphqlInvestigation.id, investigation.id); - assert.equal(graphqlInvestigation.origin.provenance, "SERVER_VERIFIED"); - assert.equal(graphqlInvestigation.corroborationCount, 0); - assert.notEqual(graphqlInvestigation.origin.serverVerifiedAt, null); }); -void test("public.getInvestigation returns CLIENT_FALLBACK without corroboration", async () => { - const caller = createCaller(); +void test("publicInvestigation returns CLIENT_FALLBACK without corroboration", async () => { const post = await seedPost({ platform: "LESSWRONG", externalId: "public-investigation-fallback-1", @@ -605,54 +753,14 @@ void test("public.getInvestigation returns CLIENT_FALLBACK without corroboration provenance: "CLIENT_FALLBACK", }); - const result = await caller.public.getInvestigation({ - investigationId: investigation.id, - }); - + const result = await queryPublicInvestigation(investigation.id); assert.ok(result); - assert.equal(result.investigation.id, investigation.id); assert.equal(result.investigation.origin.provenance, "CLIENT_FALLBACK"); assert.equal(result.investigation.corroborationCount, 0); - assert.equal(result.investigation.origin.serverVerifiedAt, undefined); - - const graphqlResult = await queryPublicGraphql<{ - publicInvestigation: { - investigation: { - origin: { - provenance: "SERVER_VERIFIED" | "CLIENT_FALLBACK"; - serverVerifiedAt: string | null; - }; - corroborationCount: number; - }; - } | null; - }>( - ` - query PublicInvestigation($investigationId: ID!) { - publicInvestigation(investigationId: $investigationId) { - investigation { - origin { - provenance - serverVerifiedAt - } - corroborationCount - } - } - } - `, - { investigationId: investigation.id }, - ); - - assert.ok(graphqlResult.publicInvestigation); - assert.equal( - graphqlResult.publicInvestigation.investigation.origin.provenance, - "CLIENT_FALLBACK", - ); - assert.equal(graphqlResult.publicInvestigation.investigation.corroborationCount, 0); - assert.equal(graphqlResult.publicInvestigation.investigation.origin.serverVerifiedAt, null); + assert.equal(result.investigation.origin.serverVerifiedAt, null); }); -void test("public.getInvestigation reports corroborationCount for CLIENT_FALLBACK investigations", async () => { - const caller = createCaller(); +void test("publicInvestigation reports corroborationCount for CLIENT_FALLBACK investigations", async () => { const post = await seedPost({ platform: "LESSWRONG", externalId: "public-investigation-corroborated-1", @@ -668,10 +776,7 @@ void test("public.getInvestigation reports corroborationCount for CLIENT_FALLBAC await seedClaimWithSource(investigation.id, 1); await seedCorroborationCredits(investigation.id, 3); - const result = await caller.public.getInvestigation({ - investigationId: investigation.id, - }); - + const result = await queryPublicInvestigation(investigation.id); assert.ok(result); assert.equal(result.investigation.id, investigation.id); assert.equal(result.investigation.origin.provenance, "CLIENT_FALLBACK"); @@ -679,8 +784,7 @@ void test("public.getInvestigation reports corroborationCount for CLIENT_FALLBAC assert.equal(result.claims.length, 1); }); -void test("public.getPostInvestigations lists all complete investigations for a post", async () => { - const caller = createCaller(); +void test("postInvestigations lists all complete investigations for a post", async () => { const post = await seedPost({ platform: "LESSWRONG", externalId: "public-post-investigations-all-complete-1", @@ -706,44 +810,24 @@ void test("public.getPostInvestigations lists all complete investigations for a }); await seedClaimWithSource(serverVerifiedInvestigation.id, 1); - const result = await caller.public.getPostInvestigations({ - platform: post.platform, - externalId: post.externalId, - }); - - assert.ok(result.post); - assert.equal(result.post.platform, post.platform); - assert.equal(result.post.externalId, post.externalId); - assert.equal(result.investigations.length, 2); - - const byId = new Map( - result.investigations.map((item) => [item.id, item]), - ); - const fallback = byId.get(fallbackInvestigation.id); - assert.ok(fallback); - assert.equal(fallback.origin.provenance, "CLIENT_FALLBACK"); - assert.equal(fallback.corroborationCount, 2); - - const serverVerified = byId.get(serverVerifiedInvestigation.id); - assert.ok(serverVerified); - assert.equal(serverVerified.origin.provenance, "SERVER_VERIFIED"); - assert.equal(serverVerified.claimCount, 1); - - const graphqlResult = await queryPublicGraphql<{ + const result = await queryPublicGraphql<{ postInvestigations: { + post: { platform: Platform; externalId: string } | null; investigations: { id: string; - origin: { - provenance: "SERVER_VERIFIED" | "CLIENT_FALLBACK"; - serverVerifiedAt: string | null; - }; + origin: GraphqlOrigin; corroborationCount: number; + claimCount: number; }[]; }; }>( ` query PostInvestigations($platform: Platform!, $externalId: String!) { postInvestigations(platform: $platform, externalId: $externalId) { + post { + platform + externalId + } investigations { id origin { @@ -751,6 +835,7 @@ void test("public.getPostInvestigations lists all complete investigations for a serverVerifiedAt } corroborationCount + claimCount } } } @@ -761,12 +846,25 @@ void test("public.getPostInvestigations lists all complete investigations for a }, ); - assert.equal(graphqlResult.postInvestigations.investigations.length, 2); -}); + assert.deepEqual(result.postInvestigations.post, { + platform: post.platform, + externalId: post.externalId, + }); + const byId = new Map(result.postInvestigations.investigations.map((item) => [item.id, item])); + assert.equal(byId.size, 2); -void test("public.searchInvestigations filters by query/platform and includes fallback matches", async () => { - const caller = createCaller(); + const fallback = byId.get(fallbackInvestigation.id); + assert.ok(fallback); + assert.equal(fallback.origin.provenance, "CLIENT_FALLBACK"); + assert.equal(fallback.corroborationCount, 2); + + const serverVerified = byId.get(serverVerifiedInvestigation.id); + assert.ok(serverVerified); + assert.equal(serverVerified.origin.provenance, "SERVER_VERIFIED"); + assert.equal(serverVerified.claimCount, 1); +}); +void test("searchInvestigations filters by query/platform and includes fallback matches", async () => { const moonMarker = "graphql-search-marker-astronomy-moon"; const moonPost = await seedPost({ platform: "LESSWRONG", @@ -823,121 +921,33 @@ void test("public.searchInvestigations filters by query/platform and includes fa provenance: "CLIENT_FALLBACK", }); - const queryResult = await caller.public.searchInvestigations({ - query: moonMarker, - limit: 20, - offset: 0, - }); - - const queryIds = new Set(queryResult.investigations.map((item) => item.id)); + const queryResult = await querySearchInvestigations({ query: moonMarker }); + const queryIds = new Set(queryResult.map((item) => item.id)); assert.equal(queryIds.has(moonInvestigation.id), true); assert.equal(queryIds.has(multiClaimMoonInvestigation.id), true); assert.equal(queryIds.has(fallbackMoonInvestigation.id), true); assert.equal( - queryResult.investigations.every((item) => item.platform === "LESSWRONG"), + queryResult.every((item) => item.platform === "LESSWRONG"), true, ); - const minClaimCountResult = await caller.public.searchInvestigations({ + const minClaimCountResult = await querySearchInvestigations({ query: moonMarker, minClaimCount: 2, - limit: 20, - offset: 0, }); - assert.deepEqual( - minClaimCountResult.investigations.map((item) => item.id), + minClaimCountResult.map((item) => item.id), [multiClaimMoonInvestigation.id], ); - const platformResult = await caller.public.searchInvestigations({ - platform: "X", - limit: 20, - offset: 0, - }); - - const platformIds = new Set(platformResult.investigations.map((item) => item.id)); + const platformIds = new Set( + (await querySearchInvestigations({ platform: "X" })).map((item) => item.id), + ); assert.equal(platformIds.has(xInvestigation.id), true); assert.equal(platformIds.has(moonInvestigation.id), false); - - const graphqlResult = await queryPublicGraphql<{ - searchInvestigations: { - investigations: { - id: string; - origin: { - provenance: "SERVER_VERIFIED" | "CLIENT_FALLBACK"; - serverVerifiedAt: string | null; - }; - }[]; - }; - }>( - ` - query SearchInvestigations($query: String!, $limit: Int!, $offset: Int!) { - searchInvestigations(query: $query, limit: $limit, offset: $offset) { - investigations { - id - origin { - provenance - serverVerifiedAt - } - } - } - } - `, - { - query: moonMarker, - limit: 20, - offset: 0, - }, - ); - const graphqlIds = new Set( - graphqlResult.searchInvestigations.investigations.map((item) => item.id), - ); - assert.equal(graphqlIds.has(moonInvestigation.id), true); - assert.equal(graphqlIds.has(multiClaimMoonInvestigation.id), true); - assert.equal(graphqlIds.has(fallbackMoonInvestigation.id), true); - - const minClaimCountGraphqlResult = await queryPublicGraphql<{ - searchInvestigations: { - investigations: { - id: string; - }[]; - }; - }>( - ` - query SearchInvestigationsWithMinClaimCount( - $query: String! - $minClaimCount: Int! - $limit: Int! - $offset: Int! - ) { - searchInvestigations( - query: $query - minClaimCount: $minClaimCount - limit: $limit - offset: $offset - ) { - investigations { - id - } - } - } - `, - { - query: moonMarker, - minClaimCount: 2, - limit: 20, - offset: 0, - }, - ); - assert.deepEqual( - minClaimCountGraphqlResult.searchInvestigations.investigations.map((item) => item.id), - [multiClaimMoonInvestigation.id], - ); }); -void test("public.getMetrics counts all complete investigations and honors filters", async () => { - const caller = createCaller(); +void test("publicMetrics counts all complete investigations and honors filters", async () => { const metricsWindowStart = "2026-02-23T00:00:00.000Z"; const metricsWindowEnd = "2026-02-23T23:59:59.999Z"; @@ -985,56 +995,26 @@ void test("public.getMetrics counts all complete investigations and honors filte }); await seedCorroborationCredits(fallbackInvestigation.id, 1); - const allMetrics = await caller.public.getMetrics({ - windowStart: metricsWindowStart, - windowEnd: metricsWindowEnd, - }); - assert.equal(allMetrics.totalInvestigatedPosts, 3); - assert.equal(allMetrics.investigatedPostsWithFlags, 1); - assert.equal(allMetrics.factCheckIncidence, 1 / 3); - - const xMetrics = await caller.public.getMetrics({ - platform: "X", - windowStart: metricsWindowStart, - windowEnd: metricsWindowEnd, - }); - assert.equal(xMetrics.totalInvestigatedPosts, 2); - assert.equal(xMetrics.investigatedPostsWithFlags, 1); - assert.equal(xMetrics.factCheckIncidence, 0.5); - - const emptyWindowMetrics = await caller.public.getMetrics({ - windowStart: "2026-02-24T00:00:00.000Z", - windowEnd: "2026-02-24T23:59:59.999Z", - }); - assert.equal(emptyWindowMetrics.totalInvestigatedPosts, 0); - assert.equal(emptyWindowMetrics.investigatedPostsWithFlags, 0); - assert.equal(emptyWindowMetrics.factCheckIncidence, 0); - - const graphqlResult = await queryPublicGraphql<{ - publicMetrics: { - totalInvestigatedPosts: number; - investigatedPostsWithFlags: number; - factCheckIncidence: number; - }; - }>( - ` - query PublicMetrics($windowStart: DateTime, $windowEnd: DateTime, $platform: Platform) { - publicMetrics(windowStart: $windowStart, windowEnd: $windowEnd, platform: $platform) { - totalInvestigatedPosts - investigatedPostsWithFlags - factCheckIncidence - } - } - `, - { + assert.deepEqual( + await queryPublicMetrics({ windowStart: metricsWindowStart, windowEnd: metricsWindowEnd }), + { totalInvestigatedPosts: 3, investigatedPostsWithFlags: 1, factCheckIncidence: 1 / 3 }, + ); + assert.deepEqual( + await queryPublicMetrics({ windowStart: metricsWindowStart, windowEnd: metricsWindowEnd, platform: "X", - }, + }), + { totalInvestigatedPosts: 2, investigatedPostsWithFlags: 1, factCheckIncidence: 0.5 }, + ); + // No investigations in the window: incidence is undefined, not 0. + assert.deepEqual( + await queryPublicMetrics({ + windowStart: "2026-02-24T00:00:00.000Z", + windowEnd: "2026-02-24T23:59:59.999Z", + }), + { totalInvestigatedPosts: 0, investigatedPostsWithFlags: 0, factCheckIncidence: null }, ); - assert.equal(graphqlResult.publicMetrics.totalInvestigatedPosts, 2); - assert.equal(graphqlResult.publicMetrics.investigatedPostsWithFlags, 1); - assert.equal(graphqlResult.publicMetrics.factCheckIncidence, 0.5); }); void test("post.validateSettings reports instance api-key acceptance", async () => { @@ -1065,7 +1045,6 @@ void test("createContext authenticates active instance API keys from database", ); assert.equal(context.isAuthenticated, true); - assert.equal(context.canInvestigate, true); }); void test("createContext rejects unknown and revoked instance API keys", async () => { @@ -1098,10 +1077,5 @@ void test("createContext rejects unknown and revoked instance API keys", async ( false, `Expected unauthenticated context for ${rejectedKeyCase.label}`, ); - assert.equal( - context.canInvestigate, - false, - `Expected non-investigating context for ${rejectedKeyCase.label}`, - ); } }); diff --git a/src/typescript/api/test/integration/api-endpoints.record-and-investigate.content-and-versioning.integration.test.ts b/src/typescript/api/test/integration/api-endpoints.record-and-investigate.content-and-versioning.integration.test.ts index 3aace8c..30ba1f1 100644 --- a/src/typescript/api/test/integration/api-endpoints.record-and-investigate.content-and-versioning.integration.test.ts +++ b/src/typescript/api/test/integration/api-endpoints.record-and-investigate.content-and-versioning.integration.test.ts @@ -4,7 +4,6 @@ import { INTEGRATION_LESSWRONG_FIXTURE_KEYS, InvestigatorExecutionError, MINIMUM_SUPPORTED_EXTENSION_VERSION, - OpenAIInvestigator, WORD_COUNT_LIMIT, appRouter, assert, @@ -13,13 +12,13 @@ import { buildHealthResponse, buildLesswrongViewInput, buildSucceededAttemptAudit, + buildSucceededInvestigatorOutput, buildXViewInput, closeQueueUtils, createCaller, createContext, createDeterministicRandom, createMockRequestEvent, - ensureInvestigationQueued, ensurePostVersionForSeed, errorHasOpenErrataCode, getPrisma, @@ -55,6 +54,10 @@ import { withIntegrationPrefix, withMockLesswrongCanonicalHtml, withMockLesswrongFetch, + LESSWRONG_MOCK_SERVER_AUTHOR_NAME, + LESSWRONG_MOCK_SERVER_AUTHOR_SLUG, + LESSWRONG_MOCK_SERVER_SLUG, + LESSWRONG_MOCK_SERVER_TITLE, } from "./api-endpoints.integration.shared.js"; import { createInvestigateNowFuzzRoundScenario } from "./helpers/investigate-now-scenario-dsl.js"; @@ -64,7 +67,6 @@ void [ INTEGRATION_LESSWRONG_FIXTURE_KEYS, InvestigatorExecutionError, MINIMUM_SUPPORTED_EXTENSION_VERSION, - OpenAIInvestigator, WORD_COUNT_LIMIT, appRouter, assert, @@ -73,13 +75,13 @@ void [ buildHealthResponse, buildLesswrongViewInput, buildSucceededAttemptAudit, + buildSucceededInvestigatorOutput, buildXViewInput, closeQueueUtils, createCaller, createContext, createDeterministicRandom, createMockRequestEvent, - ensureInvestigationQueued, ensurePostVersionForSeed, errorHasOpenErrataCode, getPrisma, @@ -289,6 +291,7 @@ void test("post.registerObservedVersion corrects Wikipedia identity to server-ve return new Response( JSON.stringify({ parse: { + title: "OpenErrata", text: "

Server canonical article text.

", pageid: 99999, revid: 67890, @@ -413,6 +416,7 @@ void test("post.registerObservedVersion enriches existing Wikipedia version html return new Response( JSON.stringify({ parse: { + title: "OpenErrata", text: canonicalHtml, pageid: Number(pageId), revid: 888888, @@ -470,7 +474,13 @@ void test("post.registerObservedVersion keeps LessWrong mutable version metadata }); const firstInput = { ...firstInputBase, - observedImageUrls: ["https://images.example.test/lesswrong-old.png"], + observedImageOccurrences: [ + { + originalIndex: 0, + normalizedTextOffset: 0, + sourceUrl: "https://images.example.test/lesswrong.png", + }, + ], metadata: { ...firstInputBase.metadata, slug: `${firstInputBase.externalId}-old-slug`, @@ -493,7 +503,13 @@ void test("post.registerObservedVersion keeps LessWrong mutable version metadata }); const secondInput = { ...secondInputBase, - observedImageUrls: ["https://images.example.test/lesswrong-new.png"], + observedImageOccurrences: [ + { + originalIndex: 0, + normalizedTextOffset: 0, + sourceUrl: "https://images.example.test/lesswrong.png", + }, + ], metadata: { ...secondInputBase.metadata, slug: `${secondInputBase.externalId}-new-slug`, @@ -532,12 +548,17 @@ void test("post.registerObservedVersion keeps LessWrong mutable version metadata }); assert.ok(meta); assert.equal(meta.slug, secondInput.metadata.slug); - assert.equal(meta.title, secondInput.metadata.title); - assert.equal(meta.authorName, secondInput.metadata.authorName); - assert.equal(meta.authorSlug, secondInput.metadata.authorSlug); + // Server-verified posts take title and author from LessWrong, not the client. + assert.equal(meta.title, LESSWRONG_MOCK_SERVER_TITLE); + assert.equal(meta.authorName, LESSWRONG_MOCK_SERVER_AUTHOR_NAME); + assert.equal(meta.authorSlug, LESSWRONG_MOCK_SERVER_AUTHOR_SLUG); assert.deepEqual(meta.tags, secondInput.metadata.tags); assert.equal(meta.publishedAt?.toISOString(), "2026-02-21T00:00:00.000Z"); - assert.deepEqual(meta.imageUrls, secondInput.observedImageUrls); + // Image URLs are derived from the version's image occurrences. + assert.deepEqual( + meta.imageUrls, + secondInput.observedImageOccurrences.map((occurrence) => occurrence.sourceUrl), + ); assert.equal(meta.serverHtmlBlob?.htmlContent, canonicalHtml); // Client HTML remains first-write-wins. assert.equal(meta.clientHtmlBlob?.htmlContent ?? null, firstClientHtml); @@ -555,7 +576,13 @@ void test("post.registerObservedVersion keeps Substack mutable version metadata externalId, url: "https://openerrata-integration.substack.com/p/mutable-version-meta", observedContentText: stableContent, - observedImageUrls: ["https://images.example.test/substack-old.png"], + observedImageOccurrences: [ + { + originalIndex: 0, + normalizedTextOffset: 0, + sourceUrl: "https://images.example.test/substack.png", + }, + ], metadata: { substackPostId: "40001", publicationSubdomain: "openerrata-integration", @@ -579,7 +606,13 @@ void test("post.registerObservedVersion keeps Substack mutable version metadata externalId, url: "https://openerrata-integration.substack.com/p/mutable-version-meta", observedContentText: stableContent, - observedImageUrls: ["https://images.example.test/substack-new.png"], + observedImageOccurrences: [ + { + originalIndex: 0, + normalizedTextOffset: 0, + sourceUrl: "https://images.example.test/substack.png", + }, + ], metadata: { substackPostId: "40001", publicationSubdomain: "openerrata-updated", @@ -615,7 +648,6 @@ void test("post.registerObservedVersion keeps Substack mutable version metadata likeCount: true, commentCount: true, imageUrls: true, - serverHtmlBlob: { select: { htmlContent: true } }, clientHtmlBlob: { select: { htmlContent: true } }, }, }); @@ -629,8 +661,11 @@ void test("post.registerObservedVersion keeps Substack mutable version metadata assert.equal(meta.publishedAt?.toISOString(), "2026-02-11T00:00:00.000Z"); assert.equal(meta.likeCount, secondInput.metadata.likeCount); assert.equal(meta.commentCount, secondInput.metadata.commentCount); - assert.deepEqual(meta.imageUrls, secondInput.observedImageUrls); - assert.equal(meta.serverHtmlBlob?.htmlContent ?? null, null); + // Image URLs are derived from the version's image occurrences. + assert.deepEqual( + meta.imageUrls, + secondInput.observedImageOccurrences.map((occurrence) => occurrence.sourceUrl), + ); // Client HTML remains first-write-wins. assert.equal(meta.clientHtmlBlob?.htmlContent ?? null, firstClientHtml); }); @@ -646,7 +681,13 @@ void test("post.registerObservedVersion updates Wikipedia revision metadata for platform: "WIKIPEDIA" as const, url, observedContentText: normalizeContent("Stable article text."), - observedImageUrls: ["https://images.example.test/wiki-old.png"], + observedImageOccurrences: [ + { + originalIndex: 0, + normalizedTextOffset: 0, + sourceUrl: "https://images.example.test/wiki.png", + }, + ], metadata: { language: "en", title: "OpenErrata_revision_meta_sync", @@ -673,6 +714,7 @@ void test("post.registerObservedVersion updates Wikipedia revision metadata for return new Response( JSON.stringify({ parse: { + title: "OpenErrata", text: canonicalHtml, pageid: Number(pageId), revid: 10001, @@ -698,7 +740,13 @@ void test("post.registerObservedVersion updates Wikipedia revision metadata for platform: "WIKIPEDIA" as const, url, observedContentText: normalizeContent("Stable article text."), - observedImageUrls: ["https://images.example.test/wiki-new.png"], + observedImageOccurrences: [ + { + originalIndex: 0, + normalizedTextOffset: 0, + sourceUrl: "https://images.example.test/wiki.png", + }, + ], metadata: { language: "en", title: "OpenErrata_revision_meta_sync_v2", @@ -723,6 +771,7 @@ void test("post.registerObservedVersion updates Wikipedia revision metadata for return new Response( JSON.stringify({ parse: { + title: "OpenErrata", text: canonicalHtml, pageid: Number(pageId), revid: 10002, @@ -767,7 +816,11 @@ void test("post.registerObservedVersion updates Wikipedia revision metadata for assert.equal(meta.displayTitle, secondInput.metadata.displayTitle); assert.equal(meta.revisionId, secondInput.metadata.revisionId); assert.equal(meta.lastModifiedAt?.toISOString(), "2026-02-26T00:00:00.000Z"); - assert.deepEqual(meta.imageUrls, secondInput.observedImageUrls); + // Image URLs are derived from the version's image occurrences. + assert.deepEqual( + meta.imageUrls, + secondInput.observedImageOccurrences.map((occurrence) => occurrence.sourceUrl), + ); assert.equal(meta.serverHtmlBlob?.htmlContent, canonicalHtml); // Client HTML remains first-write-wins. assert.equal(meta.clientHtmlBlob?.htmlContent ?? null, firstClientHtml); @@ -877,3 +930,97 @@ void test("post.recordViewAndGetStatus deduplicates unique-view credit for repea }); assert.equal(creditCount, 1); }); + +void test("post.registerObservedVersion binds LessWrong URL and author to the server's answer", async () => { + const caller = createCaller(); + const html = "

Identity-bound LessWrong post body.

"; + const baseInput = buildLesswrongViewInput({ + externalId: "identity-binding-lw-1", + htmlContent: html, + }); + const verifiedInput = { + ...baseInput, + metadata: { + ...baseInput.metadata, + authorName: "Client Claimed Author", + authorSlug: withIntegrationPrefix("client-claimed-author"), + }, + }; + + await withMockLesswrongCanonicalHtml(html, () => + caller.post.registerObservedVersion(verifiedInput), + ); + + const loadPost = () => + prisma.post.findUniqueOrThrow({ + where: { + platform_externalId: { platform: "LESSWRONG", externalId: verifiedInput.externalId }, + }, + select: { + url: true, + identityVerifiedAt: true, + author: { select: { platformUserId: true, displayName: true } }, + }, + }); + const verified = await loadPost(); + const serverUrl = `https://www.lesswrong.com/posts/${verifiedInput.externalId}/${LESSWRONG_MOCK_SERVER_SLUG}`; + assert.equal(verified.url, serverUrl); + assert.notEqual(verified.identityVerifiedAt, null); + assert.deepEqual(verified.author, { + platformUserId: LESSWRONG_MOCK_SERVER_AUTHOR_SLUG, + displayName: LESSWRONG_MOCK_SERVER_AUTHOR_NAME, + }); + + // A later registration that falls back to client data (LessWrong unreachable) + // must not overwrite the verified identity. + const originalFetch = globalThis.fetch; + globalThis.fetch = async (fetchInput, fetchInit) => { + const url = + typeof fetchInput === "string" + ? fetchInput + : fetchInput instanceof URL + ? fetchInput.toString() + : fetchInput.url; + if (url === "https://www.lesswrong.com/graphql") { + return new Response("unavailable", { status: 503 }); + } + return originalFetch(fetchInput, fetchInit); + }; + try { + const fallback = await caller.post.registerObservedVersion({ + ...verifiedInput, + url: `https://www.lesswrong.com/posts/${verifiedInput.externalId}/spoofed-slug`, + metadata: { + ...verifiedInput.metadata, + htmlContent: "

Different client-observed body.

", + authorName: "Spoofed Author", + authorSlug: withIntegrationPrefix("spoofed-author"), + }, + }); + assert.equal(fallback.provenance, "CLIENT_FALLBACK"); + } finally { + globalThis.fetch = originalFetch; + } + + assert.deepEqual(await loadPost(), verified); +}); + +void test("post.registerObservedVersion rejects post URLs that are not https on the platform's host", async () => { + const caller = createCaller(); + const base = buildXViewInput({ + externalId: "observed-url-validation-1", + observedContentText: "URL validation guards what the public site links to.", + }); + for (const url of [ + ["javascript", "alert(document.cookie)"].join(":"), + `http://x.com/openerrata/status/${base.externalId}`, + `https://evil.example/openerrata/status/${base.externalId}`, + "https://x.com/openerrata/status/some-other-post", + ]) { + await assert.rejects( + caller.post.registerObservedVersion({ ...base, url }), + /Invalid X post URL/, + url, + ); + } +}); diff --git a/src/typescript/api/test/integration/api-endpoints.record-and-investigate.integration.test.ts b/src/typescript/api/test/integration/api-endpoints.record-and-investigate.integration.test.ts index 82a9051..76f6166 100644 --- a/src/typescript/api/test/integration/api-endpoints.record-and-investigate.integration.test.ts +++ b/src/typescript/api/test/integration/api-endpoints.record-and-investigate.integration.test.ts @@ -4,7 +4,6 @@ import { INTEGRATION_LESSWRONG_FIXTURE_KEYS, InvestigatorExecutionError, MINIMUM_SUPPORTED_EXTENSION_VERSION, - OpenAIInvestigator, WORD_COUNT_LIMIT, appRouter, assert, @@ -12,13 +11,13 @@ import { buildFailedAttemptAudit, buildLesswrongViewInput, buildSucceededAttemptAudit, + buildSucceededInvestigatorOutput, buildXViewInput, closeQueueUtils, createCaller, createContext, createDeterministicRandom, createMockRequestEvent, - ensureInvestigationQueued, ensurePostVersionForSeed, errorHasOpenErrataCode, getPrisma, @@ -63,7 +62,6 @@ void [ INTEGRATION_LESSWRONG_FIXTURE_KEYS, InvestigatorExecutionError, MINIMUM_SUPPORTED_EXTENSION_VERSION, - OpenAIInvestigator, WORD_COUNT_LIMIT, appRouter, assert, @@ -71,13 +69,13 @@ void [ buildFailedAttemptAudit, buildLesswrongViewInput, buildSucceededAttemptAudit, + buildSucceededInvestigatorOutput, buildXViewInput, closeQueueUtils, createCaller, createContext, createDeterministicRandom, createMockRequestEvent, - ensureInvestigationQueued, ensurePostVersionForSeed, errorHasOpenErrataCode, getPrisma, @@ -142,7 +140,7 @@ void test("post.getInvestigation returns complete investigation with claims", as assert.equal(result.checkedAt, "2026-02-19T00:00:00.000Z"); }); -void test("post.getInvestigation returns priorInvestigationResult for update investigations and null for non-updates", async () => { +void test("post.getInvestigation reports the claims carried forward to an investigating version, and none when no claim still applies", async () => { const caller = createCaller(); const post = await seedPost({ platform: "X", @@ -156,7 +154,7 @@ void test("post.getInvestigation returns priorInvestigationResult for update inv contentText: post.contentText, provenance: "SERVER_VERIFIED", }); - await seedClaimWithSource(parent.id, 1); + await seedClaimWithSource(parent.id, 1, { text: "Initial content" }); const updateContentText = normalizeContent( "Initial content for update projection coverage. Edited sentence.", @@ -173,6 +171,7 @@ void test("post.getInvestigation returns priorInvestigationResult for update inv contentDiff: "Diff summary (line context):\n- Removed lines:\nOld\n+ Added lines:\nNew", }); + // A version that no longer contains the passage the earlier claim quoted. const nonUpdateContentText = normalizeContent( "Fresh pending investigation that is not an update.", ); diff --git a/src/typescript/api/test/integration/api-endpoints.record-and-investigate.interim-source-selection.integration.test.ts b/src/typescript/api/test/integration/api-endpoints.record-and-investigate.interim-source-selection.integration.test.ts index 65bfa62..ad00828 100644 --- a/src/typescript/api/test/integration/api-endpoints.record-and-investigate.interim-source-selection.integration.test.ts +++ b/src/typescript/api/test/integration/api-endpoints.record-and-investigate.interim-source-selection.integration.test.ts @@ -4,7 +4,6 @@ import { INTEGRATION_LESSWRONG_FIXTURE_KEYS, InvestigatorExecutionError, MINIMUM_SUPPORTED_EXTENSION_VERSION, - OpenAIInvestigator, WORD_COUNT_LIMIT, appRouter, assert, @@ -12,13 +11,13 @@ import { buildFailedAttemptAudit, buildLesswrongViewInput, buildSucceededAttemptAudit, + buildSucceededInvestigatorOutput, buildXViewInput, closeQueueUtils, createCaller, createContext, createDeterministicRandom, createMockRequestEvent, - ensureInvestigationQueued, ensurePostVersionForSeed, errorHasOpenErrataCode, getPrisma, @@ -56,6 +55,10 @@ import { withMockLesswrongFetch, } from "./api-endpoints.integration.shared.js"; import { createInvestigateNowFuzzRoundScenario } from "./helpers/investigate-now-scenario-dsl.js"; +import { + findCarriedForwardClaims, + prismaInvestigationRepository, +} from "../../src/lib/trpc/routes/post/investigation-queries.js"; void [ EMPTY_IMAGE_OCCURRENCES_HASH, @@ -63,7 +66,6 @@ void [ INTEGRATION_LESSWRONG_FIXTURE_KEYS, InvestigatorExecutionError, MINIMUM_SUPPORTED_EXTENSION_VERSION, - OpenAIInvestigator, WORD_COUNT_LIMIT, appRouter, assert, @@ -71,13 +73,13 @@ void [ buildFailedAttemptAudit, buildLesswrongViewInput, buildSucceededAttemptAudit, + buildSucceededInvestigatorOutput, buildXViewInput, closeQueueUtils, createCaller, createContext, createDeterministicRandom, createMockRequestEvent, - ensureInvestigationQueued, ensurePostVersionForSeed, errorHasOpenErrataCode, getPrisma, @@ -115,7 +117,7 @@ void [ withMockLesswrongFetch, createInvestigateNowFuzzRoundScenario, ]; -void test("post.recordViewAndGetStatus returns interim old claims from latest complete server-verified investigation without queueing", async () => { +void test("post.recordViewAndGetStatus carries forward only the server-verified claims whose text is still on the page, without queueing", async () => { const caller = createCaller(); const externalId = "view-post-update-interim-1"; const previousHtml = "

The moon is made of green cheese.

"; @@ -147,8 +149,11 @@ void test("post.recordViewAndGetStatus returns interim old claims from latest co contentText: previousCanonicalText, provenance: "SERVER_VERIFIED", }); - await seedClaimWithSource(sourceInvestigation.id, 1); - await seedClaimWithSource(sourceInvestigation.id, 2); + const survivingClaim = await seedClaimWithSource(sourceInvestigation.id, 1, { + text: "The moon is made of", + }); + // The edit removed the text this claim corrects, so it no longer applies. + await seedClaimWithSource(sourceInvestigation.id, 2, { text: "made of green cheese" }); const currentInput = buildLesswrongViewInput({ externalId, @@ -163,7 +168,10 @@ void test("post.recordViewAndGetStatus returns interim old claims from latest co assert.notEqual(interimResult, null); assert.ok(interimResult); assert.equal(interimResult.sourceInvestigationId, sourceInvestigation.id); - assert.equal(interimResult.oldClaims.length, 2); + assert.deepEqual( + interimResult.oldClaims.map((claim) => claim.id), + [survivingClaim.id], + ); const currentCanonicalText = lesswrongHtmlToNormalizedText(currentHtml); const currentCanonicalHash = await hashContent(currentCanonicalText); @@ -208,7 +216,7 @@ void test("post.recordViewAndGetStatus returns interim old claims from latest co ); }); -void test("post.recordViewAndGetStatus does not reuse CLIENT_FALLBACK investigations as interim update claims", async () => { +void test("post.recordViewAndGetStatus carries forward CLIENT_FALLBACK investigations as interim claims", async () => { const caller = createCaller(); const externalId = "view-post-update-interim-fallback-source-1"; const previousHtml = "

Venus has one moon.

"; @@ -231,7 +239,9 @@ void test("post.recordViewAndGetStatus does not reuse CLIENT_FALLBACK investigat contentText: previousCanonicalText, provenance: "CLIENT_FALLBACK", }); - await seedClaimWithSource(fallbackOnlySource.id, 11); + const survivingClaim = await seedClaimWithSource(fallbackOnlySource.id, 11, { + text: "Venus has", + }); const currentInput = buildLesswrongViewInput({ externalId, @@ -242,10 +252,15 @@ void test("post.recordViewAndGetStatus does not reuse CLIENT_FALLBACK investigat ); assert.equal(result.investigationState, "NOT_INVESTIGATED"); - assert.equal(result.priorInvestigationResult, null); + assert.ok(result.priorInvestigationResult); + assert.equal(result.priorInvestigationResult.sourceInvestigationId, fallbackOnlySource.id); + assert.deepEqual( + result.priorInvestigationResult.oldClaims.map((claim) => claim.id), + [survivingClaim.id], + ); }); -void test("post.recordViewAndGetStatus reuses latest complete SERVER_VERIFIED interim claims when canonical provenance is CLIENT_FALLBACK", async () => { +void test("post.recordViewAndGetStatus carries forward SERVER_VERIFIED claims to a CLIENT_FALLBACK version", async () => { const caller = createCaller(); const externalId = "view-post-update-interim-client-fallback-canonical-1"; const previousInput = buildXViewInput({ @@ -273,7 +288,7 @@ void test("post.recordViewAndGetStatus reuses latest complete SERVER_VERIFIED in contentText: previousCanonicalText, provenance: "SERVER_VERIFIED", }); - await seedClaimWithSource(sourceInvestigation.id, 41); + await seedClaimWithSource(sourceInvestigation.id, 41, { text: "Mercury has" }); const currentInput = buildXViewInput({ externalId, @@ -287,10 +302,10 @@ void test("post.recordViewAndGetStatus reuses latest complete SERVER_VERIFIED in assert.ok(interimResult); assert.equal(interimResult.sourceInvestigationId, sourceInvestigation.id); assert.equal(interimResult.oldClaims.length, 1); - assert.equal(interimResult.oldClaims[0]?.text, "Claim 41"); + assert.equal(interimResult.oldClaims[0]?.text, "Mercury has"); }); -void test("post.recordViewAndGetStatus uses newest complete SERVER_VERIFIED investigation as interim source", async () => { +void test("post.recordViewAndGetStatus takes interim claims from the newest complete investigation of any provenance", async () => { const caller = createCaller(); const externalId = "view-post-update-interim-newest-source-1"; const baselineHtml = "

Jupiter has 79 moons.

"; @@ -323,7 +338,7 @@ void test("post.recordViewAndGetStatus uses newest complete SERVER_VERIFIED inve provenance: "SERVER_VERIFIED", checkedAt: new Date("2026-02-01T00:00:00.000Z"), }); - await seedClaimWithSource(olderSource.id, 21); + await seedClaimWithSource(olderSource.id, 21, { text: "Jupiter has" }); const newerSourceText = normalizeContent("Jupiter has exactly 92 moons."); const newerSourceHash = await hashContent(newerSourceText); @@ -331,10 +346,10 @@ void test("post.recordViewAndGetStatus uses newest complete SERVER_VERIFIED inve postId: post.id, contentHash: newerSourceHash, contentText: newerSourceText, - provenance: "SERVER_VERIFIED", + provenance: "CLIENT_FALLBACK", checkedAt: new Date("2026-02-20T00:00:00.000Z"), }); - await seedClaimWithSource(newerSource.id, 22); + await seedClaimWithSource(newerSource.id, 22, { text: "Jupiter has" }); const currentInput = buildLesswrongViewInput({ externalId, @@ -350,7 +365,7 @@ void test("post.recordViewAndGetStatus uses newest complete SERVER_VERIFIED inve assert.ok(newestInterimResult); assert.equal(newestInterimResult.sourceInvestigationId, newerSource.id); assert.equal(newestInterimResult.oldClaims.length, 1); - assert.equal(newestInterimResult.oldClaims[0]?.text, "Claim 22"); + assert.equal(newestInterimResult.oldClaims[0]?.text, "Jupiter has"); }); void test("post.recordViewAndGetStatus returns INVESTIGATED for current-version complete result even when an older interim source exists", async () => { @@ -408,3 +423,80 @@ void test("post.recordViewAndGetStatus returns INVESTIGATED for current-version assert.equal(result.claims.length, 1); assert.equal(result.claims[0]?.text, "Claim 32"); }); + +void test("post.recordViewAndGetStatus reports no interim claims when none of the source's claims is still on the page", async () => { + const caller = createCaller(); + const externalId = "view-post-update-interim-none-surviving-1"; + const previousInput = buildXViewInput({ + externalId, + observedContentText: "Neptune has sixteen moons.", + }); + await caller.post.recordViewAndGetStatus(previousInput); + const post = await prisma.post.findUniqueOrThrow({ + where: { + platform_externalId: { + platform: previousInput.platform, + externalId: previousInput.externalId, + }, + }, + select: { id: true }, + }); + const previousText = normalizeContent(previousInput.observedContentText); + const source = await seedCompleteInvestigation({ + postId: post.id, + contentHash: await hashContent(previousText), + contentText: previousText, + provenance: "CLIENT_FALLBACK", + }); + await seedClaimWithSource(source.id, 51, { text: "Neptune has sixteen moons." }); + + const result = await caller.post.recordViewAndGetStatus( + buildXViewInput({ + externalId, + observedContentText: "Neptune's moons are still being counted.", + }), + ); + + assert.equal(result.investigationState, "NOT_INVESTIGATED"); + assert.equal(result.priorInvestigationResult, null); +}); + +void test("findCarriedForwardClaims never uses the requested version's own investigation as its prior", async () => { + const post = await seedPost({ + platform: "X", + externalId: "interim-never-self-1", + url: "https://x.com/openerrata/status/interim-never-self-1", + contentText: "Uranus has 28 moons.", + }); + const olderText = normalizeContent("Uranus has 27 moons."); + const olderSource = await seedCompleteInvestigation({ + postId: post.id, + contentHash: await hashContent(olderText), + contentText: olderText, + provenance: "SERVER_VERIFIED", + checkedAt: new Date("2026-02-01T00:00:00.000Z"), + }); + const olderClaim = await seedClaimWithSource(olderSource.id, 61, { text: "Uranus has" }); + // The requested version's own investigation is the post's newest complete one. + const ownInvestigation = await seedCompleteInvestigation({ + postId: post.id, + contentHash: post.contentHash, + contentText: post.contentText, + provenance: "SERVER_VERIFIED", + checkedAt: new Date("2026-03-01T00:00:00.000Z"), + }); + await seedClaimWithSource(ownInvestigation.id, 62, { text: "Uranus has 28 moons." }); + + const carried = await findCarriedForwardClaims(prismaInvestigationRepository(prisma), { + id: post.postVersionId, + postId: post.id, + contentText: post.contentText, + }); + + assert.ok(carried); + assert.equal(carried.sourceInvestigationId, olderSource.id); + assert.deepEqual( + carried.oldClaims.map((claim) => claim.id), + [olderClaim.id], + ); +}); diff --git a/src/typescript/api/test/integration/api-endpoints.record-view-status.integration.test.ts b/src/typescript/api/test/integration/api-endpoints.record-view-status.integration.test.ts new file mode 100644 index 0000000..c7b23b8 --- /dev/null +++ b/src/typescript/api/test/integration/api-endpoints.record-view-status.integration.test.ts @@ -0,0 +1,207 @@ +import { TRPCError } from "@trpc/server"; +import { + assert, + buildXViewInput, + createCaller, + hashContent, + normalizeContent, + prisma, + seedClaimWithSource, + seedCompleteInvestigation, + seedInvestigation, + seedInvestigationForXViewInput, + seedPostForXViewInput, + test, +} from "./api-endpoints.integration.shared.js"; + +// recordViewAndGetStatus reports the investigation of the viewed post version in +// whatever state it is in (spec §2.6), with its id, so a viewer who did not +// start the investigation can still poll it to completion. + +void test("post.recordViewAndGetStatus reports a PENDING investigation as INVESTIGATING with its id and interim claims", async () => { + const caller = createCaller(); + const input = buildXViewInput({ + externalId: "record-view-status-pending-update-1", + observedContentText: "Original content for the record view status coverage.", + }); + const post = await seedPostForXViewInput(input); + const parent = await seedCompleteInvestigation({ + postId: post.id, + contentHash: post.contentHash, + contentText: post.contentText, + provenance: "SERVER_VERIFIED", + }); + const survivingClaim = await seedClaimWithSource(parent.id, 1, { + text: "Original content for the record view status coverage.", + }); + // The edit removed the passage this claim quoted, so it is not carried forward. + await seedClaimWithSource(parent.id, 2, { text: "A sentence the edit removed." }); + + const updatedText = normalizeContent( + "Original content for the record view status coverage. Edited sentence.", + ); + const pending = await seedInvestigation({ + postId: post.id, + contentHash: await hashContent(updatedText), + contentText: updatedText, + provenance: "SERVER_VERIFIED", + status: "PENDING", + promptLabel: "record-view-status-pending-update", + parentInvestigationId: parent.id, + contentDiff: "Diff summary (line context):\n- Removed lines:\nOld\n+ Added lines:\nNew", + }); + const pendingVersion = await prisma.investigation.findUniqueOrThrow({ + where: { id: pending.id }, + select: { postVersionId: true }, + }); + + const result = await caller.post.recordViewAndGetStatus({ + postVersionId: pendingVersion.postVersionId, + }); + + assert.equal(result.investigationState, "INVESTIGATING"); + assert.equal(result.investigationId, pending.id); + assert.equal(result.status, "PENDING"); + assert.equal(result.provenance, "SERVER_VERIFIED"); + assert.deepEqual(result.pendingClaims, []); + assert.deepEqual(result.confirmedClaims, []); + assert.ok(result.priorInvestigationResult); + assert.equal(result.priorInvestigationResult.sourceInvestigationId, parent.id); + assert.deepEqual( + result.priorInvestigationResult.oldClaims.map((claim) => claim.id), + [survivingClaim.id], + ); +}); + +void test("post.recordViewAndGetStatus reports no interim claims for an INVESTIGATING update when none of the parent's claims is still on the page", async () => { + const caller = createCaller(); + const input = buildXViewInput({ + externalId: "record-view-status-pending-update-none-surviving-1", + observedContentText: "A post whose every sentence the author later rewrote.", + }); + const post = await seedPostForXViewInput(input); + const parent = await seedCompleteInvestigation({ + postId: post.id, + contentHash: post.contentHash, + contentText: post.contentText, + provenance: "SERVER_VERIFIED", + }); + await seedClaimWithSource(parent.id, 1, { text: "every sentence the author later rewrote" }); + + const rewrittenText = normalizeContent("The rewritten post shares no sentence with the old one."); + const pending = await seedInvestigation({ + postId: post.id, + contentHash: await hashContent(rewrittenText), + contentText: rewrittenText, + provenance: "SERVER_VERIFIED", + status: "PENDING", + promptLabel: "record-view-status-pending-update-none-surviving", + parentInvestigationId: parent.id, + contentDiff: "Diff summary (line context):\n- Removed lines:\nOld\n+ Added lines:\nNew", + }); + const pendingVersion = await prisma.investigation.findUniqueOrThrow({ + where: { id: pending.id }, + select: { postVersionId: true }, + }); + + const result = await caller.post.recordViewAndGetStatus({ + postVersionId: pendingVersion.postVersionId, + }); + + assert.equal(result.investigationState, "INVESTIGATING"); + assert.equal(result.priorInvestigationResult, null); +}); + +void test("post.recordViewAndGetStatus reports PROCESSING progress claims from the active lease", async () => { + const caller = createCaller(); + const input = buildXViewInput({ + externalId: "record-view-status-processing-1", + observedContentText: "Processing content for the record view status coverage.", + }); + const seeded = await seedInvestigationForXViewInput({ + viewInput: input, + status: "PROCESSING", + provenance: "CLIENT_FALLBACK", + }); + const progressClaim = { + text: "Processing content", + context: "Processing content for the record view status coverage.", + summary: "Summary of a claim found so far.", + reasoning: "Reasoning for the claim found so far.", + sources: [{ url: "https://example.com/source", title: "Source", snippet: "Snippet" }], + }; + await prisma.investigationLease.update({ + where: { investigationId: seeded.investigationId }, + data: { progressClaims: { pending: [progressClaim], confirmed: [] } }, + }); + + const result = await caller.post.recordViewAndGetStatus({ + postVersionId: seeded.post.postVersionId, + }); + + assert.equal(result.investigationState, "INVESTIGATING"); + assert.equal(result.investigationId, seeded.investigationId); + assert.equal(result.status, "PROCESSING"); + assert.equal(result.provenance, "CLIENT_FALLBACK"); + assert.deepEqual(result.pendingClaims, [progressClaim]); + assert.deepEqual(result.confirmedClaims, []); + assert.equal(result.priorInvestigationResult, null); + + const viewedPost = await prisma.post.findUniqueOrThrow({ + where: { id: seeded.post.id }, + select: { viewCount: true }, + }); + assert.equal(viewedPost.viewCount, 1, "a view of an in-progress post is still recorded"); +}); + +void test("post.recordViewAndGetStatus reports FAILED investigations as FAILED rather than not investigated", async () => { + const caller = createCaller(); + const seeded = await seedInvestigationForXViewInput({ + viewInput: buildXViewInput({ + externalId: "record-view-status-failed-1", + observedContentText: "Failed content for the record view status coverage.", + }), + status: "FAILED", + provenance: "CLIENT_FALLBACK", + }); + + const result = await caller.post.recordViewAndGetStatus({ + postVersionId: seeded.post.postVersionId, + }); + + assert.deepEqual(result, { + investigationState: "FAILED", + investigationId: seeded.investigationId, + provenance: "CLIENT_FALLBACK", + }); +}); + +void test("post.recordViewAndGetStatus reports COMPLETE investigations with their id", async () => { + const caller = createCaller(); + const seeded = await seedInvestigationForXViewInput({ + viewInput: buildXViewInput({ + externalId: "record-view-status-complete-1", + observedContentText: "Complete content for the record view status coverage.", + }), + status: "COMPLETE", + provenance: "CLIENT_FALLBACK", + claimCount: 2, + }); + + const result = await caller.post.recordViewAndGetStatus({ + postVersionId: seeded.post.postVersionId, + }); + + assert.equal(result.investigationState, "INVESTIGATED"); + assert.equal(result.investigationId, seeded.investigationId); + assert.equal(result.claims.length, 2); +}); + +void test("post.recordViewAndGetStatus rejects unknown post versions", async () => { + const caller = createCaller(); + + await assert.rejects( + () => caller.post.recordViewAndGetStatus({ postVersionId: "unknown-post-version-id" }), + (error: unknown) => error instanceof TRPCError && error.code === "BAD_REQUEST", + ); +}); diff --git a/src/typescript/api/test/integration/api-endpoints.version-identity-and-retry.integration.test.ts b/src/typescript/api/test/integration/api-endpoints.version-identity-and-retry.integration.test.ts index e319663..c0b9733 100644 --- a/src/typescript/api/test/integration/api-endpoints.version-identity-and-retry.integration.test.ts +++ b/src/typescript/api/test/integration/api-endpoints.version-identity-and-retry.integration.test.ts @@ -1,11 +1,10 @@ import type { InvestigatorInput } from "../../src/lib/investigators/interface.js"; import { InvestigatorExecutionError, - OpenAIInvestigator, assert, buildFailedAttemptAudit, buildLesswrongViewInput, - buildSucceededAttemptAudit, + buildSucceededInvestigatorOutput, buildXViewInput, createCaller, orchestrateInvestigation, @@ -23,7 +22,6 @@ void test("investigateNow creates a new version and investigation when only imag const firstInput = buildXViewInput({ externalId: "investigate-now-image-only-version-change-1", observedContentText: "The text body is unchanged between views.", - observedImageUrls: [firstImageUrl], observedImageOccurrences: [ { originalIndex: 0, @@ -40,7 +38,6 @@ void test("investigateNow creates a new version and investigation when only imag const secondInput = buildXViewInput({ externalId: "investigate-now-image-only-version-change-1", observedContentText: "The text body is unchanged between views.", - observedImageUrls: [secondImageUrl], observedImageOccurrences: [ { originalIndex: 0, @@ -105,7 +102,6 @@ void test("orchestrateInvestigation retries with identical multimodal snapshot i externalId: "orchestrator-retry-multimodal-snapshot-1", htmlContent: html, }), - observedImageUrls: [imageUrl], observedImageOccurrences: [ { originalIndex: 0, @@ -123,52 +119,38 @@ void test("orchestrateInvestigation retries with identical multimodal snapshot i const capturedInputs: InvestigatorInput[] = []; let invocation = 0; - const originalInvestigateDescriptor = Object.getOwnPropertyDescriptor( - OpenAIInvestigator.prototype, - "investigate", - ); - assert.ok(originalInvestigateDescriptor); - assert.equal(typeof originalInvestigateDescriptor.value, "function"); - OpenAIInvestigator.prototype.investigate = async (input: InvestigatorInput) => { - capturedInputs.push(structuredClone(input)); - invocation += 1; - if (invocation === 1) { - throw new InvestigatorExecutionError( - "simulated transient retry path", - buildFailedAttemptAudit("multimodal-retry-first"), - new Error("simulated network timeout"), - ); - } - return { - result: { claims: [] }, - attemptAudit: buildSucceededAttemptAudit("multimodal-retry-second"), - modelVersion: "test-model-version", - }; - }; + const createInvestigator = () => ({ + investigate: async (input: InvestigatorInput) => { + capturedInputs.push(structuredClone(input)); + invocation += 1; + if (invocation === 1) { + throw new InvestigatorExecutionError( + "simulated transient retry path", + buildFailedAttemptAudit("multimodal-retry-first"), + new Error("simulated network timeout"), + ); + } + return buildSucceededInvestigatorOutput("multimodal-retry-second"); + }, + }); - try { - await orchestrateInvestigation( - queued.investigationId, - { info() {}, warn() {}, error() {} }, - { - workerIdentity: withIntegrationPrefix("worker-retry-snapshot-first"), - }, - ); + await orchestrateInvestigation( + queued.investigationId, + { info() {}, warn() {}, error() {} }, + { + workerIdentity: withIntegrationPrefix("worker-retry-snapshot-first"), + createInvestigator, + }, + ); - await orchestrateInvestigation( - queued.investigationId, - { info() {}, warn() {}, error() {} }, - { - workerIdentity: withIntegrationPrefix("worker-retry-snapshot-second"), - }, - ); - } finally { - Object.defineProperty( - OpenAIInvestigator.prototype, - "investigate", - originalInvestigateDescriptor, - ); - } + await orchestrateInvestigation( + queued.investigationId, + { info() {}, warn() {}, error() {} }, + { + workerIdentity: withIntegrationPrefix("worker-retry-snapshot-second"), + createInvestigator, + }, + ); assert.equal(capturedInputs.length, 2); const [firstAttemptInput, secondAttemptInput] = capturedInputs; @@ -176,5 +158,9 @@ void test("orchestrateInvestigation retries with identical multimodal snapshot i assert.ok(secondAttemptInput); assert.deepEqual(secondAttemptInput, firstAttemptInput); assert.match(firstAttemptInput.contentMarkdown ?? "", /\[IMAGE:0\]/); - assert.equal(firstAttemptInput.imagePlaceholders?.[0]?.matchBy, "ORIGINAL_INDEX"); + // Placeholders carry the image source URL captured at queue time, so the + // investigator matches [IMAGE:N] to the downloaded image by URL, never by position. + assert.deepEqual(firstAttemptInput.imagePlaceholders, [ + { index: 0, matchBy: "SOURCE_URL", sourceUrl: imageUrl }, + ]); }); diff --git a/src/typescript/api/test/integration/helpers/external-api-mocks.ts b/src/typescript/api/test/integration/helpers/external-api-mocks.ts new file mode 100644 index 0000000..8398d9e --- /dev/null +++ b/src/typescript/api/test/integration/helpers/external-api-mocks.ts @@ -0,0 +1,59 @@ +import { isNonNullObject } from "@openerrata/shared"; + +/** + * Stand in for the OpenAI API during integration tests so no test can reach + * it. Keys containing "rejected" get 401 (OpenAI refused the key); every other + * key is accepted. Installed once, underneath any per-test fetch mocks. + */ +export function installMockOpenAiApi(): void { + const originalFetch = globalThis.fetch; + const mockedFetch: typeof fetch = async (input, init) => { + const url = new URL( + typeof input === "string" ? input : input instanceof URL ? input.href : input.url, + ); + if (url.hostname !== "api.openai.com") { + return originalFetch(input, init); + } + const headers = new Headers(init?.headers ?? (input instanceof Request ? input.headers : {})); + const authorization = headers.get("authorization") ?? ""; + if (authorization.includes("rejected")) { + return new Response( + JSON.stringify({ + error: { message: "Incorrect API key provided", type: "invalid_request_error" }, + }), + { status: 401, headers: { "Content-Type": "application/json" } }, + ); + } + const body = url.pathname.includes("/models") + ? { id: "mock-model", object: "model", created: 0, owned_by: "openai" } + : { + id: "resp_mock", + object: "response", + status: "completed", + model: "mock-model", + output: [], + output_text: "pong", + }; + return new Response(JSON.stringify(body), { + status: 200, + headers: { "Content-Type": "application/json" }, + }); + }; + globalThis.fetch = mockedFetch; +} + +/** The post id a mocked LessWrong GraphQL request asks for. */ +export function lesswrongPostIdFromGraphqlBody(body: BodyInit | null | undefined): string { + if (typeof body !== "string") { + throw new Error("Expected LessWrong GraphQL request body to be a JSON string"); + } + const parsed: unknown = JSON.parse(body); + if ( + !isNonNullObject(parsed) || + !isNonNullObject(parsed["variables"]) || + typeof parsed["variables"]["id"] !== "string" + ) { + throw new Error("LessWrong GraphQL request is missing variables.id"); + } + return parsed["variables"]["id"]; +} diff --git a/src/typescript/api/test/integration/helpers/investigate-now-scenario-dsl.ts b/src/typescript/api/test/integration/helpers/investigate-now-scenario-dsl.ts index 63fa925..4564bde 100644 --- a/src/typescript/api/test/integration/helpers/investigate-now-scenario-dsl.ts +++ b/src/typescript/api/test/integration/helpers/investigate-now-scenario-dsl.ts @@ -83,6 +83,10 @@ function expectedStoredStatusForScenario( if (scenario === "COMPLETE") { return "COMPLETE"; } + // FAILED is terminal for a content version (SPEC §3.7); investigateNow returns it as is. + if (scenario === "FAILED") { + return "FAILED"; + } if (scenario === "PROCESSING_ACTIVE") { return "PROCESSING"; } @@ -131,7 +135,7 @@ function createCallerPlan(input: { return { jitterMs: input.jitterMs, caller: createCaller({ - userOpenAiApiKey: `sk-test-fuzz-${input.round.toString()}-${input.index.toString()}`, + userOpenAiApiKey: `sk-test-fuzz-key-${input.round.toString()}-${input.index.toString()}-0123456789`, viewerKey, ipRangeKey, }), @@ -152,7 +156,7 @@ function createCallerPlan(input: { return { jitterMs: input.jitterMs, caller: createCaller({ - userOpenAiApiKey: `sk-test-fuzz-mixed-${input.round.toString()}-${input.index.toString()}`, + userOpenAiApiKey: `sk-test-fuzz-mixed-${input.round.toString()}-${input.index.toString()}-0123456789`, viewerKey, ipRangeKey, }), diff --git a/src/typescript/api/test/integration/integration-env.ts b/src/typescript/api/test/integration/integration-env.ts index de2cb35..4d58b16 100644 --- a/src/typescript/api/test/integration/integration-env.ts +++ b/src/typescript/api/test/integration/integration-env.ts @@ -2,16 +2,18 @@ export const DEFAULT_INTEGRATION_DATABASE_URL = "postgresql://openerrata:openerrata_dev@localhost:5433/openerrata"; const INTEGRATION_ENV_OVERRIDES = { - HMAC_SECRET: "test-hmac-secret", BLOB_STORAGE_PROVIDER: "aws", BLOB_STORAGE_REGION: "us-east-1", BLOB_STORAGE_ENDPOINT: "", BLOB_STORAGE_BUCKET: "test-openerrata-images", BLOB_STORAGE_ACCESS_KEY_ID: "test-blob-access-key", BLOB_STORAGE_SECRET_ACCESS_KEY: "test-blob-secret", - BLOB_STORAGE_PUBLIC_URL_PREFIX: "https://example.test/images", DATABASE_ENCRYPTION_KEY: "integration-test-database-encryption-key", OPENAI_API_KEY: "sk-test-openai-key", + OPENAI_MAX_RESPONSE_TOOL_ROUNDS: "150", + WORKER_CONCURRENCY: "250", + SELECTOR_DAILY_BUDGET: "100", + IP_RANGE_CREDIT_CAP: "10", } as const; interface ApplyIntegrationEnvironmentOptions { diff --git a/src/typescript/api/test/integration/lesswrong-fixtures.ts b/src/typescript/api/test/integration/lesswrong-fixtures.ts index a237ca0..4b51795 100644 --- a/src/typescript/api/test/integration/lesswrong-fixtures.ts +++ b/src/typescript/api/test/integration/lesswrong-fixtures.ts @@ -37,6 +37,8 @@ interface LesswrongFixtureDefinition { fixtureKey: string; externalId: string; postUrl: string; + /** The post's author on LessWrong; the live canary checks identity extraction against it. */ + author: { slug: string; displayName: string }; } const INTEGRATION_LESSWRONG_FIXTURE_DEFINITIONS: Record< @@ -48,6 +50,7 @@ const INTEGRATION_LESSWRONG_FIXTURE_DEFINITIONS: Record< externalId: "ioZxrP7BhS5ArK59w", postUrl: "https://www.lesswrong.com/posts/ioZxrP7BhS5ArK59w/did-claude-3-opus-align-itself-via-gradient-hacking", + author: { slug: "fiora-starlight", displayName: "Fiora Starlight" }, }, }; diff --git a/src/typescript/api/test/integration/live-canary.test.ts b/src/typescript/api/test/integration/live-canary.test.ts index a7d2112..e5ac739 100644 --- a/src/typescript/api/test/integration/live-canary.test.ts +++ b/src/typescript/api/test/integration/live-canary.test.ts @@ -1,12 +1,26 @@ +/** + * Live LessWrong canary: does production extraction still work against the + * real LessWrong API? + * + * It runs the same canonical fetch registerObservedVersion uses and checks + * properties that hold for any healthy extraction of the post, whatever its + * author has since edited: the server verifies it, the text is substantial + * and free of leaked markup, identity (URL, title, author) is extracted, and + * normalization is stable across a re-fetch. It deliberately does not compare + * against the cached fixture's text, which goes stale whenever the post is + * edited (refresh it with `pnpm refresh:fixtures:lesswrong `). + */ import assert from "node:assert/strict"; import { test } from "node:test"; import { - fetchLesswrongHtmlFromLive, - hashText, INTEGRATION_LESSWRONG_FIXTURE_KEYS, readLesswrongFixture, + resolveLesswrongFixtureDefinition, } from "./lesswrong-fixtures.js"; -import { lesswrongHtmlToNormalizedText } from "../../src/lib/services/content-fetcher.js"; +import { + fetchCanonicalContent, + lesswrongHtmlToNormalizedText, +} from "../../src/lib/services/content-fetcher.js"; const fixtureKeysFromEnv = (process.env["LESSWRONG_CANARY_FIXTURE_KEYS"] ?? "") .split(",") @@ -17,20 +31,72 @@ const fixtureKeys = ? fixtureKeysFromEnv : Object.values(INTEGRATION_LESSWRONG_FIXTURE_KEYS); +/** + * Extracted text must keep at least this share of the cached fixture's text. + * Edits move the length a little; extraction failures (truncated API fields, + * a changed content field, an over-eager filter) lose most of it. + */ +const MIN_LIVE_TO_FIXTURE_TEXT_RATIO = 0.5; + +// Structural HTML elements LessWrong posts are built from. Posts may quote +// angle-bracketed text (e.g. "") legitimately, so only real HTML element +// names count as leaked markup. +const LEAKED_TAG_PATTERN = + /<\/?(?:p|div|span|a|em|strong|i|b|u|s|ul|ol|li|h[1-6]|blockquote|img|br|hr|figure|figcaption|table|thead|tbody|tr|td|th|code|pre|sup|sub|section|article)(?:\s[^<>]*)?\/?>/i; +const LEAKED_ENTITY_PATTERN = /&(?:amp|lt|gt|quot|apos|nbsp|#\d+|#x[0-9a-f]+);/i; + +async function fetchLive(externalId: string, url: string) { + const result = await fetchCanonicalContent({ platform: "LESSWRONG", externalId, url }); + if (result.provenance !== "SERVER_VERIFIED") { + assert.fail( + `LessWrong canonical fetch fell back for ${externalId}: ${result.fetchFailureReason}`, + ); + } + return result; +} + void test( - "lesswrong fixture cache matches live pages", + "live LessWrong extraction produces verified, clean, stable content and identity", { skip: fixtureKeys.length === 0 }, async () => { for (const fixtureKey of fixtureKeys) { + const definition = resolveLesswrongFixtureDefinition(fixtureKey); const fixture = await readLesswrongFixture(fixtureKey); - const liveHtml = await fetchLesswrongHtmlFromLive(fixture.externalId); - const liveHash = hashText(lesswrongHtmlToNormalizedText(liveHtml)); - const fixtureHash = hashText(lesswrongHtmlToNormalizedText(fixture.html)); + const fixtureText = lesswrongHtmlToNormalizedText(fixture.html); + + const live = await fetchLive(definition.externalId, definition.postUrl); + const label = `${fixtureKey} (${definition.externalId})`; + + assert.ok( + live.contentText.length >= fixtureText.length * MIN_LIVE_TO_FIXTURE_TEXT_RATIO, + `${label}: extracted ${live.contentText.length.toString()} chars, fixture has ${fixtureText.length.toString()}`, + ); + assert.doesNotMatch(live.contentText, LEAKED_TAG_PATTERN, `${label}: HTML tags leaked`); + assert.doesNotMatch(live.contentText, LEAKED_ENTITY_PATTERN, `${label}: entities leaked`); + assert.equal( + live.contentText, + lesswrongHtmlToNormalizedText(live.sourceHtml), + `${label}: stored text must be the normalization of the stored HTML`, + ); + + const identity = live.canonicalIdentity; + if (identity.platform !== "LESSWRONG") { + assert.fail(`${label}: canonical identity is for ${identity.platform}`); + } + assert.match( + identity.url, + new RegExp(`^https://www\\.lesswrong\\.com/posts/${definition.externalId}/[^/]+$`), + `${label}: canonical URL`, + ); + assert.ok(identity.title.trim().length > 0, `${label}: title extracted`); + assert.doesNotMatch(identity.title, LEAKED_TAG_PATTERN, `${label}: title markup`); + assert.deepEqual(identity.author, definition.author, `${label}: author`); + const refetched = await fetchLive(definition.externalId, definition.postUrl); assert.equal( - liveHash, - fixtureHash, - `Live LessWrong fixture drift detected for ${fixtureKey} (${fixture.externalId}).`, + refetched.contentHash, + live.contentHash, + `${label}: normalized text changed between two consecutive fetches`, ); } }, diff --git a/src/typescript/api/test/integration/trigger-behavior.integration.test.ts b/src/typescript/api/test/integration/trigger-behavior.integration.test.ts index 70170ec..db5e75a 100644 --- a/src/typescript/api/test/integration/trigger-behavior.integration.test.ts +++ b/src/typescript/api/test/integration/trigger-behavior.integration.test.ts @@ -124,6 +124,9 @@ async function createInvestigation( markdownSource: "NONE", markdown: null, markdownRendererVersion: null, + imagePlaceholderSourceUrls: [], + postUrl: "https://example.com/trigger-test-post", + hasVideo: false, }, }); @@ -137,7 +140,9 @@ async function createInvestigation( status, promptId, provider: "OPENAI", - model: "OPENAI_GPT_5", + model: status === "COMPLETE" ? "gpt-6.1-sol" : null, + origin: "INSTANCE_REQUEST", + admittedAt: new Date(), checkedAt: options?.checkedAt ?? (status === "COMPLETE" ? new Date() : null), }, }); @@ -605,3 +610,125 @@ describe("Investigation parent semantics trigger", () => { assert.equal(child.parentInvestigationId, parent.id); }); }); + +// ── Lease row ⇔ PROCESSING ────────────────────────────────────────────── + +function leaseRow(investigationId: string) { + const now = new Date(); + return { + investigationId, + leaseOwner: uniqueId("worker"), + leaseExpiresAt: new Date(now.getTime() + 60_000), + startedAt: now, + heartbeatAt: now, + }; +} + +describe("Investigation lease invariant triggers", () => { + test("PENDING → PROCESSING with a lease in one transaction succeeds", async () => { + const post = await createTestPost("X"); + const pv = await createPostVersion(post.id); + const investigation = await createInvestigation(pv.id, { status: "PENDING" }); + + await prisma.$transaction(async (tx) => { + await tx.investigation.update({ + where: { id: investigation.id }, + data: { status: "PROCESSING" }, + }); + await tx.investigationLease.create({ data: leaseRow(investigation.id) }); + }); + + const stored = await prisma.investigation.findUniqueOrThrow({ + where: { id: investigation.id }, + select: { status: true, lease: { select: { investigationId: true } } }, + }); + assert.equal(stored.status, "PROCESSING"); + assert.ok(stored.lease); + }); + + test("PROCESSING without a lease is rejected at commit", async () => { + const post = await createTestPost("X"); + const pv = await createPostVersion(post.id); + const investigation = await createInvestigation(pv.id, { status: "PENDING" }); + + await assert.rejects( + prisma.investigation.update({ + where: { id: investigation.id }, + data: { status: "PROCESSING" }, + }), + /lease row must exist iff status = PROCESSING/, + ); + }); + + test("a lease on a non-PROCESSING investigation is rejected at commit", async () => { + const post = await createTestPost("X"); + const pv = await createPostVersion(post.id); + const investigation = await createInvestigation(pv.id, { status: "PENDING" }); + + await assert.rejects( + prisma.investigationLease.create({ data: leaseRow(investigation.id) }), + /lease row must exist iff status = PROCESSING/, + ); + }); + + test("leaving PROCESSING without deleting the lease is rejected", async () => { + const post = await createTestPost("X"); + const pv = await createPostVersion(post.id); + const investigation = await createInvestigation(pv.id, { status: "PENDING" }); + await prisma.$transaction(async (tx) => { + await tx.investigation.update({ + where: { id: investigation.id }, + data: { status: "PROCESSING" }, + }); + await tx.investigationLease.create({ data: leaseRow(investigation.id) }); + }); + + await assert.rejects( + prisma.investigation.update({ + where: { id: investigation.id }, + data: { status: "PENDING" }, + }), + /lease row must exist iff status = PROCESSING/, + ); + }); +}); + +// ── Post identity latch ───────────────────────────────────────────────── + +describe("Post.identityVerifiedAt latch trigger", () => { + test("null → timestamp and timestamp → later timestamp succeed", async () => { + const post = await createTestPost("LESSWRONG"); + await prisma.post.update({ + where: { id: post.id }, + data: { identityVerifiedAt: new Date("2026-01-01T00:00:00.000Z") }, + }); + await prisma.post.update({ + where: { id: post.id }, + data: { identityVerifiedAt: new Date("2026-02-01T00:00:00.000Z") }, + }); + }); + + test("timestamp → null is rejected", async () => { + const post = await createTestPost("LESSWRONG"); + await prisma.post.update({ + where: { id: post.id }, + data: { identityVerifiedAt: new Date() }, + }); + await assert.rejects( + prisma.post.update({ where: { id: post.id }, data: { identityVerifiedAt: null } }), + /identityVerifiedAt cannot be cleared/, + ); + }); +}); + +// ── Substack has no server verification ───────────────────────────────── + +describe("Server-verified HTML snapshot trigger", () => { + test("a server-verified Substack version is rejected", async () => { + const post = await createTestPost("SUBSTACK"); + await assert.rejects( + createPostVersion(post.id, { serverVerifiedAt: new Date() }), + /Substack posts have no server-side verification/, + ); + }); +}); diff --git a/src/typescript/api/test/unit/attempt-audit.test.ts b/src/typescript/api/test/unit/attempt-audit.test.ts deleted file mode 100644 index e419962..0000000 --- a/src/typescript/api/test/unit/attempt-audit.test.ts +++ /dev/null @@ -1,188 +0,0 @@ -import assert from "node:assert/strict"; -import { test } from "node:test"; -import type { - InvestigatorAttemptAudit, - InvestigatorResponseAudit, -} from "../../src/lib/investigators/interface.js"; - -/** - * Invariants under test: - * - * 1. **Outcome is derived, not declared**: persistAttemptAudit derives - * "SUCCEEDED"/"FAILED" from the audit's discriminated union (error !== null - * → FAILED). There is no separate `outcome` parameter. This makes the - * inconsistent state (outcome=SUCCEEDED + error present) unrepresentable. - * - * 2. **parseInvestigatorAttemptAudit rejects untrusted payloads at the - * boundary**: The audit enters as `unknown` from the investigator. Invalid - * shapes (non-objects, malformed timestamps, invalid pairings) must be - * caught here, not at the DB layer. These are runtime validation checks on - * data we don't control (OpenAI response shapes). - * - * 3. **Paired-nullability superRefine guards**: outputItem's - * providerItemId/itemStatus and toolCall's providerToolCallId/status must - * be both-present or both-null. These are runtime refinements that the - * TypeScript type system can't express (both fields are `string | null`). - */ - -import { - investigatorOutputItemAuditSchema, - investigatorToolCallAuditSchema, - parseInvestigatorAttemptAudit, -} from "../../src/lib/investigators/interface.js"; - -// ── Test fixtures ──────────────────────────────────────────────────────────── - -const NOW = new Date().toISOString(); - -function makeMinimalResponse(): InvestigatorResponseAudit { - return { - responseId: "resp_test_1", - responseStatus: "completed", - responseModelVersion: "gpt-4o-2025-01-01", - responseOutputText: '{"claims":[]}', - outputItems: [ - { - outputIndex: 0, - providerItemId: "item_001", - itemType: "message", - itemStatus: "completed", - }, - ], - outputTextParts: [ - { - outputIndex: 0, - partIndex: 0, - partType: "output_text", - text: '{"claims":[]}', - }, - ], - outputTextAnnotations: [], - reasoningSummaries: [], - toolCalls: [], - usage: { - inputTokens: 100, - outputTokens: 50, - totalTokens: 150, - cachedInputTokens: null, - reasoningOutputTokens: null, - }, - }; -} - -function makeSucceededAudit(): InvestigatorAttemptAudit { - return { - startedAt: NOW, - completedAt: NOW, - requestModel: "gpt-4o", - requestInstructions: "system prompt", - requestInput: "user prompt", - requestReasoningEffort: "medium", - requestReasoningSummary: "auto", - requestedTools: [ - { - requestOrder: 0, - toolType: "web_search_preview", - rawDefinition: { type: "web_search_preview" }, - }, - ], - response: makeMinimalResponse(), - error: null, - }; -} - -// ── parseInvestigatorAttemptAudit: boundary validation ──────────────────────── -// -// The audit payload comes from the investigator as untyped data. These tests -// verify the runtime boundary rejects shapes that would cause DB errors or -// corrupt audit trails. - -test("parseInvestigatorAttemptAudit rejects non-object input", () => { - assert.throws(() => parseInvestigatorAttemptAudit("not an object")); - assert.throws(() => parseInvestigatorAttemptAudit(null)); - assert.throws(() => parseInvestigatorAttemptAudit(42)); -}); - -test("parseInvestigatorAttemptAudit rejects invalid ISO timestamps", () => { - assert.throws(() => - parseInvestigatorAttemptAudit({ - ...makeSucceededAudit(), - startedAt: "not-a-timestamp", - }), - ); -}); - -test("parseInvestigatorAttemptAudit rejects error=null + response=null (impossible audit state)", () => { - // The discriminated union has two branches: - // succeeded: error=null, response=ResponseAudit (non-null) - // failed: error=ErrorAudit (non-null), response=ResponseAudit|null - // error=null + response=null fits neither branch. - assert.throws(() => - parseInvestigatorAttemptAudit({ - ...makeSucceededAudit(), - response: null, - error: null, - }), - ); -}); - -// ── Paired-nullability: outputItem providerItemId ↔ itemStatus ─────────────── -// -// OpenAI sometimes returns output items without an id (e.g., intermediate -// streaming items). The superRefine guard ensures we never store a half- -// identified item (id without status or vice versa), which would break -// the audit query joins. TypeScript's `string | null` can't express this -// pairing constraint. - -test("outputItemAudit rejects providerItemId without itemStatus", () => { - const result = investigatorOutputItemAuditSchema.safeParse({ - outputIndex: 0, - providerItemId: "item_001", - itemType: "message", - itemStatus: null, - }); - assert.equal(result.success, false); -}); - -test("outputItemAudit rejects itemStatus without providerItemId", () => { - const result = investigatorOutputItemAuditSchema.safeParse({ - outputIndex: 0, - providerItemId: null, - itemType: "message", - itemStatus: "completed", - }); - assert.equal(result.success, false); -}); - -// ── Paired-nullability: toolCall providerToolCallId ↔ status ───────────────── -// -// Same invariant for tool calls. A tool call with an id but no status (or vice -// versa) indicates a parsing bug in the audit extraction layer. - -test("toolCallAudit rejects providerToolCallId without status", () => { - const result = investigatorToolCallAuditSchema.safeParse({ - outputIndex: 0, - providerToolCallId: "tc_001", - toolType: "web_search_preview", - status: null, - rawPayload: {}, - capturedAt: NOW, - providerStartedAt: null, - providerCompletedAt: null, - }); - assert.equal(result.success, false); -}); - -test("toolCallAudit rejects status without providerToolCallId", () => { - const result = investigatorToolCallAuditSchema.safeParse({ - outputIndex: 0, - providerToolCallId: null, - toolType: "web_search_preview", - status: "completed", - rawPayload: {}, - capturedAt: NOW, - providerStartedAt: null, - providerCompletedAt: null, - }); - assert.equal(result.success, false); -}); diff --git a/src/typescript/api/test/unit/canonical-resolution.test.ts b/src/typescript/api/test/unit/canonical-resolution.test.ts index 8f161ba..7aac4a5 100644 --- a/src/typescript/api/test/unit/canonical-resolution.test.ts +++ b/src/typescript/api/test/unit/canonical-resolution.test.ts @@ -6,7 +6,17 @@ import { resolveCanonicalContentVersion, type ServerVerifiedContentMismatch, } from "../../src/lib/services/canonical-resolution.js"; -import type { CanonicalFetchInput } from "../../src/lib/services/content-fetcher.js"; +import type { + CanonicalFetchInput, + CanonicalIdentity, +} from "../../src/lib/services/content-fetcher.js"; + +const SERVER_IDENTITY: CanonicalIdentity = { + platform: "LESSWRONG", + url: "https://www.lesswrong.com/posts/unit-test-post-lw-1/openerrata", + title: "OpenErrata", + author: null, +}; function buildXViewInput(observedContentText: string): Extract { return viewPostInputSchema.parse({ @@ -92,7 +102,7 @@ test("resolveCanonicalContentVersion returns server-verified canonical content", contentText: "Server canonical text", contentHash: "server-hash", sourceHtml: "

Server canonical text

", - canonicalIdentity: null, + canonicalIdentity: SERVER_IDENTITY, }; }, }); @@ -107,7 +117,7 @@ test("resolveCanonicalContentVersion returns server-verified canonical content", contentText: "Server canonical text", contentHash: "server-hash", sourceHtml: "

Server canonical text

", - canonicalIdentity: null, + canonicalIdentity: SERVER_IDENTITY, }); }); @@ -130,7 +140,7 @@ test("resolveCanonicalContentVersion uses server content even when client hash d contentText: "Server canonical text", contentHash: "server-hash", sourceHtml: "

Server canonical text

", - canonicalIdentity: null, + canonicalIdentity: SERVER_IDENTITY, }), }); @@ -139,7 +149,7 @@ test("resolveCanonicalContentVersion uses server content even when client hash d contentText: "Server canonical text", contentHash: "server-hash", sourceHtml: "

Server canonical text

", - canonicalIdentity: null, + canonicalIdentity: SERVER_IDENTITY, }); }); @@ -167,7 +177,7 @@ test("resolveCanonicalContentVersion reports mismatches for all server-verified contentText: "Server canonical text", contentHash: "server-hash", sourceHtml: "

Server canonical text

", - canonicalIdentity: null, + canonicalIdentity: SERVER_IDENTITY, }), onServerVerifiedContentMismatch: (mismatch) => { capturedMismatch = mismatch; @@ -286,7 +296,7 @@ test("resolveCanonicalContentVersion does not call onClientFallback on successfu contentText: "Observed text", contentHash: "observed-hash", sourceHtml: "

Observed text

", - canonicalIdentity: null, + canonicalIdentity: SERVER_IDENTITY, }), onClientFallback: () => { callbackCalled = true; diff --git a/src/typescript/api/test/unit/content-fetcher.test.ts b/src/typescript/api/test/unit/content-fetcher.test.ts index 622d62c..b4631ab 100644 --- a/src/typescript/api/test/unit/content-fetcher.test.ts +++ b/src/typescript/api/test/unit/content-fetcher.test.ts @@ -1,7 +1,7 @@ import assert from "node:assert/strict"; import { test } from "node:test"; import { - CONTENT_BLOCK_SEPARATOR_TAGS, + WORD_SEPARATOR_TAGS, WIKIPEDIA_EXCLUDED_SECTION_TITLES, hashContent, } from "@openerrata/shared"; @@ -23,6 +23,16 @@ test("lesswrongHtmlToNormalizedText preserves malformed literal less-than text", assert.equal(lesswrongHtmlToNormalizedText(html), "1 < 2"); }); +test("lesswrongHtmlToNormalizedText separates the words either side of a line break", () => { + const html = + "

See steg-reasoning-is-hard.
This work extends it.

A stanza
in two lines

"; + + assert.equal( + lesswrongHtmlToNormalizedText(html), + "See steg-reasoning-is-hard. This work extends it. A stanza in two lines", + ); +}); + test("lesswrongHtmlToNormalizedText removes HTML comments", () => { const html = "

Hello

world

"; @@ -102,6 +112,7 @@ function makeWikipediaParseResponse(): Response { return new Response( JSON.stringify({ parse: { + title: "OpenErrata", text: "

Server article text.

", pageid: 99999, revid: 67890, @@ -227,6 +238,7 @@ test("fetchCanonicalContent returns SERVER_VERIFIED Wikipedia content and canoni new Response( JSON.stringify({ parse: { + title: "Who Framed Roger Rabbit?", text: "

Server article text.

", pageid: 99999, revid: 67890, @@ -259,6 +271,7 @@ test("fetchCanonicalContent returns SERVER_VERIFIED Wikipedia content and canoni sourceHtml: "

Server article text.

", canonicalIdentity: { platform: "WIKIPEDIA", + url: "https://en.wikipedia.org/wiki/Who_Framed_Roger_Rabbit%3F", language: "en", pageId: "99999", revisionId: "67890", @@ -269,6 +282,97 @@ test("fetchCanonicalContent returns SERVER_VERIFIED Wikipedia content and canoni } }); +function makeLesswrongPostResponse(result: Record): Response { + return new Response(JSON.stringify({ data: { post: { result } } }), { + status: 200, + headers: { "content-type": "application/json" }, + }); +} + +const LESSWRONG_FETCH_INPUT = { + platform: "LESSWRONG" as const, + url: "https://www.lesswrong.com/posts/abc123/client-supplied-slug", + externalId: "abc123", +}; + +test("fetchCanonicalContent takes LessWrong URL, title and author from the server response", async () => { + const originalFetch = globalThis.fetch; + try { + globalThis.fetch = async (): Promise => + makeLesswrongPostResponse({ + _id: "abc123", + slug: "server-slug", + title: "Server Title", + contents: { html: "

Server body.

" }, + user: { slug: "server-author", displayName: "Server Author" }, + }); + + const result = await fetchCanonicalContent(LESSWRONG_FETCH_INPUT); + assert.equal(result.provenance, "SERVER_VERIFIED"); + assert.equal(result.contentText, "Server body."); + assert.deepEqual(result.canonicalIdentity, { + platform: "LESSWRONG", + url: "https://www.lesswrong.com/posts/abc123/server-slug", + title: "Server Title", + author: { slug: "server-author", displayName: "Server Author" }, + }); + } finally { + globalThis.fetch = originalFetch; + } +}); + +test("fetchCanonicalContent falls back when LessWrong returns a different post", async () => { + const originalFetch = globalThis.fetch; + try { + globalThis.fetch = async (): Promise => + makeLesswrongPostResponse({ + _id: "someOtherPost", + slug: "s", + title: "T", + contents: { html: "

Body.

" }, + user: null, + }); + + const result = await fetchCanonicalContent(LESSWRONG_FETCH_INPUT); + assert.equal(result.provenance, "CLIENT_FALLBACK"); + } finally { + globalThis.fetch = originalFetch; + } +}); + +test("fetchCanonicalContent falls back instead of reading an oversized canonical response", async () => { + const originalFetch = globalThis.fetch; + try { + globalThis.fetch = async (): Promise => + new Response("{}", { + status: 200, + headers: { "content-type": "application/json", "content-length": "52428800" }, + }); + + const result = await fetchCanonicalContent(WIKIPEDIA_FETCH_INPUT); + assert.equal(result.provenance, "CLIENT_FALLBACK"); + } finally { + globalThis.fetch = originalFetch; + } +}); + +test("fetchCanonicalContent passes an abort deadline to every fetch attempt", async () => { + const originalFetch = globalThis.fetch; + const signals: (AbortSignal | null | undefined)[] = []; + try { + globalThis.fetch = async (_input, init): Promise => { + signals.push(init?.signal); + return makeWikipediaParseResponse(); + }; + + await fetchCanonicalContent(WIKIPEDIA_FETCH_INPUT); + assert.equal(signals.length, 1); + assert.ok(signals[0] instanceof AbortSignal); + } finally { + globalThis.fetch = originalFetch; + } +}); + test("wikipediaHtmlToNormalizedText excludes references section and citation superscripts", () => { const html = `
@@ -391,6 +495,33 @@ test("wikipediaHtmlToNormalizedText handles Parsoid h3 sub-section exclusion und assert.equal(wikipediaHtmlToNormalizedText(html), "Main content."); }); +test("wikipediaHtmlToNormalizedText excludes the same sections when each sits in its own
element", () => { + // Parsoid read views (fr.wikipedia, 2026-10) nest every section, with its + // subsections, in a
; the Parse API returns the same article flat. + const section = (level: 2 | 3, title: string, body: string, nested = ""): string => + `
${title}
${body}${nested}
`; + const heading = (level: 2 | 3, title: string): string => + `
${title}
`; + const sectioned = `
+

Lead.

+ ${section(2, "Biographie", "

Life.

")} + ${section(2, "Notes et références", "

Cited source.

")} + ${section(2, "Voir aussi", "", section(3, "Bibliographie", "

A book.

") + section(3, "Articles connexes", "

Radium

"))} +
`; + const flat = `
+

Lead.

+ ${heading(2, "Biographie")}

Life.

+ ${heading(2, "Notes et références")}

Cited source.

+ ${heading(2, "Voir aussi")} + ${heading(3, "Bibliographie")}

A book.

+ ${heading(3, "Articles connexes")}

Radium

+
`; + + const expected = "Lead. Biographie Life. Voir aussi Articles connexes Radium"; + assert.equal(wikipediaHtmlToNormalizedText(sectioned), expected); + assert.equal(wikipediaHtmlToNormalizedText(flat), expected); +}); + test("wikipediaHtmlToNormalizedText treats div.mw-heading as a section boundary only with a direct heading child", () => { const html = `
@@ -409,25 +540,28 @@ test("wikipediaHtmlToNormalizedText treats div.mw-heading as a section boundary assert.equal(wikipediaHtmlToNormalizedText(html), "Lead paragraph. History Kept paragraph."); }); -// ── Block separator exhaustiveness ──────────────────────────────────────────── -// Every tag in CONTENT_BLOCK_SEPARATOR_TAGS must produce word-separated output +// ── Word separator exhaustiveness ───────────────────────────────────────────── +// Every tag in WORD_SEPARATOR_TAGS must produce word-separated output // when adjacent elements have no whitespace text node between them. Both // extractors (lesswrong / wikipedia) are tested because they use different // internal traversal implementations that must stay in sync. // // Table-related tags (, , ) cannot contain text as direct children // in valid HTML — parse5 foster-parents text nodes outside the element when no -// table context exists, which defeats the block-separator check. The map below -// provides minimal valid table structures that exercise each tag as a separator. -const TABLE_BLOCK_SEPARATOR_HTML: Record = { +// table context exists, which defeats the separator check; void elements +// (
,
) hold no text and separate the text around them. The map below +// provides minimal valid structures that exercise each such tag as a separator. +const SEPARATOR_HTML_OVERRIDES: Record = { tr: "
Word1
Word2
", td: "
Word1Word2
", th: "
Word1Word2
", + br: "

Word1
Word2

", + hr: "Word1
Word2", }; -test("lesswrongHtmlToNormalizedText separates text across adjacent CONTENT_BLOCK_SEPARATOR_TAGS elements", () => { - for (const tag of CONTENT_BLOCK_SEPARATOR_TAGS) { - const html = TABLE_BLOCK_SEPARATOR_HTML[tag] ?? `<${tag}>Word1<${tag}>Word2`; +test("lesswrongHtmlToNormalizedText separates text across adjacent WORD_SEPARATOR_TAGS elements", () => { + for (const tag of WORD_SEPARATOR_TAGS) { + const html = SEPARATOR_HTML_OVERRIDES[tag] ?? `<${tag}>Word1<${tag}>Word2`; assert.equal( lesswrongHtmlToNormalizedText(html), "Word1 Word2", @@ -436,9 +570,9 @@ test("lesswrongHtmlToNormalizedText separates text across adjacent CONTENT_BLOCK } }); -test("wikipediaHtmlToNormalizedText separates text across adjacent CONTENT_BLOCK_SEPARATOR_TAGS elements", () => { - for (const tag of CONTENT_BLOCK_SEPARATOR_TAGS) { - const inner = TABLE_BLOCK_SEPARATOR_HTML[tag] ?? `<${tag}>Word1<${tag}>Word2`; +test("wikipediaHtmlToNormalizedText separates text across adjacent WORD_SEPARATOR_TAGS elements", () => { + for (const tag of WORD_SEPARATOR_TAGS) { + const inner = SEPARATOR_HTML_OVERRIDES[tag] ?? `<${tag}>Word1<${tag}>Word2`; assert.equal( wikipediaHtmlToNormalizedText(`
${inner}
`), "Word1 Word2", diff --git a/src/typescript/api/test/unit/env-config.test.ts b/src/typescript/api/test/unit/env-config.test.ts index abf36d6..c5016aa 100644 --- a/src/typescript/api/test/unit/env-config.test.ts +++ b/src/typescript/api/test/unit/env-config.test.ts @@ -5,14 +5,16 @@ import { parseEnvironmentValues } from "../../src/lib/config/env.js"; function createBaseEnvironment(overrides: Partial = {}): NodeJS.ProcessEnv { return { DATABASE_URL: "postgresql://openerrata:openerrata_dev@localhost:5433/openerrata", - HMAC_SECRET: "test-hmac-secret", BLOB_STORAGE_PROVIDER: "aws", BLOB_STORAGE_REGION: "us-west-2", BLOB_STORAGE_BUCKET: "test-openerrata-images", BLOB_STORAGE_ACCESS_KEY_ID: "test-blob-access-key", BLOB_STORAGE_SECRET_ACCESS_KEY: "test-blob-secret", - BLOB_STORAGE_PUBLIC_URL_PREFIX: "https://example.test/images", DATABASE_ENCRYPTION_KEY: "integration-test-database-encryption-key", + OPENAI_MAX_RESPONSE_TOOL_ROUNDS: "150", + WORKER_CONCURRENCY: "250", + SELECTOR_DAILY_BUDGET: "100", + IP_RANGE_CREDIT_CAP: "10", ...overrides, }; } @@ -22,8 +24,6 @@ test("accepts aws blob storage configuration with an explicit region", () => { assert.equal(environment.BLOB_STORAGE_PROVIDER, "aws"); assert.equal(environment.BLOB_STORAGE_REGION, "us-west-2"); assert.equal(environment.BLOB_STORAGE_ENDPOINT, undefined); - assert.equal(environment.WORKER_CONCURRENCY, 250); - assert.equal(environment.OPENAI_MODEL_ID, "gpt-5.4"); }); test("accepts a custom worker concurrency", () => { @@ -35,6 +35,20 @@ test("accepts a custom worker concurrency", () => { assert.equal(environment.WORKER_CONCURRENCY, 32); }); +test("requires the numeric tuning variables the chart always sets", () => { + for (const name of [ + "OPENAI_MAX_RESPONSE_TOOL_ROUNDS", + "WORKER_CONCURRENCY", + "SELECTOR_DAILY_BUDGET", + "IP_RANGE_CREDIT_CAP", + ]) { + assert.throws( + () => parseEnvironmentValues(createBaseEnvironment({ [name]: undefined })), + new RegExp(name), + ); + } +}); + test("rejects aws blob storage configuration when endpoint is set", () => { assert.throws( () => @@ -85,3 +99,11 @@ test("accepts s3-compatible blob storage with endpoint and region", () => { assert.equal(environment.BLOB_STORAGE_REGION, "auto"); assert.equal(environment.BLOB_STORAGE_ENDPOINT, "https://example.r2.cloudflarestorage.com"); }); + +test("SELECTOR_DAILY_BUDGET is read as a positive integer", () => { + assert.equal( + parseEnvironmentValues(createBaseEnvironment({ SELECTOR_DAILY_BUDGET: "25" })) + .SELECTOR_DAILY_BUDGET, + 25, + ); +}); diff --git a/src/typescript/api/test/unit/env-example.test.ts b/src/typescript/api/test/unit/env-example.test.ts new file mode 100644 index 0000000..22490d8 --- /dev/null +++ b/src/typescript/api/test/unit/env-example.test.ts @@ -0,0 +1,12 @@ +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import { test } from "node:test"; +import { parse as parseDotenv } from "dotenv"; +import { parseEnvironmentValues } from "../../src/lib/config/env.js"; + +// README tells developers to copy api/.env.example to api/.env, so the example +// must always be a complete, valid configuration. +test("api/.env.example satisfies the API environment schema", () => { + const example = parseDotenv(readFileSync(new URL("../../.env.example", import.meta.url))); + assert.doesNotThrow(() => parseEnvironmentValues(example)); +}); diff --git a/src/typescript/api/test/unit/frontend-graphql-contract.test.ts b/src/typescript/api/test/unit/frontend-graphql-contract.test.ts new file mode 100644 index 0000000..0a87bb3 --- /dev/null +++ b/src/typescript/api/test/unit/frontend-graphql-contract.test.ts @@ -0,0 +1,134 @@ +/** + * Consumer contract for the public frontend: every GraphQL document the site + * sends must validate against this API's schema, and the API's responses to + * it must parse with the shared output schemas the site parses them with. + */ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { graphql, parse, validate } from "graphql"; +import { + publicInvestigationQuery, + searchInvestigationsQuery, +} from "../../../frontend/src/lib/public-queries.js"; +import { createPublicGraphqlSchema } from "../../src/lib/graphql/public-schema.js"; + +const checkedAt = new Date("2026-09-30T12:00:00.000Z"); + +/** In-process execution keeps scalars as JS values (e.g. Date); the HTTP handler sends JSON. */ +function overTheWire(data: unknown): unknown { + return JSON.parse(JSON.stringify(data)); +} + +const schema = createPublicGraphqlSchema({ + getPublicInvestigationById: async (_prisma, investigationId) => + investigationId === "missing" + ? null + : { + investigation: { + id: investigationId, + origin: { provenance: "SERVER_VERIFIED", serverVerifiedAt: checkedAt }, + corroborationCount: 1, + checkedAt, + promptVersion: "v1.0.0", + provider: "OPENAI", + model: "gpt-6.1-sol", + }, + post: { + platform: "WIKIPEDIA", + externalId: "en:42", + url: "https://en.wikipedia.org/wiki/Example", + }, + claims: [ + { + id: "claim-1", + text: "Claim text", + context: "Context around the claim text", + summary: "Correction summary", + reasoning: "Reasoning", + sources: [{ url: "https://example.org/a", title: "A", snippet: "Snippet" }], + }, + ], + }, + getPublicPostInvestigations: async () => ({ post: null, investigations: [] }), + searchPublicInvestigations: async () => ({ + investigations: [ + { + id: "inv-1", + contentHash: "b".repeat(64), + checkedAt, + platform: "X", + externalId: "1234", + url: "https://x.com/someone/status/1234", + origin: { provenance: "CLIENT_FALLBACK", serverVerifiedAt: null }, + corroborationCount: 0, + claimCount: 1, + claimSummaries: [{ id: "claim-1", summary: "Correction summary" }], + }, + { + id: "inv-2", + contentHash: "c".repeat(64), + checkedAt, + platform: "SUBSTACK", + externalId: "99", + url: "https://example.substack.com/p/post", + origin: { provenance: "CLIENT_FALLBACK", serverVerifiedAt: checkedAt }, + corroborationCount: 3, + claimCount: 0, + claimSummaries: [], + }, + ], + hasMore: true, + }), + getPublicMetrics: async () => ({ + totalInvestigatedPosts: 0, + investigatedPostsWithFlags: 0, + factCheckIncidence: 0, + }), +}); + +for (const [name, query] of Object.entries({ + searchInvestigationsQuery, + publicInvestigationQuery, +})) { + test(`frontend ${name} validates against the public GraphQL schema`, () => { + assert.deepEqual(validate(schema, parse(query.document)), []); + }); +} + +test("searchInvestigations responses parse with the frontend's shared schema", async () => { + const result = await graphql({ + schema, + source: searchInvestigationsQuery.document, + variableValues: searchInvestigationsQuery.variablesSchema.parse({ + platform: "X", + minClaimCount: 1, + }), + contextValue: { prisma: {} }, + }); + assert.equal(result.errors, undefined); + + const data = searchInvestigationsQuery.dataSchema.parse(overTheWire(result.data)); + assert.equal(data.searchInvestigations.investigations.length, 2); + assert.equal(data.searchInvestigations.hasMore, true); +}); + +async function runPublicInvestigationQuery(investigationId: string) { + const result = await graphql({ + schema, + source: publicInvestigationQuery.document, + variableValues: publicInvestigationQuery.variablesSchema.parse({ investigationId }), + contextValue: { prisma: {} }, + }); + assert.equal(result.errors, undefined); + return publicInvestigationQuery.dataSchema.parse(overTheWire(result.data)).publicInvestigation; +} + +test("publicInvestigation responses parse with the frontend's shared schema", async () => { + const investigation = await runPublicInvestigationQuery("inv-1"); + assert.equal(investigation?.investigation.model, "gpt-6.1-sol"); + assert.equal(investigation.claims.length, 1); +}); + +test("publicInvestigation answers null for an unknown investigation", async () => { + assert.equal(await runPublicInvestigationQuery("missing"), null); +}); diff --git a/src/typescript/api/test/unit/hmac.test.ts b/src/typescript/api/test/unit/hmac.test.ts deleted file mode 100644 index 6614c5a..0000000 --- a/src/typescript/api/test/unit/hmac.test.ts +++ /dev/null @@ -1,30 +0,0 @@ -import assert from "node:assert/strict"; -import { createHmac } from "node:crypto"; -import { test } from "node:test"; -import { verifyHmac } from "../../src/lib/services/hmac.js"; - -const TEST_SECRET = "test-hmac-secret"; - -function sign(body: string): string { - return createHmac("sha256", TEST_SECRET).update(body).digest("hex"); -} - -test("verifyHmac accepts signatures computed with the configured secret", async () => { - const body = '{"message":"hello"}'; - const signature = sign(body); - - const accepted = await verifyHmac(TEST_SECRET, body, signature); - assert.equal(accepted, true); -}); - -test("verifyHmac rejects wrong signatures and mismatched lengths", async () => { - const body = '{"message":"hello"}'; - const goodSignature = sign(body); - const wrongBodySignature = sign('{"message":"different"}'); - const sameLengthWrongSignature = - goodSignature.slice(0, -1) + (goodSignature.endsWith("0") ? "1" : "0"); - - assert.equal(await verifyHmac(TEST_SECRET, body, wrongBodySignature), false); - assert.equal(await verifyHmac(TEST_SECRET, body, sameLengthWrongSignature), false); - assert.equal(await verifyHmac(TEST_SECRET, body, "short"), false); -}); diff --git a/src/typescript/api/test/unit/host-safety.test.ts b/src/typescript/api/test/unit/host-safety.test.ts index 286a38e..ccfb83f 100644 --- a/src/typescript/api/test/unit/host-safety.test.ts +++ b/src/typescript/api/test/unit/host-safety.test.ts @@ -1,27 +1,136 @@ import assert from "node:assert/strict"; +import { createServer } from "node:http"; +import type { AddressInfo } from "node:net"; import { test } from "node:test"; -import { hasAddressIntersection, isPrivateIpAddress } from "../../src/lib/network/host-safety.js"; +import ipaddr from "ipaddr.js"; +import { + isLocallyScopedHostname, + isPublicUnicastAddress, + parseIpLiteral, +} from "../../src/lib/network/host-safety.js"; +import { + BlockedDestinationError, + fetchPublicHttp, + publicOnlyLookup, +} from "../../src/lib/network/public-http-fetch.js"; -test("isPrivateIpAddress blocks reserved IPv4 ranges used for SSRF bypasses", () => { - assert.equal(isPrivateIpAddress("100.64.0.1"), true); - assert.equal(isPrivateIpAddress("198.18.0.1"), true); - assert.equal(isPrivateIpAddress("240.0.0.1"), true); - assert.equal(isPrivateIpAddress("255.255.255.255"), true); +function isPublic(address: string): boolean { + return isPublicUnicastAddress(ipaddr.parse(address)); +} + +test("isPublicUnicastAddress rejects private, loopback, link-local, CGNAT and reserved IPv4", () => { + for (const address of [ + "10.0.0.1", + "172.16.5.4", + "192.168.1.1", + "127.0.0.1", + "169.254.169.254", + "100.64.0.1", + "198.18.0.1", + "192.0.0.1", + "240.0.0.1", + "224.0.0.1", + "255.255.255.255", + "0.0.0.0", + ]) { + assert.equal(isPublic(address), false, address); + } +}); + +test("isPublicUnicastAddress rejects IPv6 forms that can reach internal networks", () => { + for (const address of [ + "::1", + "::", + "fc00::1", + "fe80::1", + "ff02::1", + "::ffff:10.0.0.1", + "64:ff9b::a00:1", // NAT64 well-known prefix + "64:ff9b:1::a00:1", // NAT64 local-use prefix + "2002:a00:1::1", // 6to4 + "2001::1", // Teredo + "2001:db8::1", + ]) { + assert.equal(isPublic(address), false, address); + } }); -test("isPrivateIpAddress blocks mapped IPv4 private ranges in IPv6 notation", () => { - assert.equal(isPrivateIpAddress("::ffff:192.168.1.1"), true); - assert.equal(isPrivateIpAddress("::ffff:c0a8:0101"), true); - assert.equal(isPrivateIpAddress("[::ffff:c0a8:0101]"), true); +test("isPublicUnicastAddress accepts ordinary public addresses", () => { + assert.equal(isPublic("8.8.8.8"), true); + assert.equal(isPublic("2606:4700:4700::1111"), true); }); -test("isPrivateIpAddress allows normal public addresses", () => { - assert.equal(isPrivateIpAddress("8.8.8.8"), false); - assert.equal(isPrivateIpAddress("2606:4700:4700::1111"), false); - assert.equal(isPrivateIpAddress("[2606:4700:4700::1111]"), false); +test("parseIpLiteral recognizes bracketed and zoned IPv6 literals but not DNS names", () => { + assert.equal(parseIpLiteral("[::1]")?.toString(), "::1"); + assert.equal(parseIpLiteral("fe80::1%eth0")?.toString(), "fe80::1"); + assert.equal(parseIpLiteral("203.0.113.5")?.toString(), "203.0.113.5"); + assert.equal(parseIpLiteral("example.com"), null); +}); + +test("isLocallyScopedHostname flags localhost-style names", () => { + assert.equal(isLocallyScopedHostname("localhost"), true); + assert.equal(isLocallyScopedHostname("LOCALHOST."), true); + assert.equal(isLocallyScopedHostname("printer.local"), true); + assert.equal(isLocallyScopedHostname("app.localhost"), true); + assert.equal(isLocallyScopedHostname("example.com"), false); +}); + +function lookupAll(hostname: string): Promise<{ error: Error | null; addresses: unknown }> { + return new Promise((resolve) => { + publicOnlyLookup(hostname, { all: true }, (error, addresses) => { + resolve({ error, addresses }); + }); + }); +} + +test("publicOnlyLookup refuses a name that resolves to a non-public address", async () => { + const { error } = await lookupAll("localhost"); + assert.ok(error instanceof BlockedDestinationError); +}); + +test("publicOnlyLookup passes through public addresses in both callback shapes", async () => { + const all = await lookupAll("8.8.8.8"); + assert.equal(all.error, null); + assert.deepEqual(all.addresses, [{ address: "8.8.8.8", family: 4 }]); + + const single = await new Promise<{ address: unknown; family: unknown }>((resolve) => { + publicOnlyLookup("8.8.8.8", { all: false }, (_error, address, family) => { + resolve({ address, family }); + }); + }); + assert.deepEqual(single, { address: "8.8.8.8", family: 4 }); }); -test("hasAddressIntersection compares address sets case-insensitively", () => { - assert.equal(hasAddressIntersection(["2001:DB8::1", "203.0.113.5"], ["203.0.113.5"]), true); - assert.equal(hasAddressIntersection(["2001:DB8::1"], ["2001:db8::2"]), false); +test("fetchPublicHttp never connects to loopback, private or credentialed URLs", async () => { + let requests = 0; + const server = createServer((_request, response) => { + requests += 1; + response.end("internal"); + }); + await new Promise((resolve) => { + server.listen(0, "127.0.0.1", resolve); + }); + const { port } = server.address() as AddressInfo; + try { + for (const url of [ + `http://127.0.0.1:${port.toString()}/`, + `http://[::ffff:127.0.0.1]:${port.toString()}/`, + `http://localhost:${port.toString()}/`, + `http://user:pass@example.com/`, + `ftp://example.com/`, + ]) { + await assert.rejects( + fetchPublicHttp({ url: new URL(url), headers: {}, signal: AbortSignal.timeout(5_000) }), + BlockedDestinationError, + url, + ); + } + assert.equal(requests, 0); + } finally { + await new Promise((resolve) => { + server.close(() => { + resolve(); + }); + }); + } }); diff --git a/src/typescript/api/test/unit/html-to-markdown.test.ts b/src/typescript/api/test/unit/html-to-markdown.test.ts index e0fe959..6039762 100644 --- a/src/typescript/api/test/unit/html-to-markdown.test.ts +++ b/src/typescript/api/test/unit/html-to-markdown.test.ts @@ -11,11 +11,14 @@ import { substackHtmlToContentMarkdown, } from "../../src/lib/services/html-to-markdown.js"; +/** Base URL for resolving relative image sources. */ +const POST_URL = "https://www.lesswrong.com/posts/abc123/example-post"; + // ── Basic structural elements ──────────────────────────────────────────── test("converts headings to markdown heading syntax", () => { const html = "

Title

Subtitle

Sub-subtitle

"; - const md = lesswrongHtmlToContentMarkdown(html).markdown; + const md = lesswrongHtmlToContentMarkdown(html, POST_URL).markdown; assert.ok(md.includes("# Title")); assert.ok(md.includes("## Subtitle")); assert.ok(md.includes("### Sub-subtitle")); @@ -23,7 +26,7 @@ test("converts headings to markdown heading syntax", () => { test("converts paragraphs to double-newline-separated blocks", () => { const html = "

First paragraph.

Second paragraph.

"; - const md = lesswrongHtmlToContentMarkdown(html).markdown; + const md = lesswrongHtmlToContentMarkdown(html, POST_URL).markdown; assert.ok(md.includes("First paragraph.")); assert.ok(md.includes("Second paragraph.")); // Should have separation between paragraphs. @@ -34,7 +37,7 @@ test("converts paragraphs to double-newline-separated blocks", () => { test("converts unordered lists to dash-prefixed items", () => { const html = "
  • Item one
  • Item two
  • Item three
"; - const md = lesswrongHtmlToContentMarkdown(html).markdown; + const md = lesswrongHtmlToContentMarkdown(html, POST_URL).markdown; assert.ok(md.includes("- Item one")); assert.ok(md.includes("- Item two")); assert.ok(md.includes("- Item three")); @@ -42,7 +45,7 @@ test("converts unordered lists to dash-prefixed items", () => { test("converts ordered lists to numbered items", () => { const html = "
  1. First
  2. Second
  3. Third
"; - const md = lesswrongHtmlToContentMarkdown(html).markdown; + const md = lesswrongHtmlToContentMarkdown(html, POST_URL).markdown; assert.ok(md.includes("1. First"), `Expected "1. First" in: ${md}`); assert.ok(md.includes("2. Second"), `Expected "2. Second" in: ${md}`); assert.ok(md.includes("3. Third"), `Expected "3. Third" in: ${md}`); @@ -51,7 +54,7 @@ test("converts ordered lists to numbered items", () => { test("handles nested lists with indentation", () => { const html = "
  • Outer
    • Inner one
    • Inner two
  • Another outer
"; - const md = lesswrongHtmlToContentMarkdown(html).markdown; + const md = lesswrongHtmlToContentMarkdown(html, POST_URL).markdown; assert.ok(md.includes("- Outer"), `Expected "- Outer" in: ${md}`); assert.ok(md.includes("- Inner one"), `Expected nested "- Inner one" in: ${md}`); assert.ok(md.includes("- Inner two"), `Expected nested "- Inner two" in: ${md}`); @@ -60,7 +63,7 @@ test("handles nested lists with indentation", () => { test("converts blockquotes to > prefixed text", () => { const html = "

A wise quote.

"; - const md = lesswrongHtmlToContentMarkdown(html).markdown; + const md = lesswrongHtmlToContentMarkdown(html, POST_URL).markdown; assert.ok(md.includes("> A wise quote."), `Expected "> A wise quote." in: ${md}`); }); @@ -68,41 +71,41 @@ test("converts blockquotes to > prefixed text", () => { test("renders bold tags as plain text (no ** markers)", () => { const html = "

This is important text.

"; - const md = lesswrongHtmlToContentMarkdown(html).markdown; + const md = lesswrongHtmlToContentMarkdown(html, POST_URL).markdown; assert.ok(md.includes("important")); assert.ok(!md.includes("**important**")); }); test("renders tags as plain text (no ** markers)", () => { const html = "

Also bold text.

"; - const md = lesswrongHtmlToContentMarkdown(html).markdown; + const md = lesswrongHtmlToContentMarkdown(html, POST_URL).markdown; assert.ok(md.includes("bold")); assert.ok(!md.includes("**bold**")); }); test("renders italic tags as plain text (no * markers)", () => { const html = "

This is emphasized text.

"; - const md = lesswrongHtmlToContentMarkdown(html).markdown; + const md = lesswrongHtmlToContentMarkdown(html, POST_URL).markdown; assert.ok(md.includes("emphasized")); assert.ok(!md.includes("*emphasized*")); }); test("renders tags as plain text (no * markers)", () => { const html = "

Also italic text.

"; - const md = lesswrongHtmlToContentMarkdown(html).markdown; + const md = lesswrongHtmlToContentMarkdown(html, POST_URL).markdown; assert.ok(md.includes("italic")); assert.ok(!md.includes("*italic*")); }); test("converts links to markdown link syntax", () => { const html = '

Visit Example for more.

'; - const md = lesswrongHtmlToContentMarkdown(html).markdown; + const md = lesswrongHtmlToContentMarkdown(html, POST_URL).markdown; assert.ok(md.includes("[Example](https://example.com)")); }); test("preserves superscript text inline", () => { const html = "

Footnote1 here.

"; - const md = lesswrongHtmlToContentMarkdown(html).markdown; + const md = lesswrongHtmlToContentMarkdown(html, POST_URL).markdown; assert.ok(md.includes("Footnote1 here."), `Expected inline superscript text in: ${md}`); }); @@ -111,7 +114,7 @@ test("preserves superscript text inline", () => { test("excludes NON_CONTENT_TAGS (script, style, noscript)", () => { const html = '

Article text.

More text.

'; - const md = lesswrongHtmlToContentMarkdown(html).markdown; + const md = lesswrongHtmlToContentMarkdown(html, POST_URL).markdown; assert.ok(!md.includes("leaked")); assert.ok(!md.includes("color")); assert.ok(!md.includes("tracking")); @@ -120,29 +123,25 @@ test("excludes NON_CONTENT_TAGS (script, style, noscript)", () => { }); test("replaces img elements with [IMAGE:N] placeholders", () => { - const { markdown, imagePlaceholders } = lesswrongHtmlToContentMarkdown( + const { markdown, imageSourceUrls } = lesswrongHtmlToContentMarkdown( '

Before image.

test

After image.

', + POST_URL, ); assert.ok(!markdown.includes("test.jpg"), "src URL should not appear in markdown"); assert.ok(markdown.includes("[IMAGE:0]"), "placeholder should appear in markdown"); assert.ok(markdown.includes("Before image.")); assert.ok(markdown.includes("After image.")); - assert.equal(imagePlaceholders.length, 1); - assert.deepEqual(imagePlaceholders[0], { - index: 0, - matchBy: "SOURCE_URL", - sourceUrl: "test.jpg", - }); + assert.deepEqual(imageSourceUrls, ["https://www.lesswrong.com/posts/abc123/test.jpg"]); }); test("strips anchor wrapper when link contains only an image", () => { // Substack (and other platforms) wrap images in . // The anchor URL is redundant because the image URL is already captured in - // imagePlaceholders, so the link markup should be removed and only the + // imageSourceUrls, so the link markup should be removed and only the // [IMAGE:N] placeholder should remain. const html = '

Caption text.

After.

'; - const { markdown, imagePlaceholders } = substackHtmlToContentMarkdown(html); + const { markdown, imageSourceUrls } = substackHtmlToContentMarkdown(html, POST_URL); assert.ok(markdown.includes("[IMAGE:0]"), "placeholder should be present"); assert.ok( !markdown.includes("](https://cdn.example.com/big.jpg)"), @@ -152,12 +151,7 @@ test("strips anchor wrapper when link contains only an image", () => { !markdown.includes("[ [IMAGE:0] ]"), "placeholder should not be wrapped in link syntax", ); - assert.equal(imagePlaceholders.length, 1); - assert.deepEqual(imagePlaceholders[0], { - index: 0, - matchBy: "SOURCE_URL", - sourceUrl: "https://cdn.example.com/thumb.jpg", - }); + assert.deepEqual(imageSourceUrls, ["https://cdn.example.com/thumb.jpg"]); }); test("strips anchor wrapper for image-in-div structure (Substack CDN pattern)", () => { @@ -170,7 +164,7 @@ test("strips anchor wrapper for image-in-div structure (Substack CDN pattern)",

As you can see above.

`; - const { markdown } = substackHtmlToContentMarkdown(html); + const { markdown } = substackHtmlToContentMarkdown(html, POST_URL); assert.ok(markdown.includes("[IMAGE:0]")); assert.ok(!markdown.includes("](https://substackcdn.com/big.png)")); // No stray newlines inside the placeholder from the inner div @@ -185,7 +179,7 @@ test("strips anchor wrapper for image-in-div structure (Substack CDN pattern)", test("preserves anchor when link contains text alongside an image", () => { // An anchor with both text and an image should keep the link markup. const html = '

See chart

'; - const { markdown } = lesswrongHtmlToContentMarkdown(html); + const { markdown } = lesswrongHtmlToContentMarkdown(html, POST_URL); assert.ok( markdown.includes("[See chart [IMAGE:0] ](https://example.com)") || markdown.includes("https://example.com"), @@ -197,7 +191,7 @@ test("preserves anchor when link contains text alongside an image", () => { test("decodes HTML entities", () => { const html = "

Tea & Biscuits

"; - const md = lesswrongHtmlToContentMarkdown(html).markdown; + const md = lesswrongHtmlToContentMarkdown(html, POST_URL).markdown; assert.ok(md.includes("Tea")); assert.ok(md.includes("&")); assert.ok(md.includes("Biscuits")); @@ -214,7 +208,7 @@ test("bullet list items are separate lines, not run-on text", () => {
  • issuing an executive order
    • `; - const md = substackHtmlToContentMarkdown(html).markdown; + const md = substackHtmlToContentMarkdown(html, POST_URL).markdown; assert.ok(md.includes("- canceling the contract")); assert.ok(md.includes("- using the Defense Production Act")); assert.ok(md.includes("- issuing an executive order")); @@ -241,7 +235,7 @@ test("wikipediaHtmlToContentMarkdown excludes all WIKIPEDIA_EXCLUDED_SECTION_TIT
    `; assert.ok( - !wikipediaHtmlToContentMarkdown(html).markdown.includes("Excluded section text."), + !wikipediaHtmlToContentMarkdown(html, POST_URL).markdown.includes("Excluded section text."), `"${title}" section should be excluded (legacy heading format)`, ); } @@ -261,7 +255,7 @@ test("wikipediaHtmlToContentMarkdown excludes all WIKIPEDIA_EXCLUDED_SECTION_TIT
    `; assert.ok( - !wikipediaHtmlToContentMarkdown(html).markdown.includes("Excluded section text."), + !wikipediaHtmlToContentMarkdown(html, POST_URL).markdown.includes("Excluded section text."), `"${title}" section should be excluded (Parsoid heading format)`, ); } @@ -273,7 +267,7 @@ test("wikipedia markdown excludes citation superscripts", () => {

    A fact.[1]

    `; - const md = wikipediaHtmlToContentMarkdown(html).markdown; + const md = wikipediaHtmlToContentMarkdown(html, POST_URL).markdown; assert.ok(md.includes("A fact.")); assert.ok(!md.includes("[1]")); }); @@ -292,7 +286,7 @@ test("lesswrongHtmlToContentMarkdown handles complex post structure", () => {

    A quote from someone.

    `; - const md = lesswrongHtmlToContentMarkdown(html).markdown; + const md = lesswrongHtmlToContentMarkdown(html, POST_URL).markdown; assert.ok(md.includes("# Post Title")); assert.ok(md.includes("bold") && !md.includes("**bold**")); assert.ok(md.includes("italic") && !md.includes("*italic*")); @@ -304,7 +298,7 @@ test("lesswrongHtmlToContentMarkdown handles complex post structure", () => { test("substackHtmlToContentMarkdown renders same as lesswrong for shared elements", () => { const html = "

    Title

    Paragraph one.

    Paragraph two.

    "; - const md = substackHtmlToContentMarkdown(html).markdown; + const md = substackHtmlToContentMarkdown(html, POST_URL).markdown; assert.ok(md.includes("## Title")); assert.ok(md.includes("Paragraph one.")); assert.ok(md.includes("Paragraph two.")); @@ -316,7 +310,7 @@ test("substackHtmlToContentMarkdown renders same as lesswrong for shared element // indices skip values, repeat, or don't match the array entries, images are // silently lost or mismatched in the prompt. -test("multiple images produce sequential 0-based [IMAGE:N] placeholders with matching array entries", () => { +test("multiple images produce sequential 0-based [IMAGE:N] placeholders with matching source URLs", () => { const html = `

    First paragraph.

    Photo 1 @@ -326,69 +320,47 @@ test("multiple images produce sequential 0-based [IMAGE:N] placeholders with mat Diagram 3

    Final paragraph.

    `; - const { markdown, imagePlaceholders } = lesswrongHtmlToContentMarkdown(html); + const { markdown, imageSourceUrls } = lesswrongHtmlToContentMarkdown(html, POST_URL); - // Exactly 3 placeholders in the markdown + // Exactly 3 placeholders in the markdown, numbered 0, 1, 2 in document order const placeholderMatches = [...markdown.matchAll(/\[IMAGE:(\d+)\]/g)]; - assert.equal(placeholderMatches.length, 3); - - // Indices must be sequential: 0, 1, 2 - assert.equal(placeholderMatches[0]?.[1], "0"); - assert.equal(placeholderMatches[1]?.[1], "1"); - assert.equal(placeholderMatches[2]?.[1], "2"); + assert.deepEqual( + placeholderMatches.map((match) => match[1]), + ["0", "1", "2"], + ); - // imagePlaceholders array must have matching entries - assert.equal(imagePlaceholders.length, 3); + // imageSourceUrls[N] is the source of [IMAGE:N] + assert.deepEqual(imageSourceUrls, [ + "https://cdn.example.com/photo1.jpg", + "https://cdn.example.com/chart2.png", + "https://cdn.example.com/diagram3.svg", + ]); - const [ph0, ph1, ph2] = imagePlaceholders as [ - (typeof imagePlaceholders)[0], - (typeof imagePlaceholders)[0], - (typeof imagePlaceholders)[0], - ]; - assert.deepEqual(ph0, { - index: 0, - matchBy: "SOURCE_URL", - sourceUrl: "https://cdn.example.com/photo1.jpg", - }); - assert.deepEqual(ph1, { - index: 1, - matchBy: "SOURCE_URL", - sourceUrl: "https://cdn.example.com/chart2.png", - }); - assert.deepEqual(ph2, { - index: 2, - matchBy: "SOURCE_URL", - sourceUrl: "https://cdn.example.com/diagram3.svg", - }); - - // Placeholders must appear in document order between surrounding text - const idx0 = markdown.indexOf("[IMAGE:0]"); - const idx1 = markdown.indexOf("[IMAGE:1]"); - const idx2 = markdown.indexOf("[IMAGE:2]"); - assert.ok(idx0 < idx1); - assert.ok(idx1 < idx2); - assert.ok(markdown.indexOf("First paragraph.") < idx0); - assert.ok(idx2 < markdown.indexOf("Final paragraph.")); + assert.ok(markdown.indexOf("First paragraph.") < markdown.indexOf("[IMAGE:0]")); + assert.ok(markdown.indexOf("[IMAGE:2]") < markdown.indexOf("Final paragraph.")); }); -test("imagePlaceholders index equals array position for all images", () => { - // When buildInitialInput does `imagePlaceholders.find(p => p.index === N)`, - // it relies on each placeholder's .index matching the [IMAGE:N] in the - // markdown. This test verifies the identity: placeholder[i].index === i. +test("image sources resolve against the post URL; unfetchable sources get no placeholder", () => { + // Wikipedia HTML uses protocol-relative URLs; image occurrences reported by + // the extension are absolute, so placeholders must be absolute to match. const html = ` -
    - - - - - -
    + + + + `; - const { imagePlaceholders } = lesswrongHtmlToContentMarkdown(html); - assert.equal(imagePlaceholders.length, 5); - for (let i = 0; i < imagePlaceholders.length; i++) { - assert.equal(imagePlaceholders[i]?.index, i, `imagePlaceholders[${i}].index must equal ${i}`); - } + const { markdown, imageSourceUrls } = wikipediaHtmlToContentMarkdown( + html, + "https://en.wikipedia.org/wiki/Example", + ); + assert.deepEqual(imageSourceUrls, [ + "https://upload.wikimedia.org/a.png", + "https://en.wikipedia.org/static/b.png", + ]); + assert.deepEqual( + [...markdown.matchAll(/\[IMAGE:(\d+)\]/g)].map((match) => match[1]), + ["0", "1"], + ); }); // ── Markdown prose ⊆ normalized text invariant ───────────────────────── @@ -440,7 +412,7 @@ function assertMarkdownProseSubsetOfNormalizedText( const wrapHtml = platform === "wikipedia" ? `
    ${html}
    ` : html; - const { markdown } = toMarkdown(wrapHtml); + const { markdown } = toMarkdown(wrapHtml, POST_URL); const normalizedText = toNormalized(wrapHtml); const proseWords = extractProseWords(markdown); @@ -512,7 +484,7 @@ for (const { name, html } of PROSE_SUBSET_HTML_CASES) { // holds through the substackHtmlToContentMarkdown entry point too. for (const { name, html } of PROSE_SUBSET_HTML_CASES) { test(`markdown prose ⊆ normalized text: ${name} (substack)`, () => { - const { markdown } = substackHtmlToContentMarkdown(html); + const { markdown } = substackHtmlToContentMarkdown(html, POST_URL); const normalizedText = lesswrongHtmlToNormalizedText(html); const proseWords = extractProseWords(markdown); @@ -552,9 +524,7 @@ describe("inline emphasis/code tags never produce markdown syntax", () => { wikipediaHtmlToContentMarkdown, substackHtmlToContentMarkdown, ]) { - const { markdown } = toMarkdown( - tag === "code" ? html : html, // same HTML for all - ); + const { markdown } = toMarkdown(html, POST_URL); assert.ok( markdown.includes("wrapped content"), `<${tag}> content should appear in markdown`, diff --git a/src/typescript/api/test/unit/investigation-input.test.ts b/src/typescript/api/test/unit/investigation-input.test.ts new file mode 100644 index 0000000..6e7980f --- /dev/null +++ b/src/typescript/api/test/unit/investigation-input.test.ts @@ -0,0 +1,204 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import type { Platform } from "@openerrata/shared"; +import type { DbClient } from "../../src/lib/db/client.js"; +import { + buildInvestigationInputSnapshot, + investigationInputRow, + isLikelyVideoUrl, +} from "../../src/lib/services/investigation-input.js"; + +/** + * Invariants under test: the InvestigationInput snapshot captures, at queue + * time, everything about the post that the prompt uses and that can change + * later — markdown (with the source URL behind each image placeholder), post + * URL, author and publication time — and labels the markdown's trust tier. + */ + +interface FakePostVersion { + serverVerifiedAt: Date | null; + contentBlob: { contentHash: string }; + post: { platform: Platform; url: string; author: { displayName: string } | null }; + lesswrongVersionMeta: null | { + publishedAt: Date | null; + serverHtmlBlob: { htmlContent: string } | null; + clientHtmlBlob: { htmlContent: string } | null; + }; + xVersionMeta: null | { postedAt: Date | null; mediaUrls: string[] }; + substackVersionMeta: null | { + publishedAt: Date | null; + clientHtmlBlob: { htmlContent: string } | null; + }; + wikipediaVersionMeta: null | { + lastModifiedAt: Date | null; + serverHtmlBlob: { htmlContent: string } | null; + clientHtmlBlob: { htmlContent: string } | null; + }; +} + +function fakeDb(postVersion: FakePostVersion): DbClient { + return { + postVersion: { findUnique: async () => postVersion }, + } as unknown as DbClient; +} + +function basePostVersion(platform: Platform, url: string): FakePostVersion { + return { + serverVerifiedAt: null, + contentBlob: { contentHash: "content-hash" }, + post: { platform, url, author: null }, + lesswrongVersionMeta: null, + xVersionMeta: null, + substackVersionMeta: null, + wikipediaVersionMeta: null, + }; +} + +test("LessWrong server-verified snapshot renders server HTML and resolves image sources", async () => { + const publishedAt = new Date("2026-01-02T03:04:05.000Z"); + const snapshot = await buildInvestigationInputSnapshot( + fakeDb({ + ...basePostVersion("LESSWRONG", "https://www.lesswrong.com/posts/abc/title"), + serverVerifiedAt: new Date("2026-01-03T00:00:00.000Z"), + post: { + platform: "LESSWRONG", + url: "https://www.lesswrong.com/posts/abc/title", + author: { displayName: "Author Name" }, + }, + lesswrongVersionMeta: { + publishedAt, + serverHtmlBlob: { htmlContent: '

    Server text.

    ' }, + clientHtmlBlob: { htmlContent: "

    Client text.

    " }, + }, + }), + "pv-1", + ); + + assert.equal(snapshot.provenance, "SERVER_VERIFIED"); + assert.equal(snapshot.markdown.source, "SERVER_HTML"); + assert.ok(snapshot.markdown.markdown.includes("Server text.")); + assert.ok(snapshot.markdown.markdown.includes("[IMAGE:0]")); + assert.deepEqual(snapshot.markdown.imageSourceUrls, ["https://www.lesswrong.com/img/a.png"]); + assert.equal(snapshot.postUrl, "https://www.lesswrong.com/posts/abc/title"); + assert.equal(snapshot.authorName, "Author Name"); + assert.deepEqual(snapshot.postPublishedAt, publishedAt); + assert.equal(snapshot.hasVideo, false); +}); + +test("X snapshot has no markdown and flags video media", async () => { + const postedAt = new Date("2026-02-01T00:00:00.000Z"); + const snapshot = await buildInvestigationInputSnapshot( + fakeDb({ + ...basePostVersion("X", "https://x.com/someone/status/1"), + xVersionMeta: { + postedAt, + mediaUrls: ["https://video.twimg.com/clip.mp4?tag=12"], + }, + }), + "pv-x", + ); + + assert.equal(snapshot.provenance, "CLIENT_FALLBACK"); + assert.deepEqual(snapshot.markdown, { source: "NONE" }); + assert.equal(snapshot.hasVideo, true); + assert.deepEqual(snapshot.postPublishedAt, postedAt); + assert.equal(snapshot.authorName, null); +}); + +test("Substack and Wikipedia snapshots use client HTML and their own timestamps", async () => { + const substackPublishedAt = new Date("2026-03-01T00:00:00.000Z"); + const substack = await buildInvestigationInputSnapshot( + fakeDb({ + ...basePostVersion("SUBSTACK", "https://example.substack.com/p/post"), + substackVersionMeta: { + publishedAt: substackPublishedAt, + clientHtmlBlob: { htmlContent: "

    Substack body.

    " }, + }, + }), + "pv-s", + ); + assert.equal(substack.markdown.source, "CLIENT_HTML"); + assert.deepEqual(substack.postPublishedAt, substackPublishedAt); + + const lastModifiedAt = new Date("2026-04-01T00:00:00.000Z"); + const wikipedia = await buildInvestigationInputSnapshot( + fakeDb({ + ...basePostVersion("WIKIPEDIA", "https://en.wikipedia.org/wiki/Example"), + wikipediaVersionMeta: { + lastModifiedAt, + serverHtmlBlob: null, + clientHtmlBlob: { htmlContent: "

    Wiki body.

    " }, + }, + }), + "pv-w", + ); + assert.equal(wikipedia.markdown.source, "CLIENT_HTML"); + assert.deepEqual(wikipedia.postPublishedAt, lastModifiedAt); +}); + +test("snapshot refuses a server-verified version without server HTML", async () => { + await assert.rejects( + buildInvestigationInputSnapshot( + fakeDb({ + ...basePostVersion("WIKIPEDIA", "https://en.wikipedia.org/wiki/Example"), + serverVerifiedAt: new Date(), + wikipediaVersionMeta: { + lastModifiedAt: null, + serverHtmlBlob: null, + clientHtmlBlob: { htmlContent: "

    Wiki body.

    " }, + }, + }), + "pv-bad", + ), + /serverVerifiedAt is set but serverHtml is missing/, + ); +}); + +test("investigationInputRow stores placeholders only alongside markdown", () => { + const common = { + provenance: "CLIENT_FALLBACK" as const, + contentHash: "hash", + postUrl: "https://x.com/a/status/1", + authorName: null, + postPublishedAt: null, + hasVideo: false, + }; + const noMarkdown = investigationInputRow("inv-1", { ...common, markdown: { source: "NONE" } }); + assert.equal(noMarkdown.markdownSource, "NONE"); + assert.deepEqual(noMarkdown.imagePlaceholderSourceUrls, []); + assert.equal(noMarkdown.markdown, undefined); + + const withMarkdown = investigationInputRow("inv-2", { + ...common, + markdown: { + source: "CLIENT_HTML", + markdown: "Body [IMAGE:0]", + rendererVersion: "1.3.0", + imageSourceUrls: ["https://example.com/a.png"], + }, + }); + assert.equal(withMarkdown.markdown, "Body [IMAGE:0]"); + assert.deepEqual(withMarkdown.imagePlaceholderSourceUrls, ["https://example.com/a.png"]); +}); + +test("isLikelyVideoUrl judges by path extension, case-insensitively, ignoring the query", () => { + for (const url of [ + "https://example.com/video.mp4", + "https://example.com/video.webm", + "https://example.com/stream.m3u8", + "https://example.com/clip.mov", + "https://example.com/clip.m4v", + "https://example.com/Video.MP4", + "https://example.com/video.mp4?token=abc", + ]) { + assert.equal(isLikelyVideoUrl(url), true, url); + } + for (const url of [ + "https://example.com/image.png", + "https://example.com/page", + "https://example.com/page?file=video.mp4", + "not a url.mp4", + ]) { + assert.equal(isLikelyVideoUrl(url), false, url); + } +}); diff --git a/src/typescript/api/test/unit/investigation-lease.test.ts b/src/typescript/api/test/unit/investigation-lease.test.ts index f3f63aa..b7a1512 100644 --- a/src/typescript/api/test/unit/investigation-lease.test.ts +++ b/src/typescript/api/test/unit/investigation-lease.test.ts @@ -1,83 +1,25 @@ import assert from "node:assert/strict"; import { test } from "node:test"; import { - BASE_BACKOFF_MS, MAX_INVESTIGATION_ATTEMPTS, + retryBackoffMs, } from "../../src/lib/services/investigation-lease.js"; /** - * Invariants under test: - * - * The retry backoff schedule determines how quickly a transiently-failed - * investigation is re-attempted. Two properties must hold: - * - * 1. **Every backoff delay is strictly positive.** A zero or negative delay - * would cause immediate re-enqueue, defeating the backoff entirely and - * hammering the provider API on every failure. - * - * 2. **Delays are strictly increasing.** Each retry should wait longer than - * the last. A flat or decreasing schedule means later failures don't back - * off further, which doesn't reduce load during sustained provider issues. - * - * 3. **The maximum delay is bounded at a reasonable ceiling.** Too long and - * investigations are stuck for minutes; too short and the backoff provides - * no relief. We enforce < 120s as a sanity ceiling. - * - * These tests pin the concrete schedule (10s, 20s, 40s for attempts 1–3) so - * any change to the backoff formula or constants is a deliberate, visible - * choice rather than an accidental breakage. + * The retry backoff schedule (SPEC §3.7: 10s × 2^(attempt − 1)) determines how + * quickly a transiently failed investigation is re-attempted. Pinning the + * concrete values makes any change to the schedule or the attempt cap a + * deliberate, visible choice. */ -test("investigation retry backoff schedule: positive, strictly increasing, bounded", () => { - const retryAttempts = MAX_INVESTIGATION_ATTEMPTS - 1; // attempts that actually retry - - let prevBackoffMs = 0; - for (let attempt = 1; attempt <= retryAttempts; attempt++) { - const backoffMs = BASE_BACKOFF_MS * Math.pow(2, attempt - 1); - - assert.ok( - backoffMs > 0, - `attempt ${attempt.toString()} backoff must be positive (got ${backoffMs.toString()}ms)`, - ); - assert.ok( - backoffMs > prevBackoffMs, - `attempt ${attempt.toString()} backoff (${backoffMs.toString()}ms) must exceed attempt ${(attempt - 1).toString()} (${prevBackoffMs.toString()}ms)`, - ); - assert.ok( - backoffMs < 120_000, - `attempt ${attempt.toString()} backoff must be < 120s (got ${backoffMs.toString()}ms)`, - ); - - prevBackoffMs = backoffMs; - } -}); - -test("investigation retry backoff schedule: concrete values match spec §3.7", () => { - // Spec §3.7: exponential backoff 10s × 2^(attempt-1) - // attempt 1 → 10s, attempt 2 → 20s, attempt 3 → 40s - const expected = [10_000, 20_000, 40_000]; - const retryAttempts = MAX_INVESTIGATION_ATTEMPTS - 1; - - assert.equal( - retryAttempts, - expected.length, - "Expected retry attempt count does not match expected backoff table length", +test("retryBackoffMs follows 10s × 2^(attempt − 1) for every retried attempt", () => { + const retriedAttempts = Array.from( + { length: MAX_INVESTIGATION_ATTEMPTS - 1 }, + (_, index) => index + 1, ); - - for (let i = 0; i < retryAttempts; i++) { - const attempt = i + 1; - const backoffMs = BASE_BACKOFF_MS * Math.pow(2, attempt - 1); - assert.equal( - backoffMs, - expected[i], - `attempt ${attempt.toString()} backoff should be ${(expected[i] ?? 0).toString()}ms`, - ); - } + assert.deepEqual(retriedAttempts.map(retryBackoffMs), [10_000, 20_000, 40_000]); }); test("MAX_INVESTIGATION_ATTEMPTS is 4 (1 initial attempt + 3 retries)", () => { - // Pins the concrete value so any intentional change to the attempt cap is - // a deliberate, visible choice that breaks this test, triggering a review - // of the backoff table in the test above. assert.equal(MAX_INVESTIGATION_ATTEMPTS, 4); }); diff --git a/src/typescript/api/test/unit/investigation-lifecycle.test.ts b/src/typescript/api/test/unit/investigation-lifecycle.test.ts deleted file mode 100644 index a7cabfc..0000000 --- a/src/typescript/api/test/unit/investigation-lifecycle.test.ts +++ /dev/null @@ -1,98 +0,0 @@ -import assert from "node:assert/strict"; -import { test } from "node:test"; -import { WORD_COUNT_LIMIT } from "@openerrata/shared"; -import { - wordCount, - InvestigationWordLimitError, -} from "../../src/lib/services/investigation-lifecycle.js"; - -/** - * Invariants under test: - * - * 1. **wordCount is the gatekeeper for the word limit**: The lifecycle rejects - * posts where wordCount(text) > WORD_COUNT_LIMIT. These tests verify the - * counting algorithm handles edge cases (mixed whitespace, non-breaking - * spaces, Unicode) and that the boundary condition is correct (exactly - * WORD_COUNT_LIMIT words passes, WORD_COUNT_LIMIT + 1 fails). - * - * 2. **InvestigationWordLimitError carries metadata for the API response**: - * The observedWordCount and limit fields are what the caller returns to - * the extension to explain the rejection. - */ - -// ── wordCount ──────────────────────────────────────────────────────────────── - -test("wordCount counts words separated by single spaces", () => { - assert.equal(wordCount("hello world"), 2); -}); - -test("wordCount counts words across mixed whitespace", () => { - assert.equal(wordCount("hello world\t\tfoo\nbar"), 4); -}); - -test("wordCount returns 0 for empty string", () => { - assert.equal(wordCount(""), 0); -}); - -test("wordCount returns 0 for whitespace-only string", () => { - assert.equal(wordCount(" \t\n "), 0); -}); - -test("wordCount counts hyphenated and contracted words as single tokens", () => { - assert.equal(wordCount("mother-in-law it's don't"), 3); -}); - -test("wordCount counts punctuation-adjacent text as single tokens", () => { - assert.equal(wordCount("Hello, world!"), 2); -}); - -test("wordCount handles non-breaking space as a word separator", () => { - // \u00A0 is a non-breaking space; \s+ matches it in JavaScript - assert.equal(wordCount("hello\u00A0world"), 2); -}); - -test("wordCount handles single word", () => { - assert.equal(wordCount("hello"), 1); -}); - -// ── wordCount + WORD_COUNT_LIMIT boundary ──────────────────────────────────── - -test("wordCount of exactly WORD_COUNT_LIMIT words is at the boundary", () => { - const text = Array.from({ length: WORD_COUNT_LIMIT }, (_, i) => `word${i.toString()}`).join(" "); - assert.equal(wordCount(text), WORD_COUNT_LIMIT); -}); - -test("wordCount boundary: WORD_COUNT_LIMIT + 1 exceeds the limit", () => { - const count = WORD_COUNT_LIMIT + 1; - const text = Array.from({ length: count }, (_, i) => `word${i.toString()}`).join(" "); - assert.equal(wordCount(text), count); - assert.ok(wordCount(text) > WORD_COUNT_LIMIT); -}); - -// ── InvestigationWordLimitError ────────────────────────────────────────────── - -test("InvestigationWordLimitError stores observedWordCount and limit", () => { - const error = new InvestigationWordLimitError(15234, 10000); - assert.equal(error.observedWordCount, 15234); - assert.equal(error.limit, 10000); - assert.match(error.message, /10000/); -}); - -// ── wordCount edge cases ───────────────────────────────────────────────────── - -test("wordCount treats Unicode text as individual words", () => { - assert.equal(wordCount("你好 世界"), 2); -}); - -test("wordCount counts emoji as words when space-separated", () => { - assert.equal(wordCount("🎉 🎊 🎈"), 3); -}); - -test("wordCount handles very long words correctly", () => { - const longWord = "a".repeat(10_000); - assert.equal(wordCount(longWord), 1); -}); - -test("wordCount handles newline-only text", () => { - assert.equal(wordCount("\n\n\n"), 0); -}); diff --git a/src/typescript/api/test/unit/ip-prefix.test.ts b/src/typescript/api/test/unit/ip-prefix.test.ts index c6020c0..d935bc5 100644 --- a/src/typescript/api/test/unit/ip-prefix.test.ts +++ b/src/typescript/api/test/unit/ip-prefix.test.ts @@ -7,7 +7,7 @@ test("deriveIpRangePrefix normalizes IPv4 addresses to /24 prefixes", () => { assert.equal(deriveIpRangePrefix(" 198.51.100.7 "), "198.51.100"); }); -test("deriveIpRangePrefix returns normalized IPv6 prefixes", () => { +test("deriveIpRangePrefix returns normalized IPv6 /48 prefixes", () => { assert.equal(deriveIpRangePrefix("2001:0db8::1"), "2001:db8:0"); assert.equal(deriveIpRangePrefix("fe80::1%eth0"), "fe80:0:0"); }); @@ -16,13 +16,10 @@ test("deriveIpRangePrefix collapses IPv4-mapped IPv6 to IPv4 /24 prefixes", () = assert.equal(deriveIpRangePrefix("::ffff:192.168.1.77"), "192.168.1"); }); -test("deriveIpRangePrefix returns invalid marker for malformed input", () => { - assert.equal(deriveIpRangePrefix("999.1.1.1"), "invalid:999.1.1.1"); - assert.equal(deriveIpRangePrefix("2001::db8::1"), "invalid:2001::db8::1"); - assert.equal(deriveIpRangePrefix("NotAnIp"), "invalid:notanip"); -}); - -test("deriveIpRangePrefix returns unknown for empty client addresses", () => { - assert.equal(deriveIpRangePrefix(""), "unknown"); - assert.equal(deriveIpRangePrefix(" "), "unknown"); +test("deriveIpRangePrefix rejects anything that is not an IP address", () => { + // A non-IP client address means the proxy header configuration is broken; + // bucketing such clients together would silently merge their view credits. + for (const value of ["999.1.1.1", "2001::db8::1", "NotAnIp", "", " ", "10.1"]) { + assert.throws(() => deriveIpRangePrefix(value), /not an IP address/, JSON.stringify(value)); + } }); diff --git a/src/typescript/api/test/unit/markdown-resolution-placeholders.test.ts b/src/typescript/api/test/unit/markdown-resolution-placeholders.test.ts deleted file mode 100644 index 85fa2b3..0000000 --- a/src/typescript/api/test/unit/markdown-resolution-placeholders.test.ts +++ /dev/null @@ -1,65 +0,0 @@ -import assert from "node:assert/strict"; -import { test } from "node:test"; -import { extractImagePlaceholdersFromMarkdown } from "../../src/lib/services/markdown-resolution.js"; - -test("extractImagePlaceholdersFromMarkdown extracts placeholders in document order", () => { - const md = "Text [IMAGE:0] more text [IMAGE:1] end [IMAGE:2]"; - const result = extractImagePlaceholdersFromMarkdown(md); - - assert.equal(result.length, 3); - assert.equal(result[0]?.index, 0); - assert.equal(result[1]?.index, 1); - assert.equal(result[2]?.index, 2); -}); - -test("extractImagePlaceholdersFromMarkdown preserves out-of-order indices", () => { - const md = "[IMAGE:5] first [IMAGE:2] second [IMAGE:10]"; - const result = extractImagePlaceholdersFromMarkdown(md); - - assert.equal(result.length, 3); - assert.equal(result[0]?.index, 5); - assert.equal(result[1]?.index, 2); - assert.equal(result[2]?.index, 10); -}); - -test("extractImagePlaceholdersFromMarkdown returns empty for no placeholders", () => { - assert.deepEqual(extractImagePlaceholdersFromMarkdown("Just plain text with no images"), []); -}); - -test("extractImagePlaceholdersFromMarkdown returns empty for empty string", () => { - assert.deepEqual(extractImagePlaceholdersFromMarkdown(""), []); -}); - -test("extractImagePlaceholdersFromMarkdown ignores malformed placeholders", () => { - // [IMAGE:abc] has non-digit content and should not match \d+ - const md = "[IMAGE:0] [IMAGE:abc] [IMAGE:] [IMAGE:99]"; - const result = extractImagePlaceholdersFromMarkdown(md); - - assert.equal(result.length, 2); - assert.equal(result[0]?.index, 0); - assert.equal(result[1]?.index, 99); -}); - -test("extractImagePlaceholdersFromMarkdown handles duplicate indices", () => { - const md = "[IMAGE:0] duplicate [IMAGE:0] here"; - const result = extractImagePlaceholdersFromMarkdown(md); - - assert.equal(result.length, 2); - assert.equal(result[0]?.index, 0); - assert.equal(result[1]?.index, 0); -}); - -test("extractImagePlaceholdersFromMarkdown marks placeholders for index matching", () => { - // sourceUrl is not recoverable from markdown; retries match by original index. - const result = extractImagePlaceholdersFromMarkdown("[IMAGE:3]"); - assert.equal(result.length, 1); - assert.equal(result[0]?.matchBy, "ORIGINAL_INDEX"); -}); - -test("extractImagePlaceholdersFromMarkdown ignores bracket patterns that are not image placeholders", () => { - const md = "See [link](url) and [IMAGE:0] and [not-an-image:1]"; - const result = extractImagePlaceholdersFromMarkdown(md); - - assert.equal(result.length, 1); - assert.equal(result[0]?.index, 0); -}); diff --git a/src/typescript/api/test/unit/markdown-resolution.test.ts b/src/typescript/api/test/unit/markdown-resolution.test.ts index 0a51f39..919e5d0 100644 --- a/src/typescript/api/test/unit/markdown-resolution.test.ts +++ b/src/typescript/api/test/unit/markdown-resolution.test.ts @@ -5,6 +5,7 @@ import { MARKDOWN_RENDERER_VERSION } from "../../src/lib/services/html-to-markdo import { resolveMarkdownForInvestigation } from "../../src/lib/services/markdown-resolution.js"; const HTML_PLATFORMS: Platform[] = ["LESSWRONG", "SUBSTACK", "WIKIPEDIA"]; +const POST_URL = "https://example.com/posts/1"; function sampleHtml(platform: Platform): string { return `

    ${platform} title

    Body with an

    `; @@ -17,6 +18,7 @@ test("resolveMarkdownForInvestigation trust source is monotonic with serverVerif const clientFallback = resolveMarkdownForInvestigation({ platform, snapshots: { serverVerifiedAt: null, serverHtml: null, clientHtml: html }, + postUrl: POST_URL, }); assert.equal(clientFallback.source, "CLIENT_HTML"); @@ -27,13 +29,15 @@ test("resolveMarkdownForInvestigation trust source is monotonic with serverVerif serverHtml: html, clientHtml: html, }, + postUrl: POST_URL, }); assert.equal(serverVerified.source, "SERVER_HTML"); // Trust label can change from CLIENT_HTML -> SERVER_HTML, but the rendered // markdown payload should stay stable for identical HTML input. assert.equal(serverVerified.markdown, clientFallback.markdown); - assert.deepEqual(serverVerified.imagePlaceholders, clientFallback.imagePlaceholders); + assert.deepEqual(serverVerified.imageSourceUrls, clientFallback.imageSourceUrls); + assert.deepEqual(serverVerified.imageSourceUrls, ["https://example.com/image.png"]); assert.equal(serverVerified.rendererVersion, MARKDOWN_RENDERER_VERSION); assert.equal(clientFallback.rendererVersion, MARKDOWN_RENDERER_VERSION); } @@ -47,6 +51,7 @@ test("resolveMarkdownForInvestigation returns NONE for missing html regardless o resolveMarkdownForInvestigation({ platform, snapshots: { serverVerifiedAt: null, serverHtml: null, clientHtml: null }, + postUrl: POST_URL, }), { source: "NONE" }, ); diff --git a/src/typescript/api/test/unit/openai-attempt-audit-builder.test.ts b/src/typescript/api/test/unit/openai-attempt-audit-builder.test.ts deleted file mode 100644 index 68d77c7..0000000 --- a/src/typescript/api/test/unit/openai-attempt-audit-builder.test.ts +++ /dev/null @@ -1,108 +0,0 @@ -import assert from "node:assert/strict"; -import { test } from "node:test"; -import { - buildFailedAttemptAudit, - buildFullAttemptResponseAudit, - buildSuccessfulAttemptAudit, - createStageOneAttemptAuditBase, - createStageTwoAttemptAuditBase, -} from "../../src/lib/investigators/openai-attempt-audit-builder.js"; -import type { InvestigatorResponseAudit } from "../../src/lib/investigators/interface.js"; - -function makeResponseAudit(input: { - responseId: string; - outputCount: number; -}): InvestigatorResponseAudit { - return { - responseId: input.responseId, - responseStatus: "completed", - responseModelVersion: "test-model", - responseOutputText: null, - outputItems: Array.from({ length: input.outputCount }, (_, index) => ({ - outputIndex: index, - providerItemId: `item-${index.toString()}`, - itemType: "message", - itemStatus: "completed", - })), - outputTextParts: [], - outputTextAnnotations: [], - reasoningSummaries: [], - toolCalls: [], - usage: null, - }; -} - -test("attempt audit builders construct stage-one and stage-two request metadata", () => { - const stageOne = createStageOneAttemptAuditBase({ - startedAt: "2026-01-01T00:00:00.000Z", - openAiModelId: "test-model", - systemPrompt: "fact-check prompt", - userPrompt: "user prompt", - requestReasoning: { - effort: "medium", - summary: "detailed", - }, - requestedTools: [{ type: "web_search_preview" }], - }); - - const stageTwo = createStageTwoAttemptAuditBase({ - stageOneBase: stageOne, - userPrompt: "user prompt", - validationInputSummary: "Claim 0: approved", - }); - - assert.match(stageTwo.requestInstructions, /Stage 1/); - assert.match(stageTwo.requestInstructions, /Stage 2/); - assert.match(stageTwo.requestInput, /Claim 0: approved/); -}); - -test("buildFullAttemptResponseAudit offsets validation output indices", () => { - const factCheck = makeResponseAudit({ responseId: "resp-stage1", outputCount: 2 }); - const validationA = makeResponseAudit({ responseId: "resp-val-a", outputCount: 1 }); - const validationB = makeResponseAudit({ responseId: "resp-val-b", outputCount: 1 }); - - const merged = buildFullAttemptResponseAudit({ - factCheckResponseAudit: factCheck, - successfulValidationResponseAudits: [validationA], - failedValidationResponseAudits: [validationB], - }); - - assert.equal(merged.outputItems.length, 4); - assert.deepEqual( - merged.outputItems.map((item) => item.outputIndex), - [0, 1, 2, 3], - ); -}); - -test("buildFailedAttemptAudit and buildSuccessfulAttemptAudit set terminal fields", () => { - const base = createStageOneAttemptAuditBase({ - startedAt: "2026-01-01T00:00:00.000Z", - openAiModelId: "test-model", - systemPrompt: "prompt", - userPrompt: "user", - requestReasoning: { - effort: "medium", - summary: "detailed", - }, - requestedTools: [], - }); - - const response = makeResponseAudit({ responseId: "resp-stage1", outputCount: 1 }); - - const failed = buildFailedAttemptAudit({ - base, - response, - error: new Error("boom"), - completedAt: "2026-01-01T00:00:01.000Z", - }); - assert.notEqual(failed.error, null); - assert.equal(failed.completedAt, "2026-01-01T00:00:01.000Z"); - - const succeeded = buildSuccessfulAttemptAudit({ - base, - response, - completedAt: "2026-01-01T00:00:02.000Z", - }); - assert.equal(succeeded.error, null); - assert.equal(succeeded.completedAt, "2026-01-01T00:00:02.000Z"); -}); diff --git a/src/typescript/api/test/unit/openai-claim-tools.test.ts b/src/typescript/api/test/unit/openai-claim-tools.test.ts new file mode 100644 index 0000000..6aa0663 --- /dev/null +++ b/src/typescript/api/test/unit/openai-claim-tools.test.ts @@ -0,0 +1,140 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import type { FunctionTool } from "openai/resources/responses/responses"; +import { fetchUrlToolDefinition } from "../../src/lib/investigators/fetch-url-tool.js"; +import { + buildRetainCorrectionToolDefinition, + parseRetainCorrectionArguments, + parseSubmitCorrectionArguments, + submitCorrectionToolDefinition, +} from "../../src/lib/investigators/openai-claim-tools.js"; +import { buildFactCheckTools } from "../../src/lib/investigators/openai-request-config.js"; +import { makeClaim } from "../helpers/fake-openai.js"; + +// Keywords OpenAI strict mode accepts, per JSON Schema node type +// (https://platform.openai.com/docs/guides/structured-outputs#supported-schemas). +const STRICT_MODE_KEYWORDS: Record> = { + object: new Set(["type", "description", "properties", "required", "additionalProperties"]), + array: new Set(["type", "description", "items", "minItems", "maxItems"]), + string: new Set(["type", "description", "enum", "pattern", "format"]), +}; +const STRICT_MODE_STRING_FORMATS = new Set([ + "date-time", + "time", + "date", + "duration", + "email", + "hostname", + "ipv4", + "ipv6", + "uuid", +]); + +/** Throws describing the first strict-mode violation under `path`. */ +function assertStrictModeSchema(schema: Record, path: string): void { + const type = schema["type"]; + assert.equal(typeof type, "string", `${path}: every node needs a single type`); + const allowed = STRICT_MODE_KEYWORDS[String(type)]; + assert.ok(allowed, `${path}: unsupported type ${String(type)}`); + for (const keyword of Object.keys(schema)) { + if (keyword === "$schema" && path === "parameters") continue; + assert.ok(allowed.has(keyword), `${path}: keyword "${keyword}" is not allowed in strict mode`); + } + if (type === "string" && schema["format"] !== undefined) { + assert.ok(STRICT_MODE_STRING_FORMATS.has(schema["format"] as string), `${path}: format`); + } + if (type === "object") { + const properties = schema["properties"] as Record>; + assert.equal(schema["additionalProperties"], false, `${path}: additionalProperties`); + assert.deepEqual(schema["required"], Object.keys(properties), `${path}: all fields required`); + for (const [name, property] of Object.entries(properties)) { + assertStrictModeSchema(property, `${path}.${name}`); + } + } + if (type === "array") { + assertStrictModeSchema(schema["items"] as Record, `${path}[]`); + } +} + +test("every function tool offered to the fact-check is valid under OpenAI strict mode", () => { + const functionTools = buildFactCheckTools(["claim-1", "claim-2"]).filter( + (tool): tool is FunctionTool => tool.type === "function", + ); + assert.deepEqual( + functionTools.map((tool) => tool.name), + [fetchUrlToolDefinition.name, "submit_correction", "retain_correction"], + ); + for (const tool of functionTools) { + assert.equal(tool.strict, true, tool.name); + assert.ok(tool.parameters); + assertStrictModeSchema(tool.parameters, "parameters"); + } +}); + +test("submit_correction advertises the shared claim payload's shape", () => { + const parameters = submitCorrectionToolDefinition.parameters; + assert.ok(parameters); + assert.deepEqual(Object.keys(parameters["properties"] as object), [ + "text", + "context", + "summary", + "reasoning", + "sources", + ]); + const sources = (parameters["properties"] as Record>)["sources"]; + assert.ok(sources); + assert.equal(sources["minItems"], 1); + assert.deepEqual( + Object.keys((sources["items"] as Record)["properties"] as object), + ["url", "title", "snippet"], + ); +}); + +test("parseSubmitCorrectionArguments accepts a valid claim", () => { + const claim = makeClaim("Alpha"); + assert.deepEqual(parseSubmitCorrectionArguments(JSON.stringify(claim)), { + kind: "valid", + value: claim, + }); +}); + +test("parseSubmitCorrectionArguments rejects non-http(s) source URLs", () => { + for (const url of ["data:text/plain,hi", "ftp://example.com/file", "example.com/page"]) { + const claim = { ...makeClaim("Alpha"), sources: [{ url, title: "T", snippet: "S" }] }; + assert.equal(parseSubmitCorrectionArguments(JSON.stringify(claim)).kind, "invalid", url); + } +}); + +test("parseSubmitCorrectionArguments rejects claims the shared schema rejects", () => { + const cases = [ + { ...makeClaim("Alpha"), sources: [] }, + { ...makeClaim("Alpha"), text: "" }, + { ...makeClaim("Alpha"), extra: "field" }, + ]; + for (const claim of cases) { + assert.equal(parseSubmitCorrectionArguments(JSON.stringify(claim)).kind, "invalid"); + } + assert.deepEqual(parseSubmitCorrectionArguments("{not json"), { + kind: "invalid", + error: "Arguments are not valid JSON", + }); +}); + +test("retain_correction accepts exactly the retainable claim ids", () => { + const ids = ["claim-1", "claim-2"] as const; + const definition = buildRetainCorrectionToolDefinition(ids); + assert.deepEqual( + (definition.parameters?.["properties"] as Record>)["id"]?.[ + "enum" + ], + ["claim-1", "claim-2"], + ); + assert.deepEqual(parseRetainCorrectionArguments(JSON.stringify({ id: "claim-2" }), ids), { + kind: "valid", + value: "claim-2", + }); + assert.equal( + parseRetainCorrectionArguments(JSON.stringify({ id: "claim-3" }), ids).kind, + "invalid", + ); +}); diff --git a/src/typescript/api/test/unit/openai-claim-validation-scheduler.test.ts b/src/typescript/api/test/unit/openai-claim-validation-scheduler.test.ts index 910c48c..99a7144 100644 --- a/src/typescript/api/test/unit/openai-claim-validation-scheduler.test.ts +++ b/src/typescript/api/test/unit/openai-claim-validation-scheduler.test.ts @@ -6,7 +6,8 @@ import { createInvestigationRunState, getConfirmedClaims, } from "../../src/lib/investigators/openai-investigation-run-state.js"; -import type { PerClaimValidationResult } from "../../src/lib/investigators/openai-claim-validator.js"; +import type { ClaimValidationResult } from "../../src/lib/investigators/openai-claim-validator.js"; +import { makeValidationRequestAudit } from "../helpers/fake-openai.js"; function makeClaim(label: string): InvestigationResult["claims"][number] { return { @@ -24,23 +25,11 @@ function makeClaim(label: string): InvestigationResult["claims"][number] { }; } -function makeValidationResult(claimIndex: number, approved: boolean): PerClaimValidationResult { +function makeValidationResult(claimIndex: number, approved: boolean): ClaimValidationResult { return { + kind: approved ? "approved" : "rejected", claimIndex, - approved, - responseAudit: { - responseId: `resp-${claimIndex.toString()}`, - responseStatus: "completed", - responseModelVersion: "test-model", - responseOutputText: JSON.stringify({ approved }), - outputItems: [], - outputTextParts: [], - outputTextAnnotations: [], - reasoningSummaries: [], - toolCalls: [], - usage: null, - }, - error: null, + request: makeValidationRequestAudit(claimIndex), }; } @@ -48,13 +37,13 @@ test("claim validation scheduler preserves submission ordering in confirmed clai const claimA = makeClaim("alpha"); const claimB = makeClaim("beta"); - const resolvers: ((result: PerClaimValidationResult) => void)[] = []; + const resolvers: ((result: ClaimValidationResult) => void)[] = []; const scheduler = createClaimValidationScheduler({ initialState: createInvestigationRunState({}), validationLimiter: async (task) => task(), runValidation: async (claimIndex) => - new Promise((resolve) => { + new Promise((resolve) => { resolvers[claimIndex] = resolve; }), }); diff --git a/src/typescript/api/test/unit/openai-claim-validator.test.ts b/src/typescript/api/test/unit/openai-claim-validator.test.ts index 81c6df8..fd35548 100644 --- a/src/typescript/api/test/unit/openai-claim-validator.test.ts +++ b/src/typescript/api/test/unit/openai-claim-validator.test.ts @@ -1,274 +1,99 @@ import assert from "node:assert/strict"; import { test } from "node:test"; -import type OpenAI from "openai"; import { InvestigatorIncompleteResponseError, - validateClaim, -} from "../../src/lib/investigators/openai-claim-validator.js"; -import type { InvestigationResult } from "@openerrata/shared"; - -/** - * Invariants under test: - * - * 1. **InvestigatorIncompleteResponseError carries diagnostic metadata**: The - * orchestrator uses responseStatus, responseId, incompleteReason, and - * outputTextLength to classify failures as transient vs. terminal. If any - * of these are silently dropped, the retry logic makes wrong decisions. - * - * 2. **Error message is grep-able**: Operator log triage needs all four - * diagnostic fields in the message string. Null fields must render as - * "unknown", not crash the constructor. - * - * 3. **validateClaim routes through discriminated result paths**: Success - * returns approved + audit + null error; API exception returns error result; - * incomplete response returns error result with responseAudit preserved. - */ - -function makeClaim(): InvestigationResult["claims"][number] { - return { - text: "The earth is flat.", - context: "In the article the author states the earth is flat.", - summary: "The earth is not flat.", - reasoning: "Scientific consensus establishes the earth as an oblate spheroid.", - sources: [{ url: "https://example.com", title: "Source", snippet: "Evidence" }], - }; -} - -function makeCompletedResponse(approved: boolean): Record { - return { - id: "resp_test_123", - status: "completed", - model: "gpt-4.1", - output: [ - { - type: "message", - id: "msg_1", - status: "completed", - role: "assistant", - content: [{ type: "output_text", text: JSON.stringify({ approved }) }], - }, - ], - output_text: JSON.stringify({ approved }), - usage: { - input_tokens: 100, - output_tokens: 50, - total_tokens: 150, - }, - }; -} - -function makeMockClient(responseOrError: Record | Error): OpenAI { - const responsesCreate = - responseOrError instanceof Error - ? () => Promise.reject(responseOrError) - : () => Promise.resolve(responseOrError); - - return { - responses: { create: responsesCreate }, - } as unknown as OpenAI; + InvestigatorStructuredOutputError, +} from "../../src/lib/investigators/errors.js"; +import { validateClaim } from "../../src/lib/investigators/openai-claim-validator.js"; +import { INVESTIGATION_REQUEST_CONFIG } from "../../src/lib/investigators/openai-request-config.js"; +import { INVESTIGATION_VALIDATION_SYSTEM_PROMPT } from "../../src/lib/investigators/prompt.js"; +import { + createFakeOpenAiClient, + makeClaim, + makeMessage, + makeResponse, + makeVerdictResponse, + type FakeOpenAiReply, +} from "../helpers/fake-openai.js"; + +async function validate(reply: FakeOpenAiReply) { + const { client, requests } = createFakeOpenAiClient(() => reply); + const result = await validateClaim({ + client, + requestConfig: INVESTIGATION_REQUEST_CONFIG, + claimIndex: 3, + claim: makeClaim("Alpha"), + contentText: "The post text.", + imageContextNotes: undefined, + signal: new AbortController().signal, + }); + return { result, requests }; } -const defaultReasoning = { effort: "low" as const, summary: "auto" as const }; - -test("InvestigatorIncompleteResponseError stores all diagnostic fields", () => { - const error = new InvestigatorIncompleteResponseError({ - responseStatus: "incomplete", - responseId: "resp_abc123", - incompleteReason: "max_output_tokens", - outputTextLength: 1024, +test("validateClaim sends a structured yes/no request without tools", async () => { + const { requests } = await validate({ + kind: "response", + response: makeVerdictResponse("resp_v", true), }); - assert.equal(error.responseStatus, "incomplete"); - assert.equal(error.responseId, "resp_abc123"); - assert.equal(error.incompleteReason, "max_output_tokens"); - assert.equal(error.outputTextLength, 1024); + const [request] = requests; + assert.ok(request); + assert.equal(request.body.model, "gpt-6.1-sol"); + assert.equal(request.body.instructions, INVESTIGATION_VALIDATION_SYSTEM_PROMPT); + assert.equal(request.body.tools, undefined); + assert.equal(request.body.text?.format?.type, "json_schema"); + assert.equal(typeof request.body.input, "string"); + assert.match(request.body.input as string, /Incorrect claim: Alpha/); }); -test("InvestigatorIncompleteResponseError message contains all diagnostic fields", () => { - const error = new InvestigatorIncompleteResponseError({ - responseStatus: "incomplete", - responseId: "resp_abc123", - incompleteReason: "max_output_tokens", - outputTextLength: 512, +test("validateClaim reports approved and rejected verdicts with the request audit", async () => { + const approved = await validate({ + kind: "response", + response: makeVerdictResponse("resp_yes", true), }); + assert.equal(approved.result.kind, "approved"); + assert.deepEqual(approved.result.request.subject, { kind: "CLAIM_VALIDATION", claimIndex: 3 }); + assert.equal(approved.result.request.response?.providerResponseId, "resp_yes"); - assert.match(error.message, /incomplete/); - assert.match(error.message, /resp_abc123/); - assert.match(error.message, /max_output_tokens/); - assert.match(error.message, /512/); -}); - -test("InvestigatorIncompleteResponseError renders null fields as 'unknown'", () => { - const error = new InvestigatorIncompleteResponseError({ - responseStatus: null, - responseId: null, - incompleteReason: null, - outputTextLength: 0, + const rejected = await validate({ + kind: "response", + response: makeVerdictResponse("resp_no", false), }); - - // Must not crash, and the message must be informative for log grep. - assert.match(error.message, /unknown/); - assert.equal(error.name, "InvestigatorIncompleteResponseError"); -}); - -// --- validateClaim --- - -test("validateClaim returns approved=true with responseAudit when model approves", async () => { - const client = makeMockClient(makeCompletedResponse(true)); - const result = await validateClaim( - client, - "gpt-4.1", - 0, - makeClaim(), - "Some post content.", - undefined, - defaultReasoning, - ); - - assert.equal(result.claimIndex, 0); - assert.equal(result.approved, true); - assert.equal(result.error, null); - assert.notEqual(result.responseAudit, null); - assert.equal(result.responseAudit.responseId, "resp_test_123"); - assert.equal(result.responseAudit.responseStatus, "completed"); -}); - -test("validateClaim returns approved=false with responseAudit when model rejects", async () => { - const client = makeMockClient(makeCompletedResponse(false)); - const result = await validateClaim( - client, - "gpt-4.1", - 3, - makeClaim(), - "Some post content.", - undefined, - defaultReasoning, - ); - - assert.equal(result.claimIndex, 3); - assert.equal(result.approved, false); - assert.equal(result.error, null); - assert.notEqual(result.responseAudit, null); + assert.equal(rejected.result.kind, "rejected"); }); -test("validateClaim returns error result when API call throws", async () => { - const client = makeMockClient(new Error("API rate limit exceeded")); - const result = await validateClaim( - client, - "gpt-4.1", - 1, - makeClaim(), - "Some content.", - undefined, - defaultReasoning, - ); +test("validateClaim reports a failed request with no response audit", async () => { + const { result } = await validate({ kind: "http_error", status: 500, message: "boom" }); - assert.equal(result.claimIndex, 1); - assert.equal(result.approved, false); - assert.notEqual(result.error, null); - assert.ok(result.error !== null); - assert.match(result.error.message, /API rate limit exceeded/); - // responseAudit is null when the API call itself fails - assert.equal(result.responseAudit, null); + assert.equal(result.kind, "failed"); + assert.equal(result.request.response, null); + assert.equal(result.error.message, "500 boom"); }); -test("validateClaim returns error result for incomplete response", async () => { - const incompleteResponse: Record = { - id: "resp_incomplete", - status: "incomplete", - model: "gpt-4.1", - output: [], - output_text: null, - incomplete_details: { reason: "max_output_tokens" }, - usage: { input_tokens: 100, output_tokens: 50, total_tokens: 150 }, - }; - const client = makeMockClient(incompleteResponse); - const result = await validateClaim( - client, - "gpt-4.1", - 2, - makeClaim(), - "Some content.", - undefined, - defaultReasoning, - ); +test("validateClaim fails on an incomplete response", async () => { + const { result } = await validate({ + kind: "response", + response: makeResponse({ + id: "resp_cut", + status: "incomplete", + incompleteReason: "max_output_tokens", + output: [], + }), + }); - assert.equal(result.claimIndex, 2); - assert.equal(result.approved, false); - assert.notEqual(result.error, null); + assert.equal(result.kind, "failed"); assert.ok(result.error instanceof InvestigatorIncompleteResponseError); - // responseAudit should still be captured even on incomplete response - assert.notEqual(result.responseAudit, null); - assert.equal(result.responseAudit?.responseId, "resp_incomplete"); -}); - -test("validateClaim returns error result when output_text is invalid JSON", async () => { - const response: Record = { - id: "resp_bad_json", - status: "completed", - model: "gpt-4.1", - output: [ - { - type: "message", - id: "msg_1", - status: "completed", - role: "assistant", - content: [{ type: "output_text", text: "not valid json" }], - }, - ], - output_text: "not valid json", - usage: { input_tokens: 50, output_tokens: 10, total_tokens: 60 }, - }; - const client = makeMockClient(response); - const result = await validateClaim( - client, - "gpt-4.1", - 0, - makeClaim(), - "Some content.", - undefined, - defaultReasoning, - ); - - assert.equal(result.approved, false); - assert.notEqual(result.error, null); - // responseAudit is preserved even on JSON parse error - assert.notEqual(result.responseAudit, null); + assert.equal(result.request.response?.status, "incomplete"); }); -test("validateClaim captures responseAudit even when structured output is malformed JSON object", async () => { - // Model returns valid JSON but with wrong structure — the error path must - // still preserve the responseAudit for operator debugging - const response: Record = { - id: "resp_bad_schema", - status: "completed", - model: "gpt-4.1", - output: [ - { - type: "message", - id: "msg_1", - status: "completed", - role: "assistant", - content: [{ type: "output_text", text: JSON.stringify({ wrong_field: true }) }], - }, - ], - output_text: JSON.stringify({ wrong_field: true }), - usage: { input_tokens: 50, output_tokens: 10, total_tokens: 60 }, - }; - const client = makeMockClient(response); - const result = await validateClaim( - client, - "gpt-4.1", - 0, - makeClaim(), - "Some content.", - undefined, - defaultReasoning, - ); - - assert.equal(result.approved, false); - assert.notEqual(result.error, null); - // Key invariant: responseAudit is preserved for logging even when parse fails - assert.notEqual(result.responseAudit, null); - assert.equal(result.responseAudit?.responseId, "resp_bad_schema"); +test("validateClaim fails when the verdict is not valid structured output", async () => { + for (const text of ["not json", JSON.stringify({ approved: "yes" }), ""]) { + const { result } = await validate({ + kind: "response", + response: makeResponse({ id: "resp_bad", output: [makeMessage("msg", text)] }), + }); + assert.equal(result.kind, "failed", text); + assert.ok(result.error instanceof InvestigatorStructuredOutputError); + assert.equal(result.request.response?.providerResponseId, "resp_bad"); + } }); diff --git a/src/typescript/api/test/unit/openai-input-builder.test.ts b/src/typescript/api/test/unit/openai-input-builder.test.ts index b25cdda..699df7b 100644 --- a/src/typescript/api/test/unit/openai-input-builder.test.ts +++ b/src/typescript/api/test/unit/openai-input-builder.test.ts @@ -89,7 +89,7 @@ test("buildInitialInput interleaves images at [IMAGE:N] placeholders with duplic ]; const contentOffset = userPrompt.indexOf(contentString); - const input = buildInitialInput( + const { request: input } = buildInitialInput( userPrompt, contentString, contentOffset, @@ -152,7 +152,7 @@ test("buildInitialInput falls back to text-offset interleaving when placeholders ]; const contentOffset = userPrompt.indexOf(contentString); - const input = buildInitialInput( + const { request: input } = buildInitialInput( userPrompt, contentString, contentOffset, @@ -244,6 +244,61 @@ test("buildInitialInput yields equivalent multimodal payload on retry placeholde ); }); +test("buildInitialInput audits image parts by content hash instead of data URI", () => { + const contentString = "Alpha Beta"; + const userPrompt = `prefix\n${contentString}\nsuffix`; + const imageOccurrences: InvestigatorImageOccurrence[] = [ + { + originalIndex: 0, + normalizedTextOffset: 6, + sourceUrl: "https://example.com/a.png", + resolution: "resolved", + imageDataUri: "data:image/png;base64,AAA", + contentHash: "hash-a", + }, + ]; + + const { request, audit } = buildInitialInput( + userPrompt, + contentString, + userPrompt.indexOf(contentString), + imageOccurrences, + undefined, + ); + + assert.deepEqual(request, [ + { + role: "user", + content: [ + { type: "input_text", text: "prefix\n" }, + { type: "input_text", text: "Alpha " }, + { type: "input_image", detail: "auto", image_url: "data:image/png;base64,AAA" }, + { type: "input_text", text: "Beta" }, + { type: "input_text", text: "\nsuffix" }, + ], + }, + ]); + assert.deepEqual(audit, [ + { + role: "user", + content: [ + { type: "input_text", text: "prefix\n" }, + { type: "input_text", text: "Alpha " }, + { type: "input_image", detail: "auto", imageContentHash: "hash-a" }, + { type: "input_text", text: "Beta" }, + { type: "input_text", text: "\nsuffix" }, + ], + }, + ]); +}); + +test("buildInitialInput sends and audits the plain prompt when there are no images", () => { + assert.deepEqual(buildInitialInput("prompt with content", "content", 12, [], undefined), { + request: "prompt with content", + audit: "prompt with content", + }); +}); + test("buildValidationImageContextNotes describes resolved duplicates distinctly", () => { const notes = buildValidationImageContextNotes([ { diff --git a/src/typescript/api/test/unit/openai-investigate-claim-order.test.ts b/src/typescript/api/test/unit/openai-investigate-claim-order.test.ts deleted file mode 100644 index 35ba2ba..0000000 --- a/src/typescript/api/test/unit/openai-investigate-claim-order.test.ts +++ /dev/null @@ -1,197 +0,0 @@ -import assert from "node:assert/strict"; -import { test } from "node:test"; -import type OpenAI from "openai"; -import { OpenAIInvestigator } from "../../src/lib/investigators/openai.js"; -import type { InvestigatorInput } from "../../src/lib/investigators/interface.js"; - -/** - * Invariant under test: - * - * The final `result.claims` array must be ordered by **submission order** - * (the order the model produced submit_correction / retain_correction tool - * calls), regardless of which per-claim validation promises settle first. - * - * Without explicit sorting, `confirmedClaims` is populated by `.then()` - * callbacks on bounded-concurrency promises, whose settlement order depends - * on wall-clock timing — a non-deterministic race. - */ - -function makeClaim(label: string) { - return { - text: `Incorrect claim: ${label}`, - context: `The article states ${label}`, - summary: `${label} is wrong because of evidence`, - reasoning: `Detailed reasoning for ${label}`, - sources: [ - { - url: `https://example.com/${label.toLowerCase()}`, - title: `Source ${label}`, - snippet: `Evidence for ${label}`, - }, - ], - }; -} - -function makeSubmitCorrectionFunctionCall(callId: string, claim: ReturnType) { - return { - type: "function_call", - id: `fc_${callId}`, - call_id: callId, - name: "submit_correction", - arguments: JSON.stringify(claim), - status: "completed", - }; -} - -function makeToolLoopResponse(claims: ReturnType[]) { - return { - id: "resp_round1", - status: "completed", - model: "test-model", - output: claims.map((claim, i) => - makeSubmitCorrectionFunctionCall(`call-${i.toString()}`, claim), - ), - output_text: null, - usage: { input_tokens: 200, output_tokens: 200, total_tokens: 400 }, - }; -} - -function makeFinalToolLoopResponse() { - return { - id: "resp_round2", - status: "completed", - model: "test-model", - output: [ - { type: "message", id: "msg_final", status: "completed", role: "assistant", content: [] }, - ], - output_text: null, - usage: { input_tokens: 50, output_tokens: 10, total_tokens: 60 }, - }; -} - -function makeValidationResponse(approved: boolean) { - return { - id: "resp_validation", - status: "completed", - model: "test-model", - output: [ - { - type: "message", - id: "msg_val", - status: "completed", - role: "assistant", - content: [{ type: "output_text", text: JSON.stringify({ approved }) }], - }, - ], - output_text: JSON.stringify({ approved }), - usage: { input_tokens: 50, output_tokens: 10, total_tokens: 60 }, - }; -} - -function delay(ms: number): Promise { - return new Promise((resolve) => { - setTimeout(resolve, ms); - }); -} - -const minimalInput: InvestigatorInput = { - contentText: "Some test content for fact-checking.", - platform: "LESSWRONG", - url: "https://www.lesswrong.com/posts/abc123/test-post", -}; - -test("investigate returns claims in submission order regardless of validation settlement order", async () => { - const claimA = makeClaim("Alpha"); - const claimB = makeClaim("Beta"); - const claimC = makeClaim("Gamma"); - - let toolLoopCallCount = 0; - let validationCallIndex = 0; - - const mockClient = { - responses: { - create: async (request: Record) => { - // Stage 2 validation calls use text.format (no tools array) - if (request["text"] !== undefined) { - const idx = validationCallIndex; - validationCallIndex += 1; - - // Settle in REVERSE order: claim 2 fastest, claim 0 slowest. - // With 3 claims and pLimit(4), all start immediately. - const delays = [40, 20, 5]; - await delay(delays[idx] ?? 0); - - return makeValidationResponse(true); - } - - // Stage 1 tool loop calls have a tools array - toolLoopCallCount += 1; - if (toolLoopCallCount === 1) { - return makeToolLoopResponse([claimA, claimB, claimC]); - } - return makeFinalToolLoopResponse(); - }, - }, - } as unknown as OpenAI; - - const investigator = new OpenAIInvestigator("fake-key", { - client: mockClient, - modelId: "test-model", - maxToolRounds: 10, - }); - - const output = await investigator.investigate(minimalInput); - - assert.equal(output.result.claims.length, 3); - // Claims must be in submission order (A, B, C) — not settlement order (C, B, A). - assert.equal(output.result.claims[0]?.text, claimA.text); - assert.equal(output.result.claims[1]?.text, claimB.text); - assert.equal(output.result.claims[2]?.text, claimC.text); -}); - -test("investigate preserves submission order when some validations reject", async () => { - const claimA = makeClaim("Alpha"); - const claimB = makeClaim("Beta"); - const claimC = makeClaim("Gamma"); - - let toolLoopCallCount = 0; - let validationCallIndex = 0; - - const mockClient = { - responses: { - create: async (request: Record) => { - if (request["text"] !== undefined) { - const idx = validationCallIndex; - validationCallIndex += 1; - - // Claim 0 (Alpha): slow, approved - // Claim 1 (Beta): fast, REJECTED - // Claim 2 (Gamma): medium, approved - const delays = [30, 5, 15]; - const approvals = [true, false, true]; - await delay(delays[idx] ?? 0); - return makeValidationResponse(approvals[idx] ?? false); - } - - toolLoopCallCount += 1; - if (toolLoopCallCount === 1) { - return makeToolLoopResponse([claimA, claimB, claimC]); - } - return makeFinalToolLoopResponse(); - }, - }, - } as unknown as OpenAI; - - const investigator = new OpenAIInvestigator("fake-key", { - client: mockClient, - modelId: "test-model", - maxToolRounds: 10, - }); - - const output = await investigator.investigate(minimalInput); - - // Beta was rejected; Alpha and Gamma approved — must be in submission order. - assert.equal(output.result.claims.length, 2); - assert.equal(output.result.claims[0]?.text, claimA.text); - assert.equal(output.result.claims[1]?.text, claimC.text); -}); diff --git a/src/typescript/api/test/unit/openai-investigation-run-state.test.ts b/src/typescript/api/test/unit/openai-investigation-run-state.test.ts index 8cc3de5..7459eed 100644 --- a/src/typescript/api/test/unit/openai-investigation-run-state.test.ts +++ b/src/typescript/api/test/unit/openai-investigation-run-state.test.ts @@ -9,7 +9,8 @@ import { retainOldClaim, settlePendingValidation, } from "../../src/lib/investigators/openai-investigation-run-state.js"; -import type { PerClaimValidationResult } from "../../src/lib/investigators/openai-claim-validator.js"; +import type { ClaimValidationResult } from "../../src/lib/investigators/openai-claim-validator.js"; +import { makeValidationRequestAudit } from "../helpers/fake-openai.js"; function makeClaim(label: string): InvestigationResult["claims"][number] { return { @@ -30,23 +31,11 @@ function makeClaim(label: string): InvestigationResult["claims"][number] { function makeValidationResult(input: { claimIndex: number; approved: boolean; -}): PerClaimValidationResult { +}): ClaimValidationResult { return { + kind: input.approved ? "approved" : "rejected", claimIndex: input.claimIndex, - approved: input.approved, - responseAudit: { - responseId: "resp-1", - responseStatus: "completed", - responseModelVersion: "test-model", - responseOutputText: JSON.stringify({ approved: input.approved }), - outputItems: [], - outputTextParts: [], - outputTextAnnotations: [], - reasoningSummaries: [], - toolCalls: [], - usage: null, - }, - error: null, + request: makeValidationRequestAudit(input.claimIndex), }; } @@ -119,3 +108,23 @@ test("retainOldClaim validates claim id and deduplicates retention", () => { reason: "unknown_id", }); }); + +test("settlePendingValidation rejects settling the same validation twice", () => { + const queued = enqueuePendingValidation(createInvestigationRunState({}), { + claim: makeClaim("alpha"), + promise: Promise.resolve(makeValidationResult({ claimIndex: 0, approved: true })), + }); + const settled = settlePendingValidation(queued.nextState, { + pendingIndex: queued.pendingIndex, + result: makeValidationResult({ claimIndex: 0, approved: true }), + }); + + assert.throws( + () => + settlePendingValidation(settled, { + pendingIndex: queued.pendingIndex, + result: makeValidationResult({ claimIndex: 0, approved: false }), + }), + /already settled/, + ); +}); diff --git a/src/typescript/api/test/unit/openai-investigator.test.ts b/src/typescript/api/test/unit/openai-investigator.test.ts new file mode 100644 index 0000000..517f95c --- /dev/null +++ b/src/typescript/api/test/unit/openai-investigator.test.ts @@ -0,0 +1,403 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { claimIdSchema } from "@openerrata/shared"; +import { + InvestigatorExecutionError, + InvestigatorIncompleteResponseError, + InvestigatorStructuredOutputError, +} from "../../src/lib/investigators/errors.js"; +import type { InvestigatorInput } from "../../src/lib/investigators/interface.js"; +import { OpenAIInvestigator } from "../../src/lib/investigators/openai.js"; +import { INVESTIGATION_REQUEST_CONFIG } from "../../src/lib/investigators/openai-request-config.js"; +import { + INVESTIGATION_SYSTEM_PROMPT, + INVESTIGATION_VALIDATION_SYSTEM_PROMPT, +} from "../../src/lib/investigators/prompt.js"; +import { isNonRetryableProviderError } from "../../src/lib/services/orchestrator-errors.js"; +import { + createFakeOpenAiClient, + isValidationRequest, + makeClaim, + makeFunctionCall, + makeMessage, + makeResponse, + makeVerdictResponse, + makeWebSearch, + type FakeOpenAiReply, + type RecordedOpenAiRequest, +} from "../helpers/fake-openai.js"; + +const minimalInput: InvestigatorInput = { + contentText: "Some test content for fact-checking.", + platform: "LESSWRONG", + url: "https://www.lesswrong.com/posts/abc123/test-post", +}; + +function delay(ms: number): Promise { + return new Promise((resolve) => { + setTimeout(resolve, ms); + }); +} + +function runOptions() { + return { signal: new AbortController().signal }; +} + +function createInvestigator( + reply: ( + request: RecordedOpenAiRequest, + index: number, + ) => FakeOpenAiReply | Promise, + maxToolRounds = 10, +) { + const fake = createFakeOpenAiClient(reply); + return { + investigator: new OpenAIInvestigator({ + client: fake.client, + requestConfig: INVESTIGATION_REQUEST_CONFIG, + maxToolRounds, + }), + requests: fake.requests, + }; +} + +/** Fact-check rounds answered from `rounds` in order; validations approve unless `approve` says otherwise. */ +function scriptedReplies(input: { + rounds: ReturnType[]; + approve?: (validationIndex: number) => boolean; + validationDelayMs?: (validationIndex: number) => number; +}) { + let roundIndex = 0; + let validationIndex = 0; + return async (request: RecordedOpenAiRequest): Promise => { + if (isValidationRequest(request)) { + const index = validationIndex; + validationIndex += 1; + await delay(input.validationDelayMs?.(index) ?? 0); + return { + kind: "response", + response: makeVerdictResponse( + `resp_validation_${index.toString()}`, + input.approve?.(index) ?? true, + ), + }; + } + const response = input.rounds[roundIndex]; + roundIndex += 1; + assert.ok(response, "unexpected extra fact-check round"); + return { kind: "response", response }; + }; +} + +async function rejectsWithExecutionError( + promise: Promise, +): Promise { + try { + await promise; + } catch (error) { + assert.ok(error instanceof InvestigatorExecutionError, `unexpected error: ${String(error)}`); + return error; + } + assert.fail("expected investigate() to reject"); +} + +test("fact-check rounds send the gpt-6.1-sol request shape and chain previous responses", async () => { + const { investigator, requests } = createInvestigator( + scriptedReplies({ + rounds: [ + makeResponse({ + id: "resp_round_0", + output: [makeFunctionCall("call-fetch", "fetch_url", { url: "not a url" })], + }), + makeResponse({ id: "resp_round_1", output: [makeMessage("msg_done", "Done.")] }), + ], + }), + ); + + await investigator.investigate(minimalInput, runOptions()); + + const [firstRound, secondRound] = requests; + assert.ok(firstRound && secondRound); + assert.equal(firstRound.path, "/v1/responses"); + assert.equal(firstRound.body.model, "gpt-6.1-sol"); + assert.equal(firstRound.body.instructions, INVESTIGATION_SYSTEM_PROMPT); + assert.deepEqual(firstRound.body.reasoning, { effort: "medium", summary: "detailed" }); + assert.deepEqual(firstRound.body.include, ["web_search_call.action.sources"]); + assert.deepEqual( + firstRound.body.tools?.map((tool) => (tool.type === "function" ? tool.name : tool.type)), + ["web_search", "fetch_url", "submit_correction"], + ); + assert.equal(firstRound.body.previous_response_id, undefined); + assert.equal(firstRound.body.max_output_tokens, undefined); + + assert.equal(secondRound.body.previous_response_id, "resp_round_0"); + assert.deepEqual(secondRound.body.tools, firstRound.body.tools); + const [fetchOutput] = Array.isArray(secondRound.body.input) ? secondRound.body.input : []; + assert.ok(fetchOutput?.type === "function_call_output"); + assert.equal(fetchOutput.call_id, "call-fetch"); +}); + +test("update investigations offer retain_correction limited to the prior claim ids", async () => { + const oldClaimId = claimIdSchema.parse("claim-old-1"); + const { investigator, requests } = createInvestigator( + scriptedReplies({ + rounds: [ + makeResponse({ + id: "resp_round_0", + output: [makeFunctionCall("call-retain", "retain_correction", { id: oldClaimId })], + }), + makeResponse({ id: "resp_round_1", output: [] }), + ], + }), + ); + + const output = await investigator.investigate( + { ...minimalInput, isUpdate: true, oldClaims: [{ id: oldClaimId, ...makeClaim("Old") }] }, + runOptions(), + ); + + const retainTool = requests[0]?.body.tools?.find( + (tool) => tool.type === "function" && tool.name === "retain_correction", + ); + assert.ok(retainTool?.type === "function"); + assert.deepEqual(retainTool.parameters?.["properties"], { + id: { + type: "string", + enum: [oldClaimId], + description: "The ID of the existing claim to retain.", + }, + }); + assert.deepEqual(output.result.claims, [makeClaim("Old")]); +}); + +test("investigate returns claims in submission order regardless of validation settlement order", async () => { + const claims = [makeClaim("Alpha"), makeClaim("Beta"), makeClaim("Gamma")]; + const { investigator } = createInvestigator( + scriptedReplies({ + rounds: [ + makeResponse({ + id: "resp_round_0", + output: claims.map((claim, index) => + makeFunctionCall(`call-${index.toString()}`, "submit_correction", claim), + ), + }), + makeResponse({ id: "resp_round_1", output: [] }), + ], + // Settle in reverse order: claim 2 fastest, claim 0 slowest. + validationDelayMs: (index) => [40, 20, 5][index] ?? 0, + }), + ); + + const output = await investigator.investigate(minimalInput, runOptions()); + + assert.deepEqual(output.result.claims, claims); +}); + +test("investigate preserves submission order when some validations reject", async () => { + const claims = [makeClaim("Alpha"), makeClaim("Beta"), makeClaim("Gamma")]; + const { investigator } = createInvestigator( + scriptedReplies({ + rounds: [ + makeResponse({ + id: "resp_round_0", + output: claims.map((claim, index) => + makeFunctionCall(`call-${index.toString()}`, "submit_correction", claim), + ), + }), + makeResponse({ id: "resp_round_1", output: [] }), + ], + approve: (index) => index !== 1, + validationDelayMs: (index) => [30, 5, 15][index] ?? 0, + }), + ); + + const output = await investigator.investigate(minimalInput, runOptions()); + + assert.deepEqual(output.result.claims, [claims[0], claims[2]]); +}); + +test("invalid claim submissions are rejected back to the model and never validated", async () => { + const unsafeClaim = { + ...makeClaim("Unsafe"), + sources: [{ url: "ftp://example.com/evidence.txt", title: "Source", snippet: "Snippet" }], + }; + const { investigator, requests } = createInvestigator( + scriptedReplies({ + rounds: [ + makeResponse({ + id: "resp_round_0", + output: [makeFunctionCall("call-unsafe", "submit_correction", unsafeClaim)], + }), + makeResponse({ id: "resp_round_1", output: [] }), + ], + }), + ); + + const output = await investigator.investigate(minimalInput, runOptions()); + + assert.deepEqual(output.result.claims, []); + assert.equal(requests.filter(isValidationRequest).length, 0); + const [rejection] = Array.isArray(requests[1]?.body.input) ? requests[1].body.input : []; + assert.ok(rejection?.type === "function_call_output"); + assert.equal(rejection.call_id, "call-unsafe"); + assert.equal(typeof rejection.output, "string"); + assert.match(rejection.output as string, /Invalid claim, not recorded/); +}); + +test("the attempt audit records every provider request, with model and modelVersion from stage 1", async () => { + const { investigator } = createInvestigator( + scriptedReplies({ + rounds: [ + makeResponse({ + id: "resp_round_0", + model: "gpt-6.1-sol-2026-09-01", + output: [ + makeWebSearch("ws_1", "moon distance", ["https://nasa.gov/moon"]), + makeFunctionCall("call-0", "submit_correction", makeClaim("Alpha")), + ], + }), + makeResponse({ id: "resp_round_1", model: "gpt-6.1-sol-2026-09-01", output: [] }), + ], + }), + ); + + const output = await investigator.investigate(minimalInput, runOptions()); + + assert.equal(output.model, "gpt-6.1-sol"); + assert.equal(output.modelVersion, "gpt-6.1-sol-2026-09-01"); + assert.equal(output.attemptAudit.outcome, "SUCCEEDED"); + assert.deepEqual( + output.attemptAudit.requests.map((request) => request.subject), + [ + { kind: "FACT_CHECK_ROUND", round: 0 }, + { kind: "FACT_CHECK_ROUND", round: 1 }, + { kind: "CLAIM_VALIDATION", claimIndex: 0 }, + ], + ); + const [round0, round1, validation] = output.attemptAudit.requests; + assert.ok(round0 && round1 && validation); + assert.equal(typeof round0.input, "string"); + assert.match(round0.input as string, /Some test content for fact-checking\./); + assert.equal(round0.previousResponseId, null); + assert.equal(round1.previousResponseId, "resp_round_0"); + assert.deepEqual(round0.include, ["web_search_call.action.sources"]); + assert.equal(round0.response?.outputItems[0]?.content.kind, "TOOL_CALL"); + assert.equal(validation.instructions, INVESTIGATION_VALIDATION_SYSTEM_PROMPT); + assert.equal(typeof validation.input, "string"); + assert.match(validation.input as string, /Incorrect claim: Alpha/); + assert.deepEqual(validation.tools, []); + assert.equal(validation.response?.providerResponseId, "resp_validation_0"); +}); + +test("the round limit fails the attempt without running the last round's tool calls", async () => { + const { investigator, requests } = createInvestigator( + scriptedReplies({ + rounds: [ + makeResponse({ + id: "resp_round_0", + output: [makeFunctionCall("call-0", "submit_correction", makeClaim("Alpha"))], + }), + makeResponse({ + id: "resp_round_1", + output: [makeFunctionCall("call-1", "submit_correction", makeClaim("Beta"))], + }), + ], + }), + 2, + ); + + const error = await rejectsWithExecutionError( + investigator.investigate(minimalInput, runOptions()), + ); + + assert.ok(error.cause instanceof InvestigatorStructuredOutputError); + assert.match(error.cause.message, /round limit \(2\)/); + assert.equal(isNonRetryableProviderError(error), true); + // Only the first round's claim was validated; the second was never acknowledged. + assert.equal(requests.filter(isValidationRequest).length, 1); + assert.deepEqual( + error.attemptAudit.requests.map((request) => request.subject.kind), + ["FACT_CHECK_ROUND", "FACT_CHECK_ROUND", "CLAIM_VALIDATION"], + ); + assert.equal(error.attemptAudit.outcome, "FAILED"); +}); + +test("an incomplete fact-check response fails the attempt non-retryably", async () => { + const { investigator } = createInvestigator( + scriptedReplies({ + rounds: [ + makeResponse({ + id: "resp_round_0", + status: "incomplete", + incompleteReason: "max_output_tokens", + output: [makeMessage("msg_partial", "Partial")], + }), + ], + }), + ); + + const error = await rejectsWithExecutionError( + investigator.investigate(minimalInput, runOptions()), + ); + + assert.ok(error.cause instanceof InvestigatorIncompleteResponseError); + assert.equal(error.cause.responseStatus, "incomplete"); + assert.equal(error.cause.incompleteReason, "max_output_tokens"); + assert.equal(isNonRetryableProviderError(error), true); + assert.equal(error.attemptAudit.error.errorName, "InvestigatorIncompleteResponseError"); + assert.equal(error.attemptAudit.requests[0]?.response?.status, "incomplete"); +}); + +test("a failed validation fails the attempt after recording every request", async () => { + const { investigator } = createInvestigator(async (request) => { + if (isValidationRequest(request)) { + return { kind: "http_error", status: 500, message: "upstream failure" }; + } + return request.body.previous_response_id === undefined + ? { + kind: "response", + response: makeResponse({ + id: "resp_round_0", + output: [makeFunctionCall("call-0", "submit_correction", makeClaim("Alpha"))], + }), + } + : { kind: "response", response: makeResponse({ id: "resp_round_1", output: [] }) }; + }); + + const error = await rejectsWithExecutionError( + investigator.investigate(minimalInput, runOptions()), + ); + + assert.match(error.message, /claim indices: 0/); + assert.equal(isNonRetryableProviderError(error), false); + assert.equal(error.attemptAudit.error.statusCode, 500); + const validation = error.attemptAudit.requests[2]; + assert.deepEqual(validation?.subject, { kind: "CLAIM_VALIDATION", claimIndex: 0 }); + assert.equal(validation.response, null); +}); + +test("an aborted run fails with the audit of the request it interrupted", async () => { + const { investigator } = createInvestigator(() => ({ kind: "hang" })); + const controller = new AbortController(); + + const run = investigator.investigate(minimalInput, { signal: controller.signal }); + controller.abort(); + const error = await rejectsWithExecutionError(run); + + assert.equal(error.attemptAudit.requests.length, 1); + assert.equal(error.attemptAudit.requests[0]?.response, null); + assert.equal(error.attemptAudit.error.errorName, "APIUserAbortError"); +}); + +test("the constructor rejects a non-positive tool round budget", () => { + const { client } = createFakeOpenAiClient(() => ({ kind: "hang" })); + assert.throws( + () => + new OpenAIInvestigator({ + client, + requestConfig: INVESTIGATION_REQUEST_CONFIG, + maxToolRounds: 0, + }), + /maxToolRounds must be a positive integer/, + ); +}); diff --git a/src/typescript/api/test/unit/openai-key-validation-core.test.ts b/src/typescript/api/test/unit/openai-key-validation-core.test.ts index 73ea8f8..72c28fd 100644 --- a/src/typescript/api/test/unit/openai-key-validation-core.test.ts +++ b/src/typescript/api/test/unit/openai-key-validation-core.test.ts @@ -1,102 +1,100 @@ import assert from "node:assert/strict"; import { test } from "node:test"; -import { validateOpenAiApiKeyForSettingsWithReachability } from "../../src/lib/services/openai-key-validation-core.js"; +import { validateOpenAiApiKeyForSettingsWithClient } from "../../src/lib/services/openai-key-validation-core.js"; +import { + createFakeOpenAiClient, + makeResponse, + type FakeOpenAiReply, +} from "../helpers/fake-openai.js"; -test("validateOpenAiApiKeyForSettingsWithReachability returns missing for empty values", async () => { - const result = await validateOpenAiApiKeyForSettingsWithReachability(" ", async () => { - throw new Error("should not run"); - }); +const WELL_FORMED_KEY = "sk-test-key-abcdefghijklmnopqrstuvwxyz"; - assert.deepEqual(result, { openaiApiKeyStatus: "missing" }); -}); +async function validateWithReply(reply: FakeOpenAiReply, timeoutMs?: number) { + const fake = createFakeOpenAiClient(() => reply, timeoutMs === undefined ? {} : { timeoutMs }); + const result = await validateOpenAiApiKeyForSettingsWithClient( + WELL_FORMED_KEY, + () => fake.client, + ); + return { result, requests: fake.requests }; +} -test("validateOpenAiApiKeyForSettingsWithReachability returns format_invalid for malformed keys", async () => { - const result = await validateOpenAiApiKeyForSettingsWithReachability("invalid-key", async () => { - throw new Error("should not run"); +test("missing and malformed keys are reported without calling OpenAI", async () => { + const neverCalled = () => assert.fail("OpenAI must not be called"); + assert.deepEqual(await validateOpenAiApiKeyForSettingsWithClient(" ", neverCalled), { + openaiApiKeyStatus: "missing", }); - - assert.deepEqual(result, { + assert.deepEqual(await validateOpenAiApiKeyForSettingsWithClient("invalid-key", neverCalled), { openaiApiKeyStatus: "format_invalid", openaiApiKeyMessage: "OpenAI API keys must begin with sk- and include the full token value.", }); }); -test("validateOpenAiApiKeyForSettingsWithReachability returns valid on successful reachability check", async () => { - const result = await validateOpenAiApiKeyForSettingsWithReachability( - "sk-valid-test-key-abcdefghijklmnopqrstuvwxyz", - async () => {}, - ); +test("a key is valid when OpenAI accepts the investigation request probe", async () => { + const { result, requests } = await validateWithReply({ + kind: "response", + // The probe's tiny output cap usually leaves the response incomplete. + response: makeResponse({ id: "resp_probe", status: "incomplete", output: [] }), + }); assert.deepEqual(result, { openaiApiKeyStatus: "valid" }); + const [probe] = requests; + assert.ok(probe); + assert.equal(probe.body.model, "gpt-6.1-sol"); + assert.equal(probe.body.tool_choice, "none"); + assert.equal(probe.body.max_output_tokens, 16); + assert.deepEqual(probe.body.include, ["web_search_call.action.sources"]); + assert.deepEqual(probe.body.reasoning, { effort: "medium", summary: "detailed" }); + assert.deepEqual( + probe.body.tools?.map((tool) => (tool.type === "function" ? tool.name : tool.type)), + ["web_search", "fetch_url", "submit_correction"], + ); }); -test("validateOpenAiApiKeyForSettingsWithReachability maps known OpenAI status codes", async () => { - const invalidResult = await validateOpenAiApiKeyForSettingsWithReachability( - "sk-invalid-test-key-abcdefghijklmnopqrstuvwxyz", - async () => { - const error = new Error("unauthorized") as Error & { status: number }; - error.status = 401; - throw error; - }, +test("OpenAI authentication and permission failures map to key statuses", async () => { + assert.deepEqual( + (await validateWithReply({ kind: "http_error", status: 401, message: "Incorrect API key" })) + .result, + { openaiApiKeyStatus: "invalid", openaiApiKeyMessage: "OpenAI rejected this API key." }, ); - assert.deepEqual(invalidResult, { - openaiApiKeyStatus: "invalid", - openaiApiKeyMessage: "OpenAI rejected this API key.", - }); - - const restrictedResult = await validateOpenAiApiKeyForSettingsWithReachability( - "sk-restricted-test-key-abcdefghijklmnopqrstuvwxyz", - async () => { - const error = new Error("forbidden") as Error & { status: number }; - error.status = 403; - throw error; + assert.deepEqual( + (await validateWithReply({ kind: "http_error", status: 403, message: "Forbidden" })).result, + { + openaiApiKeyStatus: "authenticated_restricted", + openaiApiKeyMessage: + "OpenAI authenticated this key, but access is restricted for validation checks.", }, ); - assert.deepEqual(restrictedResult, { - openaiApiKeyStatus: "authenticated_restricted", - openaiApiKeyMessage: - "OpenAI authenticated this key, but access is restricted for validation checks.", - }); +}); - const modelNotFoundResult = await validateOpenAiApiKeyForSettingsWithReachability( - "sk-model-test-key-abcdefghijklmnopqrstuvwxyz", - async () => { - const error = new Error("The requested model 'gpt-5.4-thinking' does not exist.") as Error & { - status: number; - }; - error.status = 400; - throw error; - }, - ); - assert.deepEqual(modelNotFoundResult, { +test("other OpenAI rejections are reported with OpenAI's explanation", async () => { + const { result } = await validateWithReply({ + kind: "http_error", + status: 404, + message: "The model `gpt-6.1-sol` does not exist or you do not have access to it.", + }); + assert.deepEqual(result, { openaiApiKeyStatus: "error", - openaiApiKeyMessage: "The requested model 'gpt-5.4-thinking' does not exist.", + openaiApiKeyMessage: + "404 The model `gpt-6.1-sol` does not exist or you do not have access to it.", }); }); -test("validateOpenAiApiKeyForSettingsWithReachability handles timeout and generic failures", async () => { - const timeoutResult = await validateOpenAiApiKeyForSettingsWithReachability( - "sk-timeout-test-key-abcdefghijklmnopqrstuvwxyz", - async () => { - const error = new Error("timed out"); - error.name = "AbortError"; - throw error; - }, - ); - assert.deepEqual(timeoutResult, { +test("a probe that times out reports the timeout", async () => { + const { result } = await validateWithReply({ kind: "hang" }, 20); + assert.deepEqual(result, { openaiApiKeyStatus: "error", openaiApiKeyMessage: "OpenAI key validation timed out. Confirm outbound network access and retry.", }); +}); - const genericResult = await validateOpenAiApiKeyForSettingsWithReachability( - "sk-generic-test-key-abcdefghijklmnopqrstuvwxyz", - async () => { - throw new Error("network is blocked"); - }, - ); - assert.deepEqual(genericResult, { +test("a probe response that failed server-side is not a valid key", async () => { + const { result } = await validateWithReply({ + kind: "response", + response: makeResponse({ id: "resp_probe", status: "failed", output: [] }), + }); + assert.deepEqual(result, { openaiApiKeyStatus: "error", - openaiApiKeyMessage: "network is blocked", + openaiApiKeyMessage: "OpenAI probe response resp_probe ended with status failed", }); }); diff --git a/src/typescript/api/test/unit/openai-response-audit.test.ts b/src/typescript/api/test/unit/openai-response-audit.test.ts index a07ec7c..0f20f01 100644 --- a/src/typescript/api/test/unit/openai-response-audit.test.ts +++ b/src/typescript/api/test/unit/openai-response-audit.test.ts @@ -1,796 +1,194 @@ import assert from "node:assert/strict"; import { test } from "node:test"; +import OpenAI from "openai"; import { - sanitizeJsonValue, - sanitizeJsonRecord, - describeJsonValueType, - requireJsonObject, - readString, - readOptionalInteger, - readIncompleteReason, - parseTimestamp, - findTimestamp, - extractRequestedTools, - extractOutputItems, - extractOutputTextArtifacts, - extractReasoningSummaries, - extractToolCalls, - extractUsage, - extractResponseAudit, - offsetResponseAuditIndices, - aggregateUsage, - mergeResponseAudits, + INVESTIGATION_REQUEST_CONFIG, + buildFactCheckRequestParams, + buildFactCheckTools, +} from "../../src/lib/investigators/openai-request-config.js"; +import { + auditRequest, + auditResponse, buildErrorAudit, - requireCompletedOutputText, } from "../../src/lib/investigators/openai-response-audit.js"; -import { InvestigatorStructuredOutputError } from "../../src/lib/investigators/openai.js"; -import type { InvestigatorResponseAudit } from "../../src/lib/investigators/interface.js"; - -// --- sanitizeJsonValue --- - -test("sanitizeJsonValue preserves primitives and null", () => { - assert.equal(sanitizeJsonValue("hello"), "hello"); - assert.equal(sanitizeJsonValue(42), 42); - assert.equal(sanitizeJsonValue(true), true); - assert.equal(sanitizeJsonValue(null), null); -}); - -test("sanitizeJsonValue sanitizes arrays recursively", () => { - assert.deepStrictEqual(sanitizeJsonValue([1, "two", null]), [1, "two", null]); -}); - -test("sanitizeJsonValue sanitizes objects recursively", () => { - assert.deepStrictEqual(sanitizeJsonValue({ a: 1, b: "x" }), { a: 1, b: "x" }); -}); - -test("sanitizeJsonValue converts bigint to string", () => { - assert.equal(sanitizeJsonValue(123n), "123"); -}); - -test("sanitizeJsonValue converts symbol to description", () => { - assert.equal(sanitizeJsonValue(Symbol("test")), "test"); -}); - -test("sanitizeJsonValue converts function to placeholder", () => { - assert.equal( - sanitizeJsonValue(() => {}), - "[function]", - ); -}); - -test("sanitizeJsonValue truncates at max depth", () => { - assert.equal(sanitizeJsonValue({ nested: true }, 9), "[max-depth]"); -}); - -test("sanitizeJsonValue returns [unsupported] for undefined", () => { - assert.equal(sanitizeJsonValue(undefined), "[unsupported]"); -}); - -// --- sanitizeJsonRecord --- - -test("sanitizeJsonRecord sanitizes object entries", () => { - const result = sanitizeJsonRecord({ a: 1, b: "two" }); - assert.deepStrictEqual(result, { a: 1, b: "two" }); -}); - -test("sanitizeJsonRecord throws for non-object input", () => { - assert.throws(() => sanitizeJsonRecord("not an object"), InvestigatorStructuredOutputError); -}); - -// --- describeJsonValueType --- - -test("describeJsonValueType returns correct types", () => { - assert.equal(describeJsonValueType(null), "null"); - assert.equal(describeJsonValueType([]), "array"); - assert.equal(describeJsonValueType({}), "object"); - assert.equal(describeJsonValueType("str"), "string"); - assert.equal(describeJsonValueType(42), "number"); - assert.equal(describeJsonValueType(true), "boolean"); -}); - -// --- requireJsonObject --- - -test("requireJsonObject returns object for valid input", () => { - const obj = { a: 1 }; - assert.deepStrictEqual(requireJsonObject(obj, "test"), obj); -}); - -test("requireJsonObject throws for null", () => { - assert.throws( - () => requireJsonObject(null, "test"), - (error: unknown) => - error instanceof InvestigatorStructuredOutputError && error.message.includes("null"), - ); -}); - -test("requireJsonObject throws for array", () => { - assert.throws( - () => requireJsonObject([], "test"), - (error: unknown) => - error instanceof InvestigatorStructuredOutputError && error.message.includes("array"), - ); -}); - -// --- requireCompletedOutputText --- - -test("requireCompletedOutputText throws when output text is null", () => { - assert.throws( - () => - requireCompletedOutputText({ - responseAudit: makeEmptyResponseAudit({ responseOutputText: null }), - responseRecord: {}, - context: "Test", - }), - InvestigatorStructuredOutputError, - ); -}); - -test("requireCompletedOutputText throws when output text is empty", () => { - assert.throws( - () => - requireCompletedOutputText({ - responseAudit: makeEmptyResponseAudit({ responseOutputText: " " }), - responseRecord: {}, - context: "Test", - }), - InvestigatorStructuredOutputError, - ); -}); - -test("requireCompletedOutputText returns output text when present", () => { - const text = requireCompletedOutputText({ - responseAudit: makeEmptyResponseAudit({ responseOutputText: '{"claims":[]}' }), - responseRecord: {}, - context: "Test", - }); - assert.equal(text, '{"claims":[]}'); -}); - -// --- readString / readOptionalInteger --- - -test("readString returns string for string input", () => { - assert.equal(readString("hello"), "hello"); -}); - -test("readString returns null for non-string input", () => { - assert.equal(readString(42), null); - assert.equal(readString(null), null); - assert.equal(readString(undefined), null); -}); - -test("readOptionalInteger returns integer for integer input", () => { - assert.equal(readOptionalInteger(42), 42); -}); - -test("readOptionalInteger returns null for non-integer", () => { - assert.equal(readOptionalInteger(3.14), null); - assert.equal(readOptionalInteger("42"), null); - assert.equal(readOptionalInteger(null), null); -}); - -// --- readIncompleteReason --- - -test("readIncompleteReason extracts reason from incomplete_details", () => { - const record = { incomplete_details: { reason: "max_output_tokens" } }; - assert.equal(readIncompleteReason(record), "max_output_tokens"); -}); - -test("readIncompleteReason returns null when incomplete_details is missing", () => { - assert.equal(readIncompleteReason({}), null); -}); - -test("readIncompleteReason returns null when reason is not a string", () => { - assert.equal(readIncompleteReason({ incomplete_details: { reason: 42 } }), null); -}); - -// --- buildErrorAudit --- - -test("buildErrorAudit handles Error instance", () => { - const error = new TypeError("bad type"); - const audit = buildErrorAudit(error); - assert.equal(audit.errorName, "TypeError"); - assert.equal(audit.errorMessage, "bad type"); - assert.equal(audit.statusCode, null); -}); - -test("buildErrorAudit handles string error", () => { - const audit = buildErrorAudit("string error"); - assert.equal(audit.errorName, "UnknownError"); - assert.equal(audit.errorMessage, "string error"); -}); - -test("buildErrorAudit handles unknown value", () => { - const audit = buildErrorAudit(42); - assert.equal(audit.errorName, "UnknownError"); - assert.equal(audit.errorMessage, "unknown"); -}); - -test("buildErrorAudit reads status from error with status property", () => { - const error = Object.assign(new Error("api error"), { status: 429 }); - const audit = buildErrorAudit(error); - assert.equal(audit.statusCode, 429); -}); - -// --- parseTimestamp --- - -test("parseTimestamp parses unix seconds", () => { - const result = parseTimestamp(1700000000); - assert.ok(result !== null); - assert.ok(result.includes("2023-11-14")); -}); - -test("parseTimestamp parses unix milliseconds", () => { - const result = parseTimestamp(1700000000000); - assert.ok(result !== null); - assert.ok(result.includes("2023-11-14")); -}); - -test("parseTimestamp parses ISO string", () => { - const result = parseTimestamp("2024-01-15T10:30:00Z"); - assert.ok(result !== null); - assert.ok(result.includes("2024-01-15")); -}); - -test("parseTimestamp returns null for empty string", () => { - assert.equal(parseTimestamp(" "), null); -}); - -test("parseTimestamp returns null for non-numeric non-string", () => { - assert.equal(parseTimestamp(null), null); - assert.equal(parseTimestamp(undefined), null); - assert.equal(parseTimestamp(true), null); -}); - -test("parseTimestamp returns null for invalid date string", () => { - assert.equal(parseTimestamp("not-a-date"), null); -}); - -test("parseTimestamp returns null for NaN", () => { - assert.equal(parseTimestamp(NaN), null); -}); - -test("parseTimestamp returns null for Infinity", () => { - assert.equal(parseTimestamp(Infinity), null); -}); - -// --- findTimestamp --- - -test("findTimestamp finds timestamp at matching key", () => { - const value = { created_at: 1700000000 }; - const result = findTimestamp(value, new Set(["created_at"])); - assert.ok(result !== null); -}); - -test("findTimestamp finds nested timestamp", () => { - const value = { outer: { inner: { started_at: "2024-01-01T00:00:00Z" } } }; - const result = findTimestamp(value, new Set(["started_at"])); - assert.ok(result !== null); -}); - -test("findTimestamp returns null when no match", () => { - const result = findTimestamp({ foo: "bar" }, new Set(["started_at"])); - assert.equal(result, null); -}); - -test("findTimestamp respects max depth", () => { - const value = { a: { b: { c: { d: { e: { f: { g: { started_at: 1700000000 } } } } } } } }; - const result = findTimestamp(value, new Set(["started_at"])); - assert.equal(result, null); -}); - -test("findTimestamp searches arrays", () => { - const value = [{ started_at: 1700000000 }]; - const result = findTimestamp(value, new Set(["started_at"])); - assert.ok(result !== null); -}); - -// --- extractRequestedTools --- - -test("extractRequestedTools extracts tools from array", () => { - const tools = [{ type: "web_search_preview" }, { type: "function", name: "fetch_url" }]; - const result = extractRequestedTools(tools); - assert.equal(result.length, 2); - assert.ok(result[0]); - assert.equal(result[0].requestOrder, 0); - assert.equal(result[0].toolType, "web_search_preview"); - assert.ok(result[1]); - assert.equal(result[1].requestOrder, 1); - assert.equal(result[1].toolType, "function"); -}); - -test("extractRequestedTools returns empty for non-array", () => { - assert.deepStrictEqual(extractRequestedTools(null), []); - assert.deepStrictEqual(extractRequestedTools(undefined), []); -}); - -test("extractRequestedTools skips non-object entries", () => { - const tools = ["not-an-object", { type: "function" }]; - const result = extractRequestedTools(tools); - assert.equal(result.length, 1); -}); - -// --- extractOutputItems --- - -test("extractOutputItems extracts items with id and status", () => { - const items = [ - { type: "message", id: "msg-1", status: "completed" }, - { type: "function_call", id: "fc-1", status: "completed" }, - ]; - const result = extractOutputItems(items); - assert.equal(result.length, 2); - assert.ok(result[0]); - assert.equal(result[0].providerItemId, "msg-1"); - assert.equal(result[0].itemStatus, "completed"); -}); - -test("extractOutputItems handles missing id by nulling both id and status", () => { - const items = [{ type: "message", status: "completed" }]; - const result = extractOutputItems(items); - assert.ok(result[0]); - assert.equal(result[0].providerItemId, null); - assert.equal(result[0].itemStatus, null); -}); - -test("extractOutputItems handles non-object entries", () => { - const items = ["not-an-object"]; - const result = extractOutputItems(items); - assert.ok(result[0]); - assert.equal(result[0].itemType, "unknown"); - assert.equal(result[0].providerItemId, null); -}); - -// --- extractOutputTextArtifacts --- - -test("extractOutputTextArtifacts extracts text parts from message output", () => { - const items = [ - { - type: "message", - content: [{ type: "output_text", text: "Hello world" }], - }, - ]; - const { parts, annotations } = extractOutputTextArtifacts(items); - assert.equal(parts.length, 1); - assert.ok(parts[0]); - assert.equal(parts[0].text, "Hello world"); - assert.equal(annotations.length, 0); -}); - -test("extractOutputTextArtifacts extracts annotations", () => { - const items = [ - { - type: "message", - content: [ +import { makeResponse, makeWebSearch } from "../helpers/fake-openai.js"; + +const RECEIVED_AT = new Date("2026-10-02T12:00:00.000Z"); + +test("auditResponse nests text parts, citations, summaries and tool payloads under their items", () => { + const webSearch = makeWebSearch("ws_1", "moon distance", ["https://nasa.gov/moon"]); + const audit = auditResponse( + makeResponse({ + id: "resp_1", + model: "gpt-6.1-sol-2026-09-01", + output: [ + // Reasoning items carry an id but no status; the id is still recorded. + { type: "reasoning", id: "rs_1", summary: [{ type: "summary_text", text: "Plan." }] }, + webSearch, + { + type: "function_call", + call_id: "call-1", + name: "fetch_url", + arguments: "{}", + }, { - type: "output_text", - text: "See [link]", - annotations: [ + type: "message", + id: "msg_1", + role: "assistant", + status: "completed", + content: [ { - type: "url_citation", - start_index: 4, - end_index: 10, - url: "https://example.com", - title: "Example", + type: "output_text", + text: "The moon is far.", + annotations: [ + { + type: "url_citation", + start_index: 0, + end_index: 16, + url: "https://nasa.gov/moon", + title: "Moon", + }, + ], }, + { type: "refusal", refusal: "No." }, ], }, ], - }, - ]; - const { parts, annotations } = extractOutputTextArtifacts(items); - assert.equal(parts.length, 1); - assert.equal(annotations.length, 1); - assert.ok(annotations[0]); - assert.equal(annotations[0].url, "https://example.com"); - assert.deepStrictEqual(annotations[0].characterPosition, { start: 4, end: 10 }); -}); + }), + RECEIVED_AT, + ); -test("extractOutputTextArtifacts extracts refusal parts", () => { - const items = [ + assert.equal(audit.providerResponseId, "resp_1"); + assert.equal(audit.modelVersion, "gpt-6.1-sol-2026-09-01"); + assert.equal(audit.status, "completed"); + assert.equal(audit.receivedAt, RECEIVED_AT); + assert.deepEqual(audit.outputItems, [ { - type: "message", - content: [{ type: "refusal", refusal: "I cannot help with that" }], + providerItemId: "rs_1", + itemType: "reasoning", + itemStatus: null, + content: { kind: "REASONING", summaries: ["Plan."] }, }, - ]; - const { parts } = extractOutputTextArtifacts(items); - assert.equal(parts.length, 1); - assert.ok(parts[0]); - assert.equal(parts[0].partType, "refusal"); - assert.equal(parts[0].text, "I cannot help with that"); -}); - -test("extractOutputTextArtifacts skips non-message output items", () => { - const items = [{ type: "reasoning", summary: [] }, { type: "function_call" }]; - const { parts } = extractOutputTextArtifacts(items); - assert.equal(parts.length, 0); -}); - -// --- extractReasoningSummaries --- - -test("extractReasoningSummaries extracts summaries from reasoning items", () => { - const items = [ { - type: "reasoning", - summary: [{ text: "Step 1 reasoning" }, { text: "Step 2 reasoning" }], + providerItemId: "ws_1", + itemType: "web_search_call", + itemStatus: "completed", + content: { kind: "TOOL_CALL", rawPayload: JSON.parse(JSON.stringify(webSearch)) }, }, - ]; - const result = extractReasoningSummaries(items); - assert.equal(result.length, 2); - assert.ok(result[0]); - assert.equal(result[0].text, "Step 1 reasoning"); - assert.ok(result[1]); - assert.equal(result[1].summaryIndex, 1); -}); - -test("extractReasoningSummaries skips empty text", () => { - const items = [ { - type: "reasoning", - summary: [{ text: "" }, { text: "valid" }], - }, - ]; - const result = extractReasoningSummaries(items); - assert.equal(result.length, 1); -}); - -// --- extractToolCalls --- - -test("extractToolCalls extracts function_call items", () => { - const items = [ - { type: "message", id: "msg-1", status: "completed" }, - { - type: "function_call", - id: "fc-1", - status: "completed", - started_at: 1700000000, - completed_at: 1700000001, - }, - ]; - const result = extractToolCalls(items); - assert.equal(result.length, 1); - assert.ok(result[0]); - assert.equal(result[0].toolType, "function_call"); - assert.equal(result[0].providerToolCallId, "fc-1"); -}); - -test("extractToolCalls skips message and reasoning types", () => { - const items = [ - { type: "message", id: "msg-1", status: "completed" }, - { type: "reasoning", id: "r-1", status: "completed" }, - ]; - const result = extractToolCalls(items); - assert.equal(result.length, 0); -}); - -// --- extractUsage --- - -test("extractUsage extracts usage data", () => { - const record = { - usage: { - input_tokens: 100, - output_tokens: 50, - total_tokens: 150, - input_tokens_details: { cached_tokens: 20 }, - output_tokens_details: { reasoning_tokens: 10 }, - }, - }; - const result = extractUsage(record); - assert.ok(result !== null); - assert.equal(result.inputTokens, 100); - assert.equal(result.outputTokens, 50); - assert.equal(result.totalTokens, 150); - assert.equal(result.cachedInputTokens, 20); - assert.equal(result.reasoningOutputTokens, 10); -}); - -test("extractUsage returns null when usage is missing", () => { - assert.equal(extractUsage({}), null); -}); - -test("extractUsage returns null when required token fields are missing", () => { - assert.equal(extractUsage({ usage: { input_tokens: 100 } }), null); -}); - -// --- extractResponseAudit --- - -test("extractResponseAudit extracts a complete audit from response record", () => { - const record = { - id: "resp-123", - status: "completed", - model: "gpt-4o-2024-08-06", - output_text: '{"claims":[]}', - output: [ - { - type: "message", - id: "msg-1", - status: "completed", - content: [{ type: "output_text", text: '{"claims":[]}' }], - }, - ], - usage: { input_tokens: 10, output_tokens: 5, total_tokens: 15 }, - }; - const audit = extractResponseAudit(record); - assert.equal(audit.responseId, "resp-123"); - assert.equal(audit.responseStatus, "completed"); - assert.equal(audit.responseModelVersion, "gpt-4o-2024-08-06"); - assert.equal(audit.outputItems.length, 1); - assert.equal(audit.outputTextParts.length, 1); - assert.ok(audit.usage !== null); -}); - -test("extractResponseAudit handles empty response record", () => { - const audit = extractResponseAudit({}); - assert.equal(audit.responseId, null); - assert.equal(audit.responseStatus, null); - assert.equal(audit.outputItems.length, 0); - assert.equal(audit.usage, null); -}); - -// --- offsetResponseAuditIndices --- - -test("offsetResponseAuditIndices returns same audit when offset is 0", () => { - const audit = makeEmptyResponseAudit(); - assert.strictEqual(offsetResponseAuditIndices(audit, 0), audit); -}); - -test("offsetResponseAuditIndices offsets all index fields", () => { - const audit: InvestigatorResponseAudit = { - ...makeEmptyResponseAudit(), - outputItems: [{ outputIndex: 0, providerItemId: null, itemType: "message", itemStatus: null }], - outputTextParts: [{ outputIndex: 0, partIndex: 0, partType: "output_text", text: "hello" }], - outputTextAnnotations: [ - { - outputIndex: 0, - partIndex: 0, - annotationIndex: 0, - annotationType: "url_citation", - url: null, - title: null, - fileId: null, - }, - ], - reasoningSummaries: [{ outputIndex: 0, summaryIndex: 0, text: "reasoning" }], - toolCalls: [ - { - outputIndex: 0, - providerToolCallId: null, - toolType: "function_call", - status: null, - rawPayload: {}, - capturedAt: new Date().toISOString(), - providerStartedAt: null, - providerCompletedAt: null, + providerItemId: null, + itemType: "function_call", + itemStatus: null, + content: { + kind: "TOOL_CALL", + rawPayload: { + type: "function_call", + call_id: "call-1", + name: "fetch_url", + arguments: "{}", + }, }, - ], - }; - - const offset = offsetResponseAuditIndices(audit, 5); - assert.ok(offset.outputItems[0]); - assert.equal(offset.outputItems[0].outputIndex, 5); - assert.ok(offset.outputTextParts[0]); - assert.equal(offset.outputTextParts[0].outputIndex, 5); - assert.ok(offset.outputTextAnnotations[0]); - assert.equal(offset.outputTextAnnotations[0].outputIndex, 5); - assert.ok(offset.reasoningSummaries[0]); - assert.equal(offset.reasoningSummaries[0].outputIndex, 5); - assert.ok(offset.toolCalls[0]); - assert.equal(offset.toolCalls[0].outputIndex, 5); -}); - -// --- aggregateUsage --- - -test("aggregateUsage returns null for empty array", () => { - assert.equal(aggregateUsage([]), null); -}); - -test("aggregateUsage returns null for all-null array", () => { - assert.equal(aggregateUsage([null, null]), null); -}); - -test("aggregateUsage sums token counts", () => { - const result = aggregateUsage([ - { - inputTokens: 10, - outputTokens: 5, - totalTokens: 15, - cachedInputTokens: 2, - reasoningOutputTokens: 1, }, { - inputTokens: 20, - outputTokens: 10, - totalTokens: 30, - cachedInputTokens: 3, - reasoningOutputTokens: null, + providerItemId: "msg_1", + itemType: "message", + itemStatus: "completed", + content: { + kind: "MESSAGE", + textParts: [ + { + partType: "output_text", + text: "The moon is far.", + annotations: [ + { + annotationType: "url_citation", + startIndex: 0, + endIndex: 16, + url: "https://nasa.gov/moon", + title: "Moon", + fileId: null, + }, + ], + }, + { partType: "refusal", text: "No.", annotations: [] }, + ], + }, }, ]); - assert.ok(result !== null); - assert.equal(result.inputTokens, 30); - assert.equal(result.outputTokens, 15); - assert.equal(result.totalTokens, 45); - assert.equal(result.cachedInputTokens, 5); - assert.equal(result.reasoningOutputTokens, 1); + assert.deepEqual(audit.usage, { + inputTokens: 100, + outputTokens: 50, + totalTokens: 150, + cachedInputTokens: 10, + reasoningOutputTokens: 20, + }); }); -// --- mergeResponseAudits --- - -test("mergeResponseAudits throws for empty array", () => { - assert.throws(() => mergeResponseAudits([]), /Cannot merge empty response audits/); +test("auditResponse records a missing status and missing usage as null", () => { + const { + usage: _usage, + status: _status, + ...withoutUsageOrStatus + } = makeResponse({ id: "resp_1", output: [] }); + const audit = auditResponse(withoutUsageOrStatus, RECEIVED_AT); + assert.equal(audit.status, null); + assert.equal(audit.usage, null); }); -test("mergeResponseAudits uses final audit metadata and flattens arrays", () => { - const audit1 = makeEmptyResponseAudit({ - responseId: "resp-1", - responseStatus: "completed", - responseModelVersion: "model-1", - }); - audit1.outputItems = [ - { outputIndex: 0, providerItemId: null, itemType: "message", itemStatus: null }, - ]; - audit1.usage = { - inputTokens: 10, - outputTokens: 5, - totalTokens: 15, - cachedInputTokens: null, - reasoningOutputTokens: null, - }; - - const audit2 = makeEmptyResponseAudit({ - responseId: "resp-2", - responseStatus: "completed", - responseModelVersion: "model-2", +test("auditRequest records the request as sent, with the audit form of its input", () => { + const params = buildFactCheckRequestParams(INVESTIGATION_REQUEST_CONFIG, { + instructions: "instructions", + tools: buildFactCheckTools(null), + input: [ + { role: "user", content: [{ type: "input_image", detail: "auto", image_url: "data:x" }] }, + ], + previousResponseId: "resp_0", }); - audit2.outputItems = [ - { outputIndex: 1, providerItemId: null, itemType: "function_call", itemStatus: null }, - ]; - audit2.usage = { - inputTokens: 20, - outputTokens: 10, - totalTokens: 30, - cachedInputTokens: null, - reasoningOutputTokens: null, - }; - - const merged = mergeResponseAudits([audit1, audit2]); - assert.equal(merged.responseId, "resp-2"); - assert.equal(merged.responseModelVersion, "model-2"); - assert.equal(merged.outputItems.length, 2); - assert.ok(merged.usage !== null); - assert.equal(merged.usage.inputTokens, 30); -}); - -// --- offset + merge pipeline (multi-round global index uniqueness) --- -// The investigation pipeline calls offsetResponseAuditIndices on each API round -// with a cumulative offset equal to the running total of output items from prior -// rounds, then merges all audits. If the offset is computed incorrectly (e.g. -// off-by-one, uses wrong count), outputIndex values will collide across rounds, -// corrupting the audit trail. - -test("offset + merge pipeline produces globally unique outputIndex values across rounds", () => { - // Simulate a 3-round tool loop where each round has different output sizes. - const round1 = makeEmptyResponseAudit({ responseId: "resp-round-1" }); - round1.outputItems = [ - { outputIndex: 0, providerItemId: "msg-1", itemType: "message", itemStatus: "completed" }, - { outputIndex: 1, providerItemId: "fc-1", itemType: "function_call", itemStatus: "completed" }, - { outputIndex: 2, providerItemId: "fc-2", itemType: "function_call", itemStatus: "completed" }, - ]; - round1.toolCalls = [ - { - outputIndex: 1, - providerToolCallId: "fc-1", - toolType: "function_call", - status: "completed", - rawPayload: {}, - capturedAt: new Date().toISOString(), - providerStartedAt: null, - providerCompletedAt: null, - }, - { - outputIndex: 2, - providerToolCallId: "fc-2", - toolType: "function_call", - status: "completed", - rawPayload: {}, - capturedAt: new Date().toISOString(), - providerStartedAt: null, - providerCompletedAt: null, - }, - ]; - round1.outputTextParts = [ - { outputIndex: 0, partIndex: 0, partType: "output_text", text: "round 1 text" }, - ]; - - const round2 = makeEmptyResponseAudit({ responseId: "resp-round-2" }); - round2.outputItems = [ - { outputIndex: 0, providerItemId: "msg-2", itemType: "message", itemStatus: "completed" }, - { - outputIndex: 1, - providerItemId: "ws-1", - itemType: "web_search_call", - itemStatus: "completed", - }, - ]; - round2.toolCalls = [ - { - outputIndex: 1, - providerToolCallId: "ws-1", - toolType: "web_search_call", - status: "completed", - rawPayload: {}, - capturedAt: new Date().toISOString(), - providerStartedAt: null, - providerCompletedAt: null, - }, + const auditInput = [ + { role: "user", content: [{ type: "input_image", detail: "auto", imageContentHash: "h" }] }, ]; - const round3 = makeEmptyResponseAudit({ - responseId: "resp-round-3", - responseStatus: "completed", + const audit = auditRequest({ + subject: { kind: "FACT_CHECK_ROUND", round: 1 }, + params, + auditInput, + response: null, }); - round3.outputItems = [ - { outputIndex: 0, providerItemId: "msg-3", itemType: "message", itemStatus: "completed" }, - ]; - round3.outputTextParts = [ - { outputIndex: 0, partIndex: 0, partType: "output_text", text: "final result" }, - ]; - - // Apply cumulative offsets (same pattern as openai.ts investigate loop). - let offset = 0; - const offsetRound1 = offsetResponseAuditIndices(round1, offset); - offset += round1.outputItems.length; // offset = 3 - - const offsetRound2 = offsetResponseAuditIndices(round2, offset); - offset += round2.outputItems.length; // offset = 5 - - const offsetRound3 = offsetResponseAuditIndices(round3, offset); - - const merged = mergeResponseAudits([offsetRound1, offsetRound2, offsetRound3]); - // All indices must be unique — no collisions between rounds. - // Note: some indices may repeat across sub-record types within the same round - // (e.g. outputItems[0] and outputTextParts[0] share outputIndex=0 because - // they refer to the same output item). The invariant is that the SAME - // sub-record type has no duplicate indices after merge. - const outputItemIndices = merged.outputItems.map((item) => item.outputIndex); - assert.equal( - new Set(outputItemIndices).size, - outputItemIndices.length, - "outputItems must have unique outputIndex values after offset + merge", - ); - - const toolCallIndices = merged.toolCalls.map((call) => call.outputIndex); - assert.equal( - new Set(toolCallIndices).size, - toolCallIndices.length, - "toolCalls must have unique outputIndex values after offset + merge", + assert.deepEqual(audit.subject, { kind: "FACT_CHECK_ROUND", round: 1 }); + assert.equal(audit.model, "gpt-6.1-sol"); + assert.equal(audit.instructions, "instructions"); + assert.equal(audit.input, auditInput); + assert.equal(audit.previousResponseId, "resp_0"); + assert.equal(audit.reasoningEffort, "medium"); + assert.equal(audit.reasoningSummary, "detailed"); + assert.deepEqual(audit.include, ["web_search_call.action.sources"]); + assert.deepEqual( + audit.tools.map((tool) => tool.toolType), + ["web_search", "function", "function"], ); + assert.deepEqual(audit.tools[0]?.rawDefinition, { type: "web_search" }); + assert.equal(audit.response, null); +}); - // Verify specific offset arithmetic: round 1 items at 0,1,2; round 2 at 3,4; round 3 at 5. - assert.deepStrictEqual( - merged.outputItems.map((item) => item.outputIndex), - [0, 1, 2, 3, 4, 5], +test("buildErrorAudit records the HTTP status of OpenAI API errors only", () => { + const apiError = OpenAI.APIError.generate( + 429, + { error: { message: "Rate limit reached" } }, + undefined, + new Headers(), ); + assert.deepEqual(buildErrorAudit(apiError), { + errorName: "RateLimitError", + errorMessage: "429 Rate limit reached", + statusCode: 429, + }); - // Metadata comes from the final audit. - assert.equal(merged.responseId, "resp-round-3"); - assert.equal(merged.responseStatus, "completed"); + const lookalike = Object.assign(new Error("not from OpenAI"), { status: 401 }); + assert.equal(buildErrorAudit(lookalike).statusCode, null); + assert.deepEqual(buildErrorAudit("plain string"), { + errorName: "UnknownError", + errorMessage: "plain string", + statusCode: null, + }); }); - -// --- helpers --- - -function makeEmptyResponseAudit( - overrides?: Partial, -): InvestigatorResponseAudit { - return { - responseId: null, - responseStatus: null, - responseModelVersion: null, - responseOutputText: null, - outputItems: [], - outputTextParts: [], - outputTextAnnotations: [], - reasoningSummaries: [], - toolCalls: [], - usage: null, - ...overrides, - }; -} diff --git a/src/typescript/api/test/unit/openai-schemas.test.ts b/src/typescript/api/test/unit/openai-schemas.test.ts deleted file mode 100644 index 94f78c5..0000000 --- a/src/typescript/api/test/unit/openai-schemas.test.ts +++ /dev/null @@ -1,64 +0,0 @@ -import assert from "node:assert/strict"; -import { test } from "node:test"; -import { ZodError } from "zod"; -import { - claimValidationResultSchema, - providerStructuredSourceUrlSchema, - providerStructuredInvestigationClaimPayloadSchema, -} from "../../src/lib/investigators/openai-schemas.js"; - -test("claimValidationResultSchema accepts { approved: true }", () => { - const result = claimValidationResultSchema.parse({ approved: true }); - assert.deepStrictEqual(result, { approved: true }); -}); - -test("claimValidationResultSchema accepts { approved: false }", () => { - const result = claimValidationResultSchema.parse({ approved: false }); - assert.deepStrictEqual(result, { approved: false }); -}); - -test("claimValidationResultSchema rejects extra keys (strict mode)", () => { - assert.throws( - () => claimValidationResultSchema.parse({ approved: true, extra: "nope" }), - ZodError, - ); -}); - -test("providerStructuredSourceUrlSchema accepts valid http url", () => { - const url = providerStructuredSourceUrlSchema.parse("https://example.com/page"); - assert.equal(url, "https://example.com/page"); -}); - -test("providerStructuredSourceUrlSchema rejects non-http url", () => { - assert.throws(() => providerStructuredSourceUrlSchema.parse("ftp://example.com"), ZodError); -}); - -test("providerStructuredSourceUrlSchema rejects empty string", () => { - assert.throws(() => providerStructuredSourceUrlSchema.parse(""), ZodError); -}); - -test("providerStructuredInvestigationClaimPayloadSchema accepts valid claim", () => { - const claim = providerStructuredInvestigationClaimPayloadSchema.parse({ - text: "claim text", - context: "claim context", - summary: "summary", - reasoning: "reasoning", - sources: [{ url: "https://example.com", title: "Title", snippet: "Snippet" }], - }); - assert.equal(claim.text, "claim text"); - assert.equal(claim.sources.length, 1); -}); - -test("providerStructuredInvestigationClaimPayloadSchema rejects empty sources", () => { - assert.throws( - () => - providerStructuredInvestigationClaimPayloadSchema.parse({ - text: "claim text", - context: "claim context", - summary: "summary", - reasoning: "reasoning", - sources: [], - }), - ZodError, - ); -}); diff --git a/src/typescript/api/test/unit/openai-tool-dispatch.test.ts b/src/typescript/api/test/unit/openai-tool-dispatch.test.ts index 76cc061..42f03f1 100644 --- a/src/typescript/api/test/unit/openai-tool-dispatch.test.ts +++ b/src/typescript/api/test/unit/openai-tool-dispatch.test.ts @@ -1,111 +1,95 @@ import assert from "node:assert/strict"; import { test } from "node:test"; import { - extractPendingFunctionToolCalls, - deduplicateFunctionToolCalls, - buildRetainCorrectionToolDefinition, - isClaimToolCall, - SUBMIT_CORRECTION_TOOL_NAME, - RETAIN_CORRECTION_TOOL_NAME, + buildFunctionCallOutput, + dispatchFunctionToolCalls, + executeFunctionToolCall, + extractFunctionToolCalls, type PendingFunctionToolCall, } from "../../src/lib/investigators/openai-tool-dispatch.js"; +import { + makeFunctionCall, + makeMessage, + makeReasoning, + makeResponse, + makeWebSearch, +} from "../helpers/fake-openai.js"; -test("extractPendingFunctionToolCalls extracts valid function calls", () => { - const record = { - output: [ - { type: "message", id: "msg-1", status: "completed" }, - { - type: "function_call", - call_id: "call-1", - name: "fetch_url", - arguments: '{"url":"https://example.com"}', - }, - ], - }; - const calls = extractPendingFunctionToolCalls(record); - assert.equal(calls.length, 1); - assert.ok(calls[0]); - assert.equal(calls[0].callId, "call-1"); - assert.equal(calls[0].name, "fetch_url"); - assert.equal(calls[0].argumentsJson, '{"url":"https://example.com"}'); -}); - -test("extractPendingFunctionToolCalls skips non-function_call items", () => { - const record = { - output: [{ type: "message", id: "msg-1", status: "completed" }, { type: "reasoning" }], - }; - const calls = extractPendingFunctionToolCalls(record); - assert.equal(calls.length, 0); -}); - -test("extractPendingFunctionToolCalls skips entries missing required fields", () => { - const record = { +test("extractFunctionToolCalls returns function calls in emission order and nothing else", () => { + const response = makeResponse({ + id: "resp_1", output: [ - { type: "function_call", call_id: "", name: "fetch_url", arguments: "{}" }, - { type: "function_call", call_id: "call-2", name: "", arguments: "{}" }, - { type: "function_call", call_id: "call-3", name: "fetch_url", arguments: null }, + makeReasoning("rs_1", []), + makeFunctionCall("call-1", "fetch_url", { url: "https://example.com" }), + makeWebSearch("ws_1", "query", []), + makeMessage("msg_1", "text"), + makeFunctionCall("call-2", "submit_correction", { text: "x" }), ], - }; - const calls = extractPendingFunctionToolCalls(record); - assert.equal(calls.length, 0); -}); - -test("extractPendingFunctionToolCalls handles missing output", () => { - assert.deepStrictEqual(extractPendingFunctionToolCalls({}), []); -}); + }); -test("extractPendingFunctionToolCalls handles non-array output", () => { - assert.deepStrictEqual(extractPendingFunctionToolCalls({ output: "not-array" }), []); + assert.deepEqual(extractFunctionToolCalls(response), [ + { callId: "call-1", name: "fetch_url", argumentsJson: '{"url":"https://example.com"}' }, + { callId: "call-2", name: "submit_correction", argumentsJson: '{"text":"x"}' }, + ]); }); -test("deduplicateFunctionToolCalls removes duplicate callIds", () => { +test("dispatchFunctionToolCalls routes each call and answers in call order", async () => { const calls: PendingFunctionToolCall[] = [ - { callId: "call-1", name: "fetch_url", argumentsJson: '{"url":"a"}' }, - { callId: "call-1", name: "fetch_url", argumentsJson: '{"url":"b"}' }, - { callId: "call-2", name: "fetch_url", argumentsJson: '{"url":"c"}' }, + { callId: "c1", name: "fetch_url", argumentsJson: "{}" }, + { callId: "c2", name: "submit_correction", argumentsJson: "{}" }, + { callId: "c3", name: "retain_correction", argumentsJson: "{}" }, + { callId: "c4", name: "submit_correction", argumentsJson: "{}" }, ]; - const result = deduplicateFunctionToolCalls(calls); - assert.equal(result.length, 2); - assert.ok(result[0]); - assert.equal(result[0].callId, "call-1"); - assert.equal(result[0].argumentsJson, '{"url":"a"}'); - assert.ok(result[1]); - assert.equal(result[1].callId, "call-2"); -}); - -test("deduplicateFunctionToolCalls handles empty array", () => { - assert.deepStrictEqual(deduplicateFunctionToolCalls([]), []); -}); + const handled: string[] = []; -// --- isClaimToolCall --- + const outputs = await dispatchFunctionToolCalls(calls, { + submitCorrection: (call) => { + handled.push(`submit:${call.callId}`); + return buildFunctionCallOutput(call.callId, "submitted"); + }, + retainCorrection: (call) => { + handled.push(`retain:${call.callId}`); + return buildFunctionCallOutput(call.callId, "retained"); + }, + research: async (call) => { + handled.push(`research:${call.callId}`); + return buildFunctionCallOutput(call.callId, "researched"); + }, + }); -test("isClaimToolCall distinguishes claim tools from research tools", () => { - // Claim tools → true (these get routed to the incremental claim pipeline) - assert.equal( - isClaimToolCall({ callId: "c1", name: SUBMIT_CORRECTION_TOOL_NAME, argumentsJson: "{}" }), - true, - ); - assert.equal( - isClaimToolCall({ callId: "c2", name: RETAIN_CORRECTION_TOOL_NAME, argumentsJson: "{}" }), - true, - ); - // Research tools → false (these get dispatched to executeFunctionToolCall) - assert.equal(isClaimToolCall({ callId: "c3", name: "fetch_url", argumentsJson: "{}" }), false); - assert.equal( - isClaimToolCall({ callId: "c4", name: "web_search_preview", argumentsJson: "{}" }), - false, + // Claim tools are handled synchronously in emission order (submission order). + assert.deepEqual(handled, ["research:c1", "submit:c2", "retain:c3", "submit:c4"]); + assert.deepEqual( + outputs.map((output) => [output.call_id, output.output]), + [ + ["c1", "researched"], + ["c2", "submitted"], + ["c3", "retained"], + ["c4", "submitted"], + ], ); }); -// --- buildRetainCorrectionToolDefinition --- - -test("buildRetainCorrectionToolDefinition constrains id to exactly the provided claim IDs", () => { - const tool = buildRetainCorrectionToolDefinition(["claim-a", "claim-b", "claim-c"]); - // The dynamic enum is the key behavior — it constrains the model to only retain existing claims - assert.deepStrictEqual(tool.parameters.properties.id.enum, ["claim-a", "claim-b", "claim-c"]); +test("executeFunctionToolCall answers unknown tools with an error output", async () => { + const output = await executeFunctionToolCall( + { callId: "c1", name: "made_up_tool", argumentsJson: "{}" }, + new AbortController().signal, + ); + assert.deepEqual(output, { + type: "function_call_output", + call_id: "c1", + output: JSON.stringify({ ok: false, error: "Unknown function tool: made_up_tool" }), + }); }); -test("buildRetainCorrectionToolDefinition single-element enum still constrains correctly", () => { - const tool = buildRetainCorrectionToolDefinition(["only-claim"]); - assert.deepStrictEqual(tool.parameters.properties.id.enum, ["only-claim"]); +test("executeFunctionToolCall does not start work for an aborted run", async () => { + const controller = new AbortController(); + controller.abort(); + await assert.rejects( + executeFunctionToolCall( + { callId: "c1", name: "fetch_url", argumentsJson: '{"url":"https://example.com"}' }, + controller.signal, + ), + { name: "AbortError" }, + ); }); diff --git a/src/typescript/api/test/unit/openai-tool-loop.test.ts b/src/typescript/api/test/unit/openai-tool-loop.test.ts index 30ff30c..04da158 100644 --- a/src/typescript/api/test/unit/openai-tool-loop.test.ts +++ b/src/typescript/api/test/unit/openai-tool-loop.test.ts @@ -1,203 +1,151 @@ import assert from "node:assert/strict"; import { test } from "node:test"; -import type OpenAI from "openai"; +import { INVESTIGATION_REQUEST_CONFIG } from "../../src/lib/investigators/openai-request-config.js"; import { - runToolLoop, - ToolLoopExecutionError, -} from "../../src/lib/investigators/openai-tool-loop.js"; -import type { - FunctionCallOutput, - PendingFunctionToolCall, + buildFunctionCallOutput, + type PendingFunctionToolCall, } from "../../src/lib/investigators/openai-tool-dispatch.js"; +import { runToolLoop } from "../../src/lib/investigators/openai-tool-loop.js"; +import { + createFakeOpenAiClient, + makeFunctionCall, + makeMessage, + makeResponse, + type FakeOpenAiReply, + type RecordedOpenAiRequest, +} from "../helpers/fake-openai.js"; -function makeFunctionCall(call: { callId: string; name: string; argumentsJson: string }) { - return { - type: "function_call", - id: `fc_${call.callId}`, - call_id: call.callId, - name: call.name, - arguments: call.argumentsJson, - status: "completed", - }; +function runLoop(input: { + reply: (request: RecordedOpenAiRequest, index: number) => FakeOpenAiReply; + maxRounds: number; + onCalls?: (calls: PendingFunctionToolCall[]) => void; +}) { + const { client, requests } = createFakeOpenAiClient(input.reply); + const handled: PendingFunctionToolCall[][] = []; + const result = runToolLoop({ + client, + requestConfig: INVESTIGATION_REQUEST_CONFIG, + maxRounds: input.maxRounds, + instructions: "test instructions", + tools: [{ type: "web_search" }], + initialInput: { request: "initial prompt", audit: "initial prompt" }, + signal: new AbortController().signal, + handleFunctionCalls: async (calls) => { + handled.push(calls); + input.onCalls?.(calls); + return calls.map((call) => buildFunctionCallOutput(call.callId, `output-${call.callId}`)); + }, + }); + return { result, requests, handled }; } -test("runToolLoop routes submit/retain/research calls and collects response audits", async () => { - const routed: { - submitted: PendingFunctionToolCall[]; - retained: PendingFunctionToolCall[]; - research: PendingFunctionToolCall[]; - } = { - submitted: [], - retained: [], - research: [], - }; +function toolCallRound(id: string, callId: string) { + return makeResponse({ id, output: [makeFunctionCall(callId, "fetch_url", { url: "x" })] }); +} - let callCount = 0; - const client = { - responses: { - create: async () => { - callCount += 1; - if (callCount === 1) { - return { - id: "resp-1", - status: "completed", - model: "test-model", - output: [ - makeFunctionCall({ - callId: "submit-1", - name: "submit_correction", - argumentsJson: JSON.stringify({ - text: "claim", - context: "ctx", - summary: "sum", - reasoning: "reason", - sources: [ - { - url: "https://example.com", - title: "Example", - snippet: "Snippet", - }, - ], - }), - }), - makeFunctionCall({ - callId: "retain-1", - name: "retain_correction", - argumentsJson: JSON.stringify({ id: "old-1" }), - }), - makeFunctionCall({ - callId: "research-1", - name: "fetch_url", - argumentsJson: JSON.stringify({ url: "https://example.com" }), - }), - ], - output_text: null, - usage: { input_tokens: 10, output_tokens: 10, total_tokens: 20 }, - }; - } +test("the loop answers function calls until the model stops calling tools", async () => { + const { result, requests, handled } = runLoop({ + maxRounds: 5, + reply: (_request, index) => ({ + kind: "response", + response: + index === 0 + ? toolCallRound("resp_0", "call-a") + : makeResponse({ id: "resp_1", output: [makeMessage("msg", "done")] }), + }), + }); - return { - id: "resp-2", - status: "completed", - model: "test-model", - output: [ - { - type: "message", - id: "msg-final", - status: "completed", - role: "assistant", - content: [], - }, - ], - output_text: null, - usage: { input_tokens: 5, output_tokens: 5, total_tokens: 10 }, - }; - }, - }, - } as unknown as OpenAI; + const outcome = await result; + assert.equal(outcome.kind, "completed"); + assert.equal(outcome.finalResponse.id, "resp_1"); + assert.deepEqual( + handled.map((calls) => calls.map((call) => call.callId)), + [["call-a"]], + ); + assert.equal(requests[1]?.body.previous_response_id, "resp_0"); + assert.deepEqual(requests[1].body.input, [ + { type: "function_call_output", call_id: "call-a", output: "output-call-a" }, + ]); + assert.deepEqual( + outcome.rounds.map((round) => [round.subject, round.previousResponseId, round.input]), + [ + [{ kind: "FACT_CHECK_ROUND", round: 0 }, null, "initial prompt"], + [ + { kind: "FACT_CHECK_ROUND", round: 1 }, + "resp_0", + [{ type: "function_call_output", call_id: "call-a", output: "output-call-a" }], + ], + ], + ); +}); - const output = await runToolLoop({ - client, - maxResponseToolRounds: 5, - baseResponseRequest: { - model: "test-model", - stream: false, - instructions: "test instructions", - tools: [], - reasoning: { - effort: "medium", - summary: "detailed", - }, - }, - initialInput: "input", - handleSubmittedClaims: async (calls) => { - routed.submitted = calls; - return [ - { - type: "function_call_output", - call_id: calls[0]?.callId ?? "none", - output: '{"acknowledged":true}', - }, - ] as FunctionCallOutput[]; - }, - handleRetainedClaims: async (calls) => { - routed.retained = calls; - return [ - { - type: "function_call_output", - call_id: calls[0]?.callId ?? "none", - output: '{"acknowledged":true}', - }, - ]; - }, - handleResearchCalls: async (calls) => { - routed.research = calls; - return [ - { - type: "function_call_output", - call_id: calls[0]?.callId ?? "none", - output: '{"ok":true}', - }, - ]; - }, +test("the loop stops at the round limit without handling the last round's calls", async () => { + const { result, requests, handled } = runLoop({ + maxRounds: 2, + reply: (_request, index) => ({ + kind: "response", + response: toolCallRound(`resp_${index.toString()}`, `call-${index.toString()}`), + }), }); - assert.equal(routed.submitted.length, 1); - assert.equal(routed.submitted[0]?.name, "submit_correction"); - assert.equal(routed.retained.length, 1); - assert.equal(routed.retained[0]?.name, "retain_correction"); - assert.equal(routed.research.length, 1); - assert.equal(routed.research[0]?.name, "fetch_url"); - assert.equal(output.responseAudits.length, 2); - assert.equal(output.latestResponseRecord?.["id"], "resp-2"); + const outcome = await result; + assert.equal(outcome.kind, "round_limit"); + assert.equal(requests.length, 2); + assert.equal(outcome.rounds.length, 2); + assert.deepEqual( + handled.map((calls) => calls.map((call) => call.callId)), + [["call-0"]], + ); }); -test("runToolLoop throws ToolLoopExecutionError when tool calls have no response id", async () => { - const client = { - responses: { - create: async () => ({ - id: null, - status: "completed", - model: "test-model", - output: [ - makeFunctionCall({ - callId: "submit-1", - name: "submit_correction", - argumentsJson: "{}", - }), - ], - output_text: null, +test("a response that did not complete ends the loop before its calls are handled", async () => { + const { result, handled } = runLoop({ + maxRounds: 5, + reply: () => ({ + kind: "response", + response: makeResponse({ + id: "resp_0", + status: "incomplete", + incompleteReason: "content_filter", + output: [makeFunctionCall("call-a", "fetch_url", {})], }), - }, - } as unknown as OpenAI; + }), + }); - await assert.rejects( - () => - runToolLoop({ - client, - maxResponseToolRounds: 2, - baseResponseRequest: { - model: "test-model", - stream: false, - instructions: "test instructions", - tools: [], - reasoning: { - effort: "medium", - summary: "detailed", - }, - }, - initialInput: "input", - handleSubmittedClaims: async () => [], - handleRetainedClaims: async () => [], - handleResearchCalls: async () => [], - }), - (error: unknown) => { - if (!(error instanceof ToolLoopExecutionError)) { - return false; - } - assert.match(error.message, /response id/); - assert.equal(error.responseAudits.length, 1); - return true; - }, + const outcome = await result; + assert.equal(outcome.kind, "response_not_completed"); + assert.equal(handled.length, 0); +}); + +test("a failed request ends the loop with that request audited without a response", async () => { + const { result } = runLoop({ + maxRounds: 5, + reply: (_request, index) => + index === 0 + ? { kind: "response", response: toolCallRound("resp_0", "call-a") } + : { kind: "http_error", status: 503, message: "overloaded" }, + }); + + const outcome = await result; + assert.equal(outcome.kind, "failed"); + assert.deepEqual( + outcome.rounds.map((round) => round.response?.providerResponseId ?? null), + ["resp_0", null], ); }); + +test("a throwing tool handler ends the loop as failed", async () => { + const { result } = runLoop({ + maxRounds: 5, + reply: () => ({ kind: "response", response: toolCallRound("resp_0", "call-a") }), + onCalls: () => { + throw new Error("tool exploded"); + }, + }); + + const outcome = await result; + assert.equal(outcome.kind, "failed"); + assert.ok(outcome.error instanceof Error); + assert.equal(outcome.error.message, "tool exploded"); + assert.equal(outcome.rounds.length, 1); +}); diff --git a/src/typescript/api/test/unit/orchestrator-errors.test.ts b/src/typescript/api/test/unit/orchestrator-errors.test.ts index b144b57..5b86342 100644 --- a/src/typescript/api/test/unit/orchestrator-errors.test.ts +++ b/src/typescript/api/test/unit/orchestrator-errors.test.ts @@ -1,5 +1,6 @@ import assert from "node:assert/strict"; import { test } from "node:test"; +import OpenAI from "openai"; import { ZodError } from "zod"; import { unwrapError, @@ -9,25 +10,22 @@ import { } from "../../src/lib/services/orchestrator-errors.js"; import { InvestigatorExecutionError, + InvestigatorIncompleteResponseError, + InvestigatorInputError, InvestigatorStructuredOutputError, -} from "../../src/lib/investigators/openai.js"; -import type { InvestigatorAttemptAudit } from "../../src/lib/investigators/interface.js"; +} from "../../src/lib/investigators/errors.js"; +import type { InvestigatorFailedAttemptAudit } from "../../src/lib/investigators/interface.js"; import { ExpiredOpenAiKeySourceError, InvalidOpenAiKeySourceError, } from "../../src/lib/services/user-key-source.js"; -function makeDummyAttemptAudit(): InvestigatorAttemptAudit { +function makeFailedAttemptAudit(): InvestigatorFailedAttemptAudit { return { - startedAt: new Date().toISOString(), - completedAt: new Date().toISOString(), - requestModel: "test-model", - requestInstructions: "test", - requestInput: "test", - requestReasoningEffort: null, - requestReasoningSummary: null, - requestedTools: [], - response: null, + outcome: "FAILED", + startedAt: new Date(), + completedAt: new Date(), + requests: [], error: { errorName: "TestError", errorMessage: "test error", @@ -36,19 +34,18 @@ function makeDummyAttemptAudit(): InvestigatorAttemptAudit { }; } +function apiError(status: number, message: string): InstanceType { + return OpenAI.APIError.generate(status, { error: { message } }, undefined, new Headers()); +} + // --- unwrapError --- test("unwrapError returns cause of InvestigatorExecutionError", () => { const cause = new TypeError("cause"); - const error = new InvestigatorExecutionError("wrapper", makeDummyAttemptAudit(), cause); + const error = new InvestigatorExecutionError("wrapper", makeFailedAttemptAudit(), cause); assert.strictEqual(unwrapError(error), cause); }); -test("unwrapError returns InvestigatorExecutionError itself when no cause", () => { - const error = new InvestigatorExecutionError("no cause", makeDummyAttemptAudit()); - assert.strictEqual(unwrapError(error), error); -}); - test("unwrapError returns plain Error directly", () => { const error = new Error("plain"); assert.strictEqual(unwrapError(error), error); @@ -73,13 +70,14 @@ test("unwrapError stringifies arrays", () => { // --- getErrorStatus --- -test("getErrorStatus returns status from error object", () => { - const error = Object.assign(new Error("fail"), { status: 500 }); - assert.equal(getErrorStatus(error), 500); +test("getErrorStatus returns the status of OpenAI API errors", () => { + assert.equal(getErrorStatus(apiError(500, "fail")), 500); }); -test("getErrorStatus returns null for error without status", () => { +test("getErrorStatus returns null for errors that are not OpenAI API errors", () => { assert.equal(getErrorStatus(new Error("no status")), null); + assert.equal(getErrorStatus(Object.assign(new Error("lookalike"), { status: 500 })), null); + assert.equal(getErrorStatus(new OpenAI.APIConnectionTimeoutError()), null); }); test("getErrorStatus returns null for string error", () => { @@ -87,16 +85,18 @@ test("getErrorStatus returns null for string error", () => { }); test("getErrorStatus unwraps InvestigatorExecutionError cause", () => { - const cause = Object.assign(new Error("api"), { status: 429 }); - const error = new InvestigatorExecutionError("wrapper", makeDummyAttemptAudit(), cause); + const error = new InvestigatorExecutionError( + "wrapper", + makeFailedAttemptAudit(), + apiError(429, "api"), + ); assert.equal(getErrorStatus(error), 429); }); // --- formatErrorForLog --- test("formatErrorForLog formats Error with status", () => { - const error = Object.assign(new Error("bad request"), { status: 400 }); - assert.equal(formatErrorForLog(error), "status=400: bad request"); + assert.equal(formatErrorForLog(apiError(400, "bad request")), "status=400: 400 bad request"); }); test("formatErrorForLog formats Error without status", () => { @@ -108,11 +108,6 @@ test("formatErrorForLog formats string", () => { assert.equal(formatErrorForLog("string error"), "string error"); }); -test("formatErrorForLog formats object with status", () => { - const error = { status: 502 }; - assert.equal(formatErrorForLog(error), "status=502"); -}); - test("formatErrorForLog formats object without status", () => { const error = { foo: "bar" }; assert.equal(formatErrorForLog(error), "unknown object error"); @@ -134,24 +129,33 @@ test("isNonRetryableProviderError returns true for InvestigatorStructuredOutputE assert.equal(isNonRetryableProviderError(error), true); }); -test("isNonRetryableProviderError returns true for 400 status", () => { - const error = Object.assign(new Error("bad request"), { status: 400 }); - assert.equal(isNonRetryableProviderError(error), true); +test("isNonRetryableProviderError returns true for request, auth and not-found statuses", () => { + for (const status of [400, 401, 403, 404, 422]) { + assert.equal(isNonRetryableProviderError(apiError(status, "rejected")), true, String(status)); + } }); -test("isNonRetryableProviderError returns true for 401 status", () => { - const error = Object.assign(new Error("unauthorized"), { status: 401 }); - assert.equal(isNonRetryableProviderError(error), true); +test("isNonRetryableProviderError returns false for rate limits and server errors", () => { + for (const status of [429, 500, 503]) { + assert.equal(isNonRetryableProviderError(apiError(status, "transient")), false, String(status)); + } }); -test("isNonRetryableProviderError returns false for 429 status (rate limit)", () => { - const error = Object.assign(new Error("rate limited"), { status: 429 }); - assert.equal(isNonRetryableProviderError(error), false); +test("isNonRetryableProviderError treats timeouts as transient", () => { + assert.equal(isNonRetryableProviderError(new OpenAI.APIConnectionTimeoutError()), false); +}); + +test("isNonRetryableProviderError returns true for incomplete responses (SPEC §3.7 PARTIAL)", () => { + const error = new InvestigatorIncompleteResponseError({ + responseStatus: "incomplete", + responseId: "resp_1", + incompleteReason: "max_output_tokens", + }); + assert.equal(isNonRetryableProviderError(error), true); }); -test("isNonRetryableProviderError returns false for 500 status (server error)", () => { - const error = Object.assign(new Error("server error"), { status: 500 }); - assert.equal(isNonRetryableProviderError(error), false); +test("isNonRetryableProviderError returns true for investigator input contract violations", () => { + assert.equal(isNonRetryableProviderError(new InvestigatorInputError("bad offsets")), true); }); test("isNonRetryableProviderError returns false for plain Error without status", () => { @@ -160,7 +164,7 @@ test("isNonRetryableProviderError returns false for plain Error without status", test("isNonRetryableProviderError unwraps InvestigatorExecutionError", () => { const cause = new InvestigatorStructuredOutputError("bad output"); - const error = new InvestigatorExecutionError("wrapper", makeDummyAttemptAudit(), cause); + const error = new InvestigatorExecutionError("wrapper", makeFailedAttemptAudit(), cause); assert.equal(isNonRetryableProviderError(error), true); }); @@ -176,6 +180,6 @@ test("isNonRetryableProviderError returns true for InvalidOpenAiKeySourceError", test("isNonRetryableProviderError detects key source errors wrapped in InvestigatorExecutionError", () => { const cause = new ExpiredOpenAiKeySourceError("run-789"); - const error = new InvestigatorExecutionError("wrapper", makeDummyAttemptAudit(), cause); + const error = new InvestigatorExecutionError("wrapper", makeFailedAttemptAudit(), cause); assert.equal(isNonRetryableProviderError(error), true); }); diff --git a/src/typescript/api/test/unit/orchestrator-persist-guard.test.ts b/src/typescript/api/test/unit/orchestrator-persist-guard.test.ts index e65b00b..4cc1dc0 100644 --- a/src/typescript/api/test/unit/orchestrator-persist-guard.test.ts +++ b/src/typescript/api/test/unit/orchestrator-persist-guard.test.ts @@ -6,7 +6,7 @@ import { releaseLeaseToRetryInTx, } from "../../src/lib/services/attempt-audit.js"; import type { Prisma } from "../../src/lib/db/prisma-client"; -import type { InvestigatorAttemptAudit } from "../../src/lib/investigators/interface.js"; +import type { InvestigatorSucceededAttemptAudit } from "../../src/lib/investigators/interface.js"; /** * Invariants under test: @@ -31,29 +31,12 @@ import type { InvestigatorAttemptAudit } from "../../src/lib/investigators/inter * inconsistent non-PROCESSING state, silently corrupting the lifecycle. */ -function makeMinimalAttemptAudit(): InvestigatorAttemptAudit { +function makeMinimalAttemptAudit(): InvestigatorSucceededAttemptAudit { return { - startedAt: new Date().toISOString(), - completedAt: new Date().toISOString(), - requestModel: "test-model", - requestInstructions: "test instructions", - requestInput: "test input", - requestReasoningEffort: null, - requestReasoningSummary: null, - requestedTools: [], - response: { - responseId: "resp_test", - responseStatus: "completed", - responseModelVersion: null, - responseOutputText: null, - outputItems: [], - outputTextParts: [], - outputTextAnnotations: [], - reasoningSummaries: [], - toolCalls: [], - usage: null, - }, - error: null, + outcome: "SUCCEEDED", + startedAt: new Date(), + completedAt: new Date(), + requests: [], }; } @@ -69,7 +52,7 @@ function makeClaim() { test("persistCompletedInvestigation returns false and writes nothing when guard fails", async () => { let claimCreateCalled = false; - let attemptUpsertCalled = false; + let attemptCreateCalled = false; // Mock transaction client where investigationLease.deleteMany returns 0 rows // (guard fails). All other methods throw if called — the test fails if any @@ -84,8 +67,8 @@ test("persistCompletedInvestigation returns false and writes nothing when guard }, }, investigationAttempt: { - upsert: async () => { - attemptUpsertCalled = true; + create: async () => { + attemptCreateCalled = true; throw new Error("Attempt audit write leaked past guard"); }, }, @@ -103,12 +86,13 @@ test("persistCompletedInvestigation returns false and writes nothing when guard claims: [makeClaim(), makeClaim()], attemptNumber: 1, attemptAudit: makeMinimalAttemptAudit(), - modelVersion: null, + model: "gpt-6.1-sol", + modelVersion: "gpt-6.1-sol-2026-09-01", }); assert.equal(result, false, "Should return false when guard matches 0 rows"); assert.equal(claimCreateCalled, false, "claim.create must not be called when guard fails"); - assert.equal(attemptUpsertCalled, false, "attemptAudit must not be written when guard fails"); + assert.equal(attemptCreateCalled, false, "attemptAudit must not be written when guard fails"); }); test("persistCompletedInvestigation proceeds to claim writes when guard succeeds", async () => { @@ -117,23 +101,25 @@ test("persistCompletedInvestigation proceeds to claim writes when guard succeeds // investigation.updateMany is called (which happens immediately after the // lease guard). Full claim-write verification is covered by integration tests. let investigationUpdateManyReached = false; + let completionData: unknown = null; const mockTx = { investigationLease: { deleteMany: async () => ({ count: 1 }), }, investigation: { - updateMany: async () => { + updateMany: async (args: { data: unknown }) => { investigationUpdateManyReached = true; + completionData = args.data; // Return count=1 to indicate the status transition succeeded, then // throw on the next call to short-circuit the deeply-mocked path. return { count: 1 }; }, }, - // persistAttemptAudit calls upsert first — intercept it to stop execution - // after verifying the guard passed and updateMany was called. + // persistAttemptAudit creates the attempt first — intercept it to stop + // execution after verifying the guard passed and updateMany was called. investigationAttempt: { - upsert: async () => { + create: async () => { throw new Error("Mock: stopping after guard verification"); }, }, @@ -146,7 +132,8 @@ test("persistCompletedInvestigation proceeds to claim writes when guard succeeds claims: [makeClaim()], attemptNumber: 1, attemptAudit: makeMinimalAttemptAudit(), - modelVersion: null, + model: "gpt-6.1-sol", + modelVersion: "gpt-6.1-sol-2026-09-01", }), /Mock: stopping after guard verification/, ); @@ -156,6 +143,16 @@ test("persistCompletedInvestigation proceeds to claim writes when guard succeeds true, "When guard succeeds (count=1), execution must proceed past the guard to investigation.updateMany", ); + // INV-INV-MODEL-AT-COMPLETION: the model that ran is recorded with the COMPLETE transition. + assert.deepEqual( + { ...(completionData as Record), checkedAt: "" }, + { + status: "COMPLETE", + checkedAt: "", + model: "gpt-6.1-sol", + modelVersion: "gpt-6.1-sol-2026-09-01", + }, + ); }); // ── Step-2 invariant: lease deleted but investigation not PROCESSING ────────── @@ -182,7 +179,8 @@ test("persistCompletedInvestigation throws invariant error when lease deleted bu claims: [], attemptNumber: 1, attemptAudit: makeMinimalAttemptAudit(), - modelVersion: null, + model: "gpt-6.1-sol", + modelVersion: "gpt-6.1-sol-2026-09-01", }), /Invariant violation.*inv-test-123/, "Must throw invariant error when lease exists but investigation is not PROCESSING", diff --git a/src/typescript/api/test/unit/post-investigation-queries.test.ts b/src/typescript/api/test/unit/post-investigation-queries.test.ts index 2abea75..27b70c8 100644 --- a/src/typescript/api/test/unit/post-investigation-queries.test.ts +++ b/src/typescript/api/test/unit/post-investigation-queries.test.ts @@ -1,17 +1,12 @@ import assert from "node:assert/strict"; import { test } from "node:test"; -import type { ResolvedPostVersion } from "../../src/lib/trpc/routes/post/content-storage.js"; import { - ensureInvestigationsWithUpdateMetadata, - findCompletedInvestigationByPostVersionId, - findLatestServerVerifiedCompleteInvestigationForPost, - investigationQueriesInternals, + carryForwardClaims, + findCarriedForwardClaims, loadInvestigationWithClaims, maybeRecordCorroboration, parseProgressClaims, requireCompleteCheckedAtIso, - selectSourceInvestigationForUpdate, - toPriorInvestigationResult, unreachableInvestigationStatus, type InvestigationRepository, } from "../../src/lib/trpc/routes/post/investigation-queries.js"; @@ -19,30 +14,20 @@ import { function nullRepo(): InvestigationRepository { return { findInvestigationWithClaims: async () => null, - findCompletedByPostVersionId: async () => null, - findLatestServerVerifiedComplete: async () => null, + findLatestCompleteOnOtherVersion: async () => null, findClientFallbackInvestigationId: async () => null, recordCorroborationCredit: async () => {}, }; } -function buildResolvedPostVersion(contentText = "new line"): ResolvedPostVersion { +function claim(id: string, text: string) { return { - id: "post-version-id", - postId: "post-id", - versionHash: "version-hash", - serverVerifiedAt: new Date(), - contentBlob: { - contentHash: "content-hash", - contentText, - wordCount: 2, - }, - post: { - id: "post-id", - platform: "X", - externalId: "external-id", - url: "https://x.com/openerrata/status/1", - }, + id, + text, + context: `Context of ${text}`, + summary: `Summary of ${text}`, + reasoning: `Reasoning about ${text}`, + sources: [{ url: `https://example.com/${id}`, title: `Source ${id}`, snippet: "Snippet" }], }; } @@ -53,84 +38,77 @@ test("requireCompleteCheckedAtIso returns ISO and throws when checkedAt is missi assert.throws(() => requireCompleteCheckedAtIso("inv-2", null), /COMPLETE with null checkedAt/); }); -test("selectSourceInvestigationForUpdate drops same-version source and keeps prior version", () => { +test("carryForwardClaims keeps exactly the claims whose text still occurs in the content", () => { const source = { id: "source-investigation-id", - postVersion: { - id: "source-post-version-id", - contentBlob: { - contentText: "old line", - }, - }, - claims: [], + claims: [ + claim("claim_kept", "The moon is made of cheese."), + claim("claim_removed", "Mars has three moons."), + ], }; + const contentText = "Intro. The moon is made of cheese. Mars has two moons."; - assert.equal(selectSourceInvestigationForUpdate(null, "current-post-version-id"), null); - assert.equal(selectSourceInvestigationForUpdate(source, "source-post-version-id"), null); - assert.equal(selectSourceInvestigationForUpdate(source, "current-post-version-id"), source); + assert.deepEqual(carryForwardClaims(source, contentText), { + sourceInvestigationId: "source-investigation-id", + oldClaims: [claim("claim_kept", "The moon is made of cheese.")], + }); }); -test("toPriorInvestigationResult maps source claims and handles null source", () => { - assert.equal(toPriorInvestigationResult(null), null); +test("carryForwardClaims is null when no claim survives", () => { + const source = { id: "source", claims: [claim("claim_1", "Removed sentence.")] }; + assert.equal(carryForwardClaims(source, "Entirely rewritten post."), null); + assert.equal(carryForwardClaims({ id: "source", claims: [] }, "Any text."), null); +}); +test("carryForwardClaims matches claim text after content normalization", () => { + // Content text is normalized (§3.8): curly quotes and dashes become ASCII + // and whitespace collapses. A claim quoting the page's original typography + // still occurs in it. const source = { - id: "source-investigation-id", - postVersion: { - id: "source-post-version-id", - contentBlob: { contentText: "old line" }, + id: "source", + claims: [claim("claim_1", "It\u2019s a \u201Ctest\u201D \u2014 really.")], + }; + const result = carryForwardClaims(source, `Prefix. It's a "test" - really. Suffix.`); + assert.equal(result?.oldClaims.length, 1); +}); + +test("carryForwardClaims never carries a claim whose text normalizes to nothing", () => { + const source = { id: "source", claims: [claim("claim_1", " \u200B ")] }; + assert.equal(carryForwardClaims(source, "Some content."), null); +}); + +test("findCarriedForwardClaims excludes the requested version and filters the latest complete source", async () => { + const lookups: { postId: string; excludedPostVersionId: string }[] = []; + const repo: InvestigationRepository = { + ...nullRepo(), + findLatestCompleteOnOtherVersion: async (postId, excludedPostVersionId) => { + lookups.push({ postId, excludedPostVersionId }); + return { + id: "source", + claims: [claim("claim_1", "Kept sentence."), claim("claim_2", "Gone sentence.")], + }; }, - claims: [ - { - id: "claim_1", - text: "Claim text", - context: "Claim context", - summary: "Claim summary", - reasoning: "Claim reasoning", - sources: [ - { - url: "https://example.com/1", - title: "Source 1", - snippet: "Snippet 1", - }, - ], - }, - ], }; - assert.deepEqual(toPriorInvestigationResult(source), { - sourceInvestigationId: "source-investigation-id", - oldClaims: [ - { - id: "claim_1", - text: "Claim text", - context: "Claim context", - summary: "Claim summary", - reasoning: "Claim reasoning", - sources: [ - { - url: "https://example.com/1", - title: "Source 1", - snippet: "Snippet 1", - }, - ], - }, - ], + const result = await findCarriedForwardClaims(repo, { + id: "requested-version", + postId: "post-1", + contentText: "Kept sentence. New sentence.", }); -}); -test("buildLineDiff reports no changes and changed line blocks", () => { - assert.equal( - investigationQueriesInternals.buildLineDiff("same\ncontent", "same\ncontent"), - "No changes detected.", + assert.deepEqual(lookups, [{ postId: "post-1", excludedPostVersionId: "requested-version" }]); + assert.deepEqual( + result?.oldClaims.map((c) => c.id), + ["claim_1"], ); - - const diff = investigationQueriesInternals.buildLineDiff( - "keep one\nremove me\nkeep tail", - "keep one\nadd me\nkeep tail", + assert.equal( + await findCarriedForwardClaims(nullRepo(), { + id: "requested-version", + postId: "post-1", + contentText: "Kept sentence.", + }), + null, ); - assert.match(diff, /Diff summary \(line context\):/); - assert.match(diff, /- Removed lines:\nremove me/); - assert.match(diff, /\+ Added lines:\nadd me/); }); test("unreachableInvestigationStatus throws explicit internal error", () => { @@ -147,11 +125,8 @@ test("parseProgressClaims fails fast on malformed progress payload", () => { ); }); -test("load and lookup helpers delegate to repository methods", async () => { - const repo = nullRepo(); - assert.equal(await loadInvestigationWithClaims(repo, "inv-1"), null); - assert.equal(await findCompletedInvestigationByPostVersionId(repo, "pv-1"), null); - assert.equal(await findLatestServerVerifiedCompleteInvestigationForPost(repo, "post-1"), null); +test("load helper delegates to repository method", async () => { + assert.equal(await loadInvestigationWithClaims(nullRepo(), "inv-1"), null); }); test("maybeRecordCorroboration gates on auth and delegates to repository", async () => { @@ -197,69 +172,3 @@ test("maybeRecordCorroboration gates on auth and delegates to repository", async assert.equal(lookupCalls, 3); assert.equal(creditCalls, 2); }); - -test("ensureInvestigationsWithUpdateMetadata forwards create and update payloads", async () => { - const ensureQueuedCalls: { - prisma: { name: string }; - postVersionId: string; - promptId: string; - parentInvestigationId?: string; - contentDiff?: string; - rejectOverWordLimitOnCreate: true; - allowRequeueFailed: true; - onPendingInvestigation?: unknown; - }[] = []; - const ensureQueued = async (input: (typeof ensureQueuedCalls)[number]) => { - ensureQueuedCalls.push(input); - return { investigation: { id: "inv-id", status: "PENDING" as const } }; - }; - const postVersion = buildResolvedPostVersion("new line\ntail"); - const prismaToken = { name: "prisma-token" }; - const onPendingInvestigation = async () => {}; - - await ensureInvestigationsWithUpdateMetadata({ - prisma: prismaToken, - promptId: "prompt-id", - postVersion, - sourceInvestigation: null, - onPendingInvestigation, - ensureQueued, - }); - - assert.deepEqual(ensureQueuedCalls[0], { - prisma: prismaToken, - postVersionId: "post-version-id", - promptId: "prompt-id", - rejectOverWordLimitOnCreate: true, - allowRequeueFailed: true, - onPendingInvestigation, - }); - - await ensureInvestigationsWithUpdateMetadata({ - prisma: prismaToken, - promptId: "prompt-id", - postVersion, - sourceInvestigation: { - id: "source-investigation-id", - postVersion: { - id: "old-post-version-id", - contentBlob: { - contentText: "old line\ntail", - }, - }, - claims: [], - }, - ensureQueued, - }); - - assert.deepEqual(ensureQueuedCalls[1], { - prisma: prismaToken, - postVersionId: "post-version-id", - promptId: "prompt-id", - parentInvestigationId: "source-investigation-id", - contentDiff: - "Diff summary (line context):\n- Removed lines:\nold line\n+ Added lines:\nnew line", - rejectOverWordLimitOnCreate: true, - allowRequeueFailed: true, - }); -}); diff --git a/src/typescript/api/test/unit/prompt-context.test.ts b/src/typescript/api/test/unit/prompt-context.test.ts deleted file mode 100644 index 015b523..0000000 --- a/src/typescript/api/test/unit/prompt-context.test.ts +++ /dev/null @@ -1,468 +0,0 @@ -import assert from "node:assert/strict"; -import { test } from "node:test"; -import { PLATFORM_VALUES, type Platform } from "@openerrata/shared"; -import { - isLikelyVideoUrl, - hasXVideoMedia, - toPromptPostContext, - resolveHtmlSnapshotsFromVersionMeta, -} from "../../src/lib/services/prompt-context.js"; - -type PromptPostContextInput = Parameters[0]; - -// --- isLikelyVideoUrl --- - -test("isLikelyVideoUrl returns true for .mp4 URL", () => { - assert.equal(isLikelyVideoUrl("https://example.com/video.mp4"), true); -}); - -test("isLikelyVideoUrl returns true for .webm URL", () => { - assert.equal(isLikelyVideoUrl("https://example.com/video.webm"), true); -}); - -test("isLikelyVideoUrl returns true for .m3u8 URL", () => { - assert.equal(isLikelyVideoUrl("https://example.com/stream.m3u8"), true); -}); - -test("isLikelyVideoUrl returns true for .mov URL", () => { - assert.equal(isLikelyVideoUrl("https://example.com/clip.mov"), true); -}); - -test("isLikelyVideoUrl returns true for .m4v URL", () => { - assert.equal(isLikelyVideoUrl("https://example.com/clip.m4v"), true); -}); - -test("isLikelyVideoUrl returns false for .png URL", () => { - assert.equal(isLikelyVideoUrl("https://example.com/image.png"), false); -}); - -test("isLikelyVideoUrl returns false for .jpg URL", () => { - assert.equal(isLikelyVideoUrl("https://example.com/photo.jpg"), false); -}); - -test("isLikelyVideoUrl returns false for URL without extension", () => { - assert.equal(isLikelyVideoUrl("https://example.com/page"), false); -}); - -test("isLikelyVideoUrl works with query strings (uses pathname only)", () => { - assert.equal(isLikelyVideoUrl("https://example.com/video.mp4?token=abc"), true); -}); - -test("isLikelyVideoUrl handles malformed URLs gracefully", () => { - // Malformed URL falls back to the raw string - assert.equal(isLikelyVideoUrl("not-a-url-video.mp4"), true); - assert.equal(isLikelyVideoUrl("not-a-url-image.png"), false); -}); - -test("isLikelyVideoUrl is case insensitive", () => { - assert.equal(isLikelyVideoUrl("https://example.com/Video.MP4"), true); - assert.equal(isLikelyVideoUrl("https://example.com/Video.WebM"), true); -}); - -// --- hasXVideoMedia --- - -test("hasXVideoMedia returns true when at least one video URL exists", () => { - assert.equal( - hasXVideoMedia(["https://example.com/photo.jpg", "https://example.com/clip.mp4"]), - true, - ); -}); - -test("hasXVideoMedia returns false when no video URLs exist", () => { - assert.equal( - hasXVideoMedia(["https://example.com/photo.jpg", "https://example.com/img.png"]), - false, - ); -}); - -test("hasXVideoMedia returns false for empty array", () => { - assert.equal(hasXVideoMedia([]), false); -}); - -// --- toPromptPostContext --- - -// Each platform stores its publication timestamp in a different version-meta field. -// This table drives exhaustive coverage: if a platform is added to PLATFORM_VALUES -// without a corresponding entry here, TypeScript will fail to compile the satisfies. - -const SAMPLE_TIMESTAMP = new Date("2026-01-02T03:04:05.000Z"); - -function sampleUrlForPlatform(platform: Platform): string { - switch (platform) { - case "LESSWRONG": - return "https://www.lesswrong.com/posts/example"; - case "X": - return "https://x.com/u/status/1"; - case "SUBSTACK": - return "https://example.substack.com/p/test"; - case "WIKIPEDIA": - return "https://en.wikipedia.org/wiki/Test"; - } -} - -const platformTimestampFixtures: Record< - Platform, - { input: PromptPostContextInput; hasVideo?: boolean } -> = { - LESSWRONG: { - input: { - serverVerifiedAt: new Date(), - contentBlob: { contentText: "unused", contentHash: "hash" }, - imageOccurrenceSet: { occurrences: [] }, - lesswrongVersionMeta: { - serverHtmlBlob: { htmlContent: "

    Version Heading

    " }, - clientHtmlBlob: { htmlContent: "

    Version Heading (client)

    " }, - publishedAt: SAMPLE_TIMESTAMP, - }, - xVersionMeta: null, - substackVersionMeta: null, - wikipediaVersionMeta: null, - post: { - platform: "LESSWRONG" as const, - url: sampleUrlForPlatform("LESSWRONG"), - author: { displayName: "Author Name" }, - }, - } satisfies PromptPostContextInput, - }, - X: { - input: { - serverVerifiedAt: null, - contentBlob: { contentText: "unused", contentHash: "hash" }, - imageOccurrenceSet: { occurrences: [] }, - lesswrongVersionMeta: null, - xVersionMeta: { - postedAt: SAMPLE_TIMESTAMP, - mediaUrls: ["https://example.com/video.mp4"], - }, - substackVersionMeta: null, - wikipediaVersionMeta: null, - post: { - platform: "X" as const, - url: sampleUrlForPlatform("X"), - author: { displayName: "Author Name" }, - }, - } satisfies PromptPostContextInput, - hasVideo: true, - }, - SUBSTACK: { - input: { - serverVerifiedAt: null, - contentBlob: { contentText: "unused", contentHash: "hash" }, - imageOccurrenceSet: { occurrences: [] }, - lesswrongVersionMeta: null, - xVersionMeta: null, - substackVersionMeta: { - publishedAt: SAMPLE_TIMESTAMP, - serverHtmlBlob: null, - clientHtmlBlob: { htmlContent: "

    Substack Version

    " }, - }, - wikipediaVersionMeta: null, - post: { - platform: "SUBSTACK" as const, - url: sampleUrlForPlatform("SUBSTACK"), - author: { displayName: "Author Name" }, - }, - } satisfies PromptPostContextInput, - }, - WIKIPEDIA: { - input: { - serverVerifiedAt: new Date(), - contentBlob: { contentText: "unused", contentHash: "hash" }, - imageOccurrenceSet: { occurrences: [] }, - lesswrongVersionMeta: null, - xVersionMeta: null, - substackVersionMeta: null, - wikipediaVersionMeta: { - lastModifiedAt: SAMPLE_TIMESTAMP, - serverHtmlBlob: { htmlContent: "

    History

    Version text.

    " }, - clientHtmlBlob: null, - }, - post: { - platform: "WIKIPEDIA" as const, - url: sampleUrlForPlatform("WIKIPEDIA"), - author: { displayName: "Author Name" }, - }, - } satisfies PromptPostContextInput, - }, -}; - -test("toPromptPostContext extracts timestamp from each platform's version meta", () => { - for (const platform of PLATFORM_VALUES) { - const { input, hasVideo } = platformTimestampFixtures[platform]; - const ctx = toPromptPostContext(input); - assert.equal(ctx.platform, platform, `platform=${platform}`); - assert.equal(ctx.postPublishedAt, SAMPLE_TIMESTAMP.toISOString(), `platform=${platform}`); - if (hasVideo !== undefined) { - assert.equal(ctx.hasVideo, hasVideo, `platform=${platform}`); - } - } -}); - -test("toPromptPostContext omits postPublishedAt when platform version meta is absent", () => { - for (const platform of PLATFORM_VALUES) { - const ctx = toPromptPostContext({ - serverVerifiedAt: null, - contentBlob: { contentText: "unused", contentHash: "hash" }, - imageOccurrenceSet: { occurrences: [] }, - lesswrongVersionMeta: null, - xVersionMeta: null, - substackVersionMeta: null, - wikipediaVersionMeta: null, - post: { - platform, - url: sampleUrlForPlatform(platform), - author: { displayName: "Author Name" }, - }, - }); - assert.equal(ctx.postPublishedAt, undefined, `platform=${platform}`); - } -}); - -// --- resolveHtmlSnapshotsFromVersionMeta --- - -type HtmlSnapshotsInput = Parameters[0]; - -test("resolveHtmlSnapshotsFromVersionMeta throws when serverVerifiedAt is set but serverHtml is missing", () => { - const HTML_PLATFORMS: Platform[] = ["LESSWRONG", "SUBSTACK", "WIKIPEDIA"]; - for (const platform of HTML_PLATFORMS) { - const metaWithMissingServerBlob = { - serverHtmlBlob: null, - clientHtmlBlob: { htmlContent: "

    client

    " }, - }; - const input: HtmlSnapshotsInput = { - serverVerifiedAt: new Date("2026-02-20T12:00:00.000Z"), - contentBlob: { contentText: "unused", contentHash: "hash" }, - imageOccurrenceSet: { occurrences: [] }, - lesswrongVersionMeta: - platform === "LESSWRONG" ? { publishedAt: null, ...metaWithMissingServerBlob } : null, - xVersionMeta: null, - substackVersionMeta: - platform === "SUBSTACK" ? { publishedAt: null, ...metaWithMissingServerBlob } : null, - wikipediaVersionMeta: - platform === "WIKIPEDIA" ? { lastModifiedAt: null, ...metaWithMissingServerBlob } : null, - post: { - platform, - url: sampleUrlForPlatform(platform), - author: { displayName: "Author" }, - }, - }; - assert.throws( - () => resolveHtmlSnapshotsFromVersionMeta(input), - /serverVerifiedAt is set but serverHtml is missing/, - `platform=${platform}`, - ); - } -}); - -// ── Additional invariants ──────────────────────────────────────────────────── -// -// 6. **Image occurrence mapping**: toPromptPostContext must faithfully forward -// image occurrences from the DB shape to the prompt shape, including the -// optional captionText field (omitted when null in DB). -// -// 7. **authorName omission**: When the post has no author, authorName must be -// absent from the returned context (not undefined or empty string). -// -// 8. **X hasVideo=false when media has no video URLs**: The X branch must -// correctly report hasVideo=false when mediaUrls contains only images. -// -// 9. **resolveHtmlSnapshotsFromVersionMeta: X always returns null/null**: -// X posts never have server/client HTML blobs. -// -// 10. **resolveHtmlSnapshotsFromVersionMeta: server-verified path returns -// typed discriminated union**: When serverVerifiedAt is non-null, the -// returned type has serverHtml: string (not null). - -test("toPromptPostContext maps image occurrences faithfully", () => { - const input: PromptPostContextInput = { - serverVerifiedAt: null, - contentBlob: { contentText: "text", contentHash: "hash" }, - imageOccurrenceSet: { - occurrences: [ - { - originalIndex: 0, - normalizedTextOffset: 42, - sourceUrl: "https://example.com/img.png", - captionText: "A caption", - }, - { - originalIndex: 1, - normalizedTextOffset: 100, - sourceUrl: "https://example.com/img2.png", - captionText: null, - }, - ], - }, - lesswrongVersionMeta: null, - xVersionMeta: null, - substackVersionMeta: { - publishedAt: null, - serverHtmlBlob: null, - clientHtmlBlob: null, - }, - wikipediaVersionMeta: null, - post: { - platform: "SUBSTACK", - url: "https://example.substack.com/p/test", - author: null, - }, - }; - - const ctx = toPromptPostContext(input); - assert.equal(ctx.imageOccurrences.length, 2); - - assert.equal(ctx.imageOccurrences[0]?.originalIndex, 0); - assert.equal(ctx.imageOccurrences[0].normalizedTextOffset, 42); - assert.equal(ctx.imageOccurrences[0].sourceUrl, "https://example.com/img.png"); - assert.equal(ctx.imageOccurrences[0].captionText, "A caption"); - - assert.equal(ctx.imageOccurrences[1]?.originalIndex, 1); - assert.equal(ctx.imageOccurrences[1].captionText, undefined, "null captionText is omitted"); -}); - -test("toPromptPostContext omits authorName when author is null", () => { - for (const platform of PLATFORM_VALUES) { - const ctx = toPromptPostContext({ - serverVerifiedAt: null, - contentBlob: { contentText: "text", contentHash: "hash" }, - imageOccurrenceSet: { occurrences: [] }, - lesswrongVersionMeta: null, - xVersionMeta: null, - substackVersionMeta: null, - wikipediaVersionMeta: null, - post: { - platform, - url: sampleUrlForPlatform(platform), - author: null, - }, - }); - assert.equal(ctx.authorName, undefined, `platform=${platform}`); - } -}); - -test("toPromptPostContext: X hasVideo=false when media contains only images", () => { - const ctx = toPromptPostContext({ - serverVerifiedAt: null, - contentBlob: { contentText: "tweet", contentHash: "hash" }, - imageOccurrenceSet: { occurrences: [] }, - lesswrongVersionMeta: null, - xVersionMeta: { - postedAt: null, - mediaUrls: ["https://pbs.twimg.com/media/photo.jpg"], - }, - substackVersionMeta: null, - wikipediaVersionMeta: null, - post: { - platform: "X", - url: "https://x.com/u/status/1", - author: { displayName: "User" }, - }, - }); - - assert.equal(ctx.hasVideo, false); -}); - -test("toPromptPostContext: X hasVideo=false when mediaUrls is empty", () => { - const ctx = toPromptPostContext({ - serverVerifiedAt: null, - contentBlob: { contentText: "tweet", contentHash: "hash" }, - imageOccurrenceSet: { occurrences: [] }, - lesswrongVersionMeta: null, - xVersionMeta: { - postedAt: null, - mediaUrls: [], - }, - substackVersionMeta: null, - wikipediaVersionMeta: null, - post: { - platform: "X", - url: "https://x.com/u/status/2", - author: null, - }, - }); - - assert.equal(ctx.hasVideo, false); -}); - -test("toPromptPostContext: non-X platforms have hasVideo=false", () => { - const NON_X: Platform[] = ["LESSWRONG", "SUBSTACK", "WIKIPEDIA"]; - for (const platform of NON_X) { - const { input } = platformTimestampFixtures[platform]; - const ctx = toPromptPostContext(input); - assert.equal(ctx.hasVideo, false, `platform=${platform}`); - } -}); - -test("resolveHtmlSnapshotsFromVersionMeta: X returns null/null regardless of serverVerifiedAt", () => { - const input: HtmlSnapshotsInput = { - serverVerifiedAt: null, - contentBlob: { contentText: "tweet", contentHash: "hash" }, - imageOccurrenceSet: { occurrences: [] }, - lesswrongVersionMeta: null, - xVersionMeta: { postedAt: null, mediaUrls: [] }, - substackVersionMeta: null, - wikipediaVersionMeta: null, - post: { - platform: "X", - url: "https://x.com/u/status/1", - author: null, - }, - }; - - const result = resolveHtmlSnapshotsFromVersionMeta(input); - assert.equal(result.serverVerifiedAt, null); - assert.equal(result.serverHtml, null); - assert.equal(result.clientHtml, null); -}); - -test("resolveHtmlSnapshotsFromVersionMeta: server-verified returns non-null serverHtml", () => { - const input: HtmlSnapshotsInput = { - serverVerifiedAt: new Date("2026-02-20T12:00:00.000Z"), - contentBlob: { contentText: "unused", contentHash: "hash" }, - imageOccurrenceSet: { occurrences: [] }, - lesswrongVersionMeta: { - publishedAt: null, - serverHtmlBlob: { htmlContent: "

    Server

    " }, - clientHtmlBlob: { htmlContent: "

    Client

    " }, - }, - xVersionMeta: null, - substackVersionMeta: null, - wikipediaVersionMeta: null, - post: { - platform: "LESSWRONG", - url: "https://www.lesswrong.com/posts/example", - author: null, - }, - }; - - const result = resolveHtmlSnapshotsFromVersionMeta(input); - assert.notEqual(result.serverVerifiedAt, null); - assert.equal(result.serverHtml, "

    Server

    "); - assert.equal(result.clientHtml, "

    Client

    "); -}); - -test("resolveHtmlSnapshotsFromVersionMeta: unverified returns nullable serverHtml", () => { - const input: HtmlSnapshotsInput = { - serverVerifiedAt: null, - contentBlob: { contentText: "unused", contentHash: "hash" }, - imageOccurrenceSet: { occurrences: [] }, - lesswrongVersionMeta: { - publishedAt: null, - serverHtmlBlob: null, - clientHtmlBlob: { htmlContent: "

    Client

    " }, - }, - xVersionMeta: null, - substackVersionMeta: null, - wikipediaVersionMeta: null, - post: { - platform: "LESSWRONG", - url: "https://www.lesswrong.com/posts/example", - author: null, - }, - }; - - const result = resolveHtmlSnapshotsFromVersionMeta(input); - assert.equal(result.serverVerifiedAt, null); - assert.equal(result.serverHtml, null); - assert.equal(result.clientHtml, "

    Client

    "); -}); diff --git a/src/typescript/api/test/unit/public-read-model.test.ts b/src/typescript/api/test/unit/public-read-model.test.ts index dfd71c5..2a34a4c 100644 --- a/src/typescript/api/test/unit/public-read-model.test.ts +++ b/src/typescript/api/test/unit/public-read-model.test.ts @@ -11,7 +11,7 @@ interface InvestigationFindFirstResult { id: string; checkedAt: Date | null; provider: string; - model: string; + model: string | null; input: { provenance: string; } | null; @@ -124,7 +124,7 @@ test("getPublicInvestigationById maps a complete SERVER_VERIFIED investigation", id: "inv_1", checkedAt, provider: "OPENAI", - model: "OPENAI_GPT_5", + model: "gpt-6.1-sol", input: { provenance: "SERVER_VERIFIED", }, @@ -176,32 +176,33 @@ test("getPublicInvestigationById maps a complete SERVER_VERIFIED investigation", provenance: "SERVER_VERIFIED", serverVerifiedAt, }); + assert.equal(result.investigation.model, "gpt-6.1-sol"); assert.equal(result.post.platform, "X"); assert.equal(result.post.externalId, "tweet_1"); assert.equal(result.claims.length, 1); assert.equal(result.claims[0]?.sources.length, 1); }); -test("getPublicInvestigationById throws on invalid lifecycle provenance data", async () => { +test("getPublicInvestigationById throws when a COMPLETE investigation has no recorded model", async () => { const prisma = createMockPrisma({ investigation: { findFirst: async () => ({ - id: "inv_invalid", + id: "inv_no_model", checkedAt: new Date("2026-02-27T09:30:00.000Z"), provider: "OPENAI", - model: "OPENAI_GPT_5", + model: null, input: { - provenance: "UNKNOWN_PROVENANCE", + provenance: "SERVER_VERIFIED", }, postVersion: { - serverVerifiedAt: null, + serverVerifiedAt: new Date("2026-02-27T09:00:00.000Z"), contentBlob: { - contentHash: "hash-invalid", + contentHash: "hash-no-model", }, post: { platform: "X", - externalId: "tweet_invalid", - url: "https://x.com/openerrata/status/tweet_invalid", + externalId: "tweet_no_model", + url: "https://x.com/openerrata/status/tweet_no_model", }, }, prompt: { @@ -217,14 +218,13 @@ test("getPublicInvestigationById throws on invalid lifecycle provenance data", a }); await assert.rejects( - getPublicInvestigationById(prisma, "inv_invalid"), + getPublicInvestigationById(prisma, "inv_no_model"), (error: unknown) => - error instanceof PublicReadModelInvariantError && - error.message.includes("invalid provenance"), + error instanceof PublicReadModelInvariantError && error.message.includes("null model"), ); }); -test("getPublicMetrics computes incidence and handles empty query rows", async () => { +test("getPublicMetrics computes incidence, and reports none when nothing was investigated", async () => { const metricsPrisma = createMockPrisma({ $queryRaw: async () => [{ total_investigated: 4, with_flags: 1 }], }); @@ -242,12 +242,19 @@ test("getPublicMetrics computes incidence and handles empty query rows", async ( }); const emptyPrisma = createMockPrisma({ - $queryRaw: async () => [], + $queryRaw: async () => [{ total_investigated: 0, with_flags: 0 }], }); const empty = await getPublicMetrics(emptyPrisma, {}); assert.deepEqual(empty, { totalInvestigatedPosts: 0, investigatedPostsWithFlags: 0, - factCheckIncidence: 0, + factCheckIncidence: null, + }); +}); + +test("getPublicMetrics fails loudly if the aggregate query returns no row", async () => { + const brokenPrisma = createMockPrisma({ + $queryRaw: async () => [], }); + await assert.rejects(getPublicMetrics(brokenPrisma, {}), PublicReadModelInvariantError); }); diff --git a/src/typescript/api/test/unit/public-schema.test.ts b/src/typescript/api/test/unit/public-schema.test.ts index 687adf9..b398aaf 100644 --- a/src/typescript/api/test/unit/public-schema.test.ts +++ b/src/typescript/api/test/unit/public-schema.test.ts @@ -1,10 +1,7 @@ import assert from "node:assert/strict"; import { test } from "node:test"; import { graphql } from "graphql"; -import { MINIMUM_SUPPORTED_EXTENSION_VERSION } from "../../src/lib/config/env.js"; import type { PrismaClient } from "../../src/lib/db/prisma-client.js"; -import { appRouter } from "../../src/lib/trpc/router.js"; -import type { Context } from "../../src/lib/trpc/context.js"; import { createPublicGraphqlSchema } from "../../src/lib/graphql/public-schema.js"; test("createPublicGraphqlSchema uses injected public read-model dependencies", async () => { @@ -39,7 +36,7 @@ test("createPublicGraphqlSchema uses injected public read-model dependencies", a checkedAt: new Date("2026-02-01T00:00:00.000Z"), promptVersion: "integration-test-prompt", provider: "OPENAI", - model: "OPENAI_GPT_5", + model: "gpt-6.1-sol", }, post: { platform: "X", externalId: "x-1", url: "https://x.com/openerrata/status/x-1" }, claims: [], @@ -155,113 +152,6 @@ test("createPublicGraphqlSchema uses injected public read-model dependencies", a }); }); -interface PublicInvestigationGraphqlResult { - investigation: { - id: string; - origin: { - provenance: "SERVER_VERIFIED" | "CLIENT_FALLBACK"; - serverVerifiedAt: Date | string | null; - }; - corroborationCount: number; - checkedAt: Date | string; - promptVersion: string; - provider: string; - model: string; - }; - post: { - platform: string; - externalId: string; - url: string; - }; - claims: { - id: string; - text: string; - context: string; - summary: string; - reasoning: string; - sources: { - url: string; - title: string; - snippet: string; - }[]; - }[]; -} - -interface PublicInvestigationTrpcResult { - investigation: { - id: string; - origin: { - provenance: "SERVER_VERIFIED" | "CLIENT_FALLBACK"; - serverVerifiedAt?: string | undefined; - }; - corroborationCount: number; - checkedAt: string; - promptVersion: string; - provider: string; - model: string; - }; - post: { - platform: string; - externalId: string; - url: string; - }; - claims: { - id: string; - text: string; - context: string; - summary: string; - reasoning: string; - sources: { - url: string; - title: string; - snippet: string; - }[]; - }[]; -} - -function normalizePublicInvestigationResult( - input: PublicInvestigationGraphqlResult | PublicInvestigationTrpcResult, -) { - const toIsoOrNull = (value: Date | string | null | undefined): string | null => { - if (value === null || value === undefined) { - return null; - } - return value instanceof Date ? value.toISOString() : value; - }; - - return { - investigation: { - id: input.investigation.id, - origin: { - provenance: input.investigation.origin.provenance, - serverVerifiedAt: toIsoOrNull(input.investigation.origin.serverVerifiedAt), - }, - corroborationCount: input.investigation.corroborationCount, - checkedAt: toIsoOrNull(input.investigation.checkedAt), - promptVersion: input.investigation.promptVersion, - provider: input.investigation.provider, - model: input.investigation.model, - }, - post: { - platform: input.post.platform, - externalId: input.post.externalId, - url: input.post.url, - }, - claims: input.claims.map((claim) => ({ - id: claim.id, - text: claim.text, - context: claim.context, - summary: claim.summary, - reasoning: claim.reasoning, - sources: claim.sources.map((source) => ({ - url: source.url, - title: source.title, - snippet: source.snippet, - })), - })), - }; -} - interface MockPublicInvestigationRow { id: string; checkedAt: Date | null; @@ -325,13 +215,13 @@ function createPublicPrismaMock( return prisma as PrismaClient; } -test("public.getInvestigation is semantically consistent between tRPC and GraphQL", async () => { +test("publicInvestigation maps trust signals and claims from the read model", async () => { const rowsByInvestigationId: Record = { "inv-server": { id: "inv-server", checkedAt: new Date("2026-02-28T09:30:00.000Z"), provider: "OPENAI", - model: "OPENAI_GPT_5", + model: "gpt-6.1-sol", input: { provenance: "SERVER_VERIFIED", }, @@ -373,7 +263,7 @@ test("public.getInvestigation is semantically consistent between tRPC and GraphQ id: "inv-fallback", checkedAt: new Date("2026-02-28T10:30:00.000Z"), provider: "OPENAI", - model: "OPENAI_GPT_5", + model: "gpt-6.1-sol", input: { provenance: "CLIENT_FALLBACK", }, @@ -398,21 +288,6 @@ test("public.getInvestigation is semantically consistent between tRPC and GraphQ }, }; const prisma = createPublicPrismaMock(rowsByInvestigationId); - const event = Object.create(null) as Context["event"]; - - const caller = appRouter.createCaller({ - event, - prisma, - viewerKey: "unit-viewer", - ipRangeKey: "unit-ip-range", - isAuthenticated: false, - canInvestigate: false, - userOpenAiApiKey: null, - hasValidAttestation: false, - extensionVersion: null, - minimumSupportedExtensionVersion: MINIMUM_SUPPORTED_EXTENSION_VERSION, - }); - const graphqlSchema = createPublicGraphqlSchema(); const graphqlQuery = ` query PublicInvestigation($investigationId: ID!) { @@ -450,15 +325,59 @@ test("public.getInvestigation is semantically consistent between tRPC and GraphQ } `; - for (const investigationId of ["inv-server", "inv-fallback"] as const) { - const trpcResult = await caller.public.getInvestigation({ - investigationId, - }); - assert.notEqual(trpcResult, null); - if (trpcResult === null) { - throw new Error("Expected tRPC public.getInvestigation result"); - } + const expected = { + "inv-server": { + investigation: { + id: "inv-server", + origin: { provenance: "SERVER_VERIFIED", serverVerifiedAt: "2026-02-28T09:00:00.000Z" }, + corroborationCount: 3, + checkedAt: "2026-02-28T09:30:00.000Z", + promptVersion: "v1.11.0", + provider: "OPENAI", + model: "gpt-6.1-sol", + }, + post: { + platform: "X", + externalId: "x-server", + url: "https://x.com/openerrata/status/x-server", + }, + claims: [ + { + id: "claim-server", + text: "Claim text", + context: "Claim context", + summary: "Claim summary", + reasoning: "Claim reasoning", + sources: [ + { + url: "https://example.com/source-server", + title: "Server Source", + snippet: "Server snippet", + }, + ], + }, + ], + }, + "inv-fallback": { + investigation: { + id: "inv-fallback", + origin: { provenance: "CLIENT_FALLBACK", serverVerifiedAt: null }, + corroborationCount: 1, + checkedAt: "2026-02-28T10:30:00.000Z", + promptVersion: "v1.11.0", + provider: "OPENAI", + model: "gpt-6.1-sol", + }, + post: { + platform: "LESSWRONG", + externalId: "lw-fallback", + url: "https://www.lesswrong.com/posts/lw-fallback", + }, + claims: [], + }, + }; + for (const investigationId of ["inv-server", "inv-fallback"] as const) { const graphqlResponse = await graphql({ schema: graphqlSchema, source: graphqlQuery, @@ -466,20 +385,8 @@ test("public.getInvestigation is semantically consistent between tRPC and GraphQ contextValue: { prisma }, }); assert.deepEqual(graphqlResponse.errors, undefined); - - const publicInvestigation = ( - graphqlResponse.data as { - publicInvestigation: PublicInvestigationGraphqlResult | null; - } - ).publicInvestigation; - assert.notEqual(publicInvestigation, null); - if (publicInvestigation === null) { - throw new Error("Expected GraphQL publicInvestigation result"); - } - - assert.deepEqual( - normalizePublicInvestigationResult(trpcResult), - normalizePublicInvestigationResult(publicInvestigation), - ); + assert.deepEqual(JSON.parse(JSON.stringify(graphqlResponse.data)), { + publicInvestigation: expected[investigationId], + }); } }); diff --git a/src/typescript/api/test/unit/queue-lifecycle.test.ts b/src/typescript/api/test/unit/queue-lifecycle.test.ts deleted file mode 100644 index ccb98c9..0000000 --- a/src/typescript/api/test/unit/queue-lifecycle.test.ts +++ /dev/null @@ -1,354 +0,0 @@ -import assert from "node:assert/strict"; -import { test } from "node:test"; -import { - QUEUE_ERROR_CODES, - QueueClosedError, - QueueConnectError, - QueueReleaseError, - createQueueManager, - type QueueErrorCode, - type Releasable, -} from "../../src/lib/services/queue-lifecycle.js"; -import { - createDeterministicRandom, - randomChance, - randomInt, - sleep, - withTimeout, -} from "../helpers/fuzz-utils.js"; - -type QueueErrorClass = - | typeof QueueClosedError - | typeof QueueConnectError - | typeof QueueReleaseError; - -function expectQueueError( - error: unknown, - expectedClass: QueueErrorClass, - expectedCode: QueueErrorCode, - expectedCause?: unknown, -): true { - assert.ok(error instanceof expectedClass); - assert.equal(error.code, expectedCode); - if (expectedCause !== undefined) { - assert.equal(error.cause, expectedCause); - } - return true; -} - -function createMockUtils(): Releasable & { released: boolean } { - return { - released: false, - async release() { - this.released = true; - }, - }; -} - -function deferred(): { - promise: Promise; - resolve: (value: T) => void; - reject: (error: Error) => void; -} { - let resolve!: (value: T) => void; - let reject!: (error: Error) => void; - const promise = new Promise((res, rej) => { - resolve = res; - reject = rej; - }); - return { promise, resolve, reject }; -} - -test("acquire returns connected utils", async () => { - const mockUtils = createMockUtils(); - const manager = createQueueManager(async () => mockUtils); - const utils = await manager.acquire(); - assert.equal(utils, mockUtils); -}); - -test("acquire coalesces concurrent callers on the same connection", async () => { - let connectCount = 0; - const mockUtils = createMockUtils(); - const manager = createQueueManager(async () => { - connectCount++; - return mockUtils; - }); - - const [a, b, c] = await Promise.all([manager.acquire(), manager.acquire(), manager.acquire()]); - - assert.equal(a, mockUtils); - assert.equal(b, mockUtils); - assert.equal(c, mockUtils); - assert.equal(connectCount, 1); -}); - -test("acquire reuses ready utils without reconnecting", async () => { - let connectCount = 0; - const mockUtils = createMockUtils(); - const manager = createQueueManager(async () => { - connectCount++; - return mockUtils; - }); - - await manager.acquire(); - await manager.acquire(); - await manager.acquire(); - assert.equal(connectCount, 1); -}); - -test("acquire throws after close", async () => { - const manager = createQueueManager(async () => createMockUtils()); - await manager.close(); - await assert.rejects( - () => manager.acquire(), - (error) => expectQueueError(error, QueueClosedError, QUEUE_ERROR_CODES.CLOSED), - ); -}); - -test("close releases ready utils", async () => { - const mockUtils = createMockUtils(); - const manager = createQueueManager(async () => mockUtils); - await manager.acquire(); - assert.equal(mockUtils.released, false); - - await manager.close(); - assert.equal(mockUtils.released, true); - await assert.rejects( - () => manager.acquire(), - (error) => expectQueueError(error, QueueClosedError, QUEUE_ERROR_CODES.CLOSED), - ); -}); - -test("close during initialization waits for init then releases", async () => { - const mockUtils = createMockUtils(); - const init = deferred(); - const manager = createQueueManager(() => init.promise); - - // Start acquisition (blocks on init) - const acquirePromise = manager.acquire(); - - // Request close while init is pending - const closePromise = manager.close(); - - // Resolve init — close should release the utils - init.resolve(mockUtils); - await closePromise; - - assert.equal(mockUtils.released, true); - await assert.rejects( - () => acquirePromise, - (error) => expectQueueError(error, QueueClosedError, QUEUE_ERROR_CODES.CLOSED), - ); -}); - -test("close during failed initialization transitions to closed", async () => { - const init = deferred(); - const manager = createQueueManager(() => init.promise); - - // Start acquisition (blocks on init) - const acquirePromise = manager.acquire(); - - // Request close while init is pending - const closePromise = manager.close(); - - // Reject init — nothing to release, close should succeed - init.reject(new Error("connection refused")); - await closePromise; - - // acquire should see "closed", not the connection error - await assert.rejects( - () => acquirePromise, - (error) => expectQueueError(error, QueueClosedError, QUEUE_ERROR_CODES.CLOSED), - ); -}); - -test("acquire swallows close errors and retries on next call", async () => { - let connectCount = 0; - const goodUtils = createMockUtils(); - const releaseFailure = new Error("release failed"); - const failingUtils: Releasable = { - async release() { - throw releaseFailure; - }, - }; - - const manager = createQueueManager(async () => { - connectCount++; - if (connectCount === 1) return failingUtils; - return goodUtils; - }); - - // First acquire succeeds - await manager.acquire(); - - // Close fails because release throws — state reverts to idle - await assert.rejects( - () => manager.close(), - (error) => - expectQueueError(error, QueueReleaseError, QUEUE_ERROR_CODES.RELEASE_FAILED, releaseFailure), - ); - - // Second acquire should reconnect (not throw "closed" or "release failed") - const utils = await manager.acquire(); - assert.equal(utils, goodUtils); - assert.equal(connectCount, 2); -}); - -test("acquire retries after connection failure", async () => { - let connectCount = 0; - const mockUtils = createMockUtils(); - const connectionFailure = new Error("connection refused"); - const manager = createQueueManager(async () => { - connectCount++; - if (connectCount === 1) throw connectionFailure; - return mockUtils; - }); - - await assert.rejects( - () => manager.acquire(), - (error) => - expectQueueError( - error, - QueueConnectError, - QUEUE_ERROR_CODES.CONNECT_FAILED, - connectionFailure, - ), - ); - - const utils = await manager.acquire(); - assert.equal(utils, mockUtils); - assert.equal(connectCount, 2); -}); - -test("concurrent close callers coalesce on the same promise", async () => { - const mockUtils = createMockUtils(); - const manager = createQueueManager(async () => mockUtils); - await manager.acquire(); - - // Both close calls should resolve without error - await Promise.all([manager.close(), manager.close()]); - assert.equal(mockUtils.released, true); -}); - -test("close is idempotent", async () => { - const manager = createQueueManager(async () => createMockUtils()); - await manager.close(); - await manager.close(); - await assert.rejects( - () => manager.acquire(), - (error) => expectQueueError(error, QueueClosedError, QUEUE_ERROR_CODES.CLOSED), - ); -}); - -test("randomized acquire/close schedule preserves lifecycle invariants", async () => { - const random = createDeterministicRandom(0x5eedc0de); - const rounds = 16; - const actionsPerRound = 80; - - for (let round = 0; round < rounds; round += 1) { - let connectInFlight = 0; - let maxConnectInFlight = 0; - let successfulConnectCount = 0; - let releaseCallCount = 0; - let closeSuccessCount = 0; - let injectReleaseFailures = true; - const seenUtils = new Set(); - - const manager = createQueueManager(async () => { - connectInFlight += 1; - maxConnectInFlight = Math.max(maxConnectInFlight, connectInFlight); - - try { - await sleep(randomInt(random, 0, 4)); - - if (randomChance(random, 0.2)) { - throw new Error(`connect failed in round ${round.toString()}`); - } - - const utils = { - id: `${round.toString()}-${successfulConnectCount.toString()}`, - async release() { - releaseCallCount += 1; - await sleep(randomInt(random, 0, 4)); - if (injectReleaseFailures && randomChance(random, 0.15)) { - throw new Error(`release failed in round ${round.toString()}`); - } - }, - }; - - successfulConnectCount += 1; - seenUtils.add(utils); - return utils; - } finally { - connectInFlight -= 1; - } - }); - - const operationPromises = Array.from({ length: actionsPerRound }, () => - (async () => { - await sleep(randomInt(random, 0, 5)); - const runAcquire = randomChance(random, 0.65); - - if (runAcquire) { - const startedAfterClose = closeSuccessCount > 0; - try { - const utils = await manager.acquire(); - assert.equal(seenUtils.has(utils), true, "acquire returned an unknown utils instance"); - if (startedAfterClose) { - assert.fail("acquire resolved successfully even though manager was already closed"); - } - } catch (error) { - assert.equal( - error instanceof QueueClosedError || error instanceof QueueConnectError, - true, - "acquire rejected with unexpected error class", - ); - } - return; - } - - try { - await manager.close(); - closeSuccessCount += 1; - } catch (error) { - assert.equal( - error instanceof QueueReleaseError, - true, - "close rejected with unexpected error class", - ); - } - })(), - ); - - await withTimeout( - Promise.all(operationPromises).then(() => undefined), - 10_000, - `queue lifecycle random round ${round.toString()} timed out`, - ); - - // Final close should deterministically terminate the manager. Release - // failures above are intentionally injected for concurrent lifecycle paths. - injectReleaseFailures = false; - await manager.close(); - - assert.equal( - maxConnectInFlight <= 1, - true, - "connect should never run concurrently across callers", - ); - assert.equal( - releaseCallCount <= successfulConnectCount, - true, - "release calls cannot exceed successful connections", - ); - - await Promise.all( - Array.from({ length: 8 }, async () => { - await assert.rejects( - () => manager.acquire(), - (error) => expectQueueError(error, QueueClosedError, QUEUE_ERROR_CODES.CLOSED), - ); - }), - ); - } -}); diff --git a/src/typescript/api/test/unit/request-identity.test.ts b/src/typescript/api/test/unit/request-identity.test.ts index 7c3e564..d32e491 100644 --- a/src/typescript/api/test/unit/request-identity.test.ts +++ b/src/typescript/api/test/unit/request-identity.test.ts @@ -2,9 +2,13 @@ import assert from "node:assert/strict"; import { test } from "node:test"; import { deriveRequestIdentity } from "../../src/lib/services/request-identity.js"; -test("deriveRequestIdentity uses authenticated viewer key path when api key is active", async () => { +const dependencies = { + hashContent: async (value: string) => `hash:${value}`, + deriveIpRangePrefix: (address: string) => `prefix:${address}`, +}; + +test("deriveRequestIdentity keys authenticated viewers by their instance API key", async () => { const lookedUpApiKeys: string[] = []; - const verifiedSignatures: { body: string; signature: string }[] = []; const identity = await deriveRequestIdentity( { @@ -12,37 +16,26 @@ test("deriveRequestIdentity uses authenticated viewer key path when api key is a userAgent: "UnitTestBrowser/1.0", instanceApiKey: " live-key ", userOpenAiApiKey: " sk-user ", - attestationSignature: "sig-v1", - attestationBody: '{"hello":"world"}', }, { - hashContent: async (value) => `hash:${value}`, + ...dependencies, findActiveInstanceApiKeyHash: async (apiKey) => { lookedUpApiKeys.push(apiKey); return apiKey === "live-key" ? "api-hash" : null; }, - deriveIpRangePrefix: (address) => `prefix:${address}`, - verifyHmac: async (body, signature) => { - verifiedSignatures.push({ body, signature }); - return true; - }, }, ); assert.deepEqual(lookedUpApiKeys, ["live-key"]); - assert.deepEqual(verifiedSignatures, [{ body: '{"hello":"world"}', signature: "sig-v1" }]); assert.deepEqual(identity, { - authenticatedApiKeyHash: "api-hash", viewerKey: "hash:apikey:api-hash", ipRangeKey: "hash:iprange:prefix:203.0.113.7", - userOpenAiApiKey: "sk-user", isAuthenticated: true, - canInvestigate: true, - hasValidAttestation: true, + userOpenAiApiKey: "sk-user", }); }); -test("deriveRequestIdentity uses anonymous viewer key path when api key is absent", async () => { +test("deriveRequestIdentity keys anonymous viewers by address and user agent", async () => { let lookupCount = 0; const identity = await deriveRequestIdentity( { @@ -50,74 +43,37 @@ test("deriveRequestIdentity uses anonymous viewer key path when api key is absen userAgent: "UnitTestBrowser/2.0", instanceApiKey: " ", userOpenAiApiKey: null, - attestationSignature: null, - attestationBody: null, }, { - hashContent: async (value) => `hash:${value}`, + ...dependencies, findActiveInstanceApiKeyHash: async () => { lookupCount += 1; return "should-not-be-used"; }, - deriveIpRangePrefix: (address) => `prefix:${address}`, - verifyHmac: async () => true, }, ); assert.equal(lookupCount, 0); assert.deepEqual(identity, { - authenticatedApiKeyHash: null, viewerKey: "hash:anon:198.51.100.11:UnitTestBrowser/2.0", ipRangeKey: "hash:iprange:prefix:198.51.100.11", - userOpenAiApiKey: null, isAuthenticated: false, - canInvestigate: false, - hasValidAttestation: false, + userOpenAiApiKey: null, }); }); -test("deriveRequestIdentity allows investigate with user-provided OpenAI key", async () => { +test("deriveRequestIdentity treats an unknown or revoked instance key as anonymous", async () => { const identity = await deriveRequestIdentity( { - clientAddress: "192.0.2.44", + clientAddress: "198.51.100.12", userAgent: "UnitTestBrowser/3.0", - instanceApiKey: null, - userOpenAiApiKey: " sk-openai ", - attestationSignature: null, - attestationBody: "", - }, - { - hashContent: async (value) => `hash:${value}`, - findActiveInstanceApiKeyHash: async () => null, - deriveIpRangePrefix: (address) => `prefix:${address}`, - verifyHmac: async () => true, + instanceApiKey: "revoked-key", + userOpenAiApiKey: "sk-user", }, + { ...dependencies, findActiveInstanceApiKeyHash: async () => null }, ); assert.equal(identity.isAuthenticated, false); - assert.equal(identity.userOpenAiApiKey, "sk-openai"); - assert.equal(identity.canInvestigate, true); -}); - -test("deriveRequestIdentity marks attestation invalid when verifier throws", async () => { - const identity = await deriveRequestIdentity( - { - clientAddress: "198.18.0.5", - userAgent: "UnitTestBrowser/4.0", - instanceApiKey: null, - userOpenAiApiKey: null, - attestationSignature: "sig-v1", - attestationBody: "payload", - }, - { - hashContent: async (value) => `hash:${value}`, - findActiveInstanceApiKeyHash: async () => null, - deriveIpRangePrefix: (address) => `prefix:${address}`, - verifyHmac: async () => { - throw new Error("boom"); - }, - }, - ); - - assert.equal(identity.hasValidAttestation, false); + assert.equal(identity.viewerKey, "hash:anon:198.51.100.12:UnitTestBrowser/3.0"); + assert.equal(identity.userOpenAiApiKey, "sk-user"); }); diff --git a/src/typescript/api/test/unit/startup-config.test.ts b/src/typescript/api/test/unit/startup-config.test.ts new file mode 100644 index 0000000..eb38c00 --- /dev/null +++ b/src/typescript/api/test/unit/startup-config.test.ts @@ -0,0 +1,36 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { assertClientAddressSourceConfigured } from "../../src/lib/config/startup.js"; + +/** + * In production the API sits behind the ingress proxy; without a trusted + * client-address header every viewer would share the proxy's IP range and the + * per-range view-credit cap would flatten selector ranking. + */ + +test("production API refuses to start without ADDRESS_HEADER", () => { + assert.throws(() => assertClientAddressSourceConfigured({}), /ADDRESS_HEADER/); + assert.throws( + () => assertClientAddressSourceConfigured({ ADDRESS_HEADER: " " }), + /ADDRESS_HEADER/, + ); +}); + +test("X-Forwarded-For needs an explicit positive XFF_DEPTH", () => { + for (const XFF_DEPTH of [undefined, "", "0", "-1", "one"]) { + assert.throws( + () => + assertClientAddressSourceConfigured({ + ADDRESS_HEADER: "X-Forwarded-For", + ...(XFF_DEPTH === undefined ? {} : { XFF_DEPTH }), + }), + /XFF_DEPTH/, + String(XFF_DEPTH), + ); + } +}); + +test("the chart's defaults and single-value headers are accepted", () => { + assertClientAddressSourceConfigured({ ADDRESS_HEADER: "x-forwarded-for", XFF_DEPTH: "1" }); + assertClientAddressSourceConfigured({ ADDRESS_HEADER: "cf-connecting-ip" }); +}); diff --git a/src/typescript/api/test/unit/update-lineage.test.ts b/src/typescript/api/test/unit/update-lineage.test.ts new file mode 100644 index 0000000..4aa97e8 --- /dev/null +++ b/src/typescript/api/test/unit/update-lineage.test.ts @@ -0,0 +1,21 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { buildLineDiff } from "../../src/lib/services/update-lineage.js"; + +test("buildLineDiff reports no changes for identical text", () => { + assert.equal(buildLineDiff("same\ncontent", "same\ncontent"), "No changes detected."); +}); + +test("buildLineDiff reports the differing middle as removed and added lines", () => { + const diff = buildLineDiff("keep one\nremove me\nkeep tail", "keep one\nadd me\nkeep tail"); + assert.equal( + diff, + "Diff summary (line context):\n- Removed lines:\nremove me\n+ Added lines:\nadd me", + ); +}); + +test("buildLineDiff marks a pure insertion with no removed lines", () => { + const diff = buildLineDiff("head\ntail", "head\nnew\ntail"); + assert.match(diff, /- Removed lines:\n\(none\)/); + assert.match(diff, /\+ Added lines:\nnew/); +}); diff --git a/src/typescript/api/test/unit/word-count.test.ts b/src/typescript/api/test/unit/word-count.test.ts new file mode 100644 index 0000000..20f4380 --- /dev/null +++ b/src/typescript/api/test/unit/word-count.test.ts @@ -0,0 +1,37 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { WORD_COUNT_LIMIT } from "@openerrata/shared"; +import { wordCount } from "../../src/lib/services/word-count.js"; + +/** + * wordCount is what ContentBlob.wordCount stores, and investigateNow and the + * selector compare it against WORD_COUNT_LIMIT, so the count must be stable + * across whitespace kinds and the limit boundary must be exact. + */ + +test("wordCount splits on any run of whitespace, including non-breaking space", () => { + assert.equal(wordCount("hello world"), 2); + assert.equal(wordCount("hello world\t\tfoo\nbar"), 4); + assert.equal(wordCount("hello world"), 2); +}); + +test("wordCount returns 0 for empty and whitespace-only text", () => { + assert.equal(wordCount(""), 0); + assert.equal(wordCount(" \t\n "), 0); + assert.equal(wordCount("\n\n\n"), 0); +}); + +test("wordCount treats punctuation-joined and non-Latin tokens as single words", () => { + assert.equal(wordCount("mother-in-law it's don't"), 3); + assert.equal(wordCount("Hello, world!"), 2); + assert.equal(wordCount("你好 世界"), 2); + assert.equal(wordCount("🎉 🎊 🎈"), 3); + assert.equal(wordCount("a".repeat(10_000)), 1); +}); + +test("wordCount is exact at the WORD_COUNT_LIMIT boundary", () => { + const words = (count: number): string => + Array.from({ length: count }, (_, i) => `word${i.toString()}`).join(" "); + assert.equal(wordCount(words(WORD_COUNT_LIMIT)), WORD_COUNT_LIMIT); + assert.equal(wordCount(words(WORD_COUNT_LIMIT + 1)), WORD_COUNT_LIMIT + 1); +}); diff --git a/src/typescript/api/tsconfig.json b/src/typescript/api/tsconfig.json index ebd3a03..d1d86cd 100644 --- a/src/typescript/api/tsconfig.json +++ b/src/typescript/api/tsconfig.json @@ -4,5 +4,5 @@ "lib": ["ES2022"], "types": ["node"] }, - "include": ["src", "test", "prisma.config.ts"] + "include": ["src", "test", "scripts", "prisma.config.ts"] } From e8b08554dec48a47467c8b813dff5aabb1fa75d6 Mon Sep 17 00:00:00 2001 From: Dean Valentine Date: Fri, 2 Oct 2026 17:06:45 -0700 Subject: [PATCH 11/15] =?UTF-8?q?extension:=200.4.0=20=E2=80=94=20reliable?= =?UTF-8?q?=20sessions=20and=20fixed=20live-site=20extraction?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Resume polling for posts investigated by others (INVESTIGATING + id) - Idempotent injection with a side-effect-free PING; distinct GET_VISIBILITY and background-driven LOCATION_CHANGED; unique session ids - Error replies delivered; only real context invalidation resets silently - One text index for adapters, claim mapping and mutation baseline; highlight marks never leave the page - Claim markdown renders no images; typed message maps end to end - Settings load as a union; an invalid API URL is an error, not a fallback - Live-site fixes: logged-out X, Substack isAccessibleForFree paywall, fr.wikipedia cachelinks, Wikipedia video detection, LessWrong /w/ tags; Substack subscribe/share/embed chrome excluded - Remove the unused attestation secret; http://*/* optional; Firefox 128+ - e2e suite runs headless (Chromium new headless, no xvfb) Co-Authored-By: Claude Opus 5.5 --- src/typescript/extension/AGENTS.md | 104 ++- src/typescript/extension/package.json | 2 +- .../extension/scripts/run-playwright-e2e.mjs | 48 +- .../src/background/api-client-core.ts | 25 +- .../extension/src/background/api-client.ts | 215 ++--- .../src/background/browser-compat.ts | 92 +- .../extension/src/background/cache-store.ts | 237 ----- .../extension/src/background/cache.ts | 47 - .../background/content-script-injection.ts | 201 ++-- .../extension/src/background/index.ts | 782 +++------------- .../src/background/investigation-polling.ts | 425 +++------ .../src/background/investigation-snapshot.ts | 92 -- .../src/background/investigation-state.ts | 48 - .../src/background/message-dispatch.ts | 29 - .../src/background/message-handlers.ts | 195 ++++ .../src/background/page-content-action.ts | 19 - .../src/background/page-content-decision.ts | 47 - .../extension/src/background/post-status.ts | 254 ++--- .../extension/src/background/tab-state.ts | 169 ++++ .../extension/src/background/toolbar-badge.ts | 18 +- .../background/upgrade-required-runtime.ts | 140 --- .../src/background/upgrade-required-state.ts | 34 - .../src/background/upgrade-required.ts | 115 +++ .../extension/src/content/adapters/index.ts | 37 +- .../src/content/adapters/lesswrong.ts | 258 ++---- .../extension/src/content/adapters/model.ts | 69 +- .../src/content/adapters/substack.ts | 191 ++-- .../extension/src/content/adapters/utils.ts | 279 ++---- .../src/content/adapters/wikipedia.ts | 280 +++--- .../extension/src/content/adapters/x.ts | 280 +++--- .../extension/src/content/annotation-dom.ts | 25 + .../src/content/annotation-lifecycle.ts | 44 +- .../extension/src/content/annotations.ts | 75 +- .../extension/src/content/annotator.ts | 212 ++--- .../extension/src/content/bootstrap.ts | 19 +- .../extension/src/content/claim-markdown.ts | 43 +- .../extension/src/content/content-text.ts | 17 + .../src/content/control-message-handler.ts | 44 - .../extension/src/content/dom-mapper.ts | 429 ++------- .../extension/src/content/dom-text-index.ts | 184 ++++ src/typescript/extension/src/content/main.ts | 33 +- .../src/content/normalized-text-index.ts | 185 ++++ .../extension/src/content/observer.ts | 83 +- .../src/content/page-session-controller.ts | 869 ++++++------------ .../extension/src/content/session-key.ts | 65 +- .../extension/src/content/session-state.ts | 184 ++-- .../src/content/sync-retry-policy.ts | 78 +- src/typescript/extension/src/content/sync.ts | 147 +-- src/typescript/extension/src/lib/messaging.ts | 180 ++++ .../extension/src/lib/page-bootstrap.ts | 34 +- .../extension/src/lib/page-locator.ts | 202 ++++ .../extension/src/lib/post-identity.ts | 71 -- .../extension/src/lib/protocol-version.ts | 37 - .../extension/src/lib/runtime-error.ts | 48 +- .../extension/src/lib/settings-core.ts | 100 +- src/typescript/extension/src/lib/settings.ts | 62 +- .../extension/src/lib/storage-keys.ts | 7 + .../extension/src/lib/substack-url.ts | 13 - .../extension/src/lib/sync-response.ts | 69 -- .../extension/src/lib/view-post-input.ts | 28 +- .../extension/src/lib/wikipedia-url.ts | 5 - src/typescript/extension/src/manifest.json | 7 +- .../extension/src/options/App.svelte | 168 ++-- src/typescript/extension/src/options/main.ts | 13 +- src/typescript/extension/src/popup/App.svelte | 294 +++--- src/typescript/extension/src/popup/main.ts | 13 +- .../extension/src/popup/status-identity.ts | 97 +- .../extension/test/e2e/extension-harness.ts | 223 +++++ .../test/e2e/extension-lifecycle.e2e.test.ts | 273 ++++++ .../wikipedia-de-marie-curie-page-html-1.json | 12 + .../wikipedia-fr-marie-curie-page-html-1.json | 12 + .../e2e/post-route-regressions.e2e.test.ts | 731 +++------------ .../test/e2e/wikipedia-browser-parity.spec.ts | 2 +- .../extension/test/e2e/wikipedia-fixtures.ts | 13 + .../extension/test/helpers/adapter-harness.ts | 2 + .../extension/test/helpers/chrome-mock.ts | 105 +++ src/typescript/extension/test/helpers/dom.ts | 86 ++ .../extension/test/helpers/statuses.ts | 89 ++ .../extension/test/unit/adapter-utils.test.ts | 153 +-- .../test/unit/api-error-code.test.ts | 4 +- .../unit/background-api-client-core.test.ts | 87 +- .../unit/background-browser-compat.test.ts | 62 +- .../background-cache-lifecycle.fuzz.test.ts | 324 ------- .../test/unit/background-cache-module.test.ts | 157 ---- .../background-investigation-polling.test.ts | 170 ++++ .../background-investigation-snapshot.test.ts | 153 --- .../background-investigation-state.test.ts | 64 -- .../unit/background-message-handlers.test.ts | 202 ++++ .../background-page-content-action.test.ts | 78 -- .../background-page-content-decision.test.ts | 230 ----- .../test/unit/background-post-status.test.ts | 202 ++-- .../test/unit/background-tab-state.test.ts | 147 +++ .../unit/background-toolbar-badge.test.ts | 102 +- ...ackground-upgrade-required-restore.test.ts | 17 + .../background-upgrade-required-state.test.ts | 60 -- .../unit/background-upgrade-required.test.ts | 41 + .../test/unit/block-separator-parity.test.ts | 39 +- .../test/unit/claim-markdown.test.ts | 30 + .../unit/content-annotation-lifecycle.test.ts | 103 +-- .../content-annotations-controller.test.ts | 291 ++---- .../test/unit/content-bootstrap.test.ts | 84 +- .../content-control-message-handler.test.ts | 75 -- .../test/unit/content-dom-mapper.test.ts | 539 ++--------- .../test/unit/content-observer.test.ts | 76 +- .../test/unit/content-session-state.test.ts | 175 ++-- .../test/unit/content-sync-listener.test.ts | 87 -- .../test/unit/content-sync-response.test.ts | 126 --- .../unit/content-sync-retry-policy.test.ts | 105 +-- .../test/unit/dom-text-index.test.ts | 74 ++ .../test/unit/lesswrong-adapter.test.ts | 51 + .../extension/test/unit/lib-messaging.test.ts | 134 +++ .../test/unit/lib-page-bootstrap.test.ts | 109 +-- .../extension/test/unit/lib-settings.test.ts | 43 +- .../test/unit/message-dispatch.test.ts | 42 - .../test/unit/normalized-text-index.test.ts | 90 ++ .../unit/platform-canonical-parity.test.ts | 45 +- .../test/unit/popup-post-view.test.ts | 7 +- .../test/unit/popup-status-identity.test.ts | 107 +-- .../test/unit/post-route-contract.test.ts | 168 ++-- .../extension/test/unit/runtime-error.test.ts | 55 +- .../extension/test/unit/session-key.test.ts | 145 +-- .../extension/test/unit/settings-core.test.ts | 42 +- .../test/unit/substack-adapter.test.ts | 100 ++ .../extension/test/unit/substack-url.test.ts | 15 - .../test/unit/view-post-input.test.ts | 36 +- .../test/unit/wikipedia-adapter.test.ts | 87 +- .../test/unit/wikipedia-boilerplate.test.ts | 284 ++++++ .../extension/test/unit/x-adapter.test.ts | 117 ++- src/typescript/extension/vite.config.ts | 9 +- 129 files changed, 7088 insertions(+), 9119 deletions(-) delete mode 100644 src/typescript/extension/src/background/cache-store.ts delete mode 100644 src/typescript/extension/src/background/cache.ts delete mode 100644 src/typescript/extension/src/background/investigation-snapshot.ts delete mode 100644 src/typescript/extension/src/background/investigation-state.ts delete mode 100644 src/typescript/extension/src/background/message-dispatch.ts create mode 100644 src/typescript/extension/src/background/message-handlers.ts delete mode 100644 src/typescript/extension/src/background/page-content-action.ts delete mode 100644 src/typescript/extension/src/background/page-content-decision.ts create mode 100644 src/typescript/extension/src/background/tab-state.ts delete mode 100644 src/typescript/extension/src/background/upgrade-required-runtime.ts delete mode 100644 src/typescript/extension/src/background/upgrade-required-state.ts create mode 100644 src/typescript/extension/src/background/upgrade-required.ts create mode 100644 src/typescript/extension/src/content/content-text.ts delete mode 100644 src/typescript/extension/src/content/control-message-handler.ts create mode 100644 src/typescript/extension/src/content/dom-text-index.ts create mode 100644 src/typescript/extension/src/content/normalized-text-index.ts create mode 100644 src/typescript/extension/src/lib/messaging.ts create mode 100644 src/typescript/extension/src/lib/page-locator.ts delete mode 100644 src/typescript/extension/src/lib/post-identity.ts delete mode 100644 src/typescript/extension/src/lib/protocol-version.ts create mode 100644 src/typescript/extension/src/lib/storage-keys.ts delete mode 100644 src/typescript/extension/src/lib/substack-url.ts delete mode 100644 src/typescript/extension/src/lib/sync-response.ts delete mode 100644 src/typescript/extension/src/lib/wikipedia-url.ts create mode 100644 src/typescript/extension/test/e2e/extension-harness.ts create mode 100644 src/typescript/extension/test/e2e/extension-lifecycle.e2e.test.ts create mode 100644 src/typescript/extension/test/e2e/fixtures/wikipedia/wikipedia-de-marie-curie-page-html-1.json create mode 100644 src/typescript/extension/test/e2e/fixtures/wikipedia/wikipedia-fr-marie-curie-page-html-1.json create mode 100644 src/typescript/extension/test/helpers/chrome-mock.ts create mode 100644 src/typescript/extension/test/helpers/dom.ts create mode 100644 src/typescript/extension/test/helpers/statuses.ts delete mode 100644 src/typescript/extension/test/unit/background-cache-lifecycle.fuzz.test.ts delete mode 100644 src/typescript/extension/test/unit/background-cache-module.test.ts create mode 100644 src/typescript/extension/test/unit/background-investigation-polling.test.ts delete mode 100644 src/typescript/extension/test/unit/background-investigation-snapshot.test.ts delete mode 100644 src/typescript/extension/test/unit/background-investigation-state.test.ts create mode 100644 src/typescript/extension/test/unit/background-message-handlers.test.ts delete mode 100644 src/typescript/extension/test/unit/background-page-content-action.test.ts delete mode 100644 src/typescript/extension/test/unit/background-page-content-decision.test.ts create mode 100644 src/typescript/extension/test/unit/background-tab-state.test.ts create mode 100644 src/typescript/extension/test/unit/background-upgrade-required-restore.test.ts delete mode 100644 src/typescript/extension/test/unit/background-upgrade-required-state.test.ts create mode 100644 src/typescript/extension/test/unit/background-upgrade-required.test.ts delete mode 100644 src/typescript/extension/test/unit/content-control-message-handler.test.ts delete mode 100644 src/typescript/extension/test/unit/content-sync-listener.test.ts delete mode 100644 src/typescript/extension/test/unit/content-sync-response.test.ts create mode 100644 src/typescript/extension/test/unit/dom-text-index.test.ts create mode 100644 src/typescript/extension/test/unit/lib-messaging.test.ts delete mode 100644 src/typescript/extension/test/unit/message-dispatch.test.ts create mode 100644 src/typescript/extension/test/unit/normalized-text-index.test.ts delete mode 100644 src/typescript/extension/test/unit/substack-url.test.ts create mode 100644 src/typescript/extension/test/unit/wikipedia-boilerplate.test.ts diff --git a/src/typescript/extension/AGENTS.md b/src/typescript/extension/AGENTS.md index 2554321..8fe041a 100644 --- a/src/typescript/extension/AGENTS.md +++ b/src/typescript/extension/AGENTS.md @@ -30,23 +30,27 @@ After building: ### Message Flow ``` -Content Script (runs on supported pages) - → extract content via platform adapter - → browser.runtime.sendMessage({ type: "PAGE_CONTENT", payload }) - → Background runtime (service worker in Chrome, scripts/event page in Firefox) - → api-client.ts: HTTP POST to /trpc/post.registerObservedVersion - → receives { postVersionId, versionHash, provenance } - → api-client.ts: HTTP POST to /trpc/post.recordViewAndGetStatus({ postVersionId }) - → optional: /trpc/post.investigateNow({ postVersionId }) when auto-investigate is enabled - → OpenErrata API - ← response with investigationState + claims - → Content Script renders annotations (or records "skipped") +Content Script (runs on supported pages; one controller per page) + → extract content via platform adapter (shared text index, src/content/dom-text-index.ts) + → PAGE_CONTENT { tabSessionId, content } (typed: src/lib/messaging.ts) + → Background runtime (service worker in Chrome, event page in Firefox) + → api-client.ts: post.registerObservedVersion → { postVersionId, ... } + → api-client.ts: post.recordViewAndGetStatus({ postVersionId }) + → caches the page session's status in storage.session (background/tab-state.ts) + → INVESTIGATING (started here or elsewhere) → polls post.getInvestigation + → optional: post.investigateNow when auto-investigate is enabled + ← reply: the cached ExtensionPostStatus; later updates arrive as STATUS_CHANGED + → Content Script renders highlights (or reports PAGE_SKIPPED) ``` -The popup communicates with the background via `browser.runtime.sendMessage` -using the same message protocol. The popup sends `GET_CACHED` to fetch the -active tab's status; the background queries `browser.tabs.query` to resolve -the active tab (since popup messages have no `sender.tab`). +The message protocol (types, payloads, responses) is one table per direction +in `shared/src/schemas/extension-protocol.ts`; see SPEC §3.8.1. The popup asks +the background for a tab's status with `GET_TAB_STATUS { tabId }` and talks to +the tab's content script directly (`GET_VISIBILITY`, `REQUEST_INVESTIGATE`, +`FOCUS_CLAIM`, ...). The background relays History API navigations to the +content script as `LOCATION_CHANGED` (content scripts run in an isolated world +and cannot see the page's `pushState`) and probes for a live content script +with the side-effect-free `PING` before injecting one. ### Content Scripts — IIFE Build Requirement @@ -67,33 +71,34 @@ the background worker and communicate via messages. ### Platform Adapters -Each adapter implements `PlatformAdapter` (defined in `adapters/lesswrong.ts`): +Each adapter implements `PlatformAdapter` (`adapters/model.ts`): -- `matches(url)` — does this URL belong to this platform? -- `extract(document)` — extract `PlatformContent` from the DOM -- `getContentRoot(document)` — return the element containing the post body +- `matches(url)` / `detectFromDom(document)` — URL-first platform selection, DOM fallback +- `pageLocator(url)` — what the URL says about the post (`lib/page-locator.ts`, the one URL parser) +- `extract(document)` — `PlatformContent` (or a not-ready reason) +- `getContentRoot(document)` + `contentExclusionFilter(root)` — the root and the + non-content subtrees; text extraction, claim matching and HTML snapshots all + go through these, so they agree with each other and with the API -The `PlatformContent` includes a `platform` field (the string union value) -and `metadata` with platform-specific fields (slug, title, author, etc.). - -**Media detection**: adapters classify posts as `text_only`, `has_images`, or -`has_video`. Image posts are investigated; `has_video` posts are skipped. -Adapters also skip private/protected/subscriber-only views with -`reason: "private_or_gated"` and do not send content to the API in that case. -The content script sends `PAGE_SKIPPED` when skipping. +**Media detection**: content carries `hasVideo` and its image occurrences +(`has_video` > `has_images` > `text_only`). Image posts are investigated; +video posts are skipped. Adapters also skip private/protected/subscriber-only +views with `reason: "private_or_gated"` and do not send content to the API in +that case. The content script sends `PAGE_SKIPPED` when skipping. ### LessWrong Adapter - URL pattern: `lesswrong.com/posts/{postId}/{slug}` -- Content selector: `.PostsPage-postContent` +- Content selector: `#postContent` inside the `.PostsPage-postContent` whose JSON-LD names the post - Media check: `img, video, iframe` inside the content element - Source: `vendor/ForumMagnum/` has the LessWrong source for selector reference ### X/Twitter Adapter - URL pattern: `x.com/{author}/status/{tweetId}` (also `twitter.com`) -- Content selector: `[data-testid="tweetText"]` -- Media check: `tweetPhoto`, `videoPlayer`, `card.wrapper` test IDs +- Content selector: `[data-testid="tweetText"]` (logged in) or the article's own + `div[dir="auto"]` (logged-out frontend, no test IDs) +- Media check: `tweetPhoto`/`card.wrapper` images, `videoPlayer` or any `