From c94ab12c133a3def3f2ae2f8835b4f83549418b1 Mon Sep 17 00:00:00 2001 From: Dean Valentine Date: Sat, 3 Oct 2026 09:35:50 -0700 Subject: [PATCH] =?UTF-8?q?Keep=200.2=E2=80=930.3.x=20extensions=20working?= =?UTF-8?q?;=20safer=20deploy=20and=20release=20workflows?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - legacy-extension-v0: time-boxed tRPC adapter for extension versions >= 0.2.0 and < 0.4.0, converting the 9ee2cae wire protocol to and from the current procedures (vendored legacy schemas; lossy mappings documented). MINIMUM_SUPPORTED_EXTENSION_VERSION back to 0.2.0. Retire on 2026-12-01 or after 14 days with no 0.3.x traffic, whichever is later. - ExtensionVersionDailyCount (migration 0026): page views per extension version per UTC day, no viewer data; runbook SQL in README. - deploy.yml: `pulumi stack select` no longer creates missing stacks. - release-extension.yml: Chrome Web Store publishing is opt-in (PUBLISH_TO_CHROME_WEB_STORE), like Firefox AMO. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/deploy.yml | 15 +- .github/workflows/release-extension.yml | 14 +- PRIVACY.md | 12 +- README.md | 31 ++ SPEC.md | 39 +- .../migration.sql | 26 + src/typescript/api/prisma/schema.prisma | 12 + src/typescript/api/src/lib/config/env.ts | 6 +- .../lib/services/extension-version-counts.ts | 20 + .../src/lib/trpc/legacy-extension-v0/index.ts | 87 ++++ .../trpc/legacy-extension-v0/procedures.ts | 198 ++++++++ .../trpc/legacy-extension-v0/wire-schemas.ts | 405 +++++++++++++++ .../api/src/lib/trpc/routes/post.ts | 15 +- .../api-endpoints.integration.shared.ts | 4 +- ...ts.legacy-extension-v0.integration.test.ts | 460 ++++++++++++++++++ 15 files changed, 1327 insertions(+), 17 deletions(-) create mode 100644 src/typescript/api/prisma/migrations/0026_extension_version_daily_count/migration.sql create mode 100644 src/typescript/api/src/lib/services/extension-version-counts.ts create mode 100644 src/typescript/api/src/lib/trpc/legacy-extension-v0/index.ts create mode 100644 src/typescript/api/src/lib/trpc/legacy-extension-v0/procedures.ts create mode 100644 src/typescript/api/src/lib/trpc/legacy-extension-v0/wire-schemas.ts create mode 100644 src/typescript/api/test/integration/api-endpoints.legacy-extension-v0.integration.test.ts diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 9e0f61d..98cc2ac 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -623,10 +623,15 @@ jobs: run: | set -euo pipefail + # Select only, never --create: a fresh stack has no state, so `pulumi + # up` would try to rebuild resources that already exist. + if ! pulumi stack select "$STACK"; then + echo "::error::Pulumi stack '$STACK' was not found. PULUMI_ACCESS_TOKEN must belong to the Pulumi org that owns the openerrata stacks. A deploy must never create a fresh stack: it would try to rebuild live resources." + exit 1 + fi + # Stack config is rebuilt from scratch on every run: this step is the # complete configuration of the hosted stacks. - pulumi stack select "$STACK" --create - if [ -z "${CI_IMAGE_REPOSITORY:-}" ] || [ -z "${CI_IMAGE_TAG:-}" ] || [ -z "${CI_IMAGE_DIGEST:-}" ]; then echo "Build image metadata is required (CI_IMAGE_REPOSITORY, CI_IMAGE_TAG, CI_IMAGE_DIGEST)." exit 1 @@ -817,7 +822,11 @@ jobs: CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} run: | set -euo pipefail - pulumi stack select "$STACK" --create + # Select only, never --create (see "Configure Pulumi stack values"). + if ! pulumi stack select "$STACK"; then + echo "::error::Pulumi stack '$STACK' was not found. PULUMI_ACCESS_TOKEN must belong to the Pulumi org that owns the openerrata stacks. A deploy must never create a fresh stack: it would try to rebuild live resources." + exit 1 + fi TARGET_NAMESPACE="openerrata-${STACK}" API_SERVICE_NAME="${TARGET_NAMESPACE}-api" PULUMI_TIMEOUT_SECONDS=1200 diff --git a/.github/workflows/release-extension.yml b/.github/workflows/release-extension.yml index 4aa4a3f..d2a489d 100644 --- a/.github/workflows/release-extension.yml +++ b/.github/workflows/release-extension.yml @@ -131,7 +131,10 @@ jobs: timeout-minutes: 20 environment: extension-store-publish steps: - # Missing credentials fail the release instead of silently skipping a store. + # Each store publishes only when its variable on the extension-store-publish + # environment is "true". An enabled store with missing credentials fails + # the release instead of silently skipping it; with neither enabled, the + # release is the GitHub release's assets alone (manual store upload). - name: Check store credentials shell: bash env: @@ -141,10 +144,16 @@ jobs: CHROME_REFRESH_TOKEN: ${{ secrets.CHROME_REFRESH_TOKEN }} FIREFOX_JWT_ISSUER: ${{ secrets.FIREFOX_JWT_ISSUER }} FIREFOX_JWT_SECRET: ${{ secrets.FIREFOX_JWT_SECRET }} + PUBLISH_TO_CHROME_WEB_STORE: ${{ vars.PUBLISH_TO_CHROME_WEB_STORE }} PUBLISH_TO_FIREFOX_AMO: ${{ vars.PUBLISH_TO_FIREFOX_AMO }} run: | set -euo pipefail - required=(CHROME_EXTENSION_ID CHROME_CLIENT_ID CHROME_CLIENT_SECRET CHROME_REFRESH_TOKEN) + required=() + if [ "${PUBLISH_TO_CHROME_WEB_STORE:-}" = "true" ]; then + required+=(CHROME_EXTENSION_ID CHROME_CLIENT_ID CHROME_CLIENT_SECRET CHROME_REFRESH_TOKEN) + else + echo "::notice::Not publishing to the Chrome Web Store (environment variable PUBLISH_TO_CHROME_WEB_STORE is not true); upload the GitHub release's Chrome zip manually." + fi if [ "${PUBLISH_TO_FIREFOX_AMO:-}" = "true" ]; then required+=(FIREFOX_JWT_ISSUER FIREFOX_JWT_SECRET) else @@ -166,6 +175,7 @@ jobs: path: . - name: Publish to Chrome Web Store + if: vars.PUBLISH_TO_CHROME_WEB_STORE == 'true' env: CHROME_EXTENSION_ID: ${{ secrets.CHROME_EXTENSION_ID }} CHROME_CLIENT_ID: ${{ secrets.CHROME_CLIENT_ID }} diff --git a/PRIVACY.md b/PRIVACY.md index 3214713..0a8e69a 100644 --- a/PRIVACY.md +++ b/PRIVACY.md @@ -44,12 +44,22 @@ anonymous identifier from a hash of your IP address range (/24 for IPv4, /48 for IPv6) and User-Agent string. This is used solely for per-day view-credit rate limiting. Your full IP address and User-Agent are not stored. +### Extension version counts + +For each UTC day, the server counts how many page views each extension version +reported (for example, "0.4.0: 1,200 views on 2026-11-02"). Only the day, the +version and the count are stored — no IP address, viewer identifier, post or +time of day — so a count cannot be linked to you or to what you read. The +counts tell us when no one uses an old extension version any more, so the +server can stop supporting it. + ## What Data We Do Not Collect - Email addresses, real names, or account credentials - Browsing history or activity outside of supported platform pages - Demographic, location, or device information -- Analytics, telemetry, or crash reports +- Analytics, telemetry, or crash reports (beyond the anonymous per-version + daily counts above) - Cookies or cross-site tracking identifiers ## How Data Is Used diff --git a/README.md b/README.md index 86dd147..2bd64ba 100644 --- a/README.md +++ b/README.md @@ -190,6 +190,23 @@ After `pnpm dev:ext`, load the built extension as an unpacked extension: For signed Firefox releases, set `FIREFOX_GECKO_ID=` before building to control the generated `browser_specific_settings.gecko.id`. +### Releasing the Extension + +Push a tag `ext-v` (e.g. `ext-v0.4.0`). The Release Extension workflow +tests and packages the extension and creates a GitHub release with the Chrome +zip and `.crx` and the Firefox zip as assets. Store publishing is opt-in per +store, through variables on the `extension-store-publish` environment: + +- `PUBLISH_TO_CHROME_WEB_STORE=true` uploads and publishes the Chrome zip; it + needs the secrets `CHROME_EXTENSION_ID`, `CHROME_CLIENT_ID`, + `CHROME_CLIENT_SECRET` and `CHROME_REFRESH_TOKEN`. +- `PUBLISH_TO_FIREFOX_AMO=true` signs and lists the Firefox zip on AMO; it needs + `FIREFOX_JWT_ISSUER` and `FIREFOX_JWT_SECRET`. + +An enabled store with a missing secret fails the release. With neither enabled, +the release finishes once the GitHub release exists; upload its assets to the +stores by hand. + ## Deployment The Helm chart at `src/helm/openerrata/` is the single deployment artifact for both on-prem and hosted environments. It does not bundle a database or blob storage — it takes a `DATABASE_URL` and an S3-compatible bucket as config. `values.yaml` documents every value; these are required: @@ -236,6 +253,20 @@ Run these in order. Steps 1–2 need admin access and are idempotent; rerun them 5. **Deploy** `staging` (a push to the `staging` branch, or a manual run of the Deploy workflow), check it, then `main`. 6. **Retire the static credentials.** Delete the repository secrets `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, `AWS_REGION` and `KUBE_CONFIG_DATA`, which no workflow reads any more, and run the cleanup commands step 1 printed. +### Legacy extension traffic + +Extensions older than 0.4.0 are served through a time-boxed adapter +(`api/src/lib/trpc/legacy-extension-v0`), to be retired on 2026-12-01 or once +14 consecutive days pass with no requests from them, whichever is later. This +query lists the days in the last 14 with legacy (0.2.x/0.3.x) page views; no +rows means the traffic is gone: + +```sql +SELECT "day", "version", "pageViewCount" FROM "ExtensionVersionDailyCount" +WHERE "version" ~ '^0\.[23](\.|$)' AND "day" > (CURRENT_TIMESTAMP AT TIME ZONE 'UTC')::date - 14 +ORDER BY "day", "version"; +``` + ## License [GNU Affero General Public License v3.0](LICENSE) diff --git a/SPEC.md b/SPEC.md index f974c50..98b6cd4 100644 --- a/SPEC.md +++ b/SPEC.md @@ -879,6 +879,25 @@ model InstanceApiKey { } ``` +### Extension version counts + +`recordViewAndGetStatus` counts each page view against the reporting extension +version (`x-openerrata-extension-version`) and the UTC day, and nothing else: no +viewer key, IP range, post or time of day, so a row describes traffic, never a +viewer (see PRIVACY.md). The counts tell when an extension line the API still +serves has gone quiet, which is the retirement condition of the legacy v0 +protocol adapter (§3.8.1). + +```prisma +model ExtensionVersionDailyCount { + day DateTime @db.Date // UTC day + version String // 1–4 numeric components (CHECK) + pageViewCount Int // > 0 (CHECK); upserted atomically per view + + @@id([day, version]) +} +``` + ### Platform metadata Platform metadata is version-scoped only. Each `PostVersion` can have one @@ -1466,10 +1485,12 @@ postRouter.registerObservedVersion Output: { platform, externalId, versionHash, postVersionId, provenance: ContentProvenance } // Record a view and return the status of this version's investigation, if any. -// Increments raw viewCount and updates uniqueViewScore. Uses postVersionId from -// registerObservedVersion for a direct primary-key lookup (no content re-derivation); -// rejects unknown post versions. Every variant about an existing investigation -// carries its id, so the client can poll an investigation it did not start. +// Increments raw viewCount, updates uniqueViewScore and counts the view against +// the request's extension version for the UTC day (ExtensionVersionDailyCount). +// Uses postVersionId from registerObservedVersion for a direct primary-key lookup +// (no content re-derivation); rejects unknown post versions. Every variant about +// an existing investigation carries its id, so the client can poll an +// investigation it did not start. postRouter.recordViewAndGetStatus Input: { postVersionId } Output: @@ -2056,7 +2077,15 @@ The protocol is not versioned: all contexts ship in one bundle, and a content script orphaned by an extension update can no longer reach the new background. API compatibility is versioned separately over HTTP (`x-openerrata-extension-version`, minimum supported version → upgrade -required). +required). Extensions from 0.2.0 up to 0.4.0 speak the legacy v0 API protocol +(the API of the 0.3 line: no investigation ids or FAILED state from +`recordViewAndGetStatus`, `observedImageUrls` in inputs). The API serves them +through a time-boxed adapter (`api/src/lib/trpc/legacy-extension-v0`) that +converts their requests to the current procedures and the answers back, so the +current procedures carry no legacy branches. It is retired on 2026-12-01 or +once the extension version counts (§3.2) show no 0.3.x traffic for 14 +consecutive days, whichever is later; the minimum supported version then +becomes 0.4.0. Future work: design a dedicated UI/UX flow for fact-checking image-only posts without relying on text-span highlighting. diff --git a/src/typescript/api/prisma/migrations/0026_extension_version_daily_count/migration.sql b/src/typescript/api/prisma/migrations/0026_extension_version_daily_count/migration.sql new file mode 100644 index 0000000..800fec0 --- /dev/null +++ b/src/typescript/api/prisma/migrations/0026_extension_version_daily_count/migration.sql @@ -0,0 +1,26 @@ +-- ============================================================================ +-- Migration 0026: per-version daily extension page-view counts +-- ============================================================================ +-- +-- One row per (UTC day, extension version): how many recordViewAndGetStatus +-- calls that version made that day. Nothing else is stored — no viewer key, IP +-- or post (PRIVACY.md). It shows when an extension line the API still serves +-- (the legacy v0 adapter for < 0.4.0) has stopped sending traffic. New table +-- only; there is no data to migrate. + +-- CreateTable +CREATE TABLE "ExtensionVersionDailyCount" ( + "day" DATE NOT NULL, + "version" TEXT NOT NULL, + "pageViewCount" INTEGER NOT NULL, + + CONSTRAINT "ExtensionVersionDailyCount_pkey" PRIMARY KEY ("day","version") +); + +-- Versions are what the API's version gate admits: 1–4 dot-separated numeric +-- components. A row exists only once a view has been counted. +ALTER TABLE "ExtensionVersionDailyCount" + ADD CONSTRAINT "ExtensionVersionDailyCount_version_format_check" + CHECK ("version" ~ '^[0-9]{1,5}(\.[0-9]{1,5}){0,3}$'), + ADD CONSTRAINT "ExtensionVersionDailyCount_page_view_count_positive_check" + CHECK ("pageViewCount" > 0); diff --git a/src/typescript/api/prisma/schema.prisma b/src/typescript/api/prisma/schema.prisma index 09d3869..f9363d7 100644 --- a/src/typescript/api/prisma/schema.prisma +++ b/src/typescript/api/prisma/schema.prisma @@ -652,3 +652,15 @@ model Source { @@index([claimId]) } + +// Page views (recordViewAndGetStatus calls) per extension version per UTC day. +// Deliberately nothing else — no viewer key, IP or post — so a row describes +// traffic, never a person (PRIVACY.md). Read it to tell when an extension line +// the API still serves (legacy-extension-v0) has gone quiet. +model ExtensionVersionDailyCount { + day DateTime @db.Date + version String + pageViewCount Int + + @@id([day, version]) +} diff --git a/src/typescript/api/src/lib/config/env.ts b/src/typescript/api/src/lib/config/env.ts index 7ceab71..ddc734b 100644 --- a/src/typescript/api/src/lib/config/env.ts +++ b/src/typescript/api/src/lib/config/env.ts @@ -6,8 +6,12 @@ import { z } from "zod"; * receive UPGRADE_REQUIRED errors. This is a property of the API code — when * the API changes in a way that breaks older extensions, bump this constant * alongside that change. + * + * Versions from here up to (not including) 0.4.0 speak the legacy v0 protocol + * and are served through the time-boxed adapter in + * `$lib/trpc/legacy-extension-v0`; this becomes "0.4.0" when it is retired. */ -export const MINIMUM_SUPPORTED_EXTENSION_VERSION = "0.4.0"; +export const MINIMUM_SUPPORTED_EXTENSION_VERSION = "0.2.0"; const positiveIntegerFromEnv = z.preprocess((value) => { if (value === undefined || value === null || value === "") return undefined; diff --git a/src/typescript/api/src/lib/services/extension-version-counts.ts b/src/typescript/api/src/lib/services/extension-version-counts.ts new file mode 100644 index 0000000..73fe3fe --- /dev/null +++ b/src/typescript/api/src/lib/services/extension-version-counts.ts @@ -0,0 +1,20 @@ +import type { DbClient } from "$lib/db/client"; + +/** + * Count one page view (one `recordViewAndGetStatus` call) against today's UTC + * day for the reporting extension version. The count is all that is kept — + * no viewer key, IP, post or time of day (PRIVACY.md) — and it exists to show + * when an extension line the API still serves has stopped sending traffic. + */ +export async function countExtensionVersionPageView( + db: DbClient, + extensionVersion: string, +): Promise { + // One atomic upsert: concurrent first views of the day cannot race on the key. + await db.$executeRaw` + INSERT INTO "ExtensionVersionDailyCount" ("day", "version", "pageViewCount") + VALUES ((CURRENT_TIMESTAMP AT TIME ZONE 'UTC')::date, ${extensionVersion}, 1) + ON CONFLICT ("day", "version") + DO UPDATE SET "pageViewCount" = "ExtensionVersionDailyCount"."pageViewCount" + 1 + `; +} diff --git a/src/typescript/api/src/lib/trpc/legacy-extension-v0/index.ts b/src/typescript/api/src/lib/trpc/legacy-extension-v0/index.ts new file mode 100644 index 0000000..b4e8521 --- /dev/null +++ b/src/typescript/api/src/lib/trpc/legacy-extension-v0/index.ts @@ -0,0 +1,87 @@ +/** + * Legacy v0 extension protocol adapter (time-boxed). + * + * Purpose: extensions older than 0.4.0 speak the extension API as it was at + * commit 9ee2cae — strict response schemas without investigation ids, no + * FAILED status from `recordViewAndGetStatus`, and `observedImageUrls` in + * `registerObservedVersion` inputs. Store installs auto-update, but manual + * installs from GitHub Releases may not; this adapter keeps them working + * without a single legacy branch in the current procedures. + * + * Serves: extension versions >= 0.2.0 and < 0.4.0 (the + * `x-openerrata-extension-version` header). 0.4.0 and newer pass through + * untouched; versions below 0.2.0 are refused by the version gate before + * reaching this middleware. + * + * How: it runs after the version gate and before input parsing. For a legacy + * request it parses the raw input with the vendored legacy schema + * (`wire-schemas.ts`), converts it to the current input, runs the current + * procedure, and converts the current output to the legacy one, validated + * against the vendored legacy output schema (`procedures.ts` documents each + * mapping, including the lossy ones). Anything not representable fails loudly. + * + * Retirement: on 2026-12-01, or once the version counts + * (`ExtensionVersionDailyCount`, see the README runbook) show zero 0.3.x + * requests for 14 consecutive days, whichever is later. Then delete this + * directory, its `.concat()` in `routes/post.ts` and its integration tests, and + * raise MINIMUM_SUPPORTED_EXTENSION_VERSION to "0.4.0". + */ + +import { initTRPC, TRPCError } from "@trpc/server"; +import { isExtensionVersionAtLeast, type ExtensionApiProcedurePath } from "@openerrata/shared"; +import { LEGACY_PROCEDURE_ADAPTERS } from "./procedures.js"; + +/** Oldest version served (the version gate's minimum). */ +const OLDEST_LEGACY_EXTENSION_VERSION = "0.2.0"; +/** First version that speaks the current protocol. */ +export const FIRST_CURRENT_PROTOCOL_EXTENSION_VERSION = "0.4.0"; + +function speaksLegacyProtocol(extensionVersion: string): boolean { + const atLeastOldest = isExtensionVersionAtLeast( + extensionVersion, + OLDEST_LEGACY_EXTENSION_VERSION, + ); + const atLeastCurrent = isExtensionVersionAtLeast( + extensionVersion, + FIRST_CURRENT_PROTOCOL_EXTENSION_VERSION, + ); + if (atLeastOldest !== true || atLeastCurrent === null) { + // The version gate admits only well-formed versions at or above its minimum. + throw new TRPCError({ + code: "INTERNAL_SERVER_ERROR", + message: `Extension version ${extensionVersion} passed the version gate but predates the legacy protocol floor ${OLDEST_LEGACY_EXTENSION_VERSION}`, + }); + } + return !atLeastCurrent; +} + +function isExtensionApiProcedurePath(path: string): path is ExtensionApiProcedurePath { + return Object.hasOwn(LEGACY_PROCEDURE_ADAPTERS, path); +} + +/** + * Procedure-builder fragment holding the adapter middleware; extension + * procedures `.concat()` it right after the version gate, whose validated + * `extensionVersion` it reads. + */ +export const legacyExtensionV0Adapter = initTRPC + .context<{ extensionVersion: string }>() + .create() + .procedure.use(async ({ ctx, path, getRawInput, next }) => { + if (!speaksLegacyProtocol(ctx.extensionVersion)) { + return next(); + } + if (!isExtensionApiProcedurePath(path)) { + throw new TRPCError({ + code: "INTERNAL_SERVER_ERROR", + message: `No legacy v0 extension adapter for procedure ${path}`, + }); + } + const adapter = LEGACY_PROCEDURE_ADAPTERS[path]; + const currentRawInput = adapter.toCurrentRawInput(await getRawInput()); + const result = await next({ getRawInput: () => Promise.resolve(currentRawInput) }); + if (!result.ok) { + return result; + } + return { ...result, data: adapter.toLegacyRawOutput(result.data) }; + }); diff --git a/src/typescript/api/src/lib/trpc/legacy-extension-v0/procedures.ts b/src/typescript/api/src/lib/trpc/legacy-extension-v0/procedures.ts new file mode 100644 index 0000000..0ab19e5 --- /dev/null +++ b/src/typescript/api/src/lib/trpc/legacy-extension-v0/procedures.ts @@ -0,0 +1,198 @@ +/** + * Per-procedure conversion between the legacy v0 wire shapes + * (`wire-schemas.ts`) and the current ones. Every extension-facing procedure + * has an entry, so adding a procedure forces a decision about legacy clients. + */ + +import { TRPCError } from "@trpc/server"; +import type { z } from "zod"; +import { + batchStatusOutputSchema, + getInvestigationOutputSchema, + investigateNowOutputSchema, + observedImageUrlsFromOccurrences, + registerObservedVersionOutputSchema, + settingsValidationOutputSchema, + viewPostOutputSchema, + type ExtensionApiProcedureContract, + type ExtensionApiProcedurePath, + type ViewPostOutput, +} from "@openerrata/shared"; +import { + legacyBatchStatusInputSchema, + legacyBatchStatusOutputSchema, + legacyGetInvestigationInputSchema, + legacyGetInvestigationOutputSchema, + legacyInvestigateNowInputSchema, + legacyInvestigateNowOutputSchema, + legacyRecordViewAndGetStatusInputSchema, + legacyRecordViewAndGetStatusOutputSchema, + legacyRegisterObservedVersionInputSchema, + legacyRegisterObservedVersionOutputSchema, + legacyValidateSettingsInputSchema, + legacyValidateSettingsOutputSchema, +} from "./wire-schemas.js"; + +type CurrentInput

= ExtensionApiProcedureContract[P]["input"]; +type CurrentOutput

= + ExtensionApiProcedureContract[P]["output"]; + +interface LegacyProcedureSpec

{ + legacyInputSchema: z.ZodType; + toCurrentInput: (input: LegacyInput) => CurrentInput

; + /** The current output schema, to type the (already validated) procedure result. */ + currentOutputSchema: z.ZodType>; + toLegacyOutput: (output: CurrentOutput

) => LegacyOutput; + legacyOutputSchema: z.ZodType; +} + +/** One procedure's legacy conversions, with its wire types erased for the middleware. */ +interface LegacyProcedureAdapter

{ + /** The procedure adapted; ties each table entry to its key. */ + procedure: P; + /** Parses a legacy raw input and converts it to the current raw input. */ + toCurrentRawInput: (legacyRawInput: unknown) => unknown; + /** Converts the current procedure output to a validated legacy output. */ + toLegacyRawOutput: (currentOutput: unknown) => unknown; +} + +function defineLegacyProcedure

( + procedure: P, + spec: LegacyProcedureSpec, +): LegacyProcedureAdapter

{ + return { + procedure, + toCurrentRawInput(legacyRawInput) { + const parsed = spec.legacyInputSchema.safeParse(legacyRawInput); + if (!parsed.success) { + // Same code tRPC's own input validation uses. + throw new TRPCError({ code: "BAD_REQUEST", cause: parsed.error }); + } + return spec.toCurrentInput(parsed.data); + }, + toLegacyRawOutput(currentOutput) { + const legacyOutput = spec.toLegacyOutput(spec.currentOutputSchema.parse(currentOutput)); + const validated = spec.legacyOutputSchema.safeParse(legacyOutput); + if (!validated.success) { + throw new TRPCError({ + code: "INTERNAL_SERVER_ERROR", + message: "Response is not representable in the legacy v0 extension protocol", + cause: validated.error, + }); + } + return validated.data; + }, + }; +} + +function unchanged(value: T): T { + return value; +} + +/** + * The legacy input carried the distinct image URLs alongside the occurrences; + * the current input derives that list from the occurrences. Every legacy + * client built both from the same page scan, so dropping the list loses + * nothing — a list naming images the occurrences do not is not representable. + */ +function withoutObservedImageUrls( + input: z.output, +): CurrentInput<"post.registerObservedVersion"> { + const { observedImageUrls, ...current } = input; + const representable = new Set(observedImageUrlsFromOccurrences(input.observedImageOccurrences)); + const unrepresentable = (observedImageUrls ?? []).filter((url) => !representable.has(url)); + if (unrepresentable.length > 0) { + throw new TRPCError({ + code: "BAD_REQUEST", + message: `observedImageUrls lists images missing from observedImageOccurrences: ${unrepresentable.join(", ")}`, + }); + } + return current; +} + +/** + * Legacy `recordViewAndGetStatus` reported only finished investigations (no + * FAILED variant, no investigation id); a client that saw NOT_INVESTIGATED + * called `investigateNow`, which returns the existing investigation's id and + * status, and polled from there. Mapping: + * - INVESTIGATED → INVESTIGATED without the id. + * - INVESTIGATING → NOT_INVESTIGATED with the same carried-forward claims, + * exactly what the legacy API answered for a running investigation. An + * INVESTIGATING answer without an id would strand a legacy client: it + * neither polls nor calls `investigateNow`. + * - FAILED → NOT_INVESTIGATED without carried-forward claims (lossy: the legacy + * API showed any carried-forward claims here; the current FAILED status does + * not compute them). `investigateNow` then reports FAILED to the client. + * Unknown post versions are not mapped: the legacy API answered + * NOT_INVESTIGATED, the current one rejects them (BAD_REQUEST), but legacy + * clients only ask about a version they registered in the same step. + */ +function toLegacyViewStatus( + output: ViewPostOutput, +): z.output { + switch (output.investigationState) { + case "NOT_INVESTIGATED": + return output; + case "INVESTIGATED": { + const { investigationId: _investigationId, ...legacy } = output; + return legacy; + } + case "INVESTIGATING": + return { + investigationState: "NOT_INVESTIGATED", + priorInvestigationResult: output.priorInvestigationResult, + }; + case "FAILED": + return { investigationState: "NOT_INVESTIGATED", priorInvestigationResult: null }; + } +} + +export const LEGACY_PROCEDURE_ADAPTERS: { + [P in ExtensionApiProcedurePath]: LegacyProcedureAdapter

; +} = { + "post.registerObservedVersion": defineLegacyProcedure("post.registerObservedVersion", { + legacyInputSchema: legacyRegisterObservedVersionInputSchema, + toCurrentInput: withoutObservedImageUrls, + currentOutputSchema: registerObservedVersionOutputSchema, + toLegacyOutput: unchanged, + legacyOutputSchema: legacyRegisterObservedVersionOutputSchema, + }), + "post.recordViewAndGetStatus": defineLegacyProcedure("post.recordViewAndGetStatus", { + legacyInputSchema: legacyRecordViewAndGetStatusInputSchema, + toCurrentInput: unchanged, + currentOutputSchema: viewPostOutputSchema, + toLegacyOutput: toLegacyViewStatus, + legacyOutputSchema: legacyRecordViewAndGetStatusOutputSchema, + }), + // The current outputs below are subsets of the legacy ones; the legacy + // schemas still validate them, so drift in the current protocol fails here + // rather than in a legacy client. + "post.getInvestigation": defineLegacyProcedure("post.getInvestigation", { + legacyInputSchema: legacyGetInvestigationInputSchema, + toCurrentInput: unchanged, + currentOutputSchema: getInvestigationOutputSchema, + toLegacyOutput: unchanged, + legacyOutputSchema: legacyGetInvestigationOutputSchema, + }), + "post.investigateNow": defineLegacyProcedure("post.investigateNow", { + legacyInputSchema: legacyInvestigateNowInputSchema, + toCurrentInput: unchanged, + currentOutputSchema: investigateNowOutputSchema, + toLegacyOutput: unchanged, + legacyOutputSchema: legacyInvestigateNowOutputSchema, + }), + "post.validateSettings": defineLegacyProcedure("post.validateSettings", { + legacyInputSchema: legacyValidateSettingsInputSchema, + toCurrentInput: unchanged, + currentOutputSchema: settingsValidationOutputSchema, + toLegacyOutput: unchanged, + legacyOutputSchema: legacyValidateSettingsOutputSchema, + }), + "post.batchStatus": defineLegacyProcedure("post.batchStatus", { + legacyInputSchema: legacyBatchStatusInputSchema, + toCurrentInput: unchanged, + currentOutputSchema: batchStatusOutputSchema, + toLegacyOutput: unchanged, + legacyOutputSchema: legacyBatchStatusOutputSchema, + }), +}; diff --git a/src/typescript/api/src/lib/trpc/legacy-extension-v0/wire-schemas.ts b/src/typescript/api/src/lib/trpc/legacy-extension-v0/wire-schemas.ts new file mode 100644 index 0000000..5ff0421 --- /dev/null +++ b/src/typescript/api/src/lib/trpc/legacy-extension-v0/wire-schemas.ts @@ -0,0 +1,405 @@ +/** + * The extension-facing tRPC wire schemas of the legacy v0 protocol: what the + * API accepted and returned at commit 9ee2cae (extensions 0.2.0–0.3.3 parse + * responses with these exact strict schemas). + * + * Vendored, not imported: copied from `shared/src/schemas/{common, + * investigation,settings}.ts` and `shared/src/{constants,enums}.ts` at + * 9ee2cae, so the current schemas can keep evolving without silently changing + * what old clients are promised. Changes from the originals: only the wire + * shapes the API serves are kept, and type-level `.brand()`s are dropped + * (they never affected what parses). Do not edit these to match the current + * protocol; delete them with the adapter. + */ + +import { z } from "zod"; + +// ── shared/src/constants.ts, shared/src/enums.ts ────────────────────────── + +const MAX_BATCH_STATUS_POSTS = 100; +const MAX_OBSERVED_IMAGE_OCCURRENCES = 256; +const MAX_OBSERVED_CONTENT_TEXT_CHARS = 500_000; +const MAX_OBSERVED_CONTENT_TEXT_UTF8_BYTES = 500_000; +const PLATFORM_VALUES = ["LESSWRONG", "X", "SUBSTACK", "WIKIPEDIA"] as const; +const CONTENT_PROVENANCE_VALUES = ["SERVER_VERIFIED", "CLIENT_FALLBACK"] as const; +const WIKIPEDIA_LANGUAGE_CODE_REGEX = /^[a-z][a-z0-9-]*$/i; + +// ── shared/src/schemas/common.ts ────────────────────────────────────────── + +const platformSchema = z.enum(PLATFORM_VALUES); +const contentProvenanceSchema = z.enum(CONTENT_PROVENANCE_VALUES); + +const utf8Encoder = new TextEncoder(); + +function utf8ByteLength(input: string): number { + return utf8Encoder.encode(input).byteLength; +} + +const observedContentTextSchema = z + .string() + .min(1) + .max(MAX_OBSERVED_CONTENT_TEXT_CHARS) + .refine((value) => utf8ByteLength(value) <= MAX_OBSERVED_CONTENT_TEXT_UTF8_BYTES, { + message: `Observed content text must be at most ${MAX_OBSERVED_CONTENT_TEXT_UTF8_BYTES.toString()} UTF-8 bytes`, + }); + +const observedImageOccurrenceSchema = z + .object({ + originalIndex: z.number().int().nonnegative(), + normalizedTextOffset: z.number().int().nonnegative(), + sourceUrl: z.url(), + captionText: z.string().min(1).optional(), + }) + .strict(); + +const observedImageOccurrencesSchema = z + .array(observedImageOccurrenceSchema) + .max(MAX_OBSERVED_IMAGE_OCCURRENCES); + +const postIdSchema = z.string().min(1); +const postVersionIdSchema = z.string().min(1); +const investigationIdSchema = z.string().min(1); +const claimIdSchema = z.string().min(1); +const versionHashSchema = z.string().regex(/^[a-f0-9]{64}$/i); + +const claimSourceSchema = z + .object({ + url: z.url(), + title: z.string().min(1), + snippet: z.string().min(1), + }) + .strict(); + +const investigationClaimPayloadSchema = z + .object({ + text: z.string().min(1), + context: z.string().min(1), + summary: z.string().min(1), + reasoning: z.string().min(1), + sources: z.array(claimSourceSchema).min(1), + }) + .strict(); + +const investigationClaimSchema = investigationClaimPayloadSchema + .extend({ + id: claimIdSchema, + }) + .strict(); + +const lesswrongMetadataSchema = z + .object({ + slug: z.string().min(1), + title: z.string().min(1).optional(), + htmlContent: z.string().min(1), + authorName: z.string().min(1).optional(), + authorSlug: z.string().min(1).nullable().optional(), + tags: z.array(z.string().min(1)), + publishedAt: z.iso.datetime().optional(), + }) + .strict(); + +const xMetadataSchema = z + .object({ + authorHandle: z.string().min(1), + authorDisplayName: z.string().min(1).nullable().optional(), + text: observedContentTextSchema, + mediaUrls: z.array(z.url()), + likeCount: z.number().int().nonnegative().optional(), + retweetCount: z.number().int().nonnegative().optional(), + postedAt: z.iso.datetime().optional(), + }) + .strict(); + +const substackMetadataSchema = z + .object({ + substackPostId: z.string().regex(/^\d+$/), + publicationSubdomain: z.string().min(1), + slug: z.string().min(1), + title: z.string().min(1), + subtitle: z.string().min(1).optional(), + htmlContent: z.string().min(1).optional(), + authorName: z.string().min(1), + authorSubstackHandle: z.string().min(1).optional(), + publishedAt: z.iso.datetime().optional(), + likeCount: z.number().int().nonnegative().optional(), + commentCount: z.number().int().nonnegative().optional(), + }) + .strict(); + +const wikipediaMetadataSchema = z + .object({ + language: z.string().regex(WIKIPEDIA_LANGUAGE_CODE_REGEX), + title: z.string().min(1), + pageId: z.string().regex(/^\d+$/), + revisionId: z.string().regex(/^\d+$/), + displayTitle: z.string().min(1).optional(), + lastModifiedAt: z.iso.datetime().optional(), + htmlContent: z.string().min(1).optional(), + }) + .strict(); + +// ── shared/src/schemas/investigation.ts ─────────────────────────────────── + +const versionedPostInputSharedSchema = z + .object({ + url: z.url(), + observedImageUrls: z.array(z.url()).optional(), + observedImageOccurrences: observedImageOccurrencesSchema.optional(), + }) + .strict(); + +const nonWikipediaViewPostInputSharedSchema = versionedPostInputSharedSchema + .extend({ + externalId: postIdSchema, + }) + .strict(); + +const lesswrongViewPostInputSchema = nonWikipediaViewPostInputSharedSchema + .extend({ + platform: z.literal("LESSWRONG"), + metadata: lesswrongMetadataSchema, + }) + .strict(); + +const xViewPostInputSchema = nonWikipediaViewPostInputSharedSchema + .extend({ + platform: z.literal("X"), + observedContentText: observedContentTextSchema, + metadata: xMetadataSchema, + }) + .strict(); + +const substackViewPostInputSchema = nonWikipediaViewPostInputSharedSchema + .extend({ + platform: z.literal("SUBSTACK"), + observedContentText: observedContentTextSchema, + metadata: substackMetadataSchema, + }) + .strict(); + +const wikipediaViewPostInputSchema = versionedPostInputSharedSchema + .extend({ + platform: z.literal("WIKIPEDIA"), + observedContentText: observedContentTextSchema, + metadata: wikipediaMetadataSchema, + }) + .strict(); + +export const legacyRegisterObservedVersionInputSchema = z.discriminatedUnion("platform", [ + lesswrongViewPostInputSchema, + xViewPostInputSchema, + substackViewPostInputSchema, + wikipediaViewPostInputSchema, +]); + +export const legacyRegisterObservedVersionOutputSchema = z + .object({ + platform: platformSchema, + externalId: postIdSchema, + versionHash: versionHashSchema, + postVersionId: postVersionIdSchema, + provenance: contentProvenanceSchema, + }) + .strict(); + +const versionedPostInputSchema = z + .object({ + postVersionId: postVersionIdSchema, + }) + .strict(); + +const priorInvestigationResultSchema = z + .object({ + oldClaims: z.array(investigationClaimSchema), + sourceInvestigationId: investigationIdSchema, + }) + .strict(); + +const investigationStatusNotInvestigatedSchema = z + .object({ + investigationState: z.literal("NOT_INVESTIGATED"), + priorInvestigationResult: priorInvestigationResultSchema.nullable(), + }) + .strict(); + +const investigationStatusInvestigatingSchema = z + .object({ + investigationState: z.literal("INVESTIGATING"), + status: z.union([z.literal("PENDING"), z.literal("PROCESSING")]), + provenance: contentProvenanceSchema, + pendingClaims: z.array(investigationClaimPayloadSchema), + confirmedClaims: z.array(investigationClaimPayloadSchema), + priorInvestigationResult: priorInvestigationResultSchema.nullable(), + }) + .strict(); + +const investigationStatusFailedSchema = z + .object({ + investigationState: z.literal("FAILED"), + provenance: contentProvenanceSchema, + }) + .strict(); + +const investigationStatusInvestigatedSchema = z + .object({ + investigationState: z.literal("INVESTIGATED"), + provenance: contentProvenanceSchema, + claims: z.array(investigationClaimSchema), + }) + .strict(); + +export const legacyRecordViewAndGetStatusInputSchema = versionedPostInputSchema; + +// `viewPostOutputSchema`: no FAILED variant, and no variant carries an +// investigation id. +export const legacyRecordViewAndGetStatusOutputSchema = z.discriminatedUnion("investigationState", [ + investigationStatusNotInvestigatedSchema, + investigationStatusInvestigatingSchema, + investigationStatusInvestigatedSchema, +]); + +export const legacyGetInvestigationInputSchema = z + .object({ + investigationId: investigationIdSchema, + }) + .strict(); + +export const legacyGetInvestigationOutputSchema = z.discriminatedUnion("investigationState", [ + investigationStatusNotInvestigatedSchema + .extend({ + checkedAt: z.iso.datetime().optional(), + }) + .strict(), + investigationStatusInvestigatingSchema + .extend({ + checkedAt: z.iso.datetime().optional(), + }) + .strict(), + investigationStatusFailedSchema + .extend({ + checkedAt: z.iso.datetime().optional(), + }) + .strict(), + investigationStatusInvestigatedSchema + .extend({ + checkedAt: z.iso.datetime(), + }) + .strict(), +]); + +export const legacyInvestigateNowInputSchema = versionedPostInputSchema; + +export const legacyInvestigateNowOutputSchema = z.discriminatedUnion("status", [ + z + .object({ + investigationId: investigationIdSchema, + status: z.union([z.literal("PENDING"), z.literal("PROCESSING")]), + provenance: contentProvenanceSchema, + }) + .strict(), + z + .object({ + investigationId: investigationIdSchema, + status: z.literal("FAILED"), + provenance: contentProvenanceSchema, + }) + .strict(), + z + .object({ + investigationId: investigationIdSchema, + status: z.literal("COMPLETE"), + provenance: contentProvenanceSchema, + claims: z.array(investigationClaimSchema), + }) + .strict(), +]); + +// ── shared/src/schemas/settings.ts ──────────────────────────────────────── + +// validateSettings had no `.input()`: clients send no input at all. +export const legacyValidateSettingsInputSchema = z.undefined(); + +export const legacyValidateSettingsOutputSchema = z.discriminatedUnion("openaiApiKeyStatus", [ + z + .object({ + instanceApiKeyAccepted: z.boolean(), + openaiApiKeyStatus: z.literal("missing"), + }) + .strict(), + z + .object({ + instanceApiKeyAccepted: z.boolean(), + openaiApiKeyStatus: z.literal("valid"), + }) + .strict(), + z + .object({ + instanceApiKeyAccepted: z.boolean(), + openaiApiKeyStatus: z.literal("format_invalid"), + openaiApiKeyMessage: z.string().min(1), + }) + .strict(), + z + .object({ + instanceApiKeyAccepted: z.boolean(), + openaiApiKeyStatus: z.literal("authenticated_restricted"), + openaiApiKeyMessage: z.string().min(1), + }) + .strict(), + z + .object({ + instanceApiKeyAccepted: z.boolean(), + openaiApiKeyStatus: z.literal("invalid"), + openaiApiKeyMessage: z.string().min(1), + }) + .strict(), + z + .object({ + instanceApiKeyAccepted: z.boolean(), + openaiApiKeyStatus: z.literal("error"), + openaiApiKeyMessage: z.string().min(1), + }) + .strict(), +]); + +export const legacyBatchStatusInputSchema = z + .object({ + posts: z + .array( + z + .object({ + platform: platformSchema, + externalId: postIdSchema, + versionHash: versionHashSchema, + }) + .strict(), + ) + .min(1) + .max(MAX_BATCH_STATUS_POSTS), + }) + .strict(); + +export const legacyBatchStatusOutputSchema = z + .object({ + statuses: z.array( + z.discriminatedUnion("investigationState", [ + z + .object({ + platform: platformSchema, + externalId: postIdSchema, + investigationState: z.literal("NOT_INVESTIGATED"), + incorrectClaimCount: z.literal(0), + }) + .strict(), + z + .object({ + platform: platformSchema, + externalId: postIdSchema, + investigationState: z.literal("INVESTIGATED"), + incorrectClaimCount: z.number().int().nonnegative(), + }) + .strict(), + ]), + ), + }) + .strict(); diff --git a/src/typescript/api/src/lib/trpc/routes/post.ts b/src/typescript/api/src/lib/trpc/routes/post.ts index 4d4b2e9..62ac091 100644 --- a/src/typescript/api/src/lib/trpc/routes/post.ts +++ b/src/typescript/api/src/lib/trpc/routes/post.ts @@ -34,6 +34,8 @@ import { } from "$lib/services/investigate-now.js"; import { maybeIncrementUniqueViewScore } from "$lib/services/view-credit.js"; import { validateOpenAiApiKeyForSettings } from "$lib/services/openai-key-validation.js"; +import { countExtensionVersionPageView } from "$lib/services/extension-version-counts.js"; +import { legacyExtensionV0Adapter } from "../legacy-extension-v0/index.js"; import { registerObservedVersion, findPostVersionById } from "./post/content-storage.js"; import { loadInvestigationWithClaims, @@ -213,10 +215,14 @@ async function projectInvestigationStatus( // Router // --------------------------------------------------------------------------- -const extensionProcedure = publicProcedure.use(async ({ ctx, next }) => { - const extensionVersion = assertSupportedExtensionVersion(ctx); - return next({ ctx: { extensionVersion } }); -}); +const extensionProcedure = publicProcedure + .use(async ({ ctx, next }) => { + const extensionVersion = assertSupportedExtensionVersion(ctx); + return next({ ctx: { extensionVersion } }); + }) + // Serves extensions older than 0.4.0 in their own protocol (time-boxed; + // see legacy-extension-v0 for the retirement condition). + .concat(legacyExtensionV0Adapter); export const postRouter = router({ registerObservedVersion: extensionProcedure @@ -253,6 +259,7 @@ export const postRouter = router({ lastViewedAt: new Date(), }, }); + await countExtensionVersionPageView(ctx.prisma, ctx.extensionVersion); await maybeIncrementUniqueViewScore( ctx.prisma, diff --git a/src/typescript/api/test/integration/api-endpoints.integration.shared.ts b/src/typescript/api/test/integration/api-endpoints.integration.shared.ts index a95b24e..5eb6b76 100644 --- a/src/typescript/api/test/integration/api-endpoints.integration.shared.ts +++ b/src/typescript/api/test/integration/api-endpoints.integration.shared.ts @@ -14,6 +14,7 @@ import { } from "@openerrata/shared"; import type { RequestEventLike } from "../../src/lib/trpc/context.js"; import { MINIMUM_SUPPORTED_EXTENSION_VERSION } from "../../src/lib/config/env.js"; +import { FIRST_CURRENT_PROTOCOL_EXTENSION_VERSION } from "../../src/lib/trpc/legacy-extension-v0/index.js"; import { createDeterministicRandom, randomChance, @@ -147,7 +148,8 @@ function createCaller(options: CallerOptions = {}): AppCaller { ipRangeKey: options.ipRangeKey ?? "integration-ip-range", isAuthenticated, userOpenAiApiKey, - extensionVersion: options.extensionVersion ?? MINIMUM_SUPPORTED_EXTENSION_VERSION, + // Callers speak the current protocol unless a test opts into another version. + extensionVersion: options.extensionVersion ?? FIRST_CURRENT_PROTOCOL_EXTENSION_VERSION, minimumSupportedExtensionVersion: MINIMUM_SUPPORTED_EXTENSION_VERSION, }); diff --git a/src/typescript/api/test/integration/api-endpoints.legacy-extension-v0.integration.test.ts b/src/typescript/api/test/integration/api-endpoints.legacy-extension-v0.integration.test.ts new file mode 100644 index 0000000..d1ae6df --- /dev/null +++ b/src/typescript/api/test/integration/api-endpoints.legacy-extension-v0.integration.test.ts @@ -0,0 +1,460 @@ +import { fetchRequestHandler } from "@trpc/server/adapters/fetch"; +import { + isNonNullObject, + observedImageUrlsFromOccurrences, + type ExtensionApiProcedurePath, +} from "@openerrata/shared"; +import { + legacyBatchStatusOutputSchema, + legacyGetInvestigationOutputSchema, + legacyInvestigateNowOutputSchema, + legacyRecordViewAndGetStatusOutputSchema, + legacyRegisterObservedVersionOutputSchema, + legacyValidateSettingsOutputSchema, +} from "../../src/lib/trpc/legacy-extension-v0/wire-schemas.js"; +import { + appRouter, + assert, + buildXViewInput, + createCaller, + createContext, + hashContent, + normalizeContent, + prisma, + seedClaimWithSource, + seedCompleteInvestigation, + seedInstanceApiKey, + seedInvestigation, + seedInvestigationForXViewInput, + seedPostForXViewInput, + test, +} from "./api-endpoints.integration.shared.js"; + +// Extensions 0.2.0–0.3.x speak the legacy v0 protocol (the API at 9ee2cae) and +// are served through the legacy-extension-v0 adapter. These tests speak that +// protocol over HTTP the way the 0.3.3 client does (tRPC httpLink: GET for +// queries, JSON POST for mutations) and hold every response to the legacy +// client's strict schemas. + +const LEGACY_EXTENSION_VERSION = "0.3.3"; +const CURRENT_EXTENSION_VERSION = "0.4.0"; + +type ApiResult = { ok: true; data: unknown } | { ok: false; status: number; code: unknown }; + +async function callApi(input: { + path: ExtensionApiProcedurePath; + kind: "query" | "mutation"; + input: unknown; + extensionVersion: string; + headers?: Record; +}): Promise { + const url = new URL(`http://localhost/trpc/${input.path}`); + const headers = new Headers({ + "x-openerrata-extension-version": input.extensionVersion, + ...input.headers, + }); + let request: Request; + if (input.kind === "query") { + if (input.input !== undefined) { + url.searchParams.set("input", JSON.stringify(input.input)); + } + request = new Request(url, { method: "GET", headers }); + } else { + headers.set("content-type", "application/json"); + request = new Request(url, { method: "POST", headers, body: JSON.stringify(input.input) }); + } + + const response = await fetchRequestHandler({ + endpoint: "/trpc", + req: request, + router: appRouter, + createContext: () => createContext({ request, getClientAddress: () => "203.0.113.7" }), + }); + const body: unknown = await response.json(); + assert.ok(isNonNullObject(body)); + const result = body["result"]; + if (response.ok && isNonNullObject(result)) { + return { ok: true, data: result["data"] }; + } + const error = body["error"]; + assert.ok(isNonNullObject(error) && isNonNullObject(error["data"])); + return { ok: false, status: response.status, code: error["data"]["code"] }; +} + +async function legacyCall(input: { + path: ExtensionApiProcedurePath; + kind: "query" | "mutation"; + input: unknown; + headers?: Record; +}): Promise { + const result = await callApi({ ...input, extensionVersion: LEGACY_EXTENSION_VERSION }); + assert.ok(result.ok, `legacy ${input.path} failed: ${JSON.stringify(result)}`); + return result.data; +} + +/** A 0.3.x X post as the legacy client sends it: image URLs listed beside the occurrences. */ +function buildLegacyXViewInput(externalId: string) { + const current = buildXViewInput({ + externalId, + observedContentText: "Legacy client content with one image.", + observedImageOccurrences: [ + { originalIndex: 0, normalizedTextOffset: 0, sourceUrl: "https://pbs.twimg.com/media/a.jpg" }, + { + originalIndex: 1, + normalizedTextOffset: 6, + sourceUrl: "https://pbs.twimg.com/media/a.jpg", + }, + ], + }); + return { + ...current, + observedImageUrls: observedImageUrlsFromOccurrences(current.observedImageOccurrences), + }; +} + +async function countedPageViews(version: string): Promise { + const rows = await prisma.$queryRaw<{ pageViewCount: number }[]>` + SELECT "pageViewCount" FROM "ExtensionVersionDailyCount" + WHERE "version" = ${version} AND "day" = (CURRENT_TIMESTAMP AT TIME ZONE 'UTC')::date + `; + return rows[0]?.pageViewCount ?? 0; +} + +void test("legacy registerObservedVersion accepts observedImageUrls and answers in the legacy shape", async () => { + const input = buildLegacyXViewInput("legacy-register-1"); + + const output = legacyRegisterObservedVersionOutputSchema.parse( + await legacyCall({ path: "post.registerObservedVersion", kind: "mutation", input }), + ); + + assert.equal(output.externalId, input.externalId); + assert.equal(output.provenance, "CLIENT_FALLBACK"); + // The image list is carried by the occurrences, so dropping it loses nothing. + const version = await prisma.postVersion.findUniqueOrThrow({ + where: { id: output.postVersionId }, + select: { + imageOccurrenceSet: { + select: { occurrences: { select: { originalIndex: true, sourceUrl: true } } }, + }, + }, + }); + assert.deepEqual( + observedImageUrlsFromOccurrences(version.imageOccurrenceSet.occurrences), + input.observedImageUrls, + ); +}); + +void test("legacy registerObservedVersion rejects observedImageUrls that the occurrences do not list", async () => { + const input = { + ...buildLegacyXViewInput("legacy-register-unrepresentable-1"), + observedImageUrls: ["https://pbs.twimg.com/media/not-on-the-page.jpg"], + }; + + const result = await callApi({ + path: "post.registerObservedVersion", + kind: "mutation", + input, + extensionVersion: LEGACY_EXTENSION_VERSION, + }); + + assert.deepEqual(result, { ok: false, status: 400, code: "BAD_REQUEST" }); +}); + +void test("legacy recordViewAndGetStatus reports an uninvestigated version as NOT_INVESTIGATED", async () => { + const registered = legacyRegisterObservedVersionOutputSchema.parse( + await legacyCall({ + path: "post.registerObservedVersion", + kind: "mutation", + input: buildLegacyXViewInput("legacy-record-view-none-1"), + }), + ); + + const status = legacyRecordViewAndGetStatusOutputSchema.parse( + await legacyCall({ + path: "post.recordViewAndGetStatus", + kind: "mutation", + input: { postVersionId: registered.postVersionId }, + }), + ); + + assert.deepEqual(status, { + investigationState: "NOT_INVESTIGATED", + priorInvestigationResult: null, + }); +}); + +void test("legacy recordViewAndGetStatus reports a COMPLETE investigation without its id", async () => { + const input = buildXViewInput({ + externalId: "legacy-record-view-complete-1", + observedContentText: "Completed content seen by a legacy client.", + }); + const seeded = await seedInvestigationForXViewInput({ + viewInput: input, + status: "COMPLETE", + provenance: "CLIENT_FALLBACK", + claimCount: 1, + }); + + const status = legacyRecordViewAndGetStatusOutputSchema.parse( + await legacyCall({ + path: "post.recordViewAndGetStatus", + kind: "mutation", + input: { postVersionId: seeded.post.postVersionId }, + }), + ); + + assert.equal(status.investigationState, "INVESTIGATED"); + assert.equal(status.claims.length, 1); +}); + +void test("legacy recordViewAndGetStatus reports a running update as NOT_INVESTIGATED with its carried-forward claims", async () => { + const input = buildXViewInput({ + externalId: "legacy-record-view-pending-update-1", + observedContentText: "Original content a legacy client saw.", + }); + const post = await seedPostForXViewInput(input); + const parent = await seedCompleteInvestigation({ + postId: post.id, + contentHash: post.contentHash, + contentText: post.contentText, + provenance: "SERVER_VERIFIED", + }); + const survivingClaim = await seedClaimWithSource(parent.id, 1, { + text: "Original content a legacy client saw.", + }); + const updatedText = normalizeContent("Original content a legacy client saw. Edited sentence."); + const pending = await seedInvestigation({ + postId: post.id, + contentHash: await hashContent(updatedText), + contentText: updatedText, + provenance: "SERVER_VERIFIED", + status: "PENDING", + promptLabel: "legacy-record-view-pending-update", + parentInvestigationId: parent.id, + contentDiff: "Diff summary (line context):\n- Removed lines:\nOld\n+ Added lines:\nNew", + }); + const pendingVersion = await prisma.investigation.findUniqueOrThrow({ + where: { id: pending.id }, + select: { postVersionId: true }, + }); + + const status = legacyRecordViewAndGetStatusOutputSchema.parse( + await legacyCall({ + path: "post.recordViewAndGetStatus", + kind: "mutation", + input: { postVersionId: pendingVersion.postVersionId }, + }), + ); + + assert.equal(status.investigationState, "NOT_INVESTIGATED"); + assert.ok(status.priorInvestigationResult); + assert.equal(status.priorInvestigationResult.sourceInvestigationId, parent.id); + assert.deepEqual( + status.priorInvestigationResult.oldClaims.map((claim) => claim.id), + [survivingClaim.id], + ); +}); + +void test("legacy recordViewAndGetStatus reports a FAILED investigation as NOT_INVESTIGATED", async () => { + const input = buildXViewInput({ + externalId: "legacy-record-view-failed-1", + observedContentText: "Content whose investigation failed.", + }); + const seeded = await seedInvestigationForXViewInput({ + viewInput: input, + status: "FAILED", + provenance: "CLIENT_FALLBACK", + }); + + const status = legacyRecordViewAndGetStatusOutputSchema.parse( + await legacyCall({ + path: "post.recordViewAndGetStatus", + kind: "mutation", + input: { postVersionId: seeded.post.postVersionId }, + }), + ); + + assert.deepEqual(status, { + investigationState: "NOT_INVESTIGATED", + priorInvestigationResult: null, + }); +}); + +void test("legacy getInvestigation answers every investigation state in the legacy shape", async () => { + for (const [index, state] of (["COMPLETE", "PROCESSING", "FAILED"] as const).entries()) { + const seeded = await seedInvestigationForXViewInput({ + viewInput: buildXViewInput({ + externalId: `legacy-get-investigation-${index.toString()}`, + observedContentText: `Legacy polling content ${index.toString()}.`, + }), + status: state, + provenance: "CLIENT_FALLBACK", + claimCount: 1, + }); + + const output = legacyGetInvestigationOutputSchema.parse( + await legacyCall({ + path: "post.getInvestigation", + kind: "query", + input: { investigationId: seeded.investigationId }, + }), + ); + + const expectedState = { + COMPLETE: "INVESTIGATED", + PROCESSING: "INVESTIGATING", + FAILED: "FAILED", + }[state]; + assert.equal(output.investigationState, expectedState); + } + + const unknown = legacyGetInvestigationOutputSchema.parse( + await legacyCall({ + path: "post.getInvestigation", + kind: "query", + input: { investigationId: "legacy-unknown-investigation" }, + }), + ); + assert.equal(unknown.investigationState, "NOT_INVESTIGATED"); +}); + +void test("legacy client flow: register, view, investigateNow, then poll the investigation", async () => { + const rawKey = "legacy-v0-instance-key"; + await seedInstanceApiKey({ name: "legacy-v0", rawKey }); + const registered = legacyRegisterObservedVersionOutputSchema.parse( + await legacyCall({ + path: "post.registerObservedVersion", + kind: "mutation", + input: buildLegacyXViewInput("legacy-investigate-now-1"), + }), + ); + + const investigateNow = legacyInvestigateNowOutputSchema.parse( + await legacyCall({ + path: "post.investigateNow", + kind: "mutation", + input: { postVersionId: registered.postVersionId }, + headers: { "x-api-key": rawKey }, + }), + ); + assert.equal(investigateNow.status, "PENDING"); + + const polled = legacyGetInvestigationOutputSchema.parse( + await legacyCall({ + path: "post.getInvestigation", + kind: "query", + input: { investigationId: investigateNow.investigationId }, + }), + ); + assert.equal(polled.investigationState, "INVESTIGATING"); +}); + +void test("legacy validateSettings and batchStatus answer in the legacy shape", async () => { + const settings = legacyValidateSettingsOutputSchema.parse( + await legacyCall({ path: "post.validateSettings", kind: "query", input: undefined }), + ); + assert.deepEqual(settings, { instanceApiKeyAccepted: false, openaiApiKeyStatus: "missing" }); + + const input = buildXViewInput({ + externalId: "legacy-batch-status-1", + observedContentText: "Batch status content for a legacy client.", + }); + const seeded = await seedInvestigationForXViewInput({ + viewInput: input, + status: "COMPLETE", + provenance: "CLIENT_FALLBACK", + claimCount: 2, + }); + const batch = legacyBatchStatusOutputSchema.parse( + await legacyCall({ + path: "post.batchStatus", + kind: "query", + input: { + posts: [ + { platform: "X", externalId: input.externalId, versionHash: seeded.post.versionHash }, + ], + }, + }), + ); + assert.deepEqual(batch.statuses, [ + { + platform: "X", + externalId: input.externalId, + investigationState: "INVESTIGATED", + incorrectClaimCount: 2, + }, + ]); +}); + +void test("0.4.0 clients get the current protocol, not the legacy one", async () => { + const legacyShapedInput = buildLegacyXViewInput("current-protocol-unadapted-1"); + const rejected = await callApi({ + path: "post.registerObservedVersion", + kind: "mutation", + input: legacyShapedInput, + extensionVersion: CURRENT_EXTENSION_VERSION, + }); + assert.deepEqual(rejected, { ok: false, status: 400, code: "BAD_REQUEST" }); + + const seeded = await seedInvestigationForXViewInput({ + viewInput: buildXViewInput({ + externalId: "current-protocol-unadapted-2", + observedContentText: "Failed content seen by a current client.", + }), + status: "FAILED", + provenance: "CLIENT_FALLBACK", + }); + const status = await callApi({ + path: "post.recordViewAndGetStatus", + kind: "mutation", + input: { postVersionId: seeded.post.postVersionId }, + extensionVersion: CURRENT_EXTENSION_VERSION, + }); + assert.deepEqual(status, { + ok: true, + data: { + investigationState: "FAILED", + investigationId: seeded.investigationId, + provenance: "CLIENT_FALLBACK", + }, + }); +}); + +void test("extensions below 0.2.0 still get UPGRADE_REQUIRED", async () => { + const result = await callApi({ + path: "post.registerObservedVersion", + kind: "mutation", + input: buildLegacyXViewInput("legacy-below-floor-1"), + extensionVersion: "0.1.4", + }); + assert.deepEqual(result, { ok: false, status: 412, code: "PRECONDITION_FAILED" }); +}); + +void test("recordViewAndGetStatus counts one page view per call against its extension version and UTC day", async () => { + const legacyBefore = await countedPageViews(LEGACY_EXTENSION_VERSION); + const currentBefore = await countedPageViews(CURRENT_EXTENSION_VERSION); + + const registered = legacyRegisterObservedVersionOutputSchema.parse( + await legacyCall({ + path: "post.registerObservedVersion", + kind: "mutation", + input: buildLegacyXViewInput("version-counts-1"), + }), + ); + // Registering a version is not a page view. + assert.equal(await countedPageViews(LEGACY_EXTENSION_VERSION), legacyBefore); + + for (let view = 0; view < 2; view += 1) { + await legacyCall({ + path: "post.recordViewAndGetStatus", + kind: "mutation", + input: { postVersionId: registered.postVersionId }, + }); + } + await createCaller({ extensionVersion: CURRENT_EXTENSION_VERSION }).post.recordViewAndGetStatus({ + postVersionId: registered.postVersionId, + }); + + assert.equal(await countedPageViews(LEGACY_EXTENSION_VERSION), legacyBefore + 2); + assert.equal(await countedPageViews(CURRENT_EXTENSION_VERSION), currentBefore + 1); +});