From 5aa0c37b234edcf8b6fc67cc146d3ff9c58e3314 Mon Sep 17 00:00:00 2001 From: Tom Beckenham <34339192+tombeckenham@users.noreply.github.com> Date: Mon, 9 Nov 2026 16:28:31 +1100 Subject: [PATCH 1/2] fal: put indexed endpoints in activity OpenAPI documents (#605) ?activity= was built from live models only, so deprecated endpoints that GET /v1/schemas/fal still lists never appeared. The bare document stays live-only, and AsyncAPI rows stay out. fal-ai/whisper is missing from the active search; look it up by endpoint id and read its category from the OpenAPI document. --- src/server/provider-openapi.ts | 21 +-- src/server/provider-openapi.worker.test.ts | 148 +++++++++++++++++++- src/server/providers/fal-chat-facts.test.ts | 118 +++++++++++++++- src/server/providers/fal.test.ts | 67 +++++++++ src/server/providers/fal.ts | 112 +++++++++++++-- 5 files changed, 439 insertions(+), 27 deletions(-) diff --git a/src/server/provider-openapi.ts b/src/server/provider-openapi.ts index ccde37b2..2f0dc973 100644 --- a/src/server/provider-openapi.ts +++ b/src/server/provider-openapi.ts @@ -424,17 +424,19 @@ interface QueueModel { capabilities: unknown } -/** Live rows for a queue document, narrowed to one activity when given. */ +/** + * Rows for a queue document. `?activity=` includes deprecated models in + * that activity: the schema index still lists their endpoints, and a + * generated client needs those bodies. The bare document stays live-only. + */ async function loadQueueModels( db: Db, providerId: string, activity: Activity | undefined, ): Promise> { - const conditions: Array = [ - eq(models.providerId, providerId), - isNull(models.deprecatedAt), - ] + const conditions: Array = [eq(models.providerId, providerId)] if (activity) conditions.push(eq(models.activity, activity)) + else conditions.push(isNull(models.deprecatedAt)) return db .select({ rawId: models.rawId, @@ -1006,9 +1008,10 @@ export function operationsFor( * model-grained providers; a provider-grained document is never capped by * path count — client generation needs the whole generation surface (#560). * A model-grained queue provider (FAL) with no `?model=` is one queue - * document: `endpoint_id` enumerates every live model with an input schema; - * `?activity=` narrows the enum and inlines each model's own body schema - * (up to {@link MAX_QUEUE_SPEC_PATHS}). + * document: `endpoint_id` enumerates every model with an input schema. + * The bare document is live models only. `?activity=` includes deprecated + * models in that activity and inlines each model's own body schema + * (up to {@link MAX_QUEUE_SPEC_PATHS}). AsyncAPI rows stay out. */ export async function assembleProviderOpenApi( db: Db, @@ -1231,7 +1234,7 @@ export async function assembleProviderOpenApi( title: `${config?.displayName ?? providerId} (modelschemas)`, version: revision, description: queue - ? `Generation-only OpenAPI assembled by modelschemas from live endpoint schemas. Call this provider directly with your own key — this is not a proxy. One queue client: endpoint_id is the model raw id${lean ? "; bodies are untyped here, pass ?activity= to inline each model's own schema" : ", and each component schema is that model's own body (x-modelschemas-model)"}.` + ? `Generation-only OpenAPI assembled by modelschemas from stored endpoint schemas. Call this provider directly with your own key — this is not a proxy. One queue client: endpoint_id is the model raw id${lean ? "; bodies are untyped here, pass ?activity= to inline each model's own schema" : ", and each component schema is that model's own body (x-modelschemas-model)"}.` : `Generation-only OpenAPI assembled by modelschemas from live endpoint schemas. Call this provider directly with your own key — this is not a proxy.`, }, servers: connect.servers, diff --git a/src/server/provider-openapi.worker.test.ts b/src/server/provider-openapi.worker.test.ts index 2b6ff956..49e56786 100644 --- a/src/server/provider-openapi.worker.test.ts +++ b/src/server/provider-openapi.worker.test.ts @@ -391,6 +391,120 @@ describe('assembleProviderOpenApi', () => { expect(info.description).toContain('x-modelschemas-model') }) + it('includes a deprecated model with stored schemas in its activity document', async () => { + await db + .insert(endpoints) + .values({ + id: 'fal/fal-ai/veo3.1', + providerId: 'fal', + activity: 'video', + method: 'POST', + path: '/fal-ai/veo3.1', + }) + .onConflictDoNothing() + await db + .insert(models) + .values({ + id: 'fal-fal-ai-veo3-1', + providerId: 'fal', + rawId: 'fal-ai/veo3.1', + activity: 'video', + providerMetadata: { category: 'text-to-video' }, + deprecatedAt: NOW, + firstSeenAt: NOW, + lastSeenAt: NOW, + }) + .onConflictDoNothing() + await db + .insert(schemaVersions) + .values([ + { + id: 'fal/fal-ai/veo3.1:input', + endpointId: 'fal/fal-ai/veo3.1', + kind: 'input', + contentHash: '1'.repeat(64), + schema: JSON.stringify({ + type: 'object', + properties: { prompt: { type: 'string' } }, + }), + createdAt: NOW, + }, + { + id: 'fal/fal-ai/veo3.1:output', + endpointId: 'fal/fal-ai/veo3.1', + kind: 'output', + contentHash: '2'.repeat(64), + schema: JSON.stringify({ + type: 'object', + properties: { video: { type: 'object' } }, + }), + createdAt: NOW, + }, + ]) + .onConflictDoNothing() + + const bare = await assembleProviderOpenApi(db, 'fal') + expect(bare.ok).toBe(true) + if (!bare.ok) return + const bareEnum = ( + bare.document.paths as Record< + string, + { + post: { + parameters: Array<{ + name: string + schema?: { enum?: Array } + }> + } + } + > + )['/{endpoint_id}']?.post.parameters.find( + (parameter) => parameter.name === 'endpoint_id', + )?.schema?.enum + expect(bareEnum).not.toContain('fal-ai/veo3.1') + + const video = await assembleProviderOpenApi(db, 'fal', { + activity: 'video', + }) + expect(video.ok).toBe(true) + if (!video.ok) return + const post = ( + video.document.paths as Record< + string, + { + post?: { + parameters?: Array<{ schema?: { enum?: Array } }> + requestBody?: { + content: { + 'application/json': { schema: { $ref?: string } } + } + } + } + } + > + )['/{endpoint_id}']?.post + expect(post?.parameters?.[0]?.schema?.enum).toEqual(['fal-ai/veo3.1']) + expect(post?.requestBody?.content['application/json'].schema.$ref).toBe( + '#/components/schemas/input_fal-ai_veo3.1', + ) + const schemas = ( + video.document.components as { + schemas: Record< + string, + { 'x-modelschemas-model'?: string; properties?: unknown } + > + } + ).schemas + expect(schemas['input_fal-ai_veo3.1']).toMatchObject({ + 'x-modelschemas-model': 'fal-ai/veo3.1', + properties: { prompt: { type: 'string' } }, + }) + expect(schemas['output_fal-ai_veo3.1']).toMatchObject({ + 'x-modelschemas-model': 'fal-ai/veo3.1', + properties: { video: { type: 'object' } }, + }) + }) + it('leaves a model with no generation endpoint out of the enum', async () => { await db .insert(models) @@ -1253,12 +1367,42 @@ describe('assembleProviderOpenApi', () => { const byActivity = await assembleProviderOpenApi(db, 'fal', { activity: 'video', }) - expect(byActivity).toMatchObject({ + expect(byActivity.ok).toBe(true) + if (!byActivity.ok) return + const videoEnum = ( + byActivity.document.paths as Record< + string, + { + post?: { + parameters?: Array<{ schema?: { enum?: Array } }> + } + } + > + )['/{endpoint_id}']?.post?.parameters?.[0]?.schema?.enum + expect(videoEnum).toEqual(['fal-ai/veo3.1']) + expect(videoEnum).not.toContain('minimax/h3-max/director') + + await db + .insert(models) + .values({ + id: 'fal-async-only', + providerId: 'fal', + rawId: 'fal-ai/async-only', + activity: 'moderation', + capabilities: { asyncapi: true }, + firstSeenAt: NOW, + lastSeenAt: NOW, + }) + .onConflictDoNothing() + const onlyAsync = await assembleProviderOpenApi(db, 'fal', { + activity: 'moderation', + }) + expect(onlyAsync).toMatchObject({ ok: false, status: 400, code: 'spec_requires_selector', }) - if (!byActivity.ok) expect(byActivity.message).toContain('AsyncAPI') + if (!onlyAsync.ok) expect(onlyAsync.message).toContain('AsyncAPI') }) it('404s unknown provider and unknown model', async () => { diff --git a/src/server/providers/fal-chat-facts.test.ts b/src/server/providers/fal-chat-facts.test.ts index de1bb846..84438225 100644 --- a/src/server/providers/fal-chat-facts.test.ts +++ b/src/server/providers/fal-chat-facts.test.ts @@ -339,13 +339,19 @@ describe('fal listModels', () => { return Promise.resolve(respond(input, () => NO_TOKEN_PRICE)) }) const { models } = await falProvider.listModels({ FAL_KEY: 'k' }) - // Listing, the chat spec page, then one llms.txt per chat endpoint. - expect(urls).toHaveLength(2 + listed.length) + // Listing, the chat spec page, one llms.txt per chat endpoint, then + // the unlisted fal-ai/whisper lookup. The mock page has no whisper row. + expect(urls).toHaveLength(3 + listed.length) expect(urls[0]?.searchParams.has('expand')).toBe(false) expect(urls[1]?.searchParams.get('expand')).toBe('openapi-3.0') expect(urls[1]?.searchParams.getAll('endpoint_id').sort()).toEqual( fixtures.map((m) => m.endpoint_id).sort(), ) + const whisperLookup = urls.find( + (url) => url.searchParams.get('endpoint_id') === 'fal-ai/whisper', + ) + expect(whisperLookup?.searchParams.get('expand')).toBe('openapi-3.0') + expect(models.some((model) => model.rawId === 'fal-ai/whisper')).toBe(false) const byId = new Map(models.map((m) => [m.rawId, m])) expect(byId.get('fal-ai/flux/dev')).toMatchObject({ activity: 'image', @@ -437,6 +443,114 @@ describe('fal listModels', () => { } }) + it('indexes fal-ai/whisper from a direct lookup when the active search omits it', async () => { + const whisper = { + endpoint_id: 'fal-ai/whisper', + metadata: null, + retires_at: null, + openapi: { + openapi: '3.0.4', + info: { + title: 'Whisper', + 'x-fal-metadata': { + category: 'speech-to-text', + endpointId: 'fal-ai/whisper', + }, + }, + paths: { + '/': { + post: { + operationId: 'whisper', + responses: { '200': { description: 'ok' } }, + }, + }, + }, + }, + } + vi.stubGlobal('fetch', (input: string) => { + const url = new URL(input) + if (url.hostname === 'fal.ai') + return Promise.resolve(new Response(NO_TOKEN_PRICE)) + if (url.searchParams.get('endpoint_id') === 'fal-ai/whisper') { + return Promise.resolve(page([whisper])) + } + return Promise.resolve( + url.searchParams.has('endpoint_id') + ? page(fixtures) + : page([image, ...listed]), + ) + }) + const { models } = await falProvider.listModels({ FAL_KEY: 'k' }) + expect( + models.find((model) => model.rawId === 'fal-ai/whisper'), + ).toMatchObject({ + activity: 'audio', + deprecated: false, + displayName: null, + providerMetadata: { category: 'speech-to-text' }, + }) + expect( + models.filter((model) => model.rawId === 'fal-ai/whisper'), + ).toHaveLength(1) + }) + + it('skips fal-ai/whisper when the direct lookup 404s', async () => { + vi.stubGlobal('fetch', (input: string) => { + const url = new URL(input) + if (url.hostname === 'fal.ai') + return Promise.resolve(new Response(NO_TOKEN_PRICE)) + if (url.searchParams.get('endpoint_id') === 'fal-ai/whisper') { + return Promise.resolve(new Response('missing', { status: 404 })) + } + return Promise.resolve( + url.searchParams.has('endpoint_id') + ? page(fixtures) + : page([image, ...listed]), + ) + }) + const { models } = await falProvider.listModels({ FAL_KEY: 'k' }) + expect(models.some((model) => model.rawId === 'fal-ai/whisper')).toBe(false) + expect(models).toHaveLength(listed.length + 1) + }) + + it('marks an unlisted endpoint deprecated once retires_at is past', async () => { + vi.stubGlobal('fetch', (input: string) => { + const url = new URL(input) + if (url.hostname === 'fal.ai') + return Promise.resolve(new Response(NO_TOKEN_PRICE)) + if (url.searchParams.get('endpoint_id') === 'fal-ai/whisper') { + return Promise.resolve( + page([ + { + endpoint_id: 'fal-ai/whisper', + metadata: null, + retires_at: '2020-01-01T00:00:00.000Z', + openapi: { + info: { 'x-fal-metadata': { category: 'speech-to-text' } }, + paths: { + '/': { + post: { + responses: { '200': { description: 'ok' } }, + }, + }, + }, + }, + }, + ]), + ) + } + return Promise.resolve( + url.searchParams.has('endpoint_id') + ? page(fixtures) + : page([image, ...listed]), + ) + }) + const { models } = await falProvider.listModels({ FAL_KEY: 'k' }) + expect( + models.find((model) => model.rawId === 'fal-ai/whisper'), + ).toMatchObject({ activity: 'audio', deprecated: true }) + }) + it('fails the poll when the listing itself fails', async () => { vi.stubGlobal('fetch', () => Promise.resolve(new Response('denied', { status: 401 })), diff --git a/src/server/providers/fal.test.ts b/src/server/providers/fal.test.ts index 2d8a0a77..9f371353 100644 --- a/src/server/providers/fal.test.ts +++ b/src/server/providers/fal.test.ts @@ -3,6 +3,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import directorAsyncApi from '../ingest/fixtures/fal-minimax-h3-max-director.asyncapi.json' import { + FAL_ACTIVITY_MARKER, falAppSlug, falAsyncApiUrl, falLlmsTxtUrl, @@ -285,4 +286,70 @@ describe('fal AsyncAPI discovery', () => { expect(fetched.bundledEndpoints).toEqual([]) expect(fetched.asyncApiRawIds).toEqual([]) }) + + it('ingests fal-ai/whisper from a direct lookup when metadata is null', async () => { + const whisperOpenApi = { + openapi: '3.0.4', + info: { + title: 'Whisper', + 'x-fal-metadata': { + category: 'speech-to-text', + endpointId: 'fal-ai/whisper', + }, + }, + paths: { + '/': { + post: { + operationId: 'whisper', + responses: { '200': { description: 'ok' } }, + }, + }, + }, + } + fetchMock.mockImplementation((input) => { + const url = requestUrl(input) + if (url.startsWith('https://api.fal.ai/v1/models')) { + const parsed = new URL(url) + if (parsed.searchParams.get('endpoint_id') === 'fal-ai/whisper') { + return Promise.resolve( + jsonBody({ + models: [ + { + endpoint_id: 'fal-ai/whisper', + metadata: null, + openapi: whisperOpenApi, + }, + ], + has_more: false, + next_cursor: null, + }), + ) + } + return Promise.resolve( + jsonBody({ + models: [ + { + endpoint_id: 'fal-ai/flux/dev', + openapi: FLUX_OPENAPI, + metadata: { category: 'text-to-image' }, + }, + ], + has_more: false, + next_cursor: null, + }), + ) + } + return Promise.resolve(new Response('nope', { status: 404 })) + }) + + const fetched = await falProvider.fetchSpec({ FAL_KEY: 'test-key' }) + const whisper = fetched.specs.find( + (spec) => spec.info?.['x-fal-endpoint-id'] === 'fal-ai/whisper', + ) + expect(whisper).toBeDefined() + const post = whisper?.paths?.['/']?.post + expect(post?.[FAL_ACTIVITY_MARKER]).toBe('audio') + expect(fetched.specs).toHaveLength(2) + expect(fetched.asyncApiRawIds).toEqual([]) + }) }) diff --git a/src/server/providers/fal.ts b/src/server/providers/fal.ts index 1f9df694..37582ef3 100644 --- a/src/server/providers/fal.ts +++ b/src/server/providers/fal.ts @@ -51,15 +51,56 @@ export const FAL_ACTIVITY_MARKER = 'x-modelschemas-fal-activity' interface FalApiModel { endpoint_id: string openapi?: OpenApiDocument + /** + * Null on rows the active search omits. Category then lives on the + * embedded OpenAPI document (`info['x-fal-metadata'].category`), which + * arrives only when the request sets `expand=openapi-3.0`. + */ metadata: { display_name?: string - category: string + category?: string description?: string status?: 'active' | 'inactive' | 'deprecated' /** Listing/release date, ISO 8601. */ date?: string [key: string]: unknown + } | null + retires_at?: string | null +} + +/** + * Ids the active models search never pages, while `endpoint_id` lookup + * still serves queue OpenAPI. `fal-ai/whisper` is the default transcription + * model in TanStack AI. One id per request: a missing id 404s the batch. + */ +const FAL_UNLISTED_ENDPOINT_IDS = ['fal-ai/whisper'] + +function openApiCategory(model: FalApiModel): string | null { + const meta = model.openapi?.info?.['x-fal-metadata'] + if (typeof meta !== 'object' || meta === null || !('category' in meta)) { + return null } + const category = meta.category + return typeof category === 'string' && category.length > 0 ? category : null +} + +/** Listing category, else the category stamped on the embedded OpenAPI. */ +function modelCategory(model: FalApiModel): string | null { + const listed = model.metadata?.category + if (typeof listed === 'string' && listed.length > 0) return listed + return openApiCategory(model) +} + +/** + * A listing says `deprecated` in metadata. An unlisted row has no metadata; + * a `retires_at` already in the past is the only deprecation signal left. + */ +function modelDeprecated(model: FalApiModel): boolean { + if (model.metadata?.status === 'deprecated') return true + if (model.metadata != null) return false + if (typeof model.retires_at !== 'string') return false + const at = Date.parse(model.retires_at) + return Number.isFinite(at) && at <= Date.now() } interface FalApiResponse { @@ -202,6 +243,37 @@ async function fetchFalModels( return allModels } +/** + * Append {@link FAL_UNLISTED_ENDPOINT_IDS} that the active crawl did not + * return. A 404 (the id is gone) is skipped. Any other non-retryable + * failure is skipped the same way: the crawl already succeeded, and one + * missing side lookup must not sink the poll. A thrown retry exhaustion + * still fails the sync. + */ +async function appendUnlistedFalModels( + apiKey: string, + models: Array, + expandOpenApi: boolean, +): Promise> { + const seen = new Set(models.map((model) => model.endpoint_id)) + const extra: Array = [] + for (const id of FAL_UNLISTED_ENDPOINT_IDS) { + if (seen.has(id)) continue + let page: Array + try { + page = await fetchFalModels(apiKey, expandOpenApi, [id]) + } catch (error) { + if (error instanceof FalFetchError) continue + throw error + } + const match = page.find((model) => model.endpoint_id === id) + if (!match || seen.has(match.endpoint_id)) continue + seen.add(match.endpoint_id) + extra.push(match) + } + return extra.length === 0 ? models : [...models, ...extra] +} + function annotateOperations(spec: OpenApiDocument, activity: Activity): void { for (const operations of Object.values(spec.paths ?? {})) { for (const op of Object.values(operations)) { @@ -304,13 +376,18 @@ async function fetchSpec(env: ProviderSecrets): Promise { ...skippedResult('fal', 'FAL_KEY'), } } - const models = await fetchFalModels(apiKey, true) + const models = await appendUnlistedFalModels( + apiKey, + await fetchFalModels(apiKey, true), + true, + ) const specs: Array = [] const sources: Array = [] const openApiPosted = new Set() for (const model of models) { if (!model.openapi) continue - const activity = falCategoryActivity(model.metadata.category) + const category = modelCategory(model) + const activity = category === null ? null : falCategoryActivity(category) if (activity === null) continue const spec = model.openapi if (hasPostOperation(spec)) openApiPosted.add(model.endpoint_id) @@ -334,7 +411,8 @@ async function fetchSpec(env: ProviderSecrets): Promise { const warnings: Array = [] for (const model of models) { if (openApiPosted.has(model.endpoint_id)) continue - const activity = falCategoryActivity(model.metadata.category) + const category = modelCategory(model) + const activity = category === null ? null : falCategoryActivity(category) if (activity === null) continue try { const discovered = await discoverAsyncApiDocument(model.endpoint_id) @@ -448,14 +526,15 @@ async function listModels( if (!apiKey) { return { models: [], ...skippedResult('fal', 'FAL_KEY') } } - const falModels = await fetchFalModels(apiKey, false) + const listedModels = await fetchFalModels(apiKey, false) // The poll lists without specs (1,500 documents). Chat facts come from // the request schema, so fetch the few chat endpoints' specs by id. A // failed fetch is the chat rows' alone: they keep their stored chat // facts, and the other 1,500 rows still poll. - const chat = falModels.filter( - (m) => falCategoryActivity(m.metadata.category) === 'chat', - ) + const chat = listedModels.filter((m) => { + const category = modelCategory(m) + return category !== null && falCategoryActivity(category) === 'chat' + }) const chatIds = chat.map((m) => m.endpoint_id) const docs = docsRun() const chatFacts = await tryDocs( @@ -483,7 +562,7 @@ async function listModels( falChatFacts( { endpoint_id: m.endpoint_id, - description: m.metadata.description, + description: m.metadata?.description, openapi: chatSpecs.get(m.endpoint_id), }, FAL_MODELS_URL, @@ -501,6 +580,10 @@ async function listModels( 'requestMap', ) const tokenPrices = await loadChatTokenPrices(chatIds, docs, kv) + // After the chat spec request, so that request stays the second models + // call. The lookup expands OpenAPI: unlisted rows carry no metadata, and + // the category is on the document. + const falModels = await appendUnlistedFalModels(apiKey, listedModels, true) const models: Array = falModels.map((m) => { const facts = chatFacts?.get(m.endpoint_id) const price = tokenPrices.get(m.endpoint_id) @@ -516,13 +599,14 @@ async function listModels( ...(card ? { pricing: card.source } : {}), } const { factSources: _listedSources, ...rest } = facts ?? {} + const category = modelCategory(m) return { rawId: m.endpoint_id, - displayName: m.metadata.display_name ?? null, - activity: falCategoryActivity(m.metadata.category), - deprecated: m.metadata.status === 'deprecated', - releasedAt: isoToEpochSeconds(m.metadata.date), - providerMetadata: { category: m.metadata.category }, + displayName: m.metadata?.display_name ?? null, + activity: category === null ? null : falCategoryActivity(category), + deprecated: modelDeprecated(m), + releasedAt: isoToEpochSeconds(m.metadata?.date), + ...(category ? { providerMetadata: { category } } : {}), // A chat row with a request schema also carries the flag list every // other provider's chat rows do. ...rest, From 31577bf5201942c0aa52c05512b65960d28096fd Mon Sep 17 00:00:00 2001 From: Tom Beckenham <34339192+tombeckenham@users.noreply.github.com> Date: Mon, 9 Nov 2026 16:38:35 +1100 Subject: [PATCH 2/2] fal: skip only a 404 on the unlisted endpoint lookup Any other failure dropped the row from the listing, so the poller marked fal-ai/whisper removed and emitted a false model.removed. --- src/server/providers/fal.test.ts | 28 ++++++++++++++++++++++++++++ src/server/providers/fal.ts | 23 ++++++++++++++++------- 2 files changed, 44 insertions(+), 7 deletions(-) diff --git a/src/server/providers/fal.test.ts b/src/server/providers/fal.test.ts index 9f371353..04f4b1a7 100644 --- a/src/server/providers/fal.test.ts +++ b/src/server/providers/fal.test.ts @@ -352,4 +352,32 @@ describe('fal AsyncAPI discovery', () => { expect(fetched.specs).toHaveLength(2) expect(fetched.asyncApiRawIds).toEqual([]) }) + + it('throws when the unlisted lookup fails with anything but 404', async () => { + fetchMock.mockImplementation((input) => { + const url = requestUrl(input) + if (url.startsWith('https://api.fal.ai/v1/models')) { + if (new URL(url).searchParams.get('endpoint_id') === 'fal-ai/whisper') { + return Promise.resolve(new Response('forbidden', { status: 403 })) + } + return Promise.resolve( + jsonBody({ + models: [ + { + endpoint_id: 'fal-ai/flux/dev', + openapi: FLUX_OPENAPI, + metadata: { category: 'text-to-image' }, + }, + ], + has_more: false, + next_cursor: null, + }), + ) + } + return Promise.resolve(new Response('nope', { status: 404 })) + }) + await expect( + falProvider.fetchSpec({ FAL_KEY: 'test-key' }), + ).rejects.toThrow('403') + }) }) diff --git a/src/server/providers/fal.ts b/src/server/providers/fal.ts index 37582ef3..10fa749d 100644 --- a/src/server/providers/fal.ts +++ b/src/server/providers/fal.ts @@ -175,7 +175,14 @@ const FETCH_TIMEOUT_MS = 60_000 const ENDPOINT_FILTER_MAX = 50 /** Non-retryable upstream failure — thrown through the retry loop as-is. */ -class FalFetchError extends Error {} +class FalFetchError extends Error { + constructor( + message: string, + readonly status: number, + ) { + super(message) + } +} /** * The paginated crawl is ~30 requests per pass and runs from both cron @@ -199,7 +206,10 @@ export async function fetchPageWithRetry( if (response.ok) return (await response.json()) as FalApiResponse lastFailure = `${String(response.status)} ${response.statusText}` if (!RETRYABLE_STATUSES.has(response.status)) { - throw new FalFetchError(`fal models fetch failed: ${lastFailure}`) + throw new FalFetchError( + `fal models fetch failed: ${lastFailure}`, + response.status, + ) } const retryAfter = Number(response.headers.get('retry-after')) if (Number.isFinite(retryAfter) && retryAfter > 0) { @@ -245,10 +255,9 @@ async function fetchFalModels( /** * Append {@link FAL_UNLISTED_ENDPOINT_IDS} that the active crawl did not - * return. A 404 (the id is gone) is skipped. Any other non-retryable - * failure is skipped the same way: the crawl already succeeded, and one - * missing side lookup must not sink the poll. A thrown retry exhaustion - * still fails the sync. + * return. Only a 404 (the id is gone) is skipped, so the row is marked + * removed. Any other failure throws: skipping it would also drop the row, + * and the poller would emit a false `model.removed`. */ async function appendUnlistedFalModels( apiKey: string, @@ -263,7 +272,7 @@ async function appendUnlistedFalModels( try { page = await fetchFalModels(apiKey, expandOpenApi, [id]) } catch (error) { - if (error instanceof FalFetchError) continue + if (error instanceof FalFetchError && error.status === 404) continue throw error } const match = page.find((model) => model.endpoint_id === id)