diff --git a/apps/sim/app/api/auth/oauth/microsoft/file/route.ts b/apps/sim/app/api/auth/oauth/microsoft/file/route.ts index 058f007427f..d0bb8a8af7d 100644 --- a/apps/sim/app/api/auth/oauth/microsoft/file/route.ts +++ b/apps/sim/app/api/auth/oauth/microsoft/file/route.ts @@ -6,7 +6,7 @@ import { authorizeCredentialUse } from '@/lib/auth/credential-access' import { validateMicrosoftGraphId } from '@/lib/core/security/input-validation' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { getCredential, refreshAccessTokenIfNeeded } from '@/app/api/auth/oauth/utils' +import { getCredential, refreshAccessTokenIfNeeded } from '@/lib/oauth/credential-service' export const dynamic = 'force-dynamic' diff --git a/apps/sim/app/api/auth/oauth/microsoft/files/route.ts b/apps/sim/app/api/auth/oauth/microsoft/files/route.ts index 7dcd342d662..8b658cd453e 100644 --- a/apps/sim/app/api/auth/oauth/microsoft/files/route.ts +++ b/apps/sim/app/api/auth/oauth/microsoft/files/route.ts @@ -6,7 +6,7 @@ import { authorizeCredentialUse } from '@/lib/auth/credential-access' import { validatePathSegment } from '@/lib/core/security/input-validation' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { getCredential, refreshAccessTokenIfNeeded } from '@/app/api/auth/oauth/utils' +import { getCredential, refreshAccessTokenIfNeeded } from '@/lib/oauth/credential-service' import { GRAPH_ID_PATTERN } from '@/tools/microsoft_excel/utils' import { assertGraphNextPageUrl, getGraphNextPageUrl } from '@/tools/sharepoint/utils' diff --git a/apps/sim/app/api/auth/oauth/token/route.test.ts b/apps/sim/app/api/auth/oauth/token/route.test.ts index e1ef6105675..17948c9f52f 100644 --- a/apps/sim/app/api/auth/oauth/token/route.test.ts +++ b/apps/sim/app/api/auth/oauth/token/route.test.ts @@ -17,7 +17,7 @@ const { mockAuthorizeCredentialUse, mockResolveServiceAccountToken } = vi.hoiste mockResolveServiceAccountToken: vi.fn(), })) -vi.mock('@/app/api/auth/oauth/utils', () => ({ +vi.mock('@/lib/oauth/credential-service', () => ({ ...authOAuthUtilsMock, resolveServiceAccountToken: mockResolveServiceAccountToken, })) diff --git a/apps/sim/app/api/auth/oauth/token/route.ts b/apps/sim/app/api/auth/oauth/token/route.ts index 302898717d0..a58591b3c1d 100644 --- a/apps/sim/app/api/auth/oauth/token/route.ts +++ b/apps/sim/app/api/auth/oauth/token/route.ts @@ -12,14 +12,14 @@ import { AuthType, checkSessionOrInternalAuth } from '@/lib/auth/hybrid' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { TokenServiceAccountValidationError } from '@/lib/credentials/token-service-accounts/errors' -import { captureServerEvent } from '@/lib/posthog/server' import { getCredential, getOAuthToken, refreshTokenIfNeeded, resolveOAuthAccountId, resolveServiceAccountToken, -} from '@/app/api/auth/oauth/utils' +} from '@/lib/oauth/credential-service' +import { captureServerEvent } from '@/lib/posthog/server' import { extractZohoDeskBaseFromScope } from '@/tools/zoho_desk/host-allowlist' export const dynamic = 'force-dynamic' diff --git a/apps/sim/app/api/auth/oauth/utils.test.ts b/apps/sim/app/api/auth/oauth/utils.test.ts index 4a22a97b8fc..e7cee6bbc33 100644 --- a/apps/sim/app/api/auth/oauth/utils.test.ts +++ b/apps/sim/app/api/auth/oauth/utils.test.ts @@ -28,17 +28,17 @@ import { db } from '@sim/db' import { __resetCoalesceLocallyForTests } from '@/lib/concurrency/singleflight' import { ZOOM_SERVICE_ACCOUNT_PROVIDER_ID } from '@/lib/credentials/client-credential-accounts/descriptors' import { refreshOAuthToken } from '@/lib/oauth' -import { - ATLASSIAN_SERVICE_ACCOUNT_PROVIDER_ID, - GOOGLE_SERVICE_ACCOUNT_PROVIDER_ID, - SLACK_CUSTOM_BOT_PROVIDER_ID, -} from '@/lib/oauth/types' import { getCredential, refreshAccessTokenIfNeeded, refreshTokenIfNeeded, resolveServiceAccountToken, -} from '@/app/api/auth/oauth/utils' +} from '@/lib/oauth/credential-service' +import { + ATLASSIAN_SERVICE_ACCOUNT_PROVIDER_ID, + GOOGLE_SERVICE_ACCOUNT_PROVIDER_ID, + SLACK_CUSTOM_BOT_PROVIDER_ID, +} from '@/lib/oauth/types' const mockDb = db as any const mockRefreshOAuthToken = refreshOAuthToken as any diff --git a/apps/sim/app/api/auth/oauth/wealthbox/item/route.ts b/apps/sim/app/api/auth/oauth/wealthbox/item/route.ts index 9e43c3bc8a0..fa871722f36 100644 --- a/apps/sim/app/api/auth/oauth/wealthbox/item/route.ts +++ b/apps/sim/app/api/auth/oauth/wealthbox/item/route.ts @@ -6,7 +6,7 @@ import { authorizeCredentialUse } from '@/lib/auth/credential-access' import { validateEnum, validatePathSegment } from '@/lib/core/security/input-validation' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { refreshAccessTokenIfNeeded } from '@/app/api/auth/oauth/utils' +import { refreshAccessTokenIfNeeded } from '@/lib/oauth/credential-service' export const dynamic = 'force-dynamic' diff --git a/apps/sim/app/api/auth/oauth/wealthbox/items/route.ts b/apps/sim/app/api/auth/oauth/wealthbox/items/route.ts index 6a31bcf3b9f..f8781057cfe 100644 --- a/apps/sim/app/api/auth/oauth/wealthbox/items/route.ts +++ b/apps/sim/app/api/auth/oauth/wealthbox/items/route.ts @@ -6,7 +6,7 @@ import { authorizeCredentialUse } from '@/lib/auth/credential-access' import { validatePathSegment } from '@/lib/core/security/input-validation' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { refreshAccessTokenIfNeeded } from '@/app/api/auth/oauth/utils' +import { refreshAccessTokenIfNeeded } from '@/lib/oauth/credential-service' export const dynamic = 'force-dynamic' diff --git a/apps/sim/app/api/auth/oauth2/callback/instagram/route.ts b/apps/sim/app/api/auth/oauth2/callback/instagram/route.ts index 6b6b9b76a7b..4aea1372f83 100644 --- a/apps/sim/app/api/auth/oauth2/callback/instagram/route.ts +++ b/apps/sim/app/api/auth/oauth2/callback/instagram/route.ts @@ -18,13 +18,13 @@ import { getBaseUrl } from '@/lib/core/utils/urls' import { isSameOrigin } from '@/lib/core/utils/validation' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { processCredentialDraft } from '@/lib/credentials/draft-processor' +import { safeAccountInsert } from '@/lib/oauth/credential-service' import { parseInstagramLongLivedToken, parseInstagramProfile, parseInstagramShortLivedToken, } from '@/lib/oauth/instagram' import { getCanonicalScopesForProvider } from '@/lib/oauth/utils' -import { safeAccountInsert } from '@/app/api/auth/oauth/utils' import { INSTAGRAM_GRAPH_BASE } from '@/tools/instagram/constants' const logger = createLogger('InstagramCallback') diff --git a/apps/sim/app/api/auth/oauth2/shopify/store/route.ts b/apps/sim/app/api/auth/oauth2/shopify/store/route.ts index 4d26c178f5b..182989f917a 100644 --- a/apps/sim/app/api/auth/oauth2/shopify/store/route.ts +++ b/apps/sim/app/api/auth/oauth2/shopify/store/route.ts @@ -12,7 +12,7 @@ import { getBaseUrl } from '@/lib/core/utils/urls' import { isSameOrigin } from '@/lib/core/utils/validation' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { processCredentialDraft } from '@/lib/credentials/draft-processor' -import { safeAccountInsert } from '@/app/api/auth/oauth/utils' +import { safeAccountInsert } from '@/lib/oauth/credential-service' import { SHOPIFY_API_VERSION } from '@/tools/shopify/constants' const logger = createLogger('ShopifyStore') diff --git a/apps/sim/app/api/auth/trello/store/route.ts b/apps/sim/app/api/auth/trello/store/route.ts index 156ed9a65d6..f4fb16feece 100644 --- a/apps/sim/app/api/auth/trello/store/route.ts +++ b/apps/sim/app/api/auth/trello/store/route.ts @@ -9,8 +9,8 @@ import { getSession } from '@/lib/auth' import { env } from '@/lib/core/config/env' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { processCredentialDraft } from '@/lib/credentials/draft-processor' +import { safeAccountInsert } from '@/lib/oauth/credential-service' import { getCanonicalScopesForProvider } from '@/lib/oauth/utils' -import { safeAccountInsert } from '@/app/api/auth/oauth/utils' const logger = createLogger('TrelloStore') diff --git a/apps/sim/app/api/cron/renew-subscriptions/route.test.ts b/apps/sim/app/api/cron/renew-subscriptions/route.test.ts index 6bdd19602ac..86579f7f037 100644 --- a/apps/sim/app/api/cron/renew-subscriptions/route.test.ts +++ b/apps/sim/app/api/cron/renew-subscriptions/route.test.ts @@ -21,7 +21,7 @@ vi.mock('@/lib/auth/internal', () => ({ verifyCronAuth: mockVerifyCronAuth, })) -vi.mock('@/app/api/auth/oauth/utils', () => authOAuthUtilsMock) +vi.mock('@/lib/oauth/credential-service', () => authOAuthUtilsMock) import { GET } from './route' diff --git a/apps/sim/app/api/cron/renew-subscriptions/route.ts b/apps/sim/app/api/cron/renew-subscriptions/route.ts index 97f00a26ff2..8818d5462b1 100644 --- a/apps/sim/app/api/cron/renew-subscriptions/route.ts +++ b/apps/sim/app/api/cron/renew-subscriptions/route.ts @@ -8,9 +8,9 @@ import { verifyCronAuth } from '@/lib/auth/internal' import { acquireLock, releaseLock } from '@/lib/core/config/redis' import { runDetached } from '@/lib/core/utils/background' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' +import { refreshAccessTokenIfNeeded } from '@/lib/oauth/credential-service' import { deliverableWebhookPredicate } from '@/lib/webhooks/delivery-predicate' import { getCredentialOwner, getNotificationUrl } from '@/lib/webhooks/provider-subscription-utils' -import { refreshAccessTokenIfNeeded } from '@/app/api/auth/oauth/utils' const logger = createLogger('TeamsSubscriptionRenewal') diff --git a/apps/sim/app/api/knowledge/[id]/connectors/[connectorId]/documents/route.test.ts b/apps/sim/app/api/knowledge/[id]/connectors/[connectorId]/documents/route.test.ts deleted file mode 100644 index b31ec4e5ea1..00000000000 --- a/apps/sim/app/api/knowledge/[id]/connectors/[connectorId]/documents/route.test.ts +++ /dev/null @@ -1,200 +0,0 @@ -/** - * @vitest-environment node - */ -import { - auditMock, - createMockRequest, - dbChainMockFns, - hybridAuthMockFns, - knowledgeApiUtilsMock, - knowledgeApiUtilsMockFns, - requestUtilsMockFns, - resetDbChainMock, -} from '@sim/testing' -import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest' - -const mockCheckAccess = knowledgeApiUtilsMockFns.mockCheckKnowledgeBaseAccess -const mockCheckWriteAccess = knowledgeApiUtilsMockFns.mockCheckKnowledgeBaseWriteAccess - -vi.mock('@/app/api/knowledge/utils', () => knowledgeApiUtilsMock) -vi.mock('@sim/audit', () => auditMock) - -import { GET, PATCH } from '@/app/api/knowledge/[id]/connectors/[connectorId]/documents/route' - -describe('Connector Documents API Route', () => { - const mockParams = Promise.resolve({ id: 'kb-123', connectorId: 'conn-456' }) - - beforeEach(() => { - vi.clearAllMocks() - resetDbChainMock() - requestUtilsMockFns.mockGenerateRequestId.mockReturnValue('test-req-id') - }) - - afterAll(() => { - resetDbChainMock() - }) - - describe('GET', () => { - it('returns 401 when unauthenticated', async () => { - hybridAuthMockFns.mockCheckSessionOrInternalAuth.mockResolvedValue({ - success: false, - userId: null, - }) - - const req = createMockRequest('GET') - const response = await GET(req as never, { params: mockParams }) - - expect(response.status).toBe(401) - }) - - it('returns 404 when connector not found', async () => { - hybridAuthMockFns.mockCheckSessionOrInternalAuth.mockResolvedValue({ - success: true, - userId: 'user-1', - }) - mockCheckAccess.mockResolvedValue({ hasAccess: true }) - dbChainMockFns.limit.mockResolvedValueOnce([]) - - const req = createMockRequest('GET') - const response = await GET(req as never, { params: mockParams }) - - expect(response.status).toBe(404) - }) - - it('returns documents list on success', async () => { - hybridAuthMockFns.mockCheckSessionOrInternalAuth.mockResolvedValue({ - success: true, - userId: 'user-1', - }) - mockCheckAccess.mockResolvedValue({ hasAccess: true }) - - const doc = { id: 'doc-1', filename: 'test.txt', userExcluded: false } - dbChainMockFns.limit.mockResolvedValueOnce([{ id: 'conn-456' }]) - dbChainMockFns.orderBy.mockResolvedValueOnce([doc]) - - const url = 'http://localhost/api/knowledge/kb-123/connectors/conn-456/documents' - const req = createMockRequest('GET', undefined, undefined, url) - const response = await GET(req as never, { params: mockParams }) - const data = await response.json() - - expect(response.status).toBe(200) - expect(data.data.documents).toHaveLength(1) - expect(data.data.counts.active).toBe(1) - expect(data.data.counts.excluded).toBe(0) - }) - - it('includes excluded documents when includeExcluded=true', async () => { - hybridAuthMockFns.mockCheckSessionOrInternalAuth.mockResolvedValue({ - success: true, - userId: 'user-1', - }) - mockCheckAccess.mockResolvedValue({ hasAccess: true }) - - dbChainMockFns.limit.mockResolvedValueOnce([{ id: 'conn-456' }]) - dbChainMockFns.orderBy - .mockResolvedValueOnce([{ id: 'doc-1', userExcluded: false }]) - .mockResolvedValueOnce([{ id: 'doc-2', userExcluded: true }]) - - const url = - 'http://localhost/api/knowledge/kb-123/connectors/conn-456/documents?includeExcluded=true' - const req = createMockRequest('GET', undefined, undefined, url) - const response = await GET(req as never, { params: mockParams }) - const data = await response.json() - - expect(response.status).toBe(200) - expect(data.data.documents).toHaveLength(2) - expect(data.data.counts.active).toBe(1) - expect(data.data.counts.excluded).toBe(1) - }) - }) - - describe('PATCH', () => { - it('returns 401 when unauthenticated', async () => { - hybridAuthMockFns.mockCheckSessionOrInternalAuth.mockResolvedValue({ - success: false, - userId: null, - }) - - const req = createMockRequest('PATCH', { operation: 'restore', documentIds: ['doc-1'] }) - const response = await PATCH(req as never, { params: mockParams }) - - expect(response.status).toBe(401) - }) - - it('returns 400 for invalid body', async () => { - hybridAuthMockFns.mockCheckSessionOrInternalAuth.mockResolvedValue({ - success: true, - userId: 'user-1', - }) - mockCheckWriteAccess.mockResolvedValue({ hasAccess: true }) - dbChainMockFns.limit.mockResolvedValueOnce([{ id: 'conn-456' }]) - - const req = createMockRequest('PATCH', { documentIds: [] }) - const response = await PATCH(req as never, { params: mockParams }) - - expect(response.status).toBe(400) - }) - - it('returns 404 when connector not found', async () => { - hybridAuthMockFns.mockCheckSessionOrInternalAuth.mockResolvedValue({ - success: true, - userId: 'user-1', - }) - mockCheckWriteAccess.mockResolvedValue({ hasAccess: true }) - dbChainMockFns.limit.mockResolvedValueOnce([]) - - const req = createMockRequest('PATCH', { operation: 'restore', documentIds: ['doc-1'] }) - const response = await PATCH(req as never, { params: mockParams }) - - expect(response.status).toBe(404) - }) - - it('returns success for restore operation', async () => { - hybridAuthMockFns.mockCheckSessionOrInternalAuth.mockResolvedValue({ - success: true, - userId: 'user-1', - userName: 'Test', - userEmail: 'test@test.com', - }) - mockCheckWriteAccess.mockResolvedValue({ - hasAccess: true, - knowledgeBase: { workspaceId: 'ws-1', name: 'Test KB' }, - }) - dbChainMockFns.limit.mockResolvedValueOnce([{ id: 'conn-456' }]) - dbChainMockFns.returning.mockResolvedValueOnce([{ id: 'doc-1' }]) - - const req = createMockRequest('PATCH', { operation: 'restore', documentIds: ['doc-1'] }) - const response = await PATCH(req as never, { params: mockParams }) - const data = await response.json() - - expect(response.status).toBe(200) - expect(data.data.restoredCount).toBe(1) - }) - - it('returns success for exclude operation', async () => { - hybridAuthMockFns.mockCheckSessionOrInternalAuth.mockResolvedValue({ - success: true, - userId: 'user-1', - userName: 'Test', - userEmail: 'test@test.com', - }) - mockCheckWriteAccess.mockResolvedValue({ - hasAccess: true, - knowledgeBase: { workspaceId: 'ws-1', name: 'Test KB' }, - }) - dbChainMockFns.limit.mockResolvedValueOnce([{ id: 'conn-456' }]) - dbChainMockFns.returning.mockResolvedValueOnce([{ id: 'doc-2' }, { id: 'doc-3' }]) - - const req = createMockRequest('PATCH', { - operation: 'exclude', - documentIds: ['doc-2', 'doc-3'], - }) - const response = await PATCH(req as never, { params: mockParams }) - const data = await response.json() - - expect(response.status).toBe(200) - expect(data.data.excludedCount).toBe(2) - expect(data.data.documentIds).toEqual(['doc-2', 'doc-3']) - }) - }) -}) diff --git a/apps/sim/app/api/knowledge/[id]/connectors/[connectorId]/documents/route.ts b/apps/sim/app/api/knowledge/[id]/connectors/[connectorId]/documents/route.ts index 9eebf944c29..68f3fd82248 100644 --- a/apps/sim/app/api/knowledge/[id]/connectors/[connectorId]/documents/route.ts +++ b/apps/sim/app/api/knowledge/[id]/connectors/[connectorId]/documents/route.ts @@ -1,247 +1,66 @@ -import { AuditAction, AuditResourceType, recordAudit } from '@sim/audit' -import { db } from '@sim/db' -import { document, knowledgeConnector } from '@sim/db/schema' -import { createLogger } from '@sim/logger' -import { and, eq, inArray, isNull } from 'drizzle-orm' -import { type NextRequest, NextResponse } from 'next/server' -import { patchKnowledgeConnectorDocumentsContract } from '@/lib/api/contracts/knowledge' -import { parseRequest } from '@/lib/api/server' -import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid' -import { generateRequestId } from '@/lib/core/utils/request' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { checkKnowledgeBaseAccess, checkKnowledgeBaseWriteAccess } from '@/app/api/knowledge/utils' - -const logger = createLogger('ConnectorDocumentsAPI') - -type RouteParams = { params: Promise<{ id: string; connectorId: string }> } - -/** - * GET /api/knowledge/[id]/connectors/[connectorId]/documents - * Returns documents for a connector, optionally including user-excluded ones. - */ -export const GET = withRouteHandler(async (request: NextRequest, { params }: RouteParams) => { - const requestId = generateRequestId() - const { id: knowledgeBaseId, connectorId } = await params - - try { - const auth = await checkSessionOrInternalAuth(request, { requireWorkflowId: false }) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) - } - - const accessCheck = await checkKnowledgeBaseAccess(knowledgeBaseId, auth.userId) - if (!accessCheck.hasAccess) { - const status = 'notFound' in accessCheck && accessCheck.notFound ? 404 : 401 - return NextResponse.json({ error: status === 404 ? 'Not found' : 'Unauthorized' }, { status }) - } - - const connectorRows = await db - .select({ id: knowledgeConnector.id }) - .from(knowledgeConnector) - .where( - and( - eq(knowledgeConnector.id, connectorId), - eq(knowledgeConnector.knowledgeBaseId, knowledgeBaseId), - isNull(knowledgeConnector.archivedAt), - isNull(knowledgeConnector.deletedAt) - ) - ) - .limit(1) - - if (connectorRows.length === 0) { - return NextResponse.json({ error: 'Connector not found' }, { status: 404 }) - } - - const includeExcluded = request.nextUrl.searchParams.get('includeExcluded') === 'true' - - const activeDocs = await db - .select({ - id: document.id, - filename: document.filename, - externalId: document.externalId, - sourceUrl: document.sourceUrl, - enabled: document.enabled, - userExcluded: document.userExcluded, - uploadedAt: document.uploadedAt, - processingStatus: document.processingStatus, - }) - .from(document) - .where( - and( - eq(document.connectorId, connectorId), - isNull(document.archivedAt), - isNull(document.deletedAt), - eq(document.userExcluded, false) - ) - ) - .orderBy(document.filename) - - const excludedDocs = includeExcluded - ? await db - .select({ - id: document.id, - filename: document.filename, - externalId: document.externalId, - sourceUrl: document.sourceUrl, - enabled: document.enabled, - userExcluded: document.userExcluded, - uploadedAt: document.uploadedAt, - processingStatus: document.processingStatus, - }) - .from(document) - .where( - and( - eq(document.connectorId, connectorId), - eq(document.userExcluded, true), - isNull(document.archivedAt), - isNull(document.deletedAt) - ) - ) - .orderBy(document.filename) - : [] - - const docs = [...activeDocs, ...excludedDocs] - const activeCount = activeDocs.length - const excludedCount = excludedDocs.length - - return NextResponse.json({ - success: true, - data: { - documents: docs, - counts: { active: activeCount, excluded: excludedCount }, - }, - }) - } catch (error) { - logger.error(`[${requestId}] Error fetching connector documents`, error) - return NextResponse.json({ error: 'Internal server error' }, { status: 500 }) - } +import { + listKnowledgeConnectorDocumentsContract, + patchKnowledgeConnectorDocumentsContract, +} from '@/lib/api/contracts/knowledge' +import { defineInternalJsonRoute, internalRateLimits } from '@/lib/api/server/routes' +import { + internalKnowledgeErrorPolicies, + internalKnowledgeSessionOrExecutorAuth, +} from '@/lib/knowledge/api/route-policies' +import { + listKnowledgeConnectorDocuments, + updateKnowledgeConnectorDocuments, +} from '@/lib/knowledge/application/connectors' +import { knowledgeOperations } from '@/lib/knowledge/application/operations' + +export const GET = defineInternalJsonRoute({ + contract: listKnowledgeConnectorDocumentsContract, + auth: internalKnowledgeSessionOrExecutorAuth, + operation: knowledgeOperations.listConnectorDocuments, + rateLimit: internalRateLimits.none({ + reason: 'Preserve existing internal connector-document list behavior', + }), + errorPolicy: internalKnowledgeErrorPolicies.connectors, + mapInput: ({ params, query }) => ({ + knowledgeBaseId: params.id, + connectorId: params.connectorId, + includeExcluded: query.includeExcluded, + limit: query.limit, + offset: query.offset, + }), + useCase: listKnowledgeConnectorDocuments, + present: ({ documents, counts }) => ({ + success: true as const, + data: { + documents: documents.map((document) => ({ + ...document, + deletedAt: null, + uploadedAt: document.uploadedAt.toISOString(), + })), + counts, + }, + }), }) -/** - * PATCH /api/knowledge/[id]/connectors/[connectorId]/documents - * Restore or exclude connector documents. - */ -export const PATCH = withRouteHandler(async (request: NextRequest, context: RouteParams) => { - const requestId = generateRequestId() - const { id: knowledgeBaseId, connectorId } = await context.params - - try { - const auth = await checkSessionOrInternalAuth(request, { requireWorkflowId: false }) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) - } - - const writeCheck = await checkKnowledgeBaseWriteAccess(knowledgeBaseId, auth.userId) - if (!writeCheck.hasAccess) { - const status = 'notFound' in writeCheck && writeCheck.notFound ? 404 : 401 - return NextResponse.json({ error: status === 404 ? 'Not found' : 'Unauthorized' }, { status }) - } - - const connectorRows = await db - .select({ id: knowledgeConnector.id }) - .from(knowledgeConnector) - .where( - and( - eq(knowledgeConnector.id, connectorId), - eq(knowledgeConnector.knowledgeBaseId, knowledgeBaseId), - isNull(knowledgeConnector.archivedAt), - isNull(knowledgeConnector.deletedAt) - ) - ) - .limit(1) - - if (connectorRows.length === 0) { - return NextResponse.json({ error: 'Connector not found' }, { status: 404 }) - } - - const parsed = await parseRequest(patchKnowledgeConnectorDocumentsContract, request, context) - if (!parsed.success) return parsed.response - - const { operation, documentIds } = parsed.data.body - - if (operation === 'restore') { - const updated = await db - .update(document) - .set({ userExcluded: false, enabled: true }) - .where( - and( - eq(document.connectorId, connectorId), - inArray(document.id, documentIds), - eq(document.userExcluded, true), - isNull(document.archivedAt), - isNull(document.deletedAt) - ) - ) - .returning({ id: document.id }) - - logger.info(`[${requestId}] Restored ${updated.length} excluded documents`, { connectorId }) - - recordAudit({ - workspaceId: writeCheck.knowledgeBase.workspaceId, - actorId: auth.userId, - actorName: auth.userName, - actorEmail: auth.userEmail, - action: AuditAction.CONNECTOR_DOCUMENT_RESTORED, - resourceType: AuditResourceType.CONNECTOR, - resourceId: connectorId, - description: `Restored ${updated.length} excluded document(s) for knowledge base "${writeCheck.knowledgeBase.name}"`, - metadata: { - knowledgeBaseId, - knowledgeBaseName: writeCheck.knowledgeBase.name, - operation: 'restore', - documentCount: updated.length, - documentIds: updated.map((d) => d.id), - }, - request, - }) - - return NextResponse.json({ - success: true, - data: { restoredCount: updated.length, documentIds: updated.map((d) => d.id) }, - }) - } - - const updated = await db - .update(document) - .set({ userExcluded: true, enabled: false }) - .where( - and( - eq(document.connectorId, connectorId), - inArray(document.id, documentIds), - eq(document.userExcluded, false), - isNull(document.archivedAt), - isNull(document.deletedAt) - ) - ) - .returning({ id: document.id }) - - logger.info(`[${requestId}] Excluded ${updated.length} documents`, { connectorId }) - - recordAudit({ - workspaceId: writeCheck.knowledgeBase.workspaceId, - actorId: auth.userId, - actorName: auth.userName, - actorEmail: auth.userEmail, - action: AuditAction.CONNECTOR_DOCUMENT_EXCLUDED, - resourceType: AuditResourceType.CONNECTOR, - resourceId: connectorId, - description: `Excluded ${updated.length} document(s) from knowledge base "${writeCheck.knowledgeBase.name}"`, - metadata: { - knowledgeBaseId, - knowledgeBaseName: writeCheck.knowledgeBase.name, - operation: 'exclude', - documentCount: updated.length, - documentIds: updated.map((d) => d.id), - }, - request, - }) - - return NextResponse.json({ - success: true, - data: { excludedCount: updated.length, documentIds: updated.map((d) => d.id) }, - }) - } catch (error) { - logger.error(`[${requestId}] Error updating connector documents`, error) - return NextResponse.json({ error: 'Internal server error' }, { status: 500 }) - } +export const PATCH = defineInternalJsonRoute({ + contract: patchKnowledgeConnectorDocumentsContract, + auth: internalKnowledgeSessionOrExecutorAuth, + operation: knowledgeOperations.updateConnectorDocuments, + rateLimit: internalRateLimits.none({ + reason: 'Preserve existing internal connector-document update behavior', + }), + errorPolicy: internalKnowledgeErrorPolicies.connectors, + mapInput: ({ params, body }) => ({ + knowledgeBaseId: params.id, + connectorId: params.connectorId, + ...body, + }), + useCase: updateKnowledgeConnectorDocuments, + present: ({ operation, count, documentIds }) => ({ + success: true as const, + data: + operation === 'restore' + ? { restoredCount: count, documentIds } + : { excludedCount: count, documentIds }, + }), }) diff --git a/apps/sim/app/api/knowledge/[id]/connectors/[connectorId]/route.test.ts b/apps/sim/app/api/knowledge/[id]/connectors/[connectorId]/route.test.ts deleted file mode 100644 index ce255768db1..00000000000 --- a/apps/sim/app/api/knowledge/[id]/connectors/[connectorId]/route.test.ts +++ /dev/null @@ -1,249 +0,0 @@ -/** - * @vitest-environment node - */ -import { - auditMock, - authOAuthUtilsMock, - createMockRequest, - dbChainMockFns, - hybridAuthMockFns, - knowledgeApiUtilsMock, - knowledgeApiUtilsMockFns, - queueTableRows, - resetDbChainMock, - schemaMock, -} from '@sim/testing' -import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest' - -const { mockHasWorkspaceLiveSyncAccess, mockValidateConfig } = vi.hoisted(() => ({ - mockHasWorkspaceLiveSyncAccess: vi.fn(), - mockValidateConfig: vi.fn(), -})) - -const mockCheckAccess = knowledgeApiUtilsMockFns.mockCheckKnowledgeBaseAccess -const mockCheckWriteAccess = knowledgeApiUtilsMockFns.mockCheckKnowledgeBaseWriteAccess - -vi.mock('@/app/api/knowledge/utils', () => knowledgeApiUtilsMock) -vi.mock('@/app/api/auth/oauth/utils', () => authOAuthUtilsMock) -vi.mock('@/connectors/registry.server', () => ({ - CONNECTOR_REGISTRY: { - jira: { validateConfig: mockValidateConfig }, - }, -})) -vi.mock('@/lib/knowledge/tags/service', () => ({ - cleanupUnusedTagDefinitions: vi.fn().mockResolvedValue(undefined), -})) -vi.mock('@/lib/knowledge/documents/service', () => ({ - deleteDocumentStorageFiles: vi.fn().mockResolvedValue(undefined), -})) -vi.mock('@/lib/billing/core/subscription', () => ({ - hasWorkspaceLiveSyncAccess: mockHasWorkspaceLiveSyncAccess, -})) -vi.mock('@sim/audit', () => auditMock) - -import { DELETE, GET, PATCH } from '@/app/api/knowledge/[id]/connectors/[connectorId]/route' - -describe('Knowledge Connector By ID API Route', () => { - const mockParams = Promise.resolve({ id: 'kb-123', connectorId: 'conn-456' }) - - beforeEach(() => { - vi.clearAllMocks() - resetDbChainMock() - }) - - afterAll(() => { - resetDbChainMock() - }) - - describe('GET', () => { - it('returns 401 when unauthenticated', async () => { - hybridAuthMockFns.mockCheckSessionOrInternalAuth.mockResolvedValue({ - success: false, - userId: null, - }) - - const req = createMockRequest('GET') - const response = await GET(req, { params: mockParams }) - - expect(response.status).toBe(401) - }) - - it('returns 404 when KB not found', async () => { - hybridAuthMockFns.mockCheckSessionOrInternalAuth.mockResolvedValue({ - success: true, - userId: 'user-1', - }) - mockCheckAccess.mockResolvedValue({ hasAccess: false, notFound: true }) - - const req = createMockRequest('GET') - const response = await GET(req, { params: mockParams }) - - expect(response.status).toBe(404) - }) - - it('returns 404 when connector not found', async () => { - hybridAuthMockFns.mockCheckSessionOrInternalAuth.mockResolvedValue({ - success: true, - userId: 'user-1', - }) - mockCheckAccess.mockResolvedValue({ hasAccess: true }) - dbChainMockFns.limit.mockResolvedValueOnce([]) - - const req = createMockRequest('GET') - const response = await GET(req, { params: mockParams }) - - expect(response.status).toBe(404) - }) - - it('returns connector with sync logs on success', async () => { - hybridAuthMockFns.mockCheckSessionOrInternalAuth.mockResolvedValue({ - success: true, - userId: 'user-1', - }) - mockCheckAccess.mockResolvedValue({ hasAccess: true }) - - const mockConnector = { id: 'conn-456', connectorType: 'jira', status: 'active' } - const mockLogs = [{ id: 'log-1', status: 'completed' }] - - dbChainMockFns.limit.mockResolvedValueOnce([mockConnector]).mockResolvedValueOnce(mockLogs) - - const req = createMockRequest('GET') - const response = await GET(req, { params: mockParams }) - const data = await response.json() - - expect(response.status).toBe(200) - expect(data.success).toBe(true) - expect(data.data.id).toBe('conn-456') - expect(data.data.syncLogs).toHaveLength(1) - }) - }) - - describe('PATCH', () => { - it('returns 401 when unauthenticated', async () => { - hybridAuthMockFns.mockCheckSessionOrInternalAuth.mockResolvedValue({ - success: false, - userId: null, - }) - - const req = createMockRequest('PATCH', { status: 'paused' }) - const response = await PATCH(req, { params: mockParams }) - - expect(response.status).toBe(401) - }) - - it('returns 400 for invalid body', async () => { - hybridAuthMockFns.mockCheckSessionOrInternalAuth.mockResolvedValue({ - success: true, - userId: 'user-1', - }) - mockCheckWriteAccess.mockResolvedValue({ hasAccess: true }) - - const req = createMockRequest('PATCH', { syncIntervalMinutes: 'not a number' }) - const response = await PATCH(req, { params: mockParams }) - const data = await response.json() - - expect(response.status).toBe(400) - expect(data.error).toBe('Validation error') - }) - - it('returns 404 when connector not found during sourceConfig validation', async () => { - hybridAuthMockFns.mockCheckSessionOrInternalAuth.mockResolvedValue({ - success: true, - userId: 'user-1', - }) - mockCheckWriteAccess.mockResolvedValue({ - hasAccess: true, - knowledgeBase: { workspaceId: 'ws-1', name: 'Test KB' }, - }) - dbChainMockFns.limit.mockResolvedValueOnce([]) - - const req = createMockRequest('PATCH', { sourceConfig: { project: 'NEW' } }) - const response = await PATCH(req, { params: mockParams }) - - expect(response.status).toBe(404) - }) - - it('allows a free external actor to enable live sync for a Max workspace', async () => { - hybridAuthMockFns.mockCheckSessionOrInternalAuth.mockResolvedValue({ - success: true, - userId: 'free-external-admin', - userName: 'Test', - userEmail: 'test@test.com', - }) - mockCheckWriteAccess.mockResolvedValue({ - hasAccess: true, - knowledgeBase: { workspaceId: 'ws-1', name: 'Test KB' }, - }) - mockHasWorkspaceLiveSyncAccess.mockResolvedValue(true) - - const updatedConnector = { id: 'conn-456', status: 'paused', syncIntervalMinutes: 5 } - dbChainMockFns.limit.mockResolvedValueOnce([{ id: 'conn-456', connectorType: 'jira' }]) - dbChainMockFns.returning.mockResolvedValueOnce([updatedConnector]) - - const req = createMockRequest('PATCH', { status: 'paused', syncIntervalMinutes: 5 }) - const response = await PATCH(req, { params: mockParams }) - const data = await response.json() - - expect(response.status).toBe(200) - expect(data.success).toBe(true) - expect(data.data.status).toBe('paused') - expect(mockHasWorkspaceLiveSyncAccess).toHaveBeenCalledWith('ws-1') - }) - - it('denies a paid actor when the knowledge base workspace is free', async () => { - hybridAuthMockFns.mockCheckSessionOrInternalAuth.mockResolvedValue({ - success: true, - userId: 'paid-external-admin', - }) - mockCheckWriteAccess.mockResolvedValue({ - hasAccess: true, - knowledgeBase: { workspaceId: 'ws-free', name: 'Free KB' }, - }) - mockHasWorkspaceLiveSyncAccess.mockResolvedValue(false) - dbChainMockFns.limit.mockResolvedValueOnce([{ id: 'conn-456', connectorType: 'jira' }]) - - const req = createMockRequest('PATCH', { syncIntervalMinutes: 5 }) - const response = await PATCH(req, { params: mockParams }) - - expect(response.status).toBe(403) - expect(mockHasWorkspaceLiveSyncAccess).toHaveBeenCalledWith('ws-free') - }) - }) - - describe('DELETE', () => { - it('returns 401 when unauthenticated', async () => { - hybridAuthMockFns.mockCheckSessionOrInternalAuth.mockResolvedValue({ - success: false, - userId: null, - }) - - const req = createMockRequest('DELETE') - const response = await DELETE(req, { params: mockParams }) - - expect(response.status).toBe(401) - }) - - it('returns 200 on successful hard-delete', async () => { - hybridAuthMockFns.mockCheckSessionOrInternalAuth.mockResolvedValue({ - success: true, - userId: 'user-1', - userName: 'Test', - userEmail: 'test@test.com', - }) - mockCheckWriteAccess.mockResolvedValue({ - hasAccess: true, - knowledgeBase: { workspaceId: 'ws-1', name: 'Test KB' }, - }) - dbChainMockFns.limit.mockResolvedValueOnce([{ id: 'conn-456', connectorType: 'jira' }]) - queueTableRows(schemaMock.document, [{ id: 'doc-1', fileUrl: '/api/uploads/test.txt' }]) - dbChainMockFns.returning.mockResolvedValueOnce([{ id: 'conn-456' }]) - - const req = createMockRequest('DELETE') - const response = await DELETE(req, { params: mockParams }) - const data = await response.json() - - expect(response.status).toBe(200) - expect(data.success).toBe(true) - }) - }) -}) diff --git a/apps/sim/app/api/knowledge/[id]/connectors/[connectorId]/route.ts b/apps/sim/app/api/knowledge/[id]/connectors/[connectorId]/route.ts index d63513af694..bfa2f27c880 100644 --- a/apps/sim/app/api/knowledge/[id]/connectors/[connectorId]/route.ts +++ b/apps/sim/app/api/knowledge/[id]/connectors/[connectorId]/route.ts @@ -1,256 +1,80 @@ -import { db } from '@sim/db' -import { knowledgeConnectorSyncLog } from '@sim/db/schema' -import { createLogger } from '@sim/logger' -import { desc, eq } from 'drizzle-orm' -import { type NextRequest, NextResponse } from 'next/server' import { deleteKnowledgeConnectorContract, + getKnowledgeConnectorContract, updateKnowledgeConnectorContract, } from '@/lib/api/contracts/knowledge' -import { parseRequest } from '@/lib/api/server' -import { decryptApiKey } from '@/lib/api-key/crypto' -import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid' +import { defineInternalJsonRoute, internalRateLimits } from '@/lib/api/server/routes' import { - messageForOrchestrationError, - statusForOrchestrationError, -} from '@/lib/core/orchestration/types' -import { generateRequestId } from '@/lib/core/utils/request' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { resolveCredentialTokenIdentity } from '@/lib/credentials/access' + internalKnowledgeAnalytics, + toInternalKnowledgeConnector, + toInternalKnowledgeConnectorDetail, +} from '@/lib/knowledge/api/internal-route' import { - getKnowledgeConnector, - type KnowledgeConnectorRow, - performDeleteKnowledgeConnector, - performUpdateKnowledgeConnector, - type SourceConfigRejection, -} from '@/lib/knowledge/orchestration' -import { refreshAccessTokenIfNeeded } from '@/app/api/auth/oauth/utils' -import { checkKnowledgeBaseAccess, checkKnowledgeBaseWriteAccess } from '@/app/api/knowledge/utils' -import { CONNECTOR_REGISTRY } from '@/connectors/registry.server' - -const logger = createLogger('KnowledgeConnectorByIdAPI') - -type RouteParams = { params: Promise<{ id: string; connectorId: string }> } - -/** - * GET /api/knowledge/[id]/connectors/[connectorId] - Get connector details with recent sync logs - */ -export const GET = withRouteHandler(async (request: NextRequest, { params }: RouteParams) => { - const requestId = generateRequestId() - const { id: knowledgeBaseId, connectorId } = await params - - try { - const auth = await checkSessionOrInternalAuth(request, { requireWorkflowId: false }) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) - } - - const accessCheck = await checkKnowledgeBaseAccess(knowledgeBaseId, auth.userId) - if (!accessCheck.hasAccess) { - const status = 'notFound' in accessCheck && accessCheck.notFound ? 404 : 401 - return NextResponse.json({ error: status === 404 ? 'Not found' : 'Unauthorized' }, { status }) - } - - const connector = await getKnowledgeConnector(knowledgeBaseId, connectorId) - if (!connector) { - return NextResponse.json({ error: 'Connector not found' }, { status: 404 }) - } - - const syncLogs = await db - .select() - .from(knowledgeConnectorSyncLog) - .where(eq(knowledgeConnectorSyncLog.connectorId, connectorId)) - .orderBy(desc(knowledgeConnectorSyncLog.startedAt)) - .limit(10) - - const { encryptedApiKey: _, ...connectorData } = connector - return NextResponse.json({ - success: true, - data: { - ...connectorData, - syncLogs, - }, - }) - } catch (error) { - logger.error(`[${requestId}] Error fetching connector`, error) - return NextResponse.json({ error: 'Internal server error' }, { status: 500 }) - } + internalKnowledgeErrorPolicies, + internalKnowledgeSessionOrExecutorAuth, +} from '@/lib/knowledge/api/route-policies' +import { + deleteKnowledgeConnector, + readKnowledgeConnector, + updateKnowledgeConnector, +} from '@/lib/knowledge/application/connectors' +import { knowledgeOperations } from '@/lib/knowledge/application/operations' + +export const GET = defineInternalJsonRoute({ + contract: getKnowledgeConnectorContract, + auth: internalKnowledgeSessionOrExecutorAuth, + operation: knowledgeOperations.readConnector, + rateLimit: internalRateLimits.none({ + reason: 'Preserve existing internal connector-read behavior', + }), + errorPolicy: internalKnowledgeErrorPolicies.connectors, + mapInput: ({ params }) => ({ + knowledgeBaseId: params.id, + connectorId: params.connectorId, + }), + useCase: readKnowledgeConnector, + present: ({ connector }) => ({ + success: true as const, + data: toInternalKnowledgeConnectorDetail(connector), + }), }) -/** - * Validates a replacement `sourceConfig` against the live source, resolving the - * connector's own token first. Returns a rejection message, or `null` to accept. - * - * Stays with the route rather than moving into orchestration because resolving - * the token needs the requesting identity: workspace credentials are shared and - * token reads are scoped to `account.userId`, so the credential's own account - * owner is used — not the knowledge base owner, and not the acting user when a - * service account mints its own token. - */ -function makeSourceConfigValidator( - actingUserId: string, - workspaceId: string | null, - connectorId: string -) { - return async ( - connector: KnowledgeConnectorRow, - sourceConfig: Record - ): Promise => { - const connectorConfig = CONNECTOR_REGISTRY[connector.connectorType] - if (!connectorConfig) { - return { - message: `Unknown connector type: ${connector.connectorType}`, - errorCode: 'validation', - } - } - - let accessToken: string | null = null - if (connectorConfig.auth.mode === 'apiKey') { - if (!connector.encryptedApiKey) { - return { - message: 'API key not found. Please reconfigure the connector.', - errorCode: 'validation', - } - } - accessToken = (await decryptApiKey(connector.encryptedApiKey)).decrypted - } else { - if (!connector.credentialId) { - return { - message: 'OAuth credential not found. Please reconfigure the connector.', - errorCode: 'validation', - } - } - if (!workspaceId) { - return { - message: 'Knowledge base is missing workspace context', - errorCode: 'conflict', - } - } - const identity = await resolveCredentialTokenIdentity(connector.credentialId, workspaceId) - if (!identity) { - return { - message: 'Credential is no longer usable in this workspace. Please reconnect it.', - errorCode: 'validation', - } - } - accessToken = await refreshAccessTokenIfNeeded( - connector.credentialId, - // Service accounts mint their own token and ignore the acting user. - identity.kind === 'oauth' ? identity.userId : actingUserId, - `patch-${connectorId}` - ) - } - - if (!accessToken) { - // A stale stored credential, not an unauthenticated caller — but the route - // has always answered 401 here, so keep that rather than silently - // reclassifying it as part of this refactor. - return { - message: 'Failed to refresh access token. Please reconnect your account.', - errorCode: 'unauthorized', - } - } - - const validation = await connectorConfig.validateConfig(accessToken, sourceConfig) - return validation.valid - ? null - : { message: validation.error || 'Invalid source configuration', errorCode: 'validation' } - } -} - -/** - * PATCH /api/knowledge/[id]/connectors/[connectorId] - Update a connector - */ -export const PATCH = withRouteHandler(async (request: NextRequest, context: RouteParams) => { - const requestId = generateRequestId() - const { id: knowledgeBaseId, connectorId } = await context.params - - const auth = await checkSessionOrInternalAuth(request, { requireWorkflowId: false }) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) - } - - const writeCheck = await checkKnowledgeBaseWriteAccess(knowledgeBaseId, auth.userId) - if (!writeCheck.hasAccess) { - const status = 'notFound' in writeCheck && writeCheck.notFound ? 404 : 401 - return NextResponse.json({ error: status === 404 ? 'Not found' : 'Unauthorized' }, { status }) - } - - const parsed = await parseRequest(updateKnowledgeConnectorContract, request, context) - if (!parsed.success) return parsed.response - - const outcome = await performUpdateKnowledgeConnector({ - knowledgeBase: { - id: knowledgeBaseId, - name: writeCheck.knowledgeBase.name, - workspaceId: writeCheck.knowledgeBase.workspaceId ?? null, - }, - connectorId, - updates: parsed.data.body, - validateSourceConfig: makeSourceConfigValidator( - auth.userId, - writeCheck.knowledgeBase.workspaceId ?? null, - connectorId - ), - userId: auth.userId, - actorName: auth.userName, - actorEmail: auth.userEmail, - source: 'ui', - requestId, - request, - }) - if (!outcome.success) { - return NextResponse.json( - { error: messageForOrchestrationError(outcome, 'Internal server error') }, - { status: statusForOrchestrationError(outcome.errorCode) } - ) - } - - return NextResponse.json({ success: true, data: outcome.connector }) +export const PATCH = defineInternalJsonRoute({ + contract: updateKnowledgeConnectorContract, + auth: internalKnowledgeSessionOrExecutorAuth, + operation: knowledgeOperations.updateConnector, + rateLimit: internalRateLimits.none({ + reason: 'Preserve existing internal connector-update behavior', + }), + errorPolicy: internalKnowledgeErrorPolicies.connectors, + mapInput: ({ params, body }) => ({ + connectorId: params.connectorId, + knowledgeBaseId: params.id, + updates: body, + source: 'ui' as const, + }), + useCase: updateKnowledgeConnector, + present: ({ connector }) => ({ + success: true as const, + data: toInternalKnowledgeConnector(connector), + }), }) -/** - * DELETE /api/knowledge/[id]/connectors/[connectorId] - Hard-delete a connector - */ -export const DELETE = withRouteHandler(async (request: NextRequest, context: RouteParams) => { - const requestId = generateRequestId() - const { id: knowledgeBaseId, connectorId } = await context.params - - const auth = await checkSessionOrInternalAuth(request, { requireWorkflowId: false }) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) - } - - const writeCheck = await checkKnowledgeBaseWriteAccess(knowledgeBaseId, auth.userId) - if (!writeCheck.hasAccess) { - const status = 'notFound' in writeCheck && writeCheck.notFound ? 404 : 401 - return NextResponse.json({ error: status === 404 ? 'Not found' : 'Unauthorized' }, { status }) - } - - const parsed = await parseRequest(deleteKnowledgeConnectorContract, request, context) - if (!parsed.success) return parsed.response - - const outcome = await performDeleteKnowledgeConnector({ - knowledgeBase: { - id: knowledgeBaseId, - name: writeCheck.knowledgeBase.name, - workspaceId: writeCheck.knowledgeBase.workspaceId ?? null, - }, - connectorId, - deleteDocuments: parsed.data.query.deleteDocuments, - userId: auth.userId, - actorName: auth.userName, - actorEmail: auth.userEmail, - source: 'ui', - requestId, - request, - }) - if (!outcome.success) { - return NextResponse.json( - { error: messageForOrchestrationError(outcome, 'Internal server error') }, - { status: statusForOrchestrationError(outcome.errorCode) } - ) - } - - return NextResponse.json({ success: true }) +export const DELETE = defineInternalJsonRoute({ + contract: deleteKnowledgeConnectorContract, + auth: internalKnowledgeSessionOrExecutorAuth, + operation: knowledgeOperations.deleteConnector, + rateLimit: internalRateLimits.none({ + reason: 'Preserve existing internal connector-delete behavior', + }), + errorPolicy: internalKnowledgeErrorPolicies.connectors, + mapInput: ({ params, query }) => ({ + connectorId: params.connectorId, + knowledgeBaseId: params.id, + deleteDocuments: query.deleteDocuments, + source: 'ui' as const, + }), + useCase: deleteKnowledgeConnector, + onSuccess: internalKnowledgeAnalytics.connectorRemoved, + present: () => ({ success: true as const }), }) diff --git a/apps/sim/app/api/knowledge/[id]/connectors/[connectorId]/sync/route.test.ts b/apps/sim/app/api/knowledge/[id]/connectors/[connectorId]/sync/route.test.ts deleted file mode 100644 index b8869013644..00000000000 --- a/apps/sim/app/api/knowledge/[id]/connectors/[connectorId]/sync/route.test.ts +++ /dev/null @@ -1,180 +0,0 @@ -/** - * @vitest-environment node - */ -import { - auditMock, - createMockRequest, - dbChainMockFns, - hybridAuthMockFns, - knowledgeApiUtilsMock, - knowledgeApiUtilsMockFns, - requestUtilsMockFns, - resetDbChainMock, -} from '@sim/testing' -import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest' - -const { mockDispatchSync, mockResolveBillingAttribution } = vi.hoisted(() => ({ - mockDispatchSync: vi.fn().mockResolvedValue(undefined), - mockResolveBillingAttribution: vi.fn(), -})) - -const mockCheckWriteAccess = knowledgeApiUtilsMockFns.mockCheckKnowledgeBaseWriteAccess - -vi.mock('@/app/api/knowledge/utils', () => knowledgeApiUtilsMock) -vi.mock('@/lib/billing/core/billing-attribution', () => ({ - requireBillingAttributionHeader: vi.fn(), - resolveBillingAttribution: mockResolveBillingAttribution, -})) -vi.mock('@/lib/knowledge/connectors/queue', () => ({ - dispatchSync: mockDispatchSync, -})) -vi.mock('@sim/audit', () => auditMock) - -import { POST } from '@/app/api/knowledge/[id]/connectors/[connectorId]/sync/route' - -describe('Connector Manual Sync API Route', () => { - const mockParams = Promise.resolve({ id: 'kb-123', connectorId: 'conn-456' }) - - beforeEach(() => { - vi.clearAllMocks() - resetDbChainMock() - requestUtilsMockFns.mockGenerateRequestId.mockReturnValue('test-req-id') - }) - - afterAll(() => { - resetDbChainMock() - }) - - it('returns 401 when unauthenticated', async () => { - hybridAuthMockFns.mockCheckSessionOrInternalAuth.mockResolvedValue({ - success: false, - userId: null, - }) - - const req = createMockRequest('POST') - const response = await POST(req as never, { params: mockParams }) - - expect(response.status).toBe(401) - }) - - it('returns 404 when connector not found', async () => { - hybridAuthMockFns.mockCheckSessionOrInternalAuth.mockResolvedValue({ - success: true, - userId: 'user-1', - }) - mockCheckWriteAccess.mockResolvedValue({ - hasAccess: true, - knowledgeBase: { workspaceId: 'ws-1', name: 'Test KB' }, - }) - dbChainMockFns.limit.mockResolvedValueOnce([]) - - const req = createMockRequest('POST') - const response = await POST(req as never, { params: mockParams }) - - expect(response.status).toBe(404) - }) - - it('returns 409 when connector is syncing', async () => { - hybridAuthMockFns.mockCheckSessionOrInternalAuth.mockResolvedValue({ - success: true, - userId: 'user-1', - }) - mockCheckWriteAccess.mockResolvedValue({ - hasAccess: true, - knowledgeBase: { workspaceId: 'ws-1', name: 'Test KB' }, - }) - dbChainMockFns.limit.mockResolvedValueOnce([{ id: 'conn-456', status: 'syncing' }]) - - const req = createMockRequest('POST') - const response = await POST(req as never, { params: mockParams }) - - expect(response.status).toBe(409) - }) - - it('dispatches sync on valid request', async () => { - const billingAttribution = { - actorUserId: 'external-admin', - workspaceId: 'ws-1', - organizationId: null, - billedAccountUserId: 'owner-1', - billingEntity: { type: 'user' as const, id: 'owner-1' }, - billingPeriod: { - start: '2026-07-01T00:00:00.000Z', - end: '2026-08-01T00:00:00.000Z', - }, - payerSubscription: null, - } - hybridAuthMockFns.mockCheckSessionOrInternalAuth.mockResolvedValue({ - success: true, - authType: 'session', - userId: 'external-admin', - userName: 'Test', - userEmail: 'test@test.com', - }) - mockCheckWriteAccess.mockResolvedValue({ - hasAccess: true, - knowledgeBase: { workspaceId: 'ws-1', name: 'Test KB' }, - }) - dbChainMockFns.limit.mockResolvedValueOnce([{ id: 'conn-456', status: 'active' }]) - mockResolveBillingAttribution.mockResolvedValue(billingAttribution) - - const req = createMockRequest('POST') - const response = await POST(req as never, { params: mockParams }) - const data = await response.json() - - expect(response.status).toBe(200) - expect(data.success).toBe(true) - expect(mockResolveBillingAttribution).toHaveBeenCalledWith({ - actorUserId: 'external-admin', - workspaceId: 'ws-1', - }) - expect(mockDispatchSync).toHaveBeenCalledWith('conn-456', { - billingAttribution, - requestId: 'test-req-id', - rehydrate: false, - }) - }) - - it('dispatches a full resync when rehydrate=true is set', async () => { - const billingAttribution = { - actorUserId: 'external-admin', - workspaceId: 'ws-1', - organizationId: null, - billedAccountUserId: 'owner-1', - billingEntity: { type: 'user' as const, id: 'owner-1' }, - billingPeriod: { - start: '2026-07-01T00:00:00.000Z', - end: '2026-08-01T00:00:00.000Z', - }, - payerSubscription: null, - } - hybridAuthMockFns.mockCheckSessionOrInternalAuth.mockResolvedValue({ - success: true, - authType: 'session', - userId: 'external-admin', - userName: 'Test', - userEmail: 'test@test.com', - }) - mockCheckWriteAccess.mockResolvedValue({ - hasAccess: true, - knowledgeBase: { workspaceId: 'ws-1', name: 'Test KB' }, - }) - dbChainMockFns.limit.mockResolvedValueOnce([{ id: 'conn-456', status: 'active' }]) - mockResolveBillingAttribution.mockResolvedValue(billingAttribution) - - const req = createMockRequest( - 'POST', - undefined, - {}, - 'http://localhost:3000/api/knowledge/kb-123/connectors/conn-456/sync?rehydrate=true' - ) - const response = await POST(req as never, { params: mockParams }) - - expect(response.status).toBe(200) - expect(mockDispatchSync).toHaveBeenCalledWith('conn-456', { - billingAttribution, - requestId: 'test-req-id', - rehydrate: true, - }) - }) -}) diff --git a/apps/sim/app/api/knowledge/[id]/connectors/[connectorId]/sync/route.ts b/apps/sim/app/api/knowledge/[id]/connectors/[connectorId]/sync/route.ts index 21e6bfdb50e..9862922bcad 100644 --- a/apps/sim/app/api/knowledge/[id]/connectors/[connectorId]/sync/route.ts +++ b/apps/sim/app/api/knowledge/[id]/connectors/[connectorId]/sync/route.ts @@ -1,76 +1,33 @@ -import { type NextRequest, NextResponse } from 'next/server' import { triggerKnowledgeConnectorSyncContract } from '@/lib/api/contracts/knowledge' -import { parseRequest } from '@/lib/api/server' -import { AuthType, checkSessionOrInternalAuth } from '@/lib/auth/hybrid' +import { defineInternalJsonRoute, internalRateLimits } from '@/lib/api/server/routes' import { - requireBillingAttributionHeader, - resolveBillingAttribution, -} from '@/lib/billing/core/billing-attribution' + internalKnowledgeAnalytics, + resolveInternalKnowledgeBillingAttribution, +} from '@/lib/knowledge/api/internal-route' import { - messageForOrchestrationError, - statusForOrchestrationError, -} from '@/lib/core/orchestration/types' -import { generateRequestId } from '@/lib/core/utils/request' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { performSyncKnowledgeConnector } from '@/lib/knowledge/orchestration' -import { checkKnowledgeBaseWriteAccess } from '@/app/api/knowledge/utils' + internalKnowledgeErrorPolicies, + internalKnowledgeSessionOrExecutorAuth, +} from '@/lib/knowledge/api/route-policies' +import { syncKnowledgeConnector } from '@/lib/knowledge/application/connectors' +import { knowledgeOperations } from '@/lib/knowledge/application/operations' -type RouteParams = { params: Promise<{ id: string; connectorId: string }> } - -/** - * POST /api/knowledge/[id]/connectors/[connectorId]/sync - Trigger a manual sync - */ -export const POST = withRouteHandler(async (request: NextRequest, context: RouteParams) => { - const requestId = generateRequestId() - const parsed = await parseRequest(triggerKnowledgeConnectorSyncContract, request, context) - if (!parsed.success) return parsed.response - const { id: knowledgeBaseId, connectorId } = parsed.data.params - const { rehydrate } = parsed.data.query - - const auth = await checkSessionOrInternalAuth(request, { requireWorkflowId: false }) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) - } - - const writeCheck = await checkKnowledgeBaseWriteAccess(knowledgeBaseId, auth.userId) - if (!writeCheck.hasAccess) { - const status = 'notFound' in writeCheck && writeCheck.notFound ? 404 : 401 - return NextResponse.json({ error: status === 404 ? 'Not found' : 'Unauthorized' }, { status }) - } - - const kbWorkspaceId = writeCheck.knowledgeBase.workspaceId ?? null - - const outcome = await performSyncKnowledgeConnector({ - knowledgeBase: { - id: knowledgeBaseId, - name: writeCheck.knowledgeBase.name, - workspaceId: kbWorkspaceId, - }, - connectorId, - resolveBillingAttribution: async () => - auth.authType === AuthType.INTERNAL_JWT - ? requireBillingAttributionHeader(request.headers, { - actorUserId: auth.userId as string, - workspaceId: kbWorkspaceId as string, - }) - : resolveBillingAttribution({ - actorUserId: auth.userId as string, - workspaceId: kbWorkspaceId as string, - }), - rehydrate, - userId: auth.userId, - actorName: auth.userName, - actorEmail: auth.userEmail, - source: 'ui', - requestId, - request, - }) - if (!outcome.success) { - return NextResponse.json( - { error: messageForOrchestrationError(outcome, 'Internal server error') }, - { status: statusForOrchestrationError(outcome.errorCode) } - ) - } - - return NextResponse.json({ success: true, message: 'Sync triggered' }) +export const POST = defineInternalJsonRoute({ + contract: triggerKnowledgeConnectorSyncContract, + auth: internalKnowledgeSessionOrExecutorAuth, + operation: knowledgeOperations.syncConnector, + rateLimit: internalRateLimits.none({ + reason: 'Preserve existing internal connector-sync behavior', + }), + errorPolicy: internalKnowledgeErrorPolicies.connectors, + mapInput: ({ params, query }, { principal, request }) => ({ + connectorId: params.connectorId, + knowledgeBaseId: params.id, + rehydrate: query.rehydrate, + resolveBillingAttribution: (workspaceId: string) => + resolveInternalKnowledgeBillingAttribution(request, principal, workspaceId), + source: 'ui' as const, + }), + useCase: syncKnowledgeConnector, + onSuccess: internalKnowledgeAnalytics.connectorSynced, + present: () => ({ success: true as const, message: 'Sync triggered' }), }) diff --git a/apps/sim/app/api/knowledge/[id]/connectors/route.test.ts b/apps/sim/app/api/knowledge/[id]/connectors/route.test.ts deleted file mode 100644 index 361a8e2ad68..00000000000 --- a/apps/sim/app/api/knowledge/[id]/connectors/route.test.ts +++ /dev/null @@ -1,182 +0,0 @@ -/** - * @vitest-environment node - */ -import { - auditMock, - authOAuthUtilsMock, - createMockRequest, - dbChainMockFns, - hybridAuthMockFns, - knowledgeApiUtilsMock, - knowledgeApiUtilsMockFns, - resetDbChainMock, -} from '@sim/testing' -import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest' - -const { - mockCaptureServerEvent, - mockDispatchSync, - mockEncryptApiKey, - mockHasWorkspaceLiveSyncAccess, - mockResolveBillingAttribution, - mockValidateConfig, -} = vi.hoisted(() => ({ - mockCaptureServerEvent: vi.fn(), - mockDispatchSync: vi.fn(), - mockEncryptApiKey: vi.fn(), - mockHasWorkspaceLiveSyncAccess: vi.fn(), - mockResolveBillingAttribution: vi.fn(), - mockValidateConfig: vi.fn(), -})) - -const mockCheckWriteAccess = knowledgeApiUtilsMockFns.mockCheckKnowledgeBaseWriteAccess - -vi.mock('@sim/audit', () => auditMock) -vi.mock('@/app/api/knowledge/utils', () => knowledgeApiUtilsMock) -vi.mock('@/app/api/auth/oauth/utils', () => authOAuthUtilsMock) -vi.mock('@/connectors/registry.server', () => ({ - CONNECTOR_REGISTRY: { - test: { - auth: { mode: 'apiKey' }, - validateConfig: mockValidateConfig, - }, - }, -})) -vi.mock('@/lib/api-key/crypto', () => ({ - encryptApiKey: mockEncryptApiKey, -})) -vi.mock('@/lib/billing/core/billing-attribution', () => ({ - requireBillingAttributionHeader: vi.fn(), - resolveBillingAttribution: mockResolveBillingAttribution, -})) -vi.mock('@/lib/billing/core/subscription', () => ({ - hasWorkspaceLiveSyncAccess: mockHasWorkspaceLiveSyncAccess, -})) -vi.mock('@/lib/knowledge/connectors/queue', () => ({ - dispatchSync: mockDispatchSync, -})) -vi.mock('@/lib/knowledge/tags/service', () => ({ - createTagDefinition: vi.fn(), -})) -vi.mock('@/lib/posthog/server', () => ({ - captureServerEvent: mockCaptureServerEvent, -})) - -import { POST } from '@/app/api/knowledge/[id]/connectors/route' - -const BILLING_ATTRIBUTION = { - actorUserId: 'free-external-admin', - workspaceId: 'workspace-paid', - organizationId: 'organization-paid', - billedAccountUserId: 'workspace-owner', - billingEntity: { type: 'organization' as const, id: 'organization-paid' }, - billingPeriod: { - start: '2026-07-01T00:00:00.000Z', - end: '2026-08-01T00:00:00.000Z', - }, - payerSubscription: { - id: 'subscription-paid', - referenceId: 'organization-paid', - plan: 'team_25000', - status: 'active', - seats: 5, - periodStart: '2026-07-01T00:00:00.000Z', - periodEnd: '2026-08-01T00:00:00.000Z', - }, -} - -describe('Knowledge Connectors API Route', () => { - const context = { params: Promise.resolve({ id: 'knowledge-base-1' }) } - - beforeEach(() => { - vi.clearAllMocks() - resetDbChainMock() - mockDispatchSync.mockResolvedValue(undefined) - mockEncryptApiKey.mockResolvedValue({ encrypted: 'encrypted-api-key' }) - mockValidateConfig.mockResolvedValue({ valid: true }) - }) - - afterAll(() => { - resetDbChainMock() - }) - - it('queues the authenticated actor with the paid workspace payer', async () => { - hybridAuthMockFns.mockCheckSessionOrInternalAuth.mockResolvedValue({ - success: true, - authType: 'session', - userId: 'free-external-admin', - userName: 'External Admin', - userEmail: 'external@example.com', - }) - mockCheckWriteAccess.mockResolvedValue({ - hasAccess: true, - knowledgeBase: { - id: 'knowledge-base-1', - name: 'Paid KB', - workspaceId: 'workspace-paid', - }, - }) - mockHasWorkspaceLiveSyncAccess.mockResolvedValue(true) - mockResolveBillingAttribution.mockResolvedValue(BILLING_ATTRIBUTION) - dbChainMockFns.limit.mockResolvedValueOnce([{ id: 'knowledge-base-1' }]) - dbChainMockFns.returning.mockResolvedValueOnce([ - { - id: 'connector-1', - knowledgeBaseId: 'knowledge-base-1', - connectorType: 'test', - status: 'active', - }, - ]) - - const request = createMockRequest('POST', { - connectorType: 'test', - apiKey: 'api-key', - sourceConfig: {}, - syncIntervalMinutes: 5, - }) - const response = await POST(request, context) - - expect(response.status).toBe(201) - expect(mockHasWorkspaceLiveSyncAccess).toHaveBeenCalledWith('workspace-paid') - expect(mockResolveBillingAttribution).toHaveBeenCalledWith({ - actorUserId: 'free-external-admin', - workspaceId: 'workspace-paid', - }) - expect(mockDispatchSync).toHaveBeenCalledWith(expect.any(String), { - billingAttribution: BILLING_ATTRIBUTION, - requestId: expect.any(String), - }) - }) - - it('denies a paid actor when the workspace payer lacks Max access', async () => { - hybridAuthMockFns.mockCheckSessionOrInternalAuth.mockResolvedValue({ - success: true, - authType: 'session', - userId: 'paid-external-admin', - }) - mockCheckWriteAccess.mockResolvedValue({ - hasAccess: true, - knowledgeBase: { - id: 'knowledge-base-1', - name: 'Free KB', - workspaceId: 'workspace-free', - }, - }) - mockHasWorkspaceLiveSyncAccess.mockResolvedValue(false) - - const request = createMockRequest('POST', { - connectorType: 'test', - apiKey: 'api-key', - sourceConfig: {}, - syncIntervalMinutes: 5, - }) - const response = await POST(request, context) - - expect(response.status).toBe(403) - expect(mockHasWorkspaceLiveSyncAccess).toHaveBeenCalledWith('workspace-free') - // The payer is resolved lazily, so a request the plan gate rejects never - // pays for the lookup. - expect(mockResolveBillingAttribution).not.toHaveBeenCalled() - expect(mockDispatchSync).not.toHaveBeenCalled() - }) -}) diff --git a/apps/sim/app/api/knowledge/[id]/connectors/route.ts b/apps/sim/app/api/knowledge/[id]/connectors/route.ts index df2f246ae1d..b28042f6481 100644 --- a/apps/sim/app/api/knowledge/[id]/connectors/route.ts +++ b/apps/sim/app/api/knowledge/[id]/connectors/route.ts @@ -1,147 +1,62 @@ -import { db } from '@sim/db' -import { knowledgeConnector } from '@sim/db/schema' -import { createLogger } from '@sim/logger' -import { and, desc, eq, isNull } from 'drizzle-orm' -import { type NextRequest, NextResponse } from 'next/server' -import { createKnowledgeConnectorContract } from '@/lib/api/contracts/knowledge' -import { parseRequest } from '@/lib/api/server' -import { AuthType, checkSessionOrInternalAuth } from '@/lib/auth/hybrid' import { - requireBillingAttributionHeader, - resolveBillingAttribution, -} from '@/lib/billing/core/billing-attribution' + createKnowledgeConnectorContract, + listKnowledgeConnectorsContract, +} from '@/lib/api/contracts/knowledge' +import { defineInternalJsonRoute, internalRateLimits } from '@/lib/api/server/routes' import { - messageForOrchestrationError, - OrchestrationError, - statusForOrchestrationError, -} from '@/lib/core/orchestration/types' -import { generateRequestId } from '@/lib/core/utils/request' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { performCreateKnowledgeConnector } from '@/lib/knowledge/orchestration' -import { getCredential } from '@/app/api/auth/oauth/utils' -import { checkKnowledgeBaseAccess, checkKnowledgeBaseWriteAccess } from '@/app/api/knowledge/utils' - -const logger = createLogger('KnowledgeConnectorsAPI') - -/** - * GET /api/knowledge/[id]/connectors - List connectors for a knowledge base - */ -export const GET = withRouteHandler( - async (request: NextRequest, { params }: { params: Promise<{ id: string }> }) => { - const requestId = generateRequestId() - const { id: knowledgeBaseId } = await params - - try { - const auth = await checkSessionOrInternalAuth(request, { requireWorkflowId: false }) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) - } - - const accessCheck = await checkKnowledgeBaseAccess(knowledgeBaseId, auth.userId) - if (!accessCheck.hasAccess) { - const status = 'notFound' in accessCheck && accessCheck.notFound ? 404 : 401 - return NextResponse.json( - { error: status === 404 ? 'Not found' : 'Unauthorized' }, - { status } - ) - } - - const connectors = await db - .select() - .from(knowledgeConnector) - .where( - and( - eq(knowledgeConnector.knowledgeBaseId, knowledgeBaseId), - isNull(knowledgeConnector.archivedAt), - isNull(knowledgeConnector.deletedAt) - ) - ) - .orderBy(desc(knowledgeConnector.createdAt)) - - return NextResponse.json({ - success: true, - data: connectors.map(({ encryptedApiKey: _, ...rest }) => rest), - }) - } catch (error) { - logger.error(`[${requestId}] Error listing connectors`, error) - return NextResponse.json({ error: 'Internal server error' }, { status: 500 }) - } - } -) - -/** - * POST /api/knowledge/[id]/connectors - Create a new connector - */ -export const POST = withRouteHandler( - async (request: NextRequest, context: { params: Promise<{ id: string }> }) => { - const requestId = generateRequestId() - const { id: knowledgeBaseId } = await context.params - - const auth = await checkSessionOrInternalAuth(request, { requireWorkflowId: false }) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) - } - - const writeCheck = await checkKnowledgeBaseWriteAccess(knowledgeBaseId, auth.userId) - if (!writeCheck.hasAccess) { - const status = 'notFound' in writeCheck && writeCheck.notFound ? 404 : 401 - return NextResponse.json({ error: status === 404 ? 'Not found' : 'Unauthorized' }, { status }) - } - - const parsed = await parseRequest(createKnowledgeConnectorContract, request, context) - if (!parsed.success) return parsed.response - - const { connectorType, credentialId, apiKey, sourceConfig, syncIntervalMinutes } = - parsed.data.body - - const kbWorkspaceId = writeCheck.knowledgeBase.workspaceId - if (!kbWorkspaceId) { - return NextResponse.json( - { error: 'Knowledge base is missing workspace billing context' }, - { status: 409 } - ) - } - - const outcome = await performCreateKnowledgeConnector({ - knowledgeBase: { - id: knowledgeBaseId, - name: writeCheck.knowledgeBase.name, - workspaceId: kbWorkspaceId, - }, - connectorType, - credentialId, - apiKey, - sourceConfig, - syncIntervalMinutes, - resolveBillingAttribution: async () => - auth.authType === AuthType.INTERNAL_JWT - ? requireBillingAttributionHeader(request.headers, { - actorUserId: auth.userId as string, - workspaceId: kbWorkspaceId, - }) - : resolveBillingAttribution({ - actorUserId: auth.userId as string, - workspaceId: kbWorkspaceId, - }), - resolveAccessToken: async (id) => { - const credential = await getCredential(requestId, id, auth.userId as string) - if (!credential) throw new OrchestrationError('validation', 'Credential not found') - return credential.accessToken ?? null - }, - userId: auth.userId, - actorName: auth.userName, - actorEmail: auth.userEmail, - source: 'ui', - requestId, - request, - }) - if (!outcome.success) { - return NextResponse.json( - { error: messageForOrchestrationError(outcome, 'Internal server error') }, - { status: statusForOrchestrationError(outcome.errorCode) } - ) - } - - return NextResponse.json({ success: true, data: outcome.connector }, { status: 201 }) - } -) + internalKnowledgeAnalytics, + resolveInternalKnowledgeBillingAttribution, + toInternalKnowledgeConnector, +} from '@/lib/knowledge/api/internal-route' +import { + internalKnowledgeErrorPolicies, + internalKnowledgeSessionOrExecutorAuth, +} from '@/lib/knowledge/api/route-policies' +import { + createKnowledgeConnector, + listKnowledgeConnectors, +} from '@/lib/knowledge/application/connectors' +import { knowledgeOperations } from '@/lib/knowledge/application/operations' + +export const GET = defineInternalJsonRoute({ + contract: listKnowledgeConnectorsContract, + auth: internalKnowledgeSessionOrExecutorAuth, + operation: knowledgeOperations.listConnectors, + rateLimit: internalRateLimits.none({ + reason: 'Preserve existing internal connector-list behavior', + }), + errorPolicy: internalKnowledgeErrorPolicies.connectors, + mapInput: ({ params }) => ({ knowledgeBaseId: params.id }), + useCase: listKnowledgeConnectors, + present: ({ connectors }) => ({ + success: true as const, + data: connectors.map(toInternalKnowledgeConnector), + }), +}) + +export const POST = defineInternalJsonRoute({ + contract: createKnowledgeConnectorContract, + auth: internalKnowledgeSessionOrExecutorAuth, + operation: knowledgeOperations.createConnector, + rateLimit: internalRateLimits.none({ + reason: 'Preserve existing internal connector-create behavior', + }), + errorPolicy: internalKnowledgeErrorPolicies.connectors, + mapInput: ({ params, body }, { principal, request }) => ({ + knowledgeBaseId: params.id, + connectorType: body.connectorType, + credentialId: body.credentialId, + apiKey: body.apiKey, + sourceConfig: body.sourceConfig, + syncIntervalMinutes: body.syncIntervalMinutes, + resolveBillingAttribution: (workspaceId: string) => + resolveInternalKnowledgeBillingAttribution(request, principal, workspaceId), + source: 'ui' as const, + }), + useCase: createKnowledgeConnector, + onSuccess: internalKnowledgeAnalytics.connectorAdded, + present: ({ connector }) => ({ + success: true as const, + data: toInternalKnowledgeConnector(connector), + }), +}) diff --git a/apps/sim/app/api/knowledge/[id]/documents/[documentId]/chunks/[chunkId]/route.ts b/apps/sim/app/api/knowledge/[id]/documents/[documentId]/chunks/[chunkId]/route.ts index 1e1ad1fed19..92e8ef049e2 100644 --- a/apps/sim/app/api/knowledge/[id]/documents/[documentId]/chunks/[chunkId]/route.ts +++ b/apps/sim/app/api/knowledge/[id]/documents/[documentId]/chunks/[chunkId]/route.ts @@ -1,243 +1,137 @@ -import { createLogger } from '@sim/logger' -import { generateId } from '@sim/utils/id' -import { type NextRequest, NextResponse } from 'next/server' -import { updateKnowledgeChunkContract } from '@/lib/api/contracts/knowledge' -import { parseRequest } from '@/lib/api/server' -import { getSession } from '@/lib/auth' -import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createDurableSecretProvenanceRegistry } from '@/lib/execution/durable-secret-provenance' -import { deleteChunk, updateChunk } from '@/lib/knowledge/chunks/service' -import { runWithKnowledgeModelInputProvenance } from '@/lib/knowledge/model-input-provenance' +import type { Principal } from '@sim/auth/principal' +import type { NextRequest } from 'next/server' import { - createKnowledgePersistedResponse, + deleteKnowledgeChunkContract, + getKnowledgeChunkContract, + updateKnowledgeChunkContract, +} from '@/lib/api/contracts/knowledge' +import { defineInternalJsonRoute, internalRateLimits } from '@/lib/api/server/routes' +import { OrchestrationError } from '@/lib/core/orchestration/types' +import { + internalKnowledgeActorUserId, + internalKnowledgeAuthType, + toInternalKnowledgeChunk, +} from '@/lib/knowledge/api/internal-route' +import { + internalKnowledgeErrorPolicies, + internalKnowledgeSessionOrExecutorAuth, +} from '@/lib/knowledge/api/route-policies' +import { + deleteKnowledgeChunk, + readKnowledgeChunk, + updateKnowledgeChunk, +} from '@/lib/knowledge/application/chunks' +import { knowledgeOperations } from '@/lib/knowledge/application/operations' +import { + finalizeKnowledgePersistedResponse, resolveKnowledgeWriteSecretProvenance, } from '@/app/api/knowledge/secret-provenance' -import { checkChunkAccess, checkChunkWriteAccess } from '@/app/api/knowledge/utils' - -const logger = createLogger('ChunkByIdAPI') - -export const GET = withRouteHandler( - async ( - req: NextRequest, - { params }: { params: Promise<{ id: string; documentId: string; chunkId: string }> } - ) => { - const requestId = generateId().slice(0, 8) - const { id: knowledgeBaseId, documentId, chunkId } = await params - - try { - const auth = await checkSessionOrInternalAuth(req, { requireWorkflowId: false }) - if (!auth.success || !auth.userId) { - logger.warn(`[${requestId}] Unauthorized chunk access attempt`) - return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) - } - const userId = auth.userId - - const accessCheck = await checkChunkAccess(knowledgeBaseId, documentId, chunkId, userId) - - if (!accessCheck.hasAccess) { - if (accessCheck.notFound) { - logger.warn( - `[${requestId}] ${accessCheck.reason}: KB=${knowledgeBaseId}, Doc=${documentId}, Chunk=${chunkId}` - ) - return NextResponse.json({ error: accessCheck.reason }, { status: 404 }) - } - logger.warn( - `[${requestId}] User ${userId} attempted unauthorized chunk access: ${accessCheck.reason}` - ) - return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) - } - - logger.info( - `[${requestId}] Retrieved chunk: ${chunkId} from document ${documentId} in knowledge base ${knowledgeBaseId}` - ) - - const responseBody = { - success: true, - data: accessCheck.chunk, - } - const workspaceId = accessCheck.knowledgeBase?.workspaceId ?? undefined - return createKnowledgePersistedResponse({ - request: req, - authType: auth.authType, - userId, - ...(workspaceId ? { workspaceId } : {}), - body: responseBody, - chunks: accessCheck.chunk - ? [ - { - id: accessCheck.chunk.id, - documentId, - content: accessCheck.chunk.content, - value: accessCheck.chunk, - }, - ] - : [], - }) - } catch (error) { - logger.error(`[${requestId}] Error fetching chunk`, error) - return NextResponse.json({ error: 'Failed to fetch chunk' }, { status: 500 }) - } - } -) - -export const PUT = withRouteHandler( - async ( - req: NextRequest, - context: { params: Promise<{ id: string; documentId: string; chunkId: string }> } - ) => { - const requestId = generateId().slice(0, 8) - const { id: knowledgeBaseId, documentId, chunkId } = await context.params - - try { - const auth = await checkSessionOrInternalAuth(req, { requireWorkflowId: false }) - if (!auth.success || !auth.userId) { - logger.warn(`[${requestId}] Unauthorized chunk update attempt`) - return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) - } - const userId = auth.userId - - const accessCheck = await checkChunkWriteAccess(knowledgeBaseId, documentId, chunkId, userId) - - if (!accessCheck.hasAccess) { - if (accessCheck.notFound) { - logger.warn( - `[${requestId}] ${accessCheck.reason}: KB=${knowledgeBaseId}, Doc=${documentId}, Chunk=${chunkId}` - ) - return NextResponse.json({ error: accessCheck.reason }, { status: 404 }) - } - logger.warn( - `[${requestId}] User ${userId} attempted unauthorized chunk update: ${accessCheck.reason}` - ) - return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) - } - - if (accessCheck.document?.connectorId) { - logger.warn( - `[${requestId}] User ${userId} attempted to update chunk on connector-synced document: Doc=${documentId}` - ) - return NextResponse.json( - { error: 'Chunks from connector-synced documents are read-only' }, - { status: 403 } - ) - } - - const parsed = await parseRequest(updateKnowledgeChunkContract, req, context) - if (!parsed.success) return parsed.response - - const validatedData = parsed.data.body - const workspaceId = accessCheck.knowledgeBase?.workspaceId ?? undefined - const writeProvenance = resolveKnowledgeWriteSecretProvenance({ - request: req, - payload: validatedData, - authType: auth.authType, - userId, - ...(workspaceId ? { workspaceId } : {}), - selectionKeys: validatedData.content === undefined ? [] : ['chunk-content'], - }) - if (!writeProvenance.success) return writeProvenance.response - const chunkProvenance = writeProvenance.provenances?.[0] - if (chunkProvenance?.status === 'unknown') { - return NextResponse.json( - { error: 'Knowledge chunk secret provenance is unavailable' }, - { status: 400 } - ) - } - const registry = chunkProvenance - ? await createDurableSecretProvenanceRegistry(chunkProvenance, { - userId, - ...(workspaceId ? { workspaceId } : {}), - }) - : undefined - - const updatedChunk = await runWithKnowledgeModelInputProvenance(registry, () => - updateChunk(chunkId, validatedData, requestId, workspaceId, chunkProvenance) - ) - - logger.info( - `[${requestId}] Chunk updated: ${chunkId} in document ${documentId} in knowledge base ${knowledgeBaseId}` - ) - - return createKnowledgePersistedResponse({ - request: req, - authType: auth.authType, - userId, - ...(workspaceId ? { workspaceId } : {}), - body: { success: true, data: updatedChunk }, - chunks: [ - { - id: updatedChunk.id, - documentId, - content: updatedChunk.content, - value: updatedChunk, - }, - ], - }) - } catch (error) { - logger.error(`[${requestId}] Error updating chunk`, error) - return NextResponse.json({ error: 'Failed to update chunk' }, { status: 500 }) - } - } -) - -export const DELETE = withRouteHandler( - async ( - req: NextRequest, - { params }: { params: Promise<{ id: string; documentId: string; chunkId: string }> } - ) => { - const requestId = generateId().slice(0, 8) - const { id: knowledgeBaseId, documentId, chunkId } = await params - - try { - const session = await getSession() - if (!session?.user?.id) { - logger.warn(`[${requestId}] Unauthorized chunk delete attempt`) - return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) - } - - const accessCheck = await checkChunkWriteAccess( - knowledgeBaseId, - documentId, - chunkId, - session.user.id - ) - - if (!accessCheck.hasAccess) { - if (accessCheck.notFound) { - logger.warn( - `[${requestId}] ${accessCheck.reason}: KB=${knowledgeBaseId}, Doc=${documentId}, Chunk=${chunkId}` - ) - return NextResponse.json({ error: accessCheck.reason }, { status: 404 }) - } - logger.warn( - `[${requestId}] User ${session.user.id} attempted unauthorized chunk deletion: ${accessCheck.reason}` - ) - return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) - } - - if (accessCheck.document?.connectorId) { - logger.warn( - `[${requestId}] User ${session.user.id} attempted to delete chunk on connector-synced document: Doc=${documentId}` - ) - return NextResponse.json( - { error: 'Chunks from connector-synced documents are read-only' }, - { status: 403 } - ) - } - - await deleteChunk(chunkId, documentId, requestId) - - logger.info( - `[${requestId}] Chunk deleted: ${chunkId} from document ${documentId} in knowledge base ${knowledgeBaseId}` - ) - return NextResponse.json({ - success: true, - data: { message: 'Chunk deleted successfully' }, - }) - } catch (error) { - logger.error(`[${requestId}] Error deleting chunk`, error) - return NextResponse.json({ error: 'Failed to delete chunk' }, { status: 500 }) - } +function resolveContentProvenance( + request: NextRequest, + principal: Principal, + payload: unknown, + workspaceId: string, + includeContent: boolean +) { + const resolved = resolveKnowledgeWriteSecretProvenance({ + request, + payload, + authType: internalKnowledgeAuthType(principal), + userId: internalKnowledgeActorUserId(principal), + workspaceId, + selectionKeys: includeContent ? ['chunk-content'] : [], + }) + if (!resolved.success) { + throw new OrchestrationError('validation', 'Invalid knowledge secret provenance') } -) + return resolved.provenances?.[0] +} + +export const GET = defineInternalJsonRoute({ + contract: getKnowledgeChunkContract, + auth: internalKnowledgeSessionOrExecutorAuth, + operation: knowledgeOperations.readChunk, + rateLimit: internalRateLimits.none({ reason: 'Preserve existing internal chunk-read behavior' }), + errorPolicy: internalKnowledgeErrorPolicies.chunks, + mapInput: ({ params }) => ({ + knowledgeBaseId: params.id, + documentId: params.documentId, + chunkId: params.chunkId, + }), + useCase: readKnowledgeChunk, + present: ({ chunk }) => ({ success: true as const, data: toInternalKnowledgeChunk(chunk) }), + finalizeResponse: ({ request, principal, result, body }) => + finalizeKnowledgePersistedResponse({ + request, + authType: internalKnowledgeAuthType(principal), + userId: internalKnowledgeActorUserId(principal), + workspaceId: result.workspaceId, + body, + chunks: [ + { + id: result.chunk.id, + documentId: result.documentId, + content: result.chunk.content, + value: result.chunk, + }, + ], + }), +}) + +export const PUT = defineInternalJsonRoute({ + contract: updateKnowledgeChunkContract, + auth: internalKnowledgeSessionOrExecutorAuth, + operation: knowledgeOperations.updateChunk, + rateLimit: internalRateLimits.none({ + reason: 'Preserve existing internal chunk-update behavior', + }), + errorPolicy: internalKnowledgeErrorPolicies.chunks, + mapInput: ({ params, body }, { principal, request }) => ({ + knowledgeBaseId: params.id, + documentId: params.documentId, + chunkId: params.chunkId, + content: body.content, + enabled: body.enabled, + resolveContentProvenance: ({ workspaceId }: { workspaceId: string }) => + resolveContentProvenance(request, principal, body, workspaceId, body.content !== undefined), + }), + useCase: updateKnowledgeChunk, + present: ({ chunk }) => ({ success: true as const, data: toInternalKnowledgeChunk(chunk) }), + finalizeResponse: ({ request, principal, result, body }) => + finalizeKnowledgePersistedResponse({ + request, + authType: internalKnowledgeAuthType(principal), + userId: internalKnowledgeActorUserId(principal), + workspaceId: result.workspaceId, + body, + chunks: [ + { + id: result.chunk.id, + documentId: result.documentId, + content: result.chunk.content, + value: result.chunk, + }, + ], + }), +}) + +export const DELETE = defineInternalJsonRoute({ + contract: deleteKnowledgeChunkContract, + auth: internalKnowledgeSessionOrExecutorAuth, + operation: knowledgeOperations.deleteChunk, + rateLimit: internalRateLimits.none({ + reason: 'Preserve existing internal chunk-delete behavior', + }), + errorPolicy: internalKnowledgeErrorPolicies.chunks, + mapInput: ({ params }) => ({ + knowledgeBaseId: params.id, + documentId: params.documentId, + chunkId: params.chunkId, + }), + useCase: deleteKnowledgeChunk, + present: () => ({ + success: true as const, + data: { message: 'Chunk deleted successfully' }, + }), +}) diff --git a/apps/sim/app/api/knowledge/[id]/documents/[documentId]/chunks/route.ts b/apps/sim/app/api/knowledge/[id]/documents/[documentId]/chunks/route.ts index bb6e3249851..44afacfe847 100644 --- a/apps/sim/app/api/knowledge/[id]/documents/[documentId]/chunks/route.ts +++ b/apps/sim/app/api/knowledge/[id]/documents/[documentId]/chunks/route.ts @@ -1,388 +1,127 @@ -import { createLogger } from '@sim/logger' -import { authorizeWorkflowByWorkspacePermission } from '@sim/platform-authz/workflow' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' +import type { Principal } from '@sim/auth/principal' +import type { NextRequest } from 'next/server' import { bulkKnowledgeChunksContract, - createChunkBodySchema, - listKnowledgeChunksQuerySchema, + createKnowledgeChunkContract, + listKnowledgeChunksContract, } from '@/lib/api/contracts/knowledge' -import { isZodError, parseJsonBody, parseRequest } from '@/lib/api/server' -import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid' -import { generateRequestId } from '@/lib/core/utils/request' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createDurableSecretProvenanceRegistry } from '@/lib/execution/durable-secret-provenance' -import { batchChunkOperation, createChunk, queryChunks } from '@/lib/knowledge/chunks/service' -import { runWithKnowledgeModelInputProvenance } from '@/lib/knowledge/model-input-provenance' +import { defineInternalJsonRoute, internalRateLimits } from '@/lib/api/server/routes' +import { OrchestrationError } from '@/lib/core/orchestration/types' import { - createKnowledgePersistedResponse, - createKnowledgeProvenanceResponse, + internalKnowledgeActorUserId, + internalKnowledgeAuthType, + toInternalKnowledgeChunk, +} from '@/lib/knowledge/api/internal-route' +import { + internalKnowledgeErrorPolicies, + internalKnowledgeSessionOrExecutorAuth, +} from '@/lib/knowledge/api/route-policies' +import { + bulkUpdateKnowledgeChunks, + createKnowledgeChunk, + listKnowledgeChunks, +} from '@/lib/knowledge/application/chunks' +import { knowledgeOperations } from '@/lib/knowledge/application/operations' +import { + finalizeKnowledgePersistedResponse, + finalizeKnowledgeProvenanceResponse, resolveKnowledgeWriteSecretProvenance, } from '@/app/api/knowledge/secret-provenance' -import { checkDocumentAccess, checkDocumentWriteAccess } from '@/app/api/knowledge/utils' -import { calculateCost } from '@/providers/utils' - -const logger = createLogger('DocumentChunksAPI') - -export const GET = withRouteHandler( - async (req: NextRequest, { params }: { params: Promise<{ id: string; documentId: string }> }) => { - const requestId = generateRequestId() - const { id: knowledgeBaseId, documentId } = await params - - try { - const auth = await checkSessionOrInternalAuth(req, { requireWorkflowId: false }) - if (!auth.success || !auth.userId) { - logger.warn(`[${requestId}] Unauthorized chunks access attempt`) - return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) - } - const userId = auth.userId - - const accessCheck = await checkDocumentAccess(knowledgeBaseId, documentId, userId) - - if (!accessCheck.hasAccess) { - if (accessCheck.notFound) { - logger.warn( - `[${requestId}] ${accessCheck.reason}: KB=${knowledgeBaseId}, Doc=${documentId}` - ) - return NextResponse.json({ error: accessCheck.reason }, { status: 404 }) - } - logger.warn( - `[${requestId}] User ${userId} attempted unauthorized chunks access: ${accessCheck.reason}` - ) - return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) - } - - const doc = accessCheck.document - if (!doc) { - logger.warn( - `[${requestId}] Document data not available: KB=${knowledgeBaseId}, Doc=${documentId}` - ) - return NextResponse.json({ error: 'Document not found' }, { status: 404 }) - } - - if (doc.processingStatus !== 'completed') { - logger.warn( - `[${requestId}] Document ${documentId} is not ready for chunk access (status: ${doc.processingStatus})` - ) - return NextResponse.json( - { - error: 'Document is not ready for access', - details: `Document status: ${doc.processingStatus}`, - retryAfter: doc.processingStatus === 'processing' ? 5 : null, - }, - { status: 400 } - ) - } - - const { searchParams } = new URL(req.url) - const queryResult = listKnowledgeChunksQuerySchema.safeParse({ - search: searchParams.get('search') || undefined, - enabled: searchParams.get('enabled') || undefined, - limit: searchParams.get('limit') || undefined, - offset: searchParams.get('offset') || undefined, - sortBy: searchParams.get('sortBy') || undefined, - sortOrder: searchParams.get('sortOrder') || undefined, - }) - if (!queryResult.success) { - return NextResponse.json( - { error: 'Invalid query parameters', details: queryResult.error.issues }, - { status: 400 } - ) - } - - const result = await queryChunks(documentId, queryResult.data, requestId) - - const responseBody = { - success: true, - data: result.chunks, - pagination: result.pagination, - } - const workspaceId = accessCheck.knowledgeBase?.workspaceId ?? undefined - return createKnowledgePersistedResponse({ - request: req, - authType: auth.authType, - userId, - ...(workspaceId ? { workspaceId } : {}), - body: responseBody, - chunks: result.chunks.map((item) => ({ - id: item.id, - documentId, - content: item.content, - value: item, - })), - }) - } catch (error) { - logger.error(`[${requestId}] Error fetching chunks`, error) - return NextResponse.json({ error: 'Failed to fetch chunks' }, { status: 500 }) - } - } -) - -export const POST = withRouteHandler( - async (req: NextRequest, { params }: { params: Promise<{ id: string; documentId: string }> }) => { - const requestId = generateRequestId() - const { id: knowledgeBaseId, documentId } = await params - - try { - const auth = await checkSessionOrInternalAuth(req, { requireWorkflowId: false }) - if (!auth.success || !auth.userId) { - logger.warn(`[${requestId}] Authentication failed: ${auth.error || 'Unauthorized'}`) - return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) - } - const userId = auth.userId - - const parsedBody = await parseJsonBody(req) - if (!parsedBody.success) return parsedBody.response - const { workflowId, ...searchParams } = parsedBody.data as Record - - if (workflowId) { - if (typeof workflowId !== 'string') { - return NextResponse.json({ error: 'workflowId must be a string' }, { status: 400 }) - } - const authorization = await authorizeWorkflowByWorkspacePermission({ - workflowId, - userId, - action: 'write', - }) - if (!authorization.allowed) { - return NextResponse.json( - { error: authorization.message || 'Access denied' }, - { status: authorization.status } - ) - } - } - - const accessCheck = await checkDocumentWriteAccess(knowledgeBaseId, documentId, userId) - - if (!accessCheck.hasAccess) { - if (accessCheck.notFound) { - logger.warn( - `[${requestId}] ${accessCheck.reason}: KB=${knowledgeBaseId}, Doc=${documentId}` - ) - return NextResponse.json({ error: accessCheck.reason }, { status: 404 }) - } - logger.warn( - `[${requestId}] User ${userId} attempted unauthorized chunk creation: ${accessCheck.reason}` - ) - return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) - } - - const doc = accessCheck.document - if (!doc) { - logger.warn( - `[${requestId}] Document data not available: KB=${knowledgeBaseId}, Doc=${documentId}` - ) - return NextResponse.json({ error: 'Document not found' }, { status: 404 }) - } - - if (doc.connectorId) { - logger.warn( - `[${requestId}] User ${userId} attempted to create chunk on connector-synced document: Doc=${documentId}` - ) - return NextResponse.json( - { error: 'Chunks from connector-synced documents are read-only' }, - { status: 403 } - ) - } - - if (doc.processingStatus === 'failed') { - logger.warn(`[${requestId}] Document ${documentId} is in failed state, cannot add chunks`) - return NextResponse.json({ error: 'Cannot add chunks to failed document' }, { status: 400 }) - } - - try { - const validatedData = createChunkBodySchema.parse(searchParams) - const workspaceId = accessCheck.knowledgeBase?.workspaceId ?? undefined - const writeProvenance = resolveKnowledgeWriteSecretProvenance({ - request: req, - payload: parsedBody.data, - authType: auth.authType, - userId, - ...(workspaceId ? { workspaceId } : {}), - selectionKeys: ['chunk-content'], - }) - if (!writeProvenance.success) return writeProvenance.response - const chunkProvenance = writeProvenance.provenances?.[0] - if (chunkProvenance?.status === 'unknown') { - return NextResponse.json( - { error: 'Knowledge chunk secret provenance is unavailable' }, - { status: 400 } - ) - } - const registry = chunkProvenance - ? await createDurableSecretProvenanceRegistry(chunkProvenance, { - userId, - ...(workspaceId ? { workspaceId } : {}), - }) - : undefined - - const docTags = { - tag1: doc.tag1 ?? null, - tag2: doc.tag2 ?? null, - tag3: doc.tag3 ?? null, - tag4: doc.tag4 ?? null, - tag5: doc.tag5 ?? null, - tag6: doc.tag6 ?? null, - tag7: doc.tag7 ?? null, - number1: doc.number1 ?? null, - number2: doc.number2 ?? null, - number3: doc.number3 ?? null, - number4: doc.number4 ?? null, - number5: doc.number5 ?? null, - date1: doc.date1 ?? null, - date2: doc.date2 ?? null, - boolean1: doc.boolean1 ?? null, - boolean2: doc.boolean2 ?? null, - boolean3: doc.boolean3 ?? null, - } - - const newChunk = await runWithKnowledgeModelInputProvenance(registry, () => - createChunk( - knowledgeBaseId, - documentId, - docTags, - validatedData, - requestId, - workspaceId, - chunkProvenance - ) - ) - - let cost = null - try { - cost = calculateCost( - accessCheck.knowledgeBase.embeddingModel, - newChunk.tokenCount, - 0, - false - ) - } catch (error) { - logger.warn(`[${requestId}] Failed to calculate cost for chunk upload`, { - error: getErrorMessage(error, 'Unknown error'), - }) - } - - return createKnowledgeProvenanceResponse({ - request: req, - authType: auth.authType, - userId, - ...(workspaceId ? { workspaceId } : {}), - provenances: chunkProvenance ? [chunkProvenance] : [], - body: { - success: true, - data: { - ...newChunk, - documentId, - documentName: doc.filename, - ...(cost - ? { - cost: { - input: cost.input, - output: cost.output, - total: cost.total, - tokens: { - prompt: newChunk.tokenCount, - completion: 0, - total: newChunk.tokenCount, - }, - model: accessCheck.knowledgeBase.embeddingModel, - pricing: cost.pricing, - }, - } - : {}), - }, - }, - }) - } catch (validationError) { - if (isZodError(validationError)) { - logger.warn(`[${requestId}] Invalid chunk creation data`, { - errors: validationError.issues, - }) - return NextResponse.json( - { error: 'Invalid request data', details: validationError.issues }, - { status: 400 } - ) - } - throw validationError - } - } catch (error) { - logger.error(`[${requestId}] Error creating chunk`, error) - return NextResponse.json({ error: 'Failed to create chunk' }, { status: 500 }) - } - } -) - -export const PATCH = withRouteHandler( - async (req: NextRequest, { params }: { params: Promise<{ id: string; documentId: string }> }) => { - const requestId = generateRequestId() - const { id: knowledgeBaseId, documentId } = await params - - try { - const auth = await checkSessionOrInternalAuth(req, { requireWorkflowId: false }) - if (!auth.success || !auth.userId) { - logger.warn(`[${requestId}] Unauthorized batch chunk operation attempt`) - return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) - } - const userId = auth.userId - - const accessCheck = await checkDocumentWriteAccess(knowledgeBaseId, documentId, userId) - - if (!accessCheck.hasAccess) { - if (accessCheck.notFound) { - logger.warn( - `[${requestId}] ${accessCheck.reason}: KB=${knowledgeBaseId}, Doc=${documentId}` - ) - return NextResponse.json({ error: accessCheck.reason }, { status: 404 }) - } - logger.warn( - `[${requestId}] User ${userId} attempted unauthorized batch chunk operation: ${accessCheck.reason}` - ) - return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) - } - - if (accessCheck.document?.connectorId) { - logger.warn( - `[${requestId}] User ${userId} attempted batch chunk operation on connector-synced document: Doc=${documentId}` - ) - return NextResponse.json( - { error: 'Chunks from connector-synced documents are read-only' }, - { status: 403 } - ) - } - - const parsed = await parseRequest( - bulkKnowledgeChunksContract, - req, - { params }, - { - validationErrorResponse: (error) => { - logger.warn(`[${requestId}] Invalid batch operation data`, { errors: error.issues }) - return NextResponse.json( - { error: 'Invalid request data', details: error.issues }, - { status: 400 } - ) - }, - } - ) - if (!parsed.success) return parsed.response - const validatedData = parsed.data.body - const { operation, chunkIds } = validatedData - - const result = await batchChunkOperation(documentId, operation, chunkIds, requestId) - return NextResponse.json({ - success: true, - data: { - operation, - successCount: result.processed, - errorCount: result.errors.length, - processed: result.processed, - errors: result.errors, - }, - }) - } catch (error) { - logger.error(`[${requestId}] Error in batch chunk operation`, error) - return NextResponse.json({ error: 'Failed to perform batch operation' }, { status: 500 }) - } +function resolveContentProvenance( + request: NextRequest, + principal: Principal, + payload: unknown, + workspaceId: string, + includeContent: boolean +) { + const resolved = resolveKnowledgeWriteSecretProvenance({ + request, + payload, + authType: internalKnowledgeAuthType(principal), + userId: internalKnowledgeActorUserId(principal), + workspaceId, + selectionKeys: includeContent ? ['chunk-content'] : [], + }) + if (!resolved.success) { + throw new OrchestrationError('validation', 'Invalid knowledge secret provenance') } -) + return resolved.provenances?.[0] +} + +export const GET = defineInternalJsonRoute({ + contract: listKnowledgeChunksContract, + auth: internalKnowledgeSessionOrExecutorAuth, + operation: knowledgeOperations.listChunks, + rateLimit: internalRateLimits.none({ reason: 'Preserve existing internal chunk-list behavior' }), + errorPolicy: internalKnowledgeErrorPolicies.chunks, + mapInput: ({ params, query }) => ({ + knowledgeBaseId: params.id, + documentId: params.documentId, + ...query, + }), + useCase: listKnowledgeChunks, + present: ({ chunks, pagination }) => ({ + success: true as const, + data: chunks.map(toInternalKnowledgeChunk), + pagination, + }), + finalizeResponse: ({ request, principal, result, body }) => + finalizeKnowledgePersistedResponse({ + request, + authType: internalKnowledgeAuthType(principal), + userId: internalKnowledgeActorUserId(principal), + workspaceId: result.workspaceId, + body, + chunks: result.chunks.map((chunk) => ({ + id: chunk.id, + documentId: result.documentId, + content: chunk.content, + value: chunk, + })), + }), +}) + +export const POST = defineInternalJsonRoute({ + contract: createKnowledgeChunkContract, + auth: internalKnowledgeSessionOrExecutorAuth, + operation: knowledgeOperations.createChunk, + rateLimit: internalRateLimits.none({ + reason: 'Preserve existing internal chunk-create behavior', + }), + errorPolicy: internalKnowledgeErrorPolicies.chunks, + mapInput: ({ params, body }, { principal, request }) => ({ + knowledgeBaseId: params.id, + documentId: params.documentId, + content: body.content, + enabled: body.enabled, + resolveContentProvenance: ({ workspaceId }: { workspaceId: string }) => + resolveContentProvenance(request, principal, body, workspaceId, true), + }), + useCase: createKnowledgeChunk, + present: ({ chunk }) => ({ success: true as const, data: toInternalKnowledgeChunk(chunk) }), + finalizeResponse: ({ request, principal, result, body }) => + finalizeKnowledgeProvenanceResponse({ + request, + authType: internalKnowledgeAuthType(principal), + userId: result.userId, + workspaceId: result.workspaceId, + body, + provenances: result.provenance ? [result.provenance] : [], + }), +}) + +export const PATCH = defineInternalJsonRoute({ + contract: bulkKnowledgeChunksContract, + auth: internalKnowledgeSessionOrExecutorAuth, + operation: knowledgeOperations.bulkChunks, + rateLimit: internalRateLimits.none({ reason: 'Preserve existing internal bulk-chunk behavior' }), + errorPolicy: internalKnowledgeErrorPolicies.chunks, + mapInput: ({ params, body }) => ({ + knowledgeBaseId: params.id, + documentId: params.documentId, + ...body, + }), + useCase: bulkUpdateKnowledgeChunks, + present: (data) => ({ success: true as const, data }), +}) diff --git a/apps/sim/app/api/knowledge/[id]/documents/[documentId]/route.test.ts b/apps/sim/app/api/knowledge/[id]/documents/[documentId]/route.test.ts deleted file mode 100644 index 84bdd9736e5..00000000000 --- a/apps/sim/app/api/knowledge/[id]/documents/[documentId]/route.test.ts +++ /dev/null @@ -1,542 +0,0 @@ -/** - * Tests for document by ID API route - * - * @vitest-environment node - */ -import { - auditMock, - authMockFns, - createMockRequest, - knowledgeApiUtilsMock, - resetDbChainMock, -} from '@sim/testing' -import { afterAll, afterEach, beforeEach, describe, expect, it, vi } from 'vitest' - -vi.mock('@/app/api/knowledge/utils', () => knowledgeApiUtilsMock) - -vi.mock('@/lib/knowledge/documents/service', () => ({ - updateDocument: vi.fn(), - deleteDocument: vi.fn(), - markDocumentAsFailedTimeout: vi.fn(), - retryDocumentProcessing: vi.fn(), - processDocumentAsync: vi.fn(), -})) - -vi.mock('@sim/audit', () => auditMock) - -import { - deleteDocument, - markDocumentAsFailedTimeout, - retryDocumentProcessing, - updateDocument, -} from '@/lib/knowledge/documents/service' -import { DELETE, GET, PUT } from '@/app/api/knowledge/[id]/documents/[documentId]/route' -import { checkDocumentAccess, checkDocumentWriteAccess } from '@/app/api/knowledge/utils' - -describe('Document By ID API Route', () => { - const mockDocument = { - id: 'doc-123', - knowledgeBaseId: 'kb-123', - filename: 'test-document.pdf', - fileUrl: 'https://example.com/test-document.pdf', - fileSize: 1024, - mimeType: 'application/pdf', - chunkCount: 5, - tokenCount: 100, - characterCount: 500, - processingStatus: 'completed' as const, - processingStartedAt: new Date('2023-01-01T10:00:00Z'), - processingCompletedAt: new Date('2023-01-01T10:05:00Z'), - processingError: null, - enabled: true, - uploadedAt: new Date('2023-01-01T09:00:00Z'), - tag1: null, - tag2: null, - tag3: null, - tag4: null, - tag5: null, - tag6: null, - tag7: null, - number1: null, - number2: null, - number3: null, - number4: null, - number5: null, - date1: null, - date2: null, - boolean1: null, - boolean2: null, - boolean3: null, - deletedAt: null, - } - - beforeEach(() => { - vi.clearAllMocks() - resetDbChainMock() - - vi.stubGlobal('crypto', { - randomUUID: vi.fn().mockReturnValue('mock-uuid-1234-5678'), - }) - }) - - afterEach(() => { - vi.clearAllMocks() - }) - - afterAll(() => { - resetDbChainMock() - }) - - describe('GET /api/knowledge/[id]/documents/[documentId]', () => { - const mockParams = Promise.resolve({ id: 'kb-123', documentId: 'doc-123' }) - - it('should retrieve document successfully for authenticated user', async () => { - authMockFns.mockGetSession.mockResolvedValue({ - user: { id: 'user-123', email: 'test@example.com' }, - }) - vi.mocked(checkDocumentAccess).mockResolvedValue({ - hasAccess: true, - document: mockDocument, - knowledgeBase: { id: 'kb-123', userId: 'user-123' }, - }) - - const req = createMockRequest('GET') - const response = await GET(req, { params: mockParams }) - const data = await response.json() - - expect(response.status).toBe(200) - expect(data.success).toBe(true) - expect(data.data.id).toBe('doc-123') - expect(data.data.filename).toBe('test-document.pdf') - expect(vi.mocked(checkDocumentAccess)).toHaveBeenCalledWith('kb-123', 'doc-123', 'user-123') - }) - - it('should return unauthorized for unauthenticated user', async () => { - authMockFns.mockGetSession.mockResolvedValue(null) - - const req = createMockRequest('GET') - const response = await GET(req, { params: mockParams }) - const data = await response.json() - - expect(response.status).toBe(401) - expect(data.error).toBe('Unauthorized') - }) - - it('should return not found for non-existent document', async () => { - authMockFns.mockGetSession.mockResolvedValue({ - user: { id: 'user-123', email: 'test@example.com' }, - }) - vi.mocked(checkDocumentAccess).mockResolvedValue({ - hasAccess: false, - notFound: true, - reason: 'Document not found', - }) - - const req = createMockRequest('GET') - const response = await GET(req, { params: mockParams }) - const data = await response.json() - - expect(response.status).toBe(404) - expect(data.error).toBe('Document not found') - }) - - it('should return unauthorized for document without access', async () => { - authMockFns.mockGetSession.mockResolvedValue({ - user: { id: 'user-123', email: 'test@example.com' }, - }) - vi.mocked(checkDocumentAccess).mockResolvedValue({ - hasAccess: false, - reason: 'Access denied', - }) - - const req = createMockRequest('GET') - const response = await GET(req, { params: mockParams }) - const data = await response.json() - - expect(response.status).toBe(401) - expect(data.error).toBe('Unauthorized') - }) - - it('should handle database errors', async () => { - authMockFns.mockGetSession.mockResolvedValue({ - user: { id: 'user-123', email: 'test@example.com' }, - }) - vi.mocked(checkDocumentAccess).mockRejectedValue(new Error('Database error')) - - const req = createMockRequest('GET') - const response = await GET(req, { params: mockParams }) - const data = await response.json() - - expect(response.status).toBe(500) - expect(data.error).toBe('Failed to fetch document') - }) - }) - - describe('PUT /api/knowledge/[id]/documents/[documentId] - Regular Updates', () => { - const mockParams = Promise.resolve({ id: 'kb-123', documentId: 'doc-123' }) - const validUpdateData = { - filename: 'updated-document.pdf', - enabled: false, - chunkCount: 10, - tokenCount: 200, - } - - it('should update document successfully', async () => { - authMockFns.mockGetSession.mockResolvedValue({ - user: { id: 'user-123', email: 'test@example.com' }, - }) - vi.mocked(checkDocumentWriteAccess).mockResolvedValue({ - hasAccess: true, - document: mockDocument, - knowledgeBase: { id: 'kb-123', userId: 'user-123' }, - }) - - const updatedDocument = { - ...mockDocument, - ...validUpdateData, - deletedAt: null, - } - vi.mocked(updateDocument).mockResolvedValue(updatedDocument) - - const req = createMockRequest('PUT', validUpdateData) - const response = await PUT(req, { params: mockParams }) - const data = await response.json() - - expect(response.status).toBe(200) - expect(data.success).toBe(true) - expect(data.data.filename).toBe('updated-document.pdf') - expect(data.data.enabled).toBe(false) - expect(vi.mocked(updateDocument)).toHaveBeenCalledWith( - 'doc-123', - validUpdateData, - expect.any(String) - ) - }) - - it('should validate update data', async () => { - authMockFns.mockGetSession.mockResolvedValue({ - user: { id: 'user-123', email: 'test@example.com' }, - }) - vi.mocked(checkDocumentWriteAccess).mockResolvedValue({ - hasAccess: true, - document: mockDocument, - knowledgeBase: { id: 'kb-123', userId: 'user-123' }, - }) - - const invalidData = { - filename: '', // Invalid: empty filename - chunkCount: -1, // Invalid: negative count - processingStatus: 'invalid', // Invalid: not in enum - } - - const req = createMockRequest('PUT', invalidData) - const response = await PUT(req, { params: mockParams }) - const data = await response.json() - - expect(response.status).toBe(400) - expect(data.error).toBe('Invalid request data') - expect(data.details).toBeDefined() - }) - }) - - describe('PUT /api/knowledge/[id]/documents/[documentId] - Mark Failed Due to Timeout', () => { - const mockParams = Promise.resolve({ id: 'kb-123', documentId: 'doc-123' }) - - it('should mark document as failed due to timeout successfully', async () => { - const processingDocument = { - ...mockDocument, - processingStatus: 'processing', - processingStartedAt: new Date(Date.now() - 200000), // 200 seconds ago - } - - authMockFns.mockGetSession.mockResolvedValue({ - user: { id: 'user-123', email: 'test@example.com' }, - }) - vi.mocked(checkDocumentWriteAccess).mockResolvedValue({ - hasAccess: true, - document: processingDocument, - knowledgeBase: { id: 'kb-123', userId: 'user-123' }, - }) - - vi.mocked(markDocumentAsFailedTimeout).mockResolvedValue({ - success: true, - processingDuration: 200000, - }) - - const req = createMockRequest('PUT', { markFailedDueToTimeout: true }) - const response = await PUT(req, { params: mockParams }) - const data = await response.json() - - expect(response.status).toBe(200) - expect(data.success).toBe(true) - expect(data.data.documentId).toBe('doc-123') - expect(data.data.status).toBe('failed') - expect(data.data.message).toBe('Document marked as failed due to timeout') - expect(vi.mocked(markDocumentAsFailedTimeout)).toHaveBeenCalledWith( - 'doc-123', - processingDocument.processingStartedAt, - expect.any(String) - ) - }) - - it('should reject marking failed for non-processing document', async () => { - authMockFns.mockGetSession.mockResolvedValue({ - user: { id: 'user-123', email: 'test@example.com' }, - }) - vi.mocked(checkDocumentWriteAccess).mockResolvedValue({ - hasAccess: true, - document: { ...mockDocument, processingStatus: 'completed' }, - knowledgeBase: { id: 'kb-123', userId: 'user-123' }, - }) - - const req = createMockRequest('PUT', { markFailedDueToTimeout: true }) - const response = await PUT(req, { params: mockParams }) - const data = await response.json() - - expect(response.status).toBe(400) - expect(data.error).toContain('Document is not in processing state') - }) - - it('should reject marking failed for recently started processing', async () => { - const recentProcessingDocument = { - ...mockDocument, - processingStatus: 'processing', - processingStartedAt: new Date(Date.now() - 60000), // 60 seconds ago - } - - authMockFns.mockGetSession.mockResolvedValue({ - user: { id: 'user-123', email: 'test@example.com' }, - }) - vi.mocked(checkDocumentWriteAccess).mockResolvedValue({ - hasAccess: true, - document: recentProcessingDocument, - knowledgeBase: { id: 'kb-123', userId: 'user-123' }, - }) - - vi.mocked(markDocumentAsFailedTimeout).mockRejectedValue( - new Error('Document has not been processing long enough to be considered dead') - ) - - const req = createMockRequest('PUT', { markFailedDueToTimeout: true }) - const response = await PUT(req, { params: mockParams }) - const data = await response.json() - - expect(response.status).toBe(400) - expect(data.error).toContain('Document has not been processing long enough') - }) - }) - - describe('PUT /api/knowledge/[id]/documents/[documentId] - Retry Processing', () => { - const mockParams = Promise.resolve({ id: 'kb-123', documentId: 'doc-123' }) - - it('should retry processing successfully', async () => { - const failedDocument = { - ...mockDocument, - processingStatus: 'failed', - processingError: 'Previous processing failed', - } - - authMockFns.mockGetSession.mockResolvedValue({ - user: { id: 'user-123', email: 'test@example.com' }, - }) - vi.mocked(checkDocumentWriteAccess).mockResolvedValue({ - hasAccess: true, - document: failedDocument, - knowledgeBase: { id: 'kb-123', userId: 'user-123' }, - }) - - vi.mocked(retryDocumentProcessing).mockResolvedValue({ - success: true, - status: 'pending', - message: 'Document retry processing started', - }) - - const req = createMockRequest('PUT', { retryProcessing: true }) - const response = await PUT(req, { params: mockParams }) - const data = await response.json() - - expect(response.status).toBe(200) - expect(data.success).toBe(true) - expect(data.data.status).toBe('pending') - expect(data.data.message).toBe('Document retry processing started') - expect(vi.mocked(retryDocumentProcessing)).toHaveBeenCalledWith( - 'kb-123', - 'doc-123', - { - filename: failedDocument.filename, - fileUrl: failedDocument.fileUrl, - fileSize: failedDocument.fileSize, - mimeType: failedDocument.mimeType, - }, - expect.any(String), - undefined - ) - }) - - it('should reject retry for non-failed document', async () => { - authMockFns.mockGetSession.mockResolvedValue({ - user: { id: 'user-123', email: 'test@example.com' }, - }) - vi.mocked(checkDocumentWriteAccess).mockResolvedValue({ - hasAccess: true, - document: { ...mockDocument, processingStatus: 'completed' }, - knowledgeBase: { id: 'kb-123', userId: 'user-123' }, - }) - - const req = createMockRequest('PUT', { retryProcessing: true }) - const response = await PUT(req, { params: mockParams }) - const data = await response.json() - - expect(response.status).toBe(400) - expect(data.error).toBe('Document is not in failed state') - }) - }) - - describe('PUT /api/knowledge/[id]/documents/[documentId] - Authentication & Authorization', () => { - const mockParams = Promise.resolve({ id: 'kb-123', documentId: 'doc-123' }) - const validUpdateData = { filename: 'updated-document.pdf' } - - it('should return unauthorized for unauthenticated user', async () => { - authMockFns.mockGetSession.mockResolvedValue(null) - - const req = createMockRequest('PUT', validUpdateData) - const response = await PUT(req, { params: mockParams }) - const data = await response.json() - - expect(response.status).toBe(401) - expect(data.error).toBe('Unauthorized') - }) - - it('should return not found for non-existent document', async () => { - authMockFns.mockGetSession.mockResolvedValue({ - user: { id: 'user-123', email: 'test@example.com' }, - }) - vi.mocked(checkDocumentWriteAccess).mockResolvedValue({ - hasAccess: false, - notFound: true, - reason: 'Document not found', - }) - - const req = createMockRequest('PUT', validUpdateData) - const response = await PUT(req, { params: mockParams }) - const data = await response.json() - - expect(response.status).toBe(404) - expect(data.error).toBe('Document not found') - }) - - it('should handle database errors during update', async () => { - authMockFns.mockGetSession.mockResolvedValue({ - user: { id: 'user-123', email: 'test@example.com' }, - }) - vi.mocked(checkDocumentWriteAccess).mockResolvedValue({ - hasAccess: true, - document: mockDocument, - knowledgeBase: { id: 'kb-123', userId: 'user-123' }, - }) - - vi.mocked(updateDocument).mockRejectedValue(new Error('Database error')) - - const req = createMockRequest('PUT', validUpdateData) - const response = await PUT(req, { params: mockParams }) - const data = await response.json() - - expect(response.status).toBe(500) - expect(data.error).toBe('Failed to update document') - }) - }) - - describe('DELETE /api/knowledge/[id]/documents/[documentId]', () => { - const mockParams = Promise.resolve({ id: 'kb-123', documentId: 'doc-123' }) - - it('should delete document successfully', async () => { - authMockFns.mockGetSession.mockResolvedValue({ - user: { id: 'user-123', email: 'test@example.com' }, - }) - vi.mocked(checkDocumentWriteAccess).mockResolvedValue({ - hasAccess: true, - document: mockDocument, - knowledgeBase: { id: 'kb-123', userId: 'user-123' }, - }) - - vi.mocked(deleteDocument).mockResolvedValue({ - success: true, - message: 'Document deleted successfully', - }) - - const req = createMockRequest('DELETE') - const response = await DELETE(req, { params: mockParams }) - const data = await response.json() - - expect(response.status).toBe(200) - expect(data.success).toBe(true) - expect(data.data.message).toBe('Document deleted successfully') - expect(vi.mocked(deleteDocument)).toHaveBeenCalledWith('doc-123', expect.any(String)) - }) - - it('should return unauthorized for unauthenticated user', async () => { - authMockFns.mockGetSession.mockResolvedValue(null) - - const req = createMockRequest('DELETE') - const response = await DELETE(req, { params: mockParams }) - const data = await response.json() - - expect(response.status).toBe(401) - expect(data.error).toBe('Unauthorized') - }) - - it('should return not found for non-existent document', async () => { - authMockFns.mockGetSession.mockResolvedValue({ - user: { id: 'user-123', email: 'test@example.com' }, - }) - vi.mocked(checkDocumentWriteAccess).mockResolvedValue({ - hasAccess: false, - notFound: true, - reason: 'Document not found', - }) - - const req = createMockRequest('DELETE') - const response = await DELETE(req, { params: mockParams }) - const data = await response.json() - - expect(response.status).toBe(404) - expect(data.error).toBe('Document not found') - }) - - it('should return unauthorized for document without access', async () => { - authMockFns.mockGetSession.mockResolvedValue({ - user: { id: 'user-123', email: 'test@example.com' }, - }) - vi.mocked(checkDocumentWriteAccess).mockResolvedValue({ - hasAccess: false, - reason: 'Access denied', - }) - - const req = createMockRequest('DELETE') - const response = await DELETE(req, { params: mockParams }) - const data = await response.json() - - expect(response.status).toBe(401) - expect(data.error).toBe('Unauthorized') - }) - - it('should handle database errors during deletion', async () => { - authMockFns.mockGetSession.mockResolvedValue({ - user: { id: 'user-123', email: 'test@example.com' }, - }) - vi.mocked(checkDocumentWriteAccess).mockResolvedValue({ - hasAccess: true, - document: mockDocument, - knowledgeBase: { id: 'kb-123', userId: 'user-123' }, - }) - vi.mocked(deleteDocument).mockRejectedValue(new Error('Database error')) - - const req = createMockRequest('DELETE') - const response = await DELETE(req, { params: mockParams }) - const data = await response.json() - - expect(response.status).toBe(500) - expect(data.error).toBe('Failed to delete document') - }) - }) -}) diff --git a/apps/sim/app/api/knowledge/[id]/documents/[documentId]/route.ts b/apps/sim/app/api/knowledge/[id]/documents/[documentId]/route.ts index e8ad31d04fa..26cea872efa 100644 --- a/apps/sim/app/api/knowledge/[id]/documents/[documentId]/route.ts +++ b/apps/sim/app/api/knowledge/[id]/documents/[documentId]/route.ts @@ -1,266 +1,115 @@ -import { createLogger } from '@sim/logger' -import { type NextRequest, NextResponse } from 'next/server' -import { updateKnowledgeDocumentContract } from '@/lib/api/contracts/knowledge' -import { parseRequest } from '@/lib/api/server' -import { AuthType, checkSessionOrInternalAuth } from '@/lib/auth/hybrid' import { - requireBillingAttributionHeader, - resolveBillingAttribution, -} from '@/lib/billing/core/billing-attribution' + deleteKnowledgeDocumentContract, + getKnowledgeDocumentContract, + updateKnowledgeDocumentContract, +} from '@/lib/api/contracts/knowledge' +import { defineInternalJsonRoute, internalRateLimits } from '@/lib/api/server/routes' import { - messageForOrchestrationError, - type OrchestrationErrorCode, - statusForOrchestrationError, -} from '@/lib/core/orchestration/types' -import { generateRequestId } from '@/lib/core/utils/request' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' + internalKnowledgeActorUserId, + internalKnowledgeAnalytics, + internalKnowledgeAuthType, + resolveInternalKnowledgeBillingAttribution, + toInternalKnowledgeDocument, +} from '@/lib/knowledge/api/internal-route' import { - performDeleteKnowledgeDocument, - performMarkKnowledgeDocumentTimedOut, - performRetryKnowledgeDocumentProcessing, - performUpdateKnowledgeDocument, -} from '@/lib/knowledge/orchestration' + internalKnowledgeErrorPolicies, + internalKnowledgeSessionOrExecutorAuth, +} from '@/lib/knowledge/api/route-policies' +import { + deleteKnowledgeDocument, + readKnowledgeDocument, + updateKnowledgeDocument, +} from '@/lib/knowledge/application/documents' +import { knowledgeOperations } from '@/lib/knowledge/application/operations' import { createKnowledgeDocumentSourceValue } from '@/lib/knowledge/secret-provenance' -import { createKnowledgePersistedResponse } from '@/app/api/knowledge/secret-provenance' -import { checkDocumentAccess, checkDocumentWriteAccess } from '@/app/api/knowledge/utils' - -const logger = createLogger('DocumentByIdAPI') - -export const GET = withRouteHandler( - async (req: NextRequest, { params }: { params: Promise<{ id: string; documentId: string }> }) => { - const requestId = generateRequestId() - const { id: knowledgeBaseId, documentId } = await params - - try { - const auth = await checkSessionOrInternalAuth(req, { requireWorkflowId: false }) - if (!auth.success || !auth.userId) { - logger.warn(`[${requestId}] Unauthorized document access attempt`) - return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) - } - const userId = auth.userId - - const accessCheck = await checkDocumentAccess(knowledgeBaseId, documentId, userId) - - if (!accessCheck.hasAccess) { - if (accessCheck.notFound) { - logger.warn( - `[${requestId}] ${accessCheck.reason}: KB=${knowledgeBaseId}, Doc=${documentId}` - ) - return NextResponse.json({ error: accessCheck.reason }, { status: 404 }) - } - logger.warn( - `[${requestId}] User ${userId} attempted unauthorized document access: ${accessCheck.reason}` - ) - return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) - } - - logger.info( - `[${requestId}] Retrieved document: ${documentId} from knowledge base ${knowledgeBaseId}` - ) - - const responseBody = { - success: true, - data: accessCheck.document, - } - const workspaceId = accessCheck.knowledgeBase?.workspaceId ?? undefined - return createKnowledgePersistedResponse({ - request: req, - authType: auth.authType, - userId, - ...(workspaceId ? { workspaceId } : {}), - body: responseBody, - documents: accessCheck.document - ? [ - { - id: accessCheck.document.id, - source: createKnowledgeDocumentSourceValue(accessCheck.document), - value: accessCheck.document, - }, - ] - : [], - }) - } catch (error) { - logger.error(`[${requestId}] Error fetching document`, error) - return NextResponse.json({ error: 'Failed to fetch document' }, { status: 500 }) - } - } -) - -export const PUT = withRouteHandler( - async (req: NextRequest, { params }: { params: Promise<{ id: string; documentId: string }> }) => { - const requestId = generateRequestId() - const { id: knowledgeBaseId, documentId } = await params - - try { - const auth = await checkSessionOrInternalAuth(req, { requireWorkflowId: false }) - if (!auth.success || !auth.userId) { - logger.warn(`[${requestId}] Unauthorized document update attempt`) - return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) - } - const userId = auth.userId - - const accessCheck = await checkDocumentWriteAccess(knowledgeBaseId, documentId, userId) - - if (!accessCheck.hasAccess) { - if (accessCheck.notFound) { - logger.warn( - `[${requestId}] ${accessCheck.reason}: KB=${knowledgeBaseId}, Doc=${documentId}` - ) - return NextResponse.json({ error: accessCheck.reason }, { status: 404 }) - } - logger.warn( - `[${requestId}] User ${userId} attempted unauthorized document update: ${accessCheck.reason}` - ) - return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseRequest( - updateKnowledgeDocumentContract, - req, - { params }, +import { finalizeKnowledgePersistedResponse } from '@/app/api/knowledge/secret-provenance' + +export const GET = defineInternalJsonRoute({ + contract: getKnowledgeDocumentContract, + auth: internalKnowledgeSessionOrExecutorAuth, + operation: knowledgeOperations.readDocument, + rateLimit: internalRateLimits.none({ + reason: 'Preserve existing internal document-read behavior', + }), + errorPolicy: internalKnowledgeErrorPolicies.documents, + mapInput: ({ params }) => ({ + knowledgeBaseId: params.id, + documentId: params.documentId, + }), + useCase: readKnowledgeDocument, + present: ({ document }) => ({ + success: true as const, + data: toInternalKnowledgeDocument(document), + }), + finalizeResponse: ({ request, principal, result, body }) => + finalizeKnowledgePersistedResponse({ + request, + authType: internalKnowledgeAuthType(principal), + userId: internalKnowledgeActorUserId(principal), + workspaceId: result.workspaceId, + body, + documents: [ { - validationErrorResponse: (error) => { - logger.warn(`[${requestId}] Invalid document update data`, { errors: error.issues }) - return NextResponse.json( - { error: 'Invalid request data', details: error.issues }, - { status: 400 } - ) - }, - } - ) - if (!parsed.success) return parsed.response - - const { markFailedDueToTimeout, retryProcessing, ...documentUpdates } = parsed.data.body - const doc = accessCheck.document - const workspaceId = accessCheck.knowledgeBase?.workspaceId ?? null - - const failed = (outcome: { error?: string; errorCode?: OrchestrationErrorCode }) => - NextResponse.json( - { error: messageForOrchestrationError(outcome, 'Failed to update document') }, - { status: statusForOrchestrationError(outcome.errorCode) } - ) - - if (markFailedDueToTimeout) { - const outcome = await performMarkKnowledgeDocumentTimedOut({ - document: doc, - requestId, - }) - if (!outcome.success) return failed(outcome) - - return NextResponse.json({ - success: true, - data: { documentId, status: outcome.status, message: outcome.message }, - }) - } - - if (retryProcessing) { - const billingAttribution = workspaceId - ? auth.authType === AuthType.INTERNAL_JWT - ? requireBillingAttributionHeader(req.headers, { - actorUserId: userId, - workspaceId, - }) - : await resolveBillingAttribution({ - actorUserId: userId, - workspaceId, - }) - : undefined - - const outcome = await performRetryKnowledgeDocumentProcessing({ - knowledgeBaseId, - document: doc, - billingAttribution, - requestId, - }) - if (!outcome.success) return failed(outcome) - - return NextResponse.json({ - success: true, - data: { documentId, status: outcome.status, message: outcome.message }, - }) - } - - const outcome = await performUpdateKnowledgeDocument({ - knowledgeBase: { - id: knowledgeBaseId, - name: accessCheck.knowledgeBase?.name, - workspaceId, + id: result.document.id, + source: createKnowledgeDocumentSourceValue(result.document), + value: result.document, }, - document: { id: documentId, filename: doc.filename }, - updates: documentUpdates, - userId, - actorName: auth.userName, - actorEmail: auth.userEmail, - source: 'ui', - requestId, - request: req, - }) - if (!outcome.success) return failed(outcome) - - return NextResponse.json({ success: true, data: outcome.document }) - } catch (error) { - logger.error(`[${requestId}] Error updating document ${documentId}`, error) - return NextResponse.json({ error: 'Failed to update document' }, { status: 500 }) + ], + }), +}) + +export const PUT = defineInternalJsonRoute({ + contract: updateKnowledgeDocumentContract, + auth: internalKnowledgeSessionOrExecutorAuth, + operation: knowledgeOperations.updateDocument, + rateLimit: internalRateLimits.none({ + reason: 'Preserve existing internal document-update behavior', + }), + errorPolicy: internalKnowledgeErrorPolicies.documents, + mapInput: ({ params, body }, { principal, request }) => { + const { markFailedDueToTimeout, retryProcessing, ...updates } = body + return { + knowledgeBaseId: params.id, + documentId: params.documentId, + updates, + markFailedDueToTimeout, + retryProcessing, + resolveBillingAttribution: (workspaceId: string) => + resolveInternalKnowledgeBillingAttribution(request, principal, workspaceId), + source: 'ui', } - } -) - -export const DELETE = withRouteHandler( - async (req: NextRequest, { params }: { params: Promise<{ id: string; documentId: string }> }) => { - const requestId = generateRequestId() - const { id: knowledgeBaseId, documentId } = await params - - try { - const auth = await checkSessionOrInternalAuth(req, { requireWorkflowId: false }) - if (!auth.success || !auth.userId) { - logger.warn(`[${requestId}] Unauthorized document delete attempt`) - return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) - } - const userId = auth.userId - - const accessCheck = await checkDocumentWriteAccess(knowledgeBaseId, documentId, userId) - - if (!accessCheck.hasAccess) { - if (accessCheck.notFound) { - logger.warn( - `[${requestId}] ${accessCheck.reason}: KB=${knowledgeBaseId}, Doc=${documentId}` - ) - return NextResponse.json({ error: accessCheck.reason }, { status: 404 }) + }, + useCase: updateKnowledgeDocument, + present: (result) => + result.kind === 'processing' + ? { + success: true as const, + data: { + documentId: result.documentId, + status: result.status, + message: result.message, + }, } - logger.warn( - `[${requestId}] User ${userId} attempted unauthorized document deletion: ${accessCheck.reason}` - ) - return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) - } - - const outcome = await performDeleteKnowledgeDocument({ - knowledgeBase: { - id: knowledgeBaseId, - name: accessCheck.knowledgeBase?.name, - workspaceId: accessCheck.knowledgeBase?.workspaceId ?? null, - }, - document: accessCheck.document, - userId, - actorName: auth.userName, - actorEmail: auth.userEmail, - source: 'ui', - requestId, - request: req, - }) - if (!outcome.success) { - return NextResponse.json( - { error: messageForOrchestrationError(outcome, 'Failed to delete document') }, - { status: statusForOrchestrationError(outcome.errorCode) } - ) - } - - return NextResponse.json({ - success: true, - data: { success: true, message: 'Document deleted successfully' }, - }) - } catch (error) { - logger.error(`[${requestId}] Error deleting document`, error) - return NextResponse.json({ error: 'Failed to delete document' }, { status: 500 }) - } - } -) + : { success: true as const, data: toInternalKnowledgeDocument(result.document) }, +}) + +export const DELETE = defineInternalJsonRoute({ + contract: deleteKnowledgeDocumentContract, + auth: internalKnowledgeSessionOrExecutorAuth, + operation: knowledgeOperations.deleteDocument, + rateLimit: internalRateLimits.none({ + reason: 'Preserve existing internal document-delete behavior', + }), + errorPolicy: internalKnowledgeErrorPolicies.documents, + mapInput: ({ params }) => ({ + knowledgeBaseId: params.id, + documentId: params.documentId, + source: 'ui', + }), + useCase: deleteKnowledgeDocument, + onSuccess: internalKnowledgeAnalytics.documentDeleted, + present: () => ({ + success: true as const, + data: { success: true, message: 'Document deleted successfully' }, + }), +}) diff --git a/apps/sim/app/api/knowledge/[id]/documents/[documentId]/tag-definitions/route.ts b/apps/sim/app/api/knowledge/[id]/documents/[documentId]/tag-definitions/route.ts index 83fa06c35eb..8cb5cc3bb65 100644 --- a/apps/sim/app/api/knowledge/[id]/documents/[documentId]/tag-definitions/route.ts +++ b/apps/sim/app/api/knowledge/[id]/documents/[documentId]/tag-definitions/route.ts @@ -1,209 +1,87 @@ -import { createLogger } from '@sim/logger' -import { generateId } from '@sim/utils/id' -import { type NextRequest, NextResponse } from 'next/server' -import { saveDocumentTagDefinitionsContract } from '@/lib/api/contracts/knowledge' -import { parseRequest } from '@/lib/api/server' -import { getSession } from '@/lib/auth' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { SUPPORTED_FIELD_TYPES } from '@/lib/knowledge/constants' import { - cleanupUnusedTagDefinitions, - createOrUpdateTagDefinitionsBulk, - deleteAllTagDefinitions, - getDocumentTagDefinitions, - KnowledgeTagProvenanceConflictError, -} from '@/lib/knowledge/tags/service' -import type { BulkTagDefinitionsData } from '@/lib/knowledge/tags/types' -import { checkDocumentAccess, checkDocumentWriteAccess } from '@/app/api/knowledge/utils' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('DocumentTagDefinitionsAPI') - -// GET /api/knowledge/[id]/documents/[documentId]/tag-definitions - Get tag definitions for a document -export const GET = withRouteHandler( - async (req: NextRequest, { params }: { params: Promise<{ id: string; documentId: string }> }) => { - const requestId = generateId().slice(0, 8) - const { id: knowledgeBaseId, documentId } = await params - - try { - logger.info(`[${requestId}] Getting tag definitions for document ${documentId}`) - - const session = await getSession() - if (!session?.user?.id) { - return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) - } - - // Verify document exists and belongs to the knowledge base - const accessCheck = await checkDocumentAccess(knowledgeBaseId, documentId, session.user.id) - if (!accessCheck.hasAccess) { - if (accessCheck.notFound) { - logger.warn( - `[${requestId}] ${accessCheck.reason}: KB=${knowledgeBaseId}, Doc=${documentId}` - ) - return NextResponse.json({ error: accessCheck.reason }, { status: 404 }) - } - logger.warn( - `[${requestId}] User ${session.user.id} attempted unauthorized document access: ${accessCheck.reason}` - ) - return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) - } - - const tagDefinitions = await getDocumentTagDefinitions(knowledgeBaseId) - - logger.info(`[${requestId}] Retrieved ${tagDefinitions.length} tag definitions`) - - return NextResponse.json({ - success: true, - data: tagDefinitions, - }) - } catch (error) { - logger.error(`[${requestId}] Error getting tag definitions`, error) - return NextResponse.json({ error: 'Failed to get tag definitions' }, { status: 500 }) - } - } -) - -// POST /api/knowledge/[id]/documents/[documentId]/tag-definitions - Create/update tag definitions -export const POST = withRouteHandler( - async (req: NextRequest, context: { params: Promise<{ id: string; documentId: string }> }) => { - const requestId = generateId().slice(0, 8) - const { id: knowledgeBaseId, documentId } = await context.params - - try { - logger.info(`[${requestId}] Creating/updating tag definitions for document ${documentId}`) - - const session = await getSession() - if (!session?.user?.id) { - return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) - } - - // Verify document exists and user has write access - const accessCheck = await checkDocumentWriteAccess( - knowledgeBaseId, - documentId, - session.user.id - ) - if (!accessCheck.hasAccess) { - if (accessCheck.notFound) { - logger.warn( - `[${requestId}] ${accessCheck.reason}: KB=${knowledgeBaseId}, Doc=${documentId}` - ) - return NextResponse.json({ error: accessCheck.reason }, { status: 404 }) - } - logger.warn( - `[${requestId}] User ${session.user.id} attempted unauthorized document write access: ${accessCheck.reason}` - ) - return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseRequest(saveDocumentTagDefinitionsContract, req, context) - if (!parsed.success) return parsed.response - - const validatedData = parsed.data.body - - for (const def of validatedData.definitions) { - /** - * Defense-in-depth runtime check: the contract types `fieldType` as a plain - * string because tightening to the field-type enum cascades into UI form - * state types. Cast here to allow `includes` to accept the wider input. - */ - if (!(SUPPORTED_FIELD_TYPES as readonly string[]).includes(def.fieldType)) { - return NextResponse.json( - { error: 'Invalid request data', details: `Unsupported field type: ${def.fieldType}` }, - { status: 400 } - ) - } - } - - const bulkData: BulkTagDefinitionsData = { - definitions: validatedData.definitions.map((def) => ({ - tagSlot: def.tagSlot, - displayName: def.displayName, - fieldType: def.fieldType, - originalDisplayName: def._originalDisplayName, - })), - } - - const result = await createOrUpdateTagDefinitionsBulk(knowledgeBaseId, bulkData, requestId) - - return NextResponse.json({ - success: true, - data: { - created: result.created, - updated: result.updated, - errors: result.errors, + deleteDocumentTagDefinitionsContract, + listDocumentTagDefinitionsContract, + saveDocumentTagDefinitionsContract, +} from '@/lib/api/contracts/knowledge' +import { defineInternalJsonRoute, internalRateLimits } from '@/lib/api/server/routes' +import { toInternalKnowledgeTag } from '@/lib/knowledge/api/internal-route' +import { + internalKnowledgeErrorPolicies, + internalKnowledgeSessionOrExecutorAuth, +} from '@/lib/knowledge/api/route-policies' +import { knowledgeOperations } from '@/lib/knowledge/application/operations' +import { + deleteKnowledgeDocumentTagDefinitions, + listKnowledgeDocumentTagDefinitions, + saveKnowledgeDocumentTagDefinitions, +} from '@/lib/knowledge/application/tags' + +const rateLimit = internalRateLimits.none({ + reason: 'Preserve existing internal document-tag behavior', +}) + +export const GET = defineInternalJsonRoute({ + contract: listDocumentTagDefinitionsContract, + auth: internalKnowledgeSessionOrExecutorAuth, + operation: knowledgeOperations.listTags, + rateLimit, + errorPolicy: internalKnowledgeErrorPolicies.tags, + mapInput: ({ params }) => ({ + knowledgeBaseId: params.id, + documentId: params.documentId, + }), + useCase: listKnowledgeDocumentTagDefinitions, + present: ({ tagDefinitions }) => ({ + success: true as const, + data: tagDefinitions.map(toInternalKnowledgeTag), + }), +}) + +export const POST = defineInternalJsonRoute({ + contract: saveDocumentTagDefinitionsContract, + auth: internalKnowledgeSessionOrExecutorAuth, + operation: knowledgeOperations.saveDocumentTagDefinitions, + rateLimit, + errorPolicy: internalKnowledgeErrorPolicies.tags, + mapInput: ({ params, body }) => ({ + knowledgeBaseId: params.id, + documentId: params.documentId, + definitions: body.definitions.map((definition) => ({ + tagSlot: definition.tagSlot, + displayName: definition.displayName, + fieldType: definition.fieldType, + originalDisplayName: definition._originalDisplayName, + })), + }), + useCase: saveKnowledgeDocumentTagDefinitions, + present: ({ created, updated, errors }) => ({ + success: true as const, + data: { + created: created.map(toInternalKnowledgeTag), + updated: updated.map(toInternalKnowledgeTag), + errors, + }, + }), +}) + +export const DELETE = defineInternalJsonRoute({ + contract: deleteDocumentTagDefinitionsContract, + auth: internalKnowledgeSessionOrExecutorAuth, + operation: knowledgeOperations.deleteDocumentTagDefinitions, + rateLimit, + errorPolicy: internalKnowledgeErrorPolicies.tags, + mapInput: ({ params, query }) => ({ + knowledgeBaseId: params.id, + documentId: params.documentId, + action: query.action, + }), + useCase: deleteKnowledgeDocumentTagDefinitions, + present: ({ action, count }) => + action === 'cleanup' + ? { success: true as const, data: { cleanedUp: count } } + : { + success: true as const, + message: 'Tag definitions deleted successfully', + data: { deleted: count }, }, - }) - } catch (error) { - logger.error(`[${requestId}] Error creating/updating tag definitions`, error) - return NextResponse.json( - { error: 'Failed to create/update tag definitions' }, - { status: 500 } - ) - } - } -) - -// DELETE /api/knowledge/[id]/documents/[documentId]/tag-definitions - Delete all tag definitions for a document -export const DELETE = withRouteHandler( - async (req: NextRequest, { params }: { params: Promise<{ id: string; documentId: string }> }) => { - const requestId = generateId().slice(0, 8) - const { id: knowledgeBaseId, documentId } = await params - const { searchParams } = new URL(req.url) - const action = searchParams.get('action') // 'cleanup' or 'all' - - try { - const session = await getSession() - if (!session?.user?.id) { - return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) - } - - // Verify document exists and user has write access - const accessCheck = await checkDocumentWriteAccess( - knowledgeBaseId, - documentId, - session.user.id - ) - if (!accessCheck.hasAccess) { - if (accessCheck.notFound) { - logger.warn( - `[${requestId}] ${accessCheck.reason}: KB=${knowledgeBaseId}, Doc=${documentId}` - ) - return NextResponse.json({ error: accessCheck.reason }, { status: 404 }) - } - logger.warn( - `[${requestId}] User ${session.user.id} attempted unauthorized document write access: ${accessCheck.reason}` - ) - return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) - } - - if (action === 'cleanup') { - // Just run cleanup - logger.info(`[${requestId}] Running cleanup for KB ${knowledgeBaseId}`) - const cleanedUpCount = await cleanupUnusedTagDefinitions(knowledgeBaseId, requestId) - - return NextResponse.json({ - success: true, - data: { cleanedUp: cleanedUpCount }, - }) - } - // Delete all tag definitions (original behavior) - logger.info(`[${requestId}] Deleting all tag definitions for KB ${knowledgeBaseId}`) - - const deletedCount = await deleteAllTagDefinitions(knowledgeBaseId, requestId) - - return NextResponse.json({ - success: true, - message: 'Tag definitions deleted successfully', - data: { deleted: deletedCount }, - }) - } catch (error) { - if (error instanceof KnowledgeTagProvenanceConflictError) { - return NextResponse.json({ error: error.message }, { status: 409 }) - } - logger.error(`[${requestId}] Error with tag definitions operation`, error) - return NextResponse.json({ error: 'Failed to process tag definitions' }, { status: 500 }) - } - } -) +}) diff --git a/apps/sim/app/api/knowledge/[id]/documents/route.test.ts b/apps/sim/app/api/knowledge/[id]/documents/route.test.ts deleted file mode 100644 index 9ed788c8a56..00000000000 --- a/apps/sim/app/api/knowledge/[id]/documents/route.test.ts +++ /dev/null @@ -1,595 +0,0 @@ -/** - * Tests for knowledge base documents API route - * - * @vitest-environment node - */ -import { - auditMock, - authMockFns, - createMockRequest, - knowledgeApiUtilsMock, - resetDbChainMock, -} from '@sim/testing' -import { afterAll, afterEach, beforeEach, describe, expect, it, vi } from 'vitest' - -vi.mock('@/app/api/knowledge/utils', () => knowledgeApiUtilsMock) - -vi.mock('@/lib/knowledge/documents/service', () => ({ - getDocuments: vi.fn(), - createSingleDocument: vi.fn(), - createDocumentRecords: vi.fn(), - processDocumentsWithQueue: vi.fn(), - getProcessingConfig: vi.fn(), - bulkDocumentOperation: vi.fn(), - updateDocument: vi.fn(), - deleteDocument: vi.fn(), - markDocumentAsFailedTimeout: vi.fn(), - retryDocumentProcessing: vi.fn(), - KnowledgeBaseFileOwnershipError: class KnowledgeBaseFileOwnershipError extends Error {}, -})) - -vi.mock('@/lib/billing/calculations/usage-monitor', () => ({ - checkActorUsageLimits: vi.fn().mockResolvedValue({ isExceeded: false }), -})) - -vi.mock('@sim/audit', () => auditMock) - -import { - createDocumentRecords, - createSingleDocument, - getDocuments, - getProcessingConfig, - processDocumentsWithQueue, -} from '@/lib/knowledge/documents/service' -import { GET, POST } from '@/app/api/knowledge/[id]/documents/route' -import { checkKnowledgeBaseAccess, checkKnowledgeBaseWriteAccess } from '@/app/api/knowledge/utils' - -describe('Knowledge Base Documents API Route', () => { - const mockDocument = { - id: 'doc-123', - knowledgeBaseId: 'kb-123', - filename: 'test-document.pdf', - fileUrl: 'https://example.com/test-document.pdf', - fileSize: 1024, - mimeType: 'application/pdf', - chunkCount: 5, - tokenCount: 100, - characterCount: 500, - processingStatus: 'completed' as const, - processingStartedAt: new Date(), - processingCompletedAt: new Date(), - processingError: null, - enabled: true, - uploadedAt: new Date(), - tag1: null, - tag2: null, - tag3: null, - tag4: null, - tag5: null, - tag6: null, - tag7: null, - number1: null, - number2: null, - number3: null, - number4: null, - number5: null, - date1: null, - date2: null, - boolean1: null, - boolean2: null, - boolean3: null, - deletedAt: null, - } - - beforeEach(() => { - vi.clearAllMocks() - resetDbChainMock() - - vi.stubGlobal('crypto', { - randomUUID: vi.fn().mockReturnValue('mock-uuid-1234-5678'), - }) - }) - - afterEach(() => { - vi.clearAllMocks() - }) - - afterAll(() => { - resetDbChainMock() - }) - - describe('GET /api/knowledge/[id]/documents', () => { - const mockParams = Promise.resolve({ id: 'kb-123' }) - - it('should retrieve documents successfully for authenticated user', async () => { - authMockFns.mockGetSession.mockResolvedValue({ - user: { id: 'user-123', email: 'test@example.com' }, - }) - vi.mocked(checkKnowledgeBaseAccess).mockResolvedValue({ - hasAccess: true, - knowledgeBase: { id: 'kb-123', userId: 'user-123' }, - }) - - vi.mocked(getDocuments).mockResolvedValue({ - documents: [mockDocument], - pagination: { - total: 1, - limit: 50, - offset: 0, - hasMore: false, - }, - }) - - const req = createMockRequest('GET') - const response = await GET(req, { params: mockParams }) - const data = await response.json() - - expect(response.status).toBe(200) - expect(data.success).toBe(true) - expect(data.data.documents).toHaveLength(1) - expect(data.data.documents[0].id).toBe('doc-123') - expect(vi.mocked(checkKnowledgeBaseAccess)).toHaveBeenCalledWith('kb-123', 'user-123') - expect(vi.mocked(getDocuments)).toHaveBeenCalledWith( - 'kb-123', - { - enabledFilter: undefined, - search: undefined, - limit: 50, - offset: 0, - }, - expect.any(String) - ) - }) - - it('should return documents with default filter', async () => { - authMockFns.mockGetSession.mockResolvedValue({ - user: { id: 'user-123', email: 'test@example.com' }, - }) - vi.mocked(checkKnowledgeBaseAccess).mockResolvedValue({ - hasAccess: true, - knowledgeBase: { id: 'kb-123', userId: 'user-123' }, - }) - - vi.mocked(getDocuments).mockResolvedValue({ - documents: [mockDocument], - pagination: { - total: 1, - limit: 50, - offset: 0, - hasMore: false, - }, - }) - - const req = createMockRequest('GET') - const response = await GET(req, { params: mockParams }) - - expect(response.status).toBe(200) - expect(vi.mocked(getDocuments)).toHaveBeenCalledWith( - 'kb-123', - { - enabledFilter: undefined, - search: undefined, - limit: 50, - offset: 0, - }, - expect.any(String) - ) - }) - - it('should filter documents by enabled status when requested', async () => { - authMockFns.mockGetSession.mockResolvedValue({ - user: { id: 'user-123', email: 'test@example.com' }, - }) - vi.mocked(checkKnowledgeBaseAccess).mockResolvedValue({ - hasAccess: true, - knowledgeBase: { id: 'kb-123', userId: 'user-123' }, - }) - - vi.mocked(getDocuments).mockResolvedValue({ - documents: [mockDocument], - pagination: { - total: 1, - limit: 50, - offset: 0, - hasMore: false, - }, - }) - - const url = 'http://localhost:3000/api/knowledge/kb-123/documents?enabledFilter=disabled' - const req = new Request(url, { method: 'GET' }) as any - - const response = await GET(req, { params: mockParams }) - - expect(response.status).toBe(200) - expect(vi.mocked(getDocuments)).toHaveBeenCalledWith( - 'kb-123', - { - enabledFilter: 'disabled', - search: undefined, - limit: 50, - offset: 0, - }, - expect.any(String) - ) - }) - - it('should return unauthorized for unauthenticated user', async () => { - authMockFns.mockGetSession.mockResolvedValue(null) - - const req = createMockRequest('GET') - const response = await GET(req, { params: mockParams }) - const data = await response.json() - - expect(response.status).toBe(401) - expect(data.error).toBe('Unauthorized') - }) - - it('should return not found for non-existent knowledge base', async () => { - authMockFns.mockGetSession.mockResolvedValue({ - user: { id: 'user-123', email: 'test@example.com' }, - }) - vi.mocked(checkKnowledgeBaseAccess).mockResolvedValue({ - hasAccess: false, - notFound: true, - }) - - const req = createMockRequest('GET') - const response = await GET(req, { params: mockParams }) - const data = await response.json() - - expect(response.status).toBe(404) - expect(data.error).toBe('Knowledge base not found') - }) - - it('should return unauthorized for knowledge base without access', async () => { - authMockFns.mockGetSession.mockResolvedValue({ - user: { id: 'user-123', email: 'test@example.com' }, - }) - vi.mocked(checkKnowledgeBaseAccess).mockResolvedValue({ hasAccess: false }) - - const req = createMockRequest('GET') - const response = await GET(req, { params: mockParams }) - const data = await response.json() - - expect(response.status).toBe(401) - expect(data.error).toBe('Unauthorized') - }) - - it('should handle database errors', async () => { - authMockFns.mockGetSession.mockResolvedValue({ - user: { id: 'user-123', email: 'test@example.com' }, - }) - vi.mocked(checkKnowledgeBaseAccess).mockResolvedValue({ - hasAccess: true, - knowledgeBase: { id: 'kb-123', userId: 'user-123' }, - }) - vi.mocked(getDocuments).mockRejectedValue(new Error('Database error')) - - const req = createMockRequest('GET') - const response = await GET(req, { params: mockParams }) - const data = await response.json() - - expect(response.status).toBe(500) - expect(data.error).toBe('Failed to fetch documents') - }) - }) - - describe('POST /api/knowledge/[id]/documents - Single Document', () => { - const mockParams = Promise.resolve({ id: 'kb-123' }) - const validDocumentData = { - filename: 'test-document.pdf', - fileUrl: 'https://example.com/test-document.pdf', - fileSize: 1024, - mimeType: 'application/pdf', - } - - it('should create single document successfully', async () => { - authMockFns.mockGetSession.mockResolvedValue({ - user: { id: 'user-123', email: 'test@example.com' }, - }) - vi.mocked(checkKnowledgeBaseWriteAccess).mockResolvedValue({ - hasAccess: true, - knowledgeBase: { id: 'kb-123', userId: 'user-123' }, - }) - - const createdDocument = { - id: 'doc-123', - knowledgeBaseId: 'kb-123', - filename: validDocumentData.filename, - fileUrl: validDocumentData.fileUrl, - fileSize: validDocumentData.fileSize, - mimeType: validDocumentData.mimeType, - chunkCount: 0, - tokenCount: 0, - characterCount: 0, - enabled: true, - uploadedAt: new Date(), - tag1: null, - tag2: null, - tag3: null, - tag4: null, - tag5: null, - tag6: null, - tag7: null, - } - vi.mocked(createSingleDocument).mockResolvedValue(createdDocument) - - const req = createMockRequest('POST', validDocumentData) - const response = await POST(req, { params: mockParams }) - const data = await response.json() - - expect(response.status).toBe(200) - expect(data.success).toBe(true) - expect(data.data.filename).toBe(validDocumentData.filename) - expect(data.data.fileUrl).toBe(validDocumentData.fileUrl) - expect(vi.mocked(createSingleDocument)).toHaveBeenCalledWith( - validDocumentData, - 'kb-123', - expect.any(String), - 'user-123', - { - filename: { status: 'exact', entries: [] }, - content: { status: 'exact', entries: [] }, - tags: [], - } - ) - }) - - it('should validate single document data', async () => { - authMockFns.mockGetSession.mockResolvedValue({ - user: { id: 'user-123', email: 'test@example.com' }, - }) - vi.mocked(checkKnowledgeBaseWriteAccess).mockResolvedValue({ - hasAccess: true, - knowledgeBase: { id: 'kb-123', userId: 'user-123' }, - }) - - const invalidData = { - filename: '', // Invalid: empty filename - fileUrl: 'invalid-url', // Invalid: not a valid URL - fileSize: 0, // Invalid: size must be > 0 - mimeType: '', // Invalid: empty mime type - } - - const req = createMockRequest('POST', invalidData) - const response = await POST(req, { params: mockParams }) - const data = await response.json() - - expect(response.status).toBe(400) - expect(data.error).toBe('Invalid request data') - expect(data.details).toBeDefined() - }) - }) - - describe('POST /api/knowledge/[id]/documents - Bulk Documents', () => { - const mockParams = Promise.resolve({ id: 'kb-123' }) - const validBulkData = { - bulk: true, - documents: [ - { - filename: 'doc1.pdf', - fileUrl: 'https://example.com/doc1.pdf', - fileSize: 1024, - mimeType: 'application/pdf', - }, - { - filename: 'doc2.pdf', - fileUrl: 'https://example.com/doc2.pdf', - fileSize: 2048, - mimeType: 'application/pdf', - }, - ], - processingOptions: { - recipe: 'default', - lang: 'en', - }, - } - - it('should create bulk documents successfully', async () => { - authMockFns.mockGetSession.mockResolvedValue({ - user: { id: 'user-123', email: 'test@example.com' }, - }) - vi.mocked(checkKnowledgeBaseWriteAccess).mockResolvedValue({ - hasAccess: true, - knowledgeBase: { id: 'kb-123', userId: 'user-123' }, - }) - - const createdDocuments = [ - { - documentId: 'doc-1', - filename: 'doc1.pdf', - fileUrl: 'https://example.com/doc1.pdf', - fileSize: 1024, - mimeType: 'application/pdf', - }, - { - documentId: 'doc-2', - filename: 'doc2.pdf', - fileUrl: 'https://example.com/doc2.pdf', - fileSize: 2048, - mimeType: 'application/pdf', - }, - ] - - vi.mocked(createDocumentRecords).mockResolvedValue(createdDocuments) - vi.mocked(processDocumentsWithQueue).mockResolvedValue(undefined) - vi.mocked(getProcessingConfig).mockReturnValue({ - maxConcurrentDocuments: 8, - batchSize: 20, - delayBetweenBatches: 100, - delayBetweenDocuments: 0, - }) - - const req = createMockRequest('POST', validBulkData) - const response = await POST(req, { params: mockParams }) - const data = await response.json() - - expect(response.status).toBe(200) - expect(data.success).toBe(true) - expect(data.data.total).toBe(2) - expect(data.data.documentsCreated).toHaveLength(2) - expect(data.data.processingMethod).toBe('background') - expect(vi.mocked(createDocumentRecords)).toHaveBeenCalledWith( - validBulkData.documents, - 'kb-123', - expect.any(String), - 'user-123', - [ - { - filename: { status: 'exact', entries: [] }, - content: { status: 'exact', entries: [] }, - tags: [], - }, - { - filename: { status: 'exact', entries: [] }, - content: { status: 'exact', entries: [] }, - tags: [], - }, - ] - ) - expect(vi.mocked(processDocumentsWithQueue)).toHaveBeenCalled() - }) - - it('should validate bulk document data', async () => { - authMockFns.mockGetSession.mockResolvedValue({ - user: { id: 'user-123', email: 'test@example.com' }, - }) - vi.mocked(checkKnowledgeBaseWriteAccess).mockResolvedValue({ - hasAccess: true, - knowledgeBase: { id: 'kb-123', userId: 'user-123' }, - }) - - const invalidBulkData = { - bulk: true, - documents: [ - { - filename: '', // Invalid: empty filename - fileUrl: 'invalid-url', - fileSize: 0, - mimeType: '', - }, - ], - processingOptions: { - recipe: 'default', - lang: 'en', - }, - } - - const req = createMockRequest('POST', invalidBulkData) - const response = await POST(req, { params: mockParams }) - const data = await response.json() - - expect(response.status).toBe(400) - expect(data.error).toBe('Invalid request data') - expect(data.details).toBeDefined() - }) - - it('should handle processing errors gracefully', async () => { - authMockFns.mockGetSession.mockResolvedValue({ - user: { id: 'user-123', email: 'test@example.com' }, - }) - vi.mocked(checkKnowledgeBaseWriteAccess).mockResolvedValue({ - hasAccess: true, - knowledgeBase: { id: 'kb-123', userId: 'user-123' }, - }) - - const createdDocuments = [ - { - documentId: 'doc-1', - filename: 'doc1.pdf', - fileUrl: 'https://example.com/doc1.pdf', - fileSize: 1024, - mimeType: 'application/pdf', - }, - ] - - vi.mocked(createDocumentRecords).mockResolvedValue(createdDocuments) - vi.mocked(processDocumentsWithQueue).mockResolvedValue(undefined) - vi.mocked(getProcessingConfig).mockReturnValue({ - maxConcurrentDocuments: 8, - batchSize: 20, - delayBetweenBatches: 100, - delayBetweenDocuments: 0, - }) - - const req = createMockRequest('POST', validBulkData) - const response = await POST(req, { params: mockParams }) - const data = await response.json() - - expect(response.status).toBe(200) - expect(data.success).toBe(true) - }) - }) - - describe('POST /api/knowledge/[id]/documents - Authentication & Authorization', () => { - const mockParams = Promise.resolve({ id: 'kb-123' }) - const validDocumentData = { - filename: 'test-document.pdf', - fileUrl: 'https://example.com/test-document.pdf', - fileSize: 1024, - mimeType: 'application/pdf', - } - - it('should return unauthorized for unauthenticated user', async () => { - authMockFns.mockGetSession.mockResolvedValue(null) - - const req = createMockRequest('POST', validDocumentData) - const response = await POST(req, { params: mockParams }) - const data = await response.json() - - expect(response.status).toBe(401) - expect(data.error).toBe('Unauthorized') - }) - - it('should return not found for non-existent knowledge base', async () => { - authMockFns.mockGetSession.mockResolvedValue({ - user: { id: 'user-123', email: 'test@example.com' }, - }) - vi.mocked(checkKnowledgeBaseWriteAccess).mockResolvedValue({ - hasAccess: false, - notFound: true, - }) - - const req = createMockRequest('POST', validDocumentData) - const response = await POST(req, { params: mockParams }) - const data = await response.json() - - expect(response.status).toBe(404) - expect(data.error).toBe('Knowledge base not found') - }) - - it('should return unauthorized for knowledge base without access', async () => { - authMockFns.mockGetSession.mockResolvedValue({ - user: { id: 'user-123', email: 'test@example.com' }, - }) - vi.mocked(checkKnowledgeBaseWriteAccess).mockResolvedValue({ hasAccess: false }) - - const req = createMockRequest('POST', validDocumentData) - const response = await POST(req, { params: mockParams }) - const data = await response.json() - - expect(response.status).toBe(401) - expect(data.error).toBe('Unauthorized') - }) - - it('should handle database errors during creation', async () => { - authMockFns.mockGetSession.mockResolvedValue({ - user: { id: 'user-123', email: 'test@example.com' }, - }) - vi.mocked(checkKnowledgeBaseWriteAccess).mockResolvedValue({ - hasAccess: true, - knowledgeBase: { id: 'kb-123', userId: 'user-123' }, - }) - vi.mocked(createSingleDocument).mockRejectedValue(new Error('Database error')) - - const req = createMockRequest('POST', validDocumentData) - const response = await POST(req, { params: mockParams }) - const data = await response.json() - - expect(response.status).toBe(500) - // An unclassified fault renders the route's own wording; the driver's - // message is logged, not returned. - expect(data.error).toBe('Failed to create document') - }) - }) -}) diff --git a/apps/sim/app/api/knowledge/[id]/documents/route.ts b/apps/sim/app/api/knowledge/[id]/documents/route.ts index 411cfe69b14..5f372b0956e 100644 --- a/apps/sim/app/api/knowledge/[id]/documents/route.ts +++ b/apps/sim/app/api/knowledge/[id]/documents/route.ts @@ -1,423 +1,154 @@ -import { createLogger } from '@sim/logger' -import { authorizeWorkflowByWorkspacePermission } from '@sim/platform-authz/workflow' -import { getErrorMessage } from '@sim/utils/errors' -import { generateId } from '@sim/utils/id' -import { type NextRequest, NextResponse } from 'next/server' import { bulkKnowledgeDocumentsContract, createKnowledgeDocumentsContract, - listKnowledgeDocumentsQuerySchema, + listKnowledgeDocumentsContract, parseDocumentTagFiltersParam, } from '@/lib/api/contracts/knowledge' -import { parseRequest } from '@/lib/api/server' -import { getSession } from '@/lib/auth' -import { AuthType, checkSessionOrInternalAuth } from '@/lib/auth/hybrid' -import { checkActorUsageLimits } from '@/lib/billing/calculations/usage-monitor' +import { defineInternalJsonRoute, internalRateLimits } from '@/lib/api/server/routes' +import { OrchestrationError } from '@/lib/core/orchestration/types' import { - checkAttributedUsageLimits, - requireBillingAttributionHeader, - resolveBillingAttribution, -} from '@/lib/billing/core/billing-attribution' + internalKnowledgeActorUserId, + internalKnowledgeAnalytics, + internalKnowledgeAuthType, + resolveInternalKnowledgeBillingAttribution, + toInternalKnowledgeDocument, +} from '@/lib/knowledge/api/internal-route' import { - messageForOrchestrationError, - statusForOrchestrationError, -} from '@/lib/core/orchestration/types' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' + internalKnowledgeErrorPolicies, + internalKnowledgeSessionOrExecutorAuth, +} from '@/lib/knowledge/api/route-policies' import { - bulkDocumentOperation, - bulkDocumentOperationByFilter, - getDocuments, - getProcessingConfig, -} from '@/lib/knowledge/documents/service' -import type { TagFilterCondition } from '@/lib/knowledge/documents/tag-filter' -import { - performUploadKnowledgeDocument, - performUploadKnowledgeDocuments, -} from '@/lib/knowledge/orchestration' + bulkUpdateKnowledgeDocuments, + createKnowledgeDocuments, + listKnowledgeDocuments, +} from '@/lib/knowledge/application/documents' +import { knowledgeOperations } from '@/lib/knowledge/application/operations' import { createKnowledgeDocumentSourceValue } from '@/lib/knowledge/secret-provenance' import { - createKnowledgePersistedResponse, - createKnowledgeProvenanceResponse, + finalizeKnowledgePersistedResponse, + finalizeKnowledgeProvenanceResponse, resolveKnowledgeDocumentWriteSecretProvenance, } from '@/app/api/knowledge/secret-provenance' -import { checkKnowledgeBaseAccess, checkKnowledgeBaseWriteAccess } from '@/app/api/knowledge/utils' - -const logger = createLogger('DocumentsAPI') - -export const GET = withRouteHandler( - async (req: NextRequest, { params }: { params: Promise<{ id: string }> }) => { - const requestId = generateId().slice(0, 8) - const { id: knowledgeBaseId } = await params +export const GET = defineInternalJsonRoute({ + contract: listKnowledgeDocumentsContract, + auth: internalKnowledgeSessionOrExecutorAuth, + operation: knowledgeOperations.listDocuments, + rateLimit: internalRateLimits.none({ + reason: 'Preserve existing internal document-list behavior', + }), + errorPolicy: internalKnowledgeErrorPolicies.documents, + mapInput: ({ params, query }) => { + let tagFilters try { - const auth = await checkSessionOrInternalAuth(req, { requireWorkflowId: false }) - if (!auth.success || !auth.userId) { - logger.warn(`[${requestId}] Unauthorized documents access attempt`) - return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) - } - const userId = auth.userId - - const accessCheck = await checkKnowledgeBaseAccess(knowledgeBaseId, userId) - - if (!accessCheck.hasAccess) { - if ('notFound' in accessCheck && accessCheck.notFound) { - logger.warn(`[${requestId}] Knowledge base not found: ${knowledgeBaseId}`) - return NextResponse.json({ error: 'Knowledge base not found' }, { status: 404 }) - } - logger.warn( - `[${requestId}] User ${userId} attempted to access unauthorized knowledge base documents ${knowledgeBaseId}` - ) - return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) - } - - const queryResult = listKnowledgeDocumentsQuerySchema.safeParse( - Object.fromEntries(new URL(req.url).searchParams.entries()) - ) - if (!queryResult.success) { - return NextResponse.json( - { error: 'Invalid query parameters', details: queryResult.error.issues }, - { status: 400 } - ) - } - const { enabledFilter, search, limit, offset, sortBy, sortOrder, tagFilters } = - queryResult.data - - let parsedTagFilters: TagFilterCondition[] | undefined - try { - parsedTagFilters = parseDocumentTagFiltersParam(tagFilters) as - | TagFilterCondition[] - | undefined - } catch { - return NextResponse.json( - { error: 'tagFilters must be a valid JSON array' }, - { status: 400 } - ) - } - - const result = await getDocuments( - knowledgeBaseId, - { - enabledFilter: enabledFilter || undefined, - search, - limit, - offset, - ...(sortBy && { sortBy }), - ...(sortOrder && { sortOrder }), - tagFilters: parsedTagFilters, - }, - requestId - ) - - const responseBody = { - success: true, - data: { - documents: result.documents, - pagination: result.pagination, - }, - } - const workspaceId = accessCheck.knowledgeBase?.workspaceId ?? undefined - return createKnowledgePersistedResponse({ - request: req, - authType: auth.authType, - userId, - ...(workspaceId ? { workspaceId } : {}), - body: responseBody, - documents: result.documents.map((item) => ({ - id: item.id, - source: createKnowledgeDocumentSourceValue(item), - value: item, - })), - }) - } catch (error) { - logger.error(`[${requestId}] Error fetching documents`, error) - return NextResponse.json({ error: 'Failed to fetch documents' }, { status: 500 }) + tagFilters = parseDocumentTagFiltersParam(query.tagFilters) + } catch { + throw new OrchestrationError('validation', 'tagFilters must be a valid JSON array') } - } -) - -export const POST = withRouteHandler( - async (req: NextRequest, { params }: { params: Promise<{ id: string }> }) => { - const requestId = generateId().slice(0, 8) - const { id: knowledgeBaseId } = await params - - try { - const auth = await checkSessionOrInternalAuth(req, { requireWorkflowId: false }) - if (!auth.success || !auth.userId) { - logger.warn(`[${requestId}] Authentication failed: ${auth.error || 'Unauthorized'}`) - return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) - } - const userId = auth.userId - - const parsed = await parseRequest( - createKnowledgeDocumentsContract, - req, - { params }, - { - validationErrorResponse: (error) => { - logger.warn(`[${requestId}] Invalid document creation request`, { - errors: error.issues, - }) - return NextResponse.json( - { error: 'Invalid request data', details: error.issues }, - { status: 400 } - ) - }, - } - ) - if (!parsed.success) return parsed.response - const body = parsed.data.body - const workflowId = body.workflowId - - logger.info(`[${requestId}] Knowledge base document creation request`, { - knowledgeBaseId, - workflowId, - hasWorkflowId: !!workflowId, - bulk: body.bulk === true, - }) - - if (workflowId) { - const authorization = await authorizeWorkflowByWorkspacePermission({ - workflowId, - userId, - action: 'write', - }) - if (!authorization.allowed) { - return NextResponse.json( - { error: authorization.message || 'Access denied' }, - { status: authorization.status } - ) - } - } - - const accessCheck = await checkKnowledgeBaseWriteAccess(knowledgeBaseId, userId) - - if (!accessCheck.hasAccess) { - if ('notFound' in accessCheck && accessCheck.notFound) { - logger.warn(`[${requestId}] Knowledge base not found: ${knowledgeBaseId}`) - return NextResponse.json({ error: 'Knowledge base not found' }, { status: 404 }) - } - logger.warn( - `[${requestId}] User ${userId} attempted to create document in unauthorized knowledge base ${knowledgeBaseId}` - ) - return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) - } - - const kbWorkspaceId = accessCheck.knowledgeBase?.workspaceId - const billingAttribution = kbWorkspaceId - ? auth.authType === AuthType.INTERNAL_JWT - ? requireBillingAttributionHeader(req.headers, { - actorUserId: userId, - workspaceId: kbWorkspaceId, - }) - : await resolveBillingAttribution({ - actorUserId: userId, - workspaceId: kbWorkspaceId, - }) - : undefined - - /** - * Gate the workspace payer and uploader before accepting indexing work. - * Legacy workspace-less KBs retain account-only enforcement; asynchronous - * connector, cron, and retry paths apply the same backstop. - */ - const usage = billingAttribution - ? await checkAttributedUsageLimits(billingAttribution) - : await checkActorUsageLimits(userId) - if (usage.isExceeded) { - return NextResponse.json( - { - error: usage.message || 'Usage limit exceeded. Please upgrade your plan to continue.', - }, - { status: 402 } - ) - } - - const provenanceDocuments = body.bulk === true ? body.documents : [body] - const writeProvenance = resolveKnowledgeDocumentWriteSecretProvenance({ - request: req, - payload: body, - authType: auth.authType, - userId, - ...(kbWorkspaceId ? { workspaceId: kbWorkspaceId } : {}), - documents: provenanceDocuments, - }) - if (!writeProvenance.success) return writeProvenance.response - - const knowledgeBase = { - id: knowledgeBaseId, - name: accessCheck.knowledgeBase?.name, - workspaceId: kbWorkspaceId ?? null, - } - const actor = { - userId, - actorName: auth.userName, - actorEmail: auth.userEmail, - source: 'ui' as const, - requestId, - request: req, - } - - if (body.bulk === true) { - const outcome = await performUploadKnowledgeDocuments({ - ...actor, - knowledgeBase, - documents: body.documents, - processingOptions: body.processingOptions, - billingAttribution, - secretProvenances: writeProvenance.provenances, - }) - if (!outcome.success) { - return NextResponse.json( - { error: messageForOrchestrationError(outcome, 'Failed to create document') }, - { status: statusForOrchestrationError(outcome.errorCode) } - ) - } - - const { batchSize, maxConcurrentDocuments } = getProcessingConfig() - return createKnowledgeProvenanceResponse({ - request: req, - authType: auth.authType, - userId, - ...(kbWorkspaceId ? { workspaceId: kbWorkspaceId } : {}), - provenances: - writeProvenance.provenances?.flatMap((provenance) => [ - provenance.filename, - ...provenance.tags.map((tag) => tag.provenance), - ]) ?? [], - body: { - success: true, - data: { - total: outcome.documents.length, - documentsCreated: outcome.documents.map((doc) => ({ - documentId: doc.documentId, - filename: doc.filename, - status: 'pending', - })), - processingMethod: 'background', - processingConfig: { - maxConcurrentDocuments, - batchSize, - totalBatches: Math.ceil(outcome.documents.length / batchSize), - }, - }, - }, - }) - } - - const { bulk: _bulk, workflowId: _workflowId, ...singleDocumentData } = body - // Indexing is deliberately not started here: this path only records the - // document, and its caller drives processing separately. - const outcome = await performUploadKnowledgeDocument({ - ...actor, - knowledgeBase, - document: singleDocumentData, - billingAttribution, - secretProvenance: writeProvenance.provenances?.[0], - }) - if (!outcome.success) { - return NextResponse.json( - { error: messageForOrchestrationError(outcome, 'Failed to create document') }, - { status: statusForOrchestrationError(outcome.errorCode) } - ) - } - - return createKnowledgeProvenanceResponse({ - request: req, - authType: auth.authType, - userId, - ...(kbWorkspaceId ? { workspaceId: kbWorkspaceId } : {}), - provenances: - writeProvenance.provenances?.flatMap((provenance) => [ - provenance.filename, - ...provenance.tags.map((tag) => tag.provenance), - ]) ?? [], - body: { success: true, data: outcome.document }, - }) - } catch (error) { - logger.error(`[${requestId}] Error creating document`, error) - return NextResponse.json( - { error: getErrorMessage(error, 'Failed to create document') }, - { status: 500 } - ) + return { + knowledgeBaseId: params.id, + enabledFilter: query.enabledFilter, + search: query.search, + limit: query.limit, + offset: query.offset, + sortBy: query.sortBy, + sortOrder: query.sortOrder, + tagFilters, } - } -) - -export const PATCH = withRouteHandler( - async (req: NextRequest, { params }: { params: Promise<{ id: string }> }) => { - const requestId = generateId().slice(0, 8) - const { id: knowledgeBaseId } = await params - - try { - const session = await getSession() - if (!session?.user?.id) { - logger.warn(`[${requestId}] Unauthorized bulk document operation attempt`) - return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) - } - - const accessCheck = await checkKnowledgeBaseWriteAccess(knowledgeBaseId, session.user.id) - - if (!accessCheck.hasAccess) { - if ('notFound' in accessCheck && accessCheck.notFound) { - logger.warn(`[${requestId}] Knowledge base not found: ${knowledgeBaseId}`) - return NextResponse.json({ error: 'Knowledge base not found' }, { status: 404 }) - } - logger.warn( - `[${requestId}] User ${session.user.id} attempted to perform bulk operation on unauthorized knowledge base ${knowledgeBaseId}` - ) - return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseRequest( - bulkKnowledgeDocumentsContract, - req, - { params }, - { - validationErrorResponse: (error) => { - logger.warn(`[${requestId}] Invalid bulk operation data`, { errors: error.issues }) - return NextResponse.json( - { error: 'Invalid request data', details: error.issues }, - { status: 400 } - ) - }, - } - ) - if (!parsed.success) return parsed.response - const validatedData = parsed.data.body - const { operation, documentIds, selectAll, enabledFilter } = validatedData - - try { - let result - if (selectAll) { - result = await bulkDocumentOperationByFilter( - knowledgeBaseId, - operation, - enabledFilter, - requestId - ) - } else if (documentIds && documentIds.length > 0) { - result = await bulkDocumentOperation(knowledgeBaseId, operation, documentIds, requestId) - } else { - return NextResponse.json({ error: 'No documents specified' }, { status: 400 }) - } - - return NextResponse.json({ - success: true, - data: { - operation, - successCount: result.successCount, - updatedDocuments: result.updatedDocuments, - }, + }, + useCase: listKnowledgeDocuments, + present: ({ documents, pagination }) => ({ + success: true as const, + data: { + documents: documents.map(toInternalKnowledgeDocument), + pagination, + }, + }), + finalizeResponse: ({ request, principal, result, body }) => + finalizeKnowledgePersistedResponse({ + request, + authType: internalKnowledgeAuthType(principal), + userId: internalKnowledgeActorUserId(principal), + workspaceId: result.workspaceId, + body, + documents: result.documents.map((document) => ({ + id: document.id, + source: createKnowledgeDocumentSourceValue(document), + value: document, + })), + }), +}) + +export const POST = defineInternalJsonRoute({ + contract: createKnowledgeDocumentsContract, + auth: internalKnowledgeSessionOrExecutorAuth, + operation: knowledgeOperations.uploadDocument, + rateLimit: internalRateLimits.none({ + reason: 'Preserve existing internal document-create behavior', + }), + errorPolicy: internalKnowledgeErrorPolicies.documents, + mapInput: ({ params, body }, { principal, request }) => { + const documents = body.bulk ? body.documents : [body] + return { + knowledgeBaseId: params.id, + documents, + bulk: body.bulk, + processingOptions: body.bulk ? body.processingOptions : undefined, + resolveBillingAttribution: (workspaceId: string) => + resolveInternalKnowledgeBillingAttribution(request, principal, workspaceId), + resolveSecretProvenances: ({ userId, workspaceId }) => { + const resolution = resolveKnowledgeDocumentWriteSecretProvenance({ + request, + payload: body, + authType: internalKnowledgeAuthType(principal), + userId, + workspaceId, + documents, }) - } catch (error) { - if (error instanceof Error && error.message === 'No valid documents found to update') { - return NextResponse.json({ error: 'No valid documents found to update' }, { status: 404 }) + if (!resolution.success) { + throw new OrchestrationError('validation', 'Invalid knowledge secret provenance') } - throw error - } - } catch (error) { - logger.error(`[${requestId}] Error in bulk document operation`, error) - return NextResponse.json({ error: 'Failed to perform bulk operation' }, { status: 500 }) + return resolution.provenances + }, + source: 'ui' as const, } - } -) + }, + useCase: createKnowledgeDocuments, + onSuccess: internalKnowledgeAnalytics.documentsUploaded, + present: (result) => ({ + success: true as const, + data: result.kind === 'bulk' ? result.data : toInternalKnowledgeDocument(result.data), + }), + finalizeResponse: ({ request, principal, result, body }) => + finalizeKnowledgeProvenanceResponse({ + request, + authType: internalKnowledgeAuthType(principal), + userId: result.userId, + workspaceId: result.workspaceId, + provenances: + result.secretProvenances?.flatMap((provenance) => [ + provenance.filename, + ...provenance.tags.map((tag) => tag.provenance), + ]) ?? [], + body, + }), +}) + +export const PATCH = defineInternalJsonRoute({ + contract: bulkKnowledgeDocumentsContract, + auth: internalKnowledgeSessionOrExecutorAuth, + operation: knowledgeOperations.bulkDocuments, + rateLimit: internalRateLimits.none({ + reason: 'Preserve existing internal bulk-document behavior', + }), + errorPolicy: internalKnowledgeErrorPolicies.documents, + mapInput: ({ params, body }) => ({ + knowledgeBaseId: params.id, + operation: body.operation, + documentIds: body.documentIds, + selectAll: body.selectAll, + enabledFilter: body.enabledFilter, + }), + useCase: bulkUpdateKnowledgeDocuments, + present: (result) => ({ success: true as const, data: result }), +}) diff --git a/apps/sim/app/api/knowledge/[id]/documents/uploads/[uploadId]/complete/route.ts b/apps/sim/app/api/knowledge/[id]/documents/uploads/[uploadId]/complete/route.ts index c426602e85c..329bdda36ed 100644 --- a/apps/sim/app/api/knowledge/[id]/documents/uploads/[uploadId]/complete/route.ts +++ b/apps/sim/app/api/knowledge/[id]/documents/uploads/[uploadId]/complete/route.ts @@ -1,70 +1,57 @@ -import { type NextRequest, NextResponse } from 'next/server' import { completeKnowledgeDocumentUploadContract } from '@/lib/api/contracts/knowledge/upload-sessions' -import { parseRequest } from '@/lib/api/server' +import { + defineInternalJsonRoute, + internalRateLimits, + internalSessionAuth, +} from '@/lib/api/server/routes' import { PlatformEvents } from '@/lib/core/telemetry' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' +import { toInternalKnowledgeDocumentUpload } from '@/lib/knowledge/api/internal-route' +import { internalKnowledgeErrorPolicies } from '@/lib/knowledge/api/route-policies' +import { knowledgeOperations } from '@/lib/knowledge/application/operations' import { completeKnowledgeDocumentUpload } from '@/lib/knowledge/application/upload-sessions' import { captureServerEvent } from '@/lib/posthog/server' -import { - knowledgeDocumentUploadErrorResponse, - requireKnowledgeDocumentUploadActor, -} from '@/app/api/knowledge/[id]/documents/uploads/utils' -import { toV2KnowledgeDocumentUpload } from '@/app/api/v2/knowledge/[id]/documents/uploads/utils' - -interface KnowledgeDocumentUploadRouteParams { - params: Promise<{ id: string; uploadId: string }> -} -export const POST = withRouteHandler( - async (request: NextRequest, context: KnowledgeDocumentUploadRouteParams) => { - const actor = await requireKnowledgeDocumentUploadActor() - if (actor instanceof NextResponse) return actor - const parsed = await parseRequest(completeKnowledgeDocumentUploadContract, request, context) - if (!parsed.success) return parsed.response - const { id: knowledgeBaseId, uploadId } = parsed.data.params - const { workspaceId } = parsed.data.query - try { - const completed = await completeKnowledgeDocumentUpload.execute({ - principal: { kind: 'session', userId: actor.id, sessionId: actor.sessionId }, - input: { - knowledgeBaseId, - assertedWorkspaceId: workspaceId, - uploadId, - uploadToken: parsed.data.headers['upload-token'], - source: 'ui', - }, - request, - }) - if (completed.value.created) { - captureServerEvent( - actor.id, - 'knowledge_base_document_uploaded', - { - knowledge_base_id: completed.knowledgeBaseId, - workspace_id: completed.workspaceId, - document_count: 1, - upload_type: 'single', - }, - { - groups: { workspace: completed.workspaceId }, - setOnce: { first_document_uploaded_at: new Date().toISOString() }, - } - ) - PlatformEvents.knowledgeBaseDocumentsUploaded({ - knowledgeBaseId: completed.knowledgeBaseId, - documentsCount: 1, - uploadType: 'single', - mimeType: completed.value.document.mimeType, - fileSize: completed.value.document.fileSize, - }) +export const POST = defineInternalJsonRoute({ + contract: completeKnowledgeDocumentUploadContract, + auth: internalSessionAuth, + operation: knowledgeOperations.uploadComplete, + rateLimit: internalRateLimits.none({ + reason: 'Preserve existing internal upload-session completion behavior', + }), + errorPolicy: internalKnowledgeErrorPolicies.uploads, + mapInput: ({ params, query, headers }) => ({ + knowledgeBaseId: params.id, + assertedWorkspaceId: query.workspaceId, + uploadId: params.uploadId, + uploadToken: headers['upload-token'], + source: 'ui' as const, + }), + useCase: completeKnowledgeDocumentUpload, + onSuccess: ({ principal, result }) => { + if (!result.value.created) return + captureServerEvent( + principal.userId, + 'knowledge_base_document_uploaded', + { + knowledge_base_id: result.knowledgeBaseId, + workspace_id: result.workspaceId, + document_count: 1, + upload_type: 'single', + }, + { + groups: { workspace: result.workspaceId }, + setOnce: { first_document_uploaded_at: new Date().toISOString() }, } - return NextResponse.json({ - data: toV2KnowledgeDocumentUpload(completed.session, completed.value.document), - }) - } catch (error) { - const classified = knowledgeDocumentUploadErrorResponse(error) - if (classified) return classified - throw error - } - } -) + ) + PlatformEvents.knowledgeBaseDocumentsUploaded({ + knowledgeBaseId: result.knowledgeBaseId, + documentsCount: 1, + uploadType: 'single', + mimeType: result.value.document.mimeType, + fileSize: result.value.document.fileSize, + }) + }, + present: (result) => ({ + data: toInternalKnowledgeDocumentUpload(result.session, result.value.document), + }), +}) diff --git a/apps/sim/app/api/knowledge/[id]/documents/uploads/[uploadId]/parts/route.ts b/apps/sim/app/api/knowledge/[id]/documents/uploads/[uploadId]/parts/route.ts index 2ae1d2bc0e7..68fa4083dbe 100644 --- a/apps/sim/app/api/knowledge/[id]/documents/uploads/[uploadId]/parts/route.ts +++ b/apps/sim/app/api/knowledge/[id]/documents/uploads/[uploadId]/parts/route.ts @@ -1,46 +1,28 @@ -import { type NextRequest, NextResponse } from 'next/server' import { createKnowledgeDocumentUploadPartUrlsContract } from '@/lib/api/contracts/knowledge/upload-sessions' -import { parseRequest } from '@/lib/api/server' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { issueKnowledgeDocumentUploadParts } from '@/lib/knowledge/application/upload-sessions' import { - knowledgeDocumentUploadErrorResponse, - requireKnowledgeDocumentUploadActor, -} from '@/app/api/knowledge/[id]/documents/uploads/utils' - -interface KnowledgeDocumentUploadRouteParams { - params: Promise<{ id: string; uploadId: string }> -} + defineInternalJsonRoute, + internalRateLimits, + internalSessionAuth, +} from '@/lib/api/server/routes' +import { internalKnowledgeErrorPolicies } from '@/lib/knowledge/api/route-policies' +import { knowledgeOperations } from '@/lib/knowledge/application/operations' +import { issueKnowledgeDocumentUploadParts } from '@/lib/knowledge/application/upload-sessions' -export const POST = withRouteHandler( - async (request: NextRequest, context: KnowledgeDocumentUploadRouteParams) => { - const actor = await requireKnowledgeDocumentUploadActor() - if (actor instanceof NextResponse) return actor - const parsed = await parseRequest( - createKnowledgeDocumentUploadPartUrlsContract, - request, - context - ) - if (!parsed.success) return parsed.response - const { id: knowledgeBaseId, uploadId } = parsed.data.params - const { workspaceId } = parsed.data.query - try { - const { parts } = await issueKnowledgeDocumentUploadParts.execute({ - principal: { kind: 'session', userId: actor.id, sessionId: actor.sessionId }, - input: { - knowledgeBaseId, - assertedWorkspaceId: workspaceId, - uploadId, - uploadToken: parsed.data.headers['upload-token'], - partNumbers: parsed.data.body.partNumbers, - }, - request, - }) - return NextResponse.json({ data: { parts } }) - } catch (error) { - const classified = knowledgeDocumentUploadErrorResponse(error) - if (classified) return classified - throw error - } - } -) +export const POST = defineInternalJsonRoute({ + contract: createKnowledgeDocumentUploadPartUrlsContract, + auth: internalSessionAuth, + operation: knowledgeOperations.uploadParts, + rateLimit: internalRateLimits.none({ + reason: 'Preserve existing internal upload-part issuance behavior', + }), + errorPolicy: internalKnowledgeErrorPolicies.uploads, + mapInput: ({ params, query, headers, body }) => ({ + knowledgeBaseId: params.id, + assertedWorkspaceId: query.workspaceId, + uploadId: params.uploadId, + uploadToken: headers['upload-token'], + partNumbers: body.partNumbers, + }), + useCase: issueKnowledgeDocumentUploadParts, + present: ({ parts }) => ({ data: { parts } }), +}) diff --git a/apps/sim/app/api/knowledge/[id]/documents/uploads/[uploadId]/route.ts b/apps/sim/app/api/knowledge/[id]/documents/uploads/[uploadId]/route.ts index 4f9f0d2c5b1..12b42bd2c94 100644 --- a/apps/sim/app/api/knowledge/[id]/documents/uploads/[uploadId]/route.ts +++ b/apps/sim/app/api/knowledge/[id]/documents/uploads/[uploadId]/route.ts @@ -1,42 +1,28 @@ -import { type NextRequest, NextResponse } from 'next/server' import { abortKnowledgeDocumentUploadContract } from '@/lib/api/contracts/knowledge/upload-sessions' -import { parseRequest } from '@/lib/api/server' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { cancelKnowledgeDocumentUpload } from '@/lib/knowledge/application/upload-sessions' import { - knowledgeDocumentUploadErrorResponse, - requireKnowledgeDocumentUploadActor, -} from '@/app/api/knowledge/[id]/documents/uploads/utils' -import { toV2KnowledgeDocumentUpload } from '@/app/api/v2/knowledge/[id]/documents/uploads/utils' - -interface KnowledgeDocumentUploadRouteParams { - params: Promise<{ id: string; uploadId: string }> -} + defineInternalJsonRoute, + internalRateLimits, + internalSessionAuth, +} from '@/lib/api/server/routes' +import { toInternalKnowledgeDocumentUpload } from '@/lib/knowledge/api/internal-route' +import { internalKnowledgeErrorPolicies } from '@/lib/knowledge/api/route-policies' +import { knowledgeOperations } from '@/lib/knowledge/application/operations' +import { cancelKnowledgeDocumentUpload } from '@/lib/knowledge/application/upload-sessions' -export const DELETE = withRouteHandler( - async (request: NextRequest, context: KnowledgeDocumentUploadRouteParams) => { - const actor = await requireKnowledgeDocumentUploadActor() - if (actor instanceof NextResponse) return actor - const parsed = await parseRequest(abortKnowledgeDocumentUploadContract, request, context) - if (!parsed.success) return parsed.response - const { id: knowledgeBaseId, uploadId } = parsed.data.params - const { workspaceId } = parsed.data.query - try { - const aborted = await cancelKnowledgeDocumentUpload.execute({ - principal: { kind: 'session', userId: actor.id, sessionId: actor.sessionId }, - input: { - knowledgeBaseId, - assertedWorkspaceId: workspaceId, - uploadId, - uploadToken: parsed.data.headers['upload-token'], - }, - request, - }) - return NextResponse.json({ data: toV2KnowledgeDocumentUpload(aborted, null) }) - } catch (error) { - const classified = knowledgeDocumentUploadErrorResponse(error) - if (classified) return classified - throw error - } - } -) +export const DELETE = defineInternalJsonRoute({ + contract: abortKnowledgeDocumentUploadContract, + auth: internalSessionAuth, + operation: knowledgeOperations.uploadCancel, + rateLimit: internalRateLimits.none({ + reason: 'Preserve existing internal upload-session cancellation behavior', + }), + errorPolicy: internalKnowledgeErrorPolicies.uploads, + mapInput: ({ params, query, headers }) => ({ + knowledgeBaseId: params.id, + assertedWorkspaceId: query.workspaceId, + uploadId: params.uploadId, + uploadToken: headers['upload-token'], + }), + useCase: cancelKnowledgeDocumentUpload, + present: (session) => ({ data: toInternalKnowledgeDocumentUpload(session, null) }), +}) diff --git a/apps/sim/app/api/knowledge/[id]/documents/uploads/control-routes.test.ts b/apps/sim/app/api/knowledge/[id]/documents/uploads/control-routes.test.ts deleted file mode 100644 index 8a4b043cfff..00000000000 --- a/apps/sim/app/api/knowledge/[id]/documents/uploads/control-routes.test.ts +++ /dev/null @@ -1,182 +0,0 @@ -/** - * @vitest-environment node - */ -import { NextRequest } from 'next/server' -import { beforeEach, describe, expect, it, vi } from 'vitest' - -const mocks = vi.hoisted(() => ({ - cancel: vi.fn(), - captureServerEvent: vi.fn(), - complete: vi.fn(), - parts: vi.fn(), - platformEvent: vi.fn(), - requireActor: vi.fn(), -})) - -vi.mock('@/lib/knowledge/application/upload-sessions', () => ({ - cancelKnowledgeDocumentUpload: { execute: mocks.cancel }, - completeKnowledgeDocumentUpload: { execute: mocks.complete }, - issueKnowledgeDocumentUploadParts: { execute: mocks.parts }, -})) - -vi.mock('@/lib/core/telemetry', () => ({ - PlatformEvents: { knowledgeBaseDocumentsUploaded: mocks.platformEvent }, -})) -vi.mock('@/lib/posthog/server', () => ({ captureServerEvent: mocks.captureServerEvent })) -vi.mock('@/app/api/knowledge/[id]/documents/uploads/utils', () => ({ - knowledgeDocumentUploadErrorResponse: vi.fn(() => null), - requireKnowledgeDocumentUploadActor: mocks.requireActor, -})) -vi.mock('@/app/api/v2/knowledge/[id]/documents/uploads/utils', () => ({ - toV2KnowledgeDocumentUpload: (_session: unknown, document: unknown) => ({ - id: 'upload-1', - knowledgeBaseId: 'kb-1', - status: document ? 'completed' : 'aborted', - name: 'guide.pdf', - contentType: 'application/pdf', - size: 1024, - expiresAt: '2026-08-05T00:00:00.000Z', - error: null, - document, - }), -})) - -import { POST as COMPLETE } from '@/app/api/knowledge/[id]/documents/uploads/[uploadId]/complete/route' -import { POST as PARTS } from '@/app/api/knowledge/[id]/documents/uploads/[uploadId]/parts/route' -import { DELETE as CANCEL } from '@/app/api/knowledge/[id]/documents/uploads/[uploadId]/route' - -const WORKSPACE_ID = '6fc7631d-88cd-46f8-9f0a-d4764daef7f8' -const PRINCIPAL = { kind: 'session', userId: 'user-1', sessionId: 'session-1' } -const SESSION = { id: 'upload-1', knowledgeBaseId: 'kb-1' } -const DOCUMENT = { - id: 'upload-1', - knowledgeBaseId: 'kb-1', - filename: 'guide.pdf', - fileSize: 1024, - mimeType: 'application/pdf', - chunkCount: 0, - tokenCount: 0, - characterCount: 0, - enabled: true, - uploadedAt: new Date('2026-08-03T21:01:00.000Z'), -} - -function routeContext() { - return { params: Promise.resolve({ id: 'kb-1', uploadId: 'upload-1' }) } -} - -function controlUrl(suffix = '') { - return `http://localhost:3000/api/knowledge/kb-1/documents/uploads/upload-1${suffix}?workspaceId=${WORKSPACE_ID}` -} - -describe('internal knowledge-document upload control routes', () => { - beforeEach(() => { - vi.clearAllMocks() - mocks.requireActor.mockResolvedValue({ id: 'user-1', sessionId: 'session-1' }) - mocks.parts.mockResolvedValue({ - parts: [ - { - partNumber: 1, - url: 'https://storage.example/1', - headers: {}, - expiresAt: '2026-08-04T21:00:00.000Z', - }, - ], - }) - mocks.cancel.mockResolvedValue(SESSION) - mocks.complete.mockResolvedValue({ - session: SESSION, - value: { document: DOCUMENT, created: true, knowledgeBaseName: 'Docs' }, - alreadyCompleted: false, - workspaceId: WORKSPACE_ID, - knowledgeBaseId: 'kb-1', - }) - }) - - it('delegates multipart part signing with the current session principal', async () => { - const request = new NextRequest(controlUrl('/parts'), { - method: 'POST', - headers: { 'content-type': 'application/json', 'upload-token': 'token' }, - body: JSON.stringify({ partNumbers: [1] }), - }) - - const response = await PARTS(request, routeContext()) - - expect(response.status).toBe(200) - expect(mocks.parts).toHaveBeenCalledWith({ - principal: PRINCIPAL, - input: { - knowledgeBaseId: 'kb-1', - assertedWorkspaceId: WORKSPACE_ID, - uploadId: 'upload-1', - uploadToken: 'token', - partNumbers: [1], - }, - request, - }) - }) - - it('delegates cancellation with the current session principal', async () => { - const request = new NextRequest(controlUrl(), { - method: 'DELETE', - headers: { 'upload-token': 'token' }, - }) - - const response = await CANCEL(request, routeContext()) - - expect(response.status).toBe(200) - expect(mocks.cancel).toHaveBeenCalledWith({ - principal: PRINCIPAL, - input: { - knowledgeBaseId: 'kb-1', - assertedWorkspaceId: WORKSPACE_ID, - uploadId: 'upload-1', - uploadToken: 'token', - }, - request, - }) - }) - - it('delegates completion and emits UI analytics only for a new document', async () => { - const request = new NextRequest(controlUrl('/complete'), { - method: 'POST', - headers: { 'upload-token': 'token' }, - }) - - const response = await COMPLETE(request, routeContext()) - - expect(response.status).toBe(200) - expect(mocks.complete).toHaveBeenCalledWith({ - principal: PRINCIPAL, - input: { - knowledgeBaseId: 'kb-1', - assertedWorkspaceId: WORKSPACE_ID, - uploadId: 'upload-1', - uploadToken: 'token', - source: 'ui', - }, - request, - }) - expect(mocks.captureServerEvent).toHaveBeenCalledTimes(1) - expect(mocks.platformEvent).toHaveBeenCalledTimes(1) - }) - - it('does not duplicate UI analytics on an idempotent completion retry', async () => { - mocks.complete.mockResolvedValue({ - session: SESSION, - value: { document: DOCUMENT, created: false, knowledgeBaseName: 'Docs' }, - alreadyCompleted: true, - workspaceId: WORKSPACE_ID, - knowledgeBaseId: 'kb-1', - }) - const request = new NextRequest(controlUrl('/complete'), { - method: 'POST', - headers: { 'upload-token': 'token' }, - }) - - await COMPLETE(request, routeContext()) - - expect(mocks.captureServerEvent).not.toHaveBeenCalled() - expect(mocks.platformEvent).not.toHaveBeenCalled() - }) -}) diff --git a/apps/sim/app/api/knowledge/[id]/documents/uploads/route.test.ts b/apps/sim/app/api/knowledge/[id]/documents/uploads/route.test.ts deleted file mode 100644 index afc86b5d24a..00000000000 --- a/apps/sim/app/api/knowledge/[id]/documents/uploads/route.test.ts +++ /dev/null @@ -1,104 +0,0 @@ -/** - * @vitest-environment node - */ -import { NextRequest } from 'next/server' -import { beforeEach, describe, expect, it, vi } from 'vitest' - -const mocks = vi.hoisted(() => ({ - createUpload: vi.fn(), - requireActor: vi.fn(), -})) - -vi.mock('@/lib/knowledge/application/upload-sessions', () => ({ - createKnowledgeDocumentUpload: { execute: mocks.createUpload }, -})) - -vi.mock('@/app/api/knowledge/[id]/documents/uploads/utils', () => ({ - knowledgeDocumentUploadErrorResponse: vi.fn(() => null), - requireKnowledgeDocumentUploadActor: mocks.requireActor, -})) - -vi.mock('@/app/api/v2/knowledge/[id]/documents/uploads/utils', () => ({ - toV2KnowledgeDocumentUpload: (session: Record) => ({ - id: session.id, - knowledgeBaseId: session.knowledgeBaseId, - status: session.status, - name: session.fileName, - contentType: session.contentType, - size: session.fileSize, - expiresAt: '2026-08-05T00:00:00.000Z', - error: null, - document: null, - }), -})) - -import { POST } from '@/app/api/knowledge/[id]/documents/uploads/route' - -const WORKSPACE_ID = '6fc7631d-88cd-46f8-9f0a-d4764daef7f8' -const SESSION = { - id: 'upload-1', - knowledgeBaseId: 'kb-1', - status: 'uploading', - fileName: 'guide.pdf', - contentType: 'application/pdf', - fileSize: 1024, - uploadToken: 'token', - transfer: { - method: 'put' as const, - url: 'https://storage.example/upload', - headers: { 'content-type': 'application/pdf' }, - }, -} - -function request() { - const request = new NextRequest('http://localhost:3000/api/knowledge/kb-1/documents/uploads', { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ - workspaceId: WORKSPACE_ID, - name: 'guide.pdf', - contentType: 'application/pdf', - size: 1024, - tag1: 'product', - }), - }) - return { - request, - response: POST(request, { params: Promise.resolve({ id: 'kb-1' }) }), - } -} - -describe('POST /api/knowledge/[id]/documents/uploads', () => { - beforeEach(() => { - vi.clearAllMocks() - mocks.requireActor.mockResolvedValue({ - id: 'user-1', - sessionId: 'session-1', - name: 'User', - email: 'user@example.com', - }) - mocks.createUpload.mockResolvedValue(SESSION) - }) - - it('constructs a server-authored session principal and delegates creation', async () => { - const call = request() - const response = await call.response - - expect(response.status).toBe(201) - expect(mocks.createUpload).toHaveBeenCalledWith({ - principal: { kind: 'session', userId: 'user-1', sessionId: 'session-1' }, - input: { - knowledgeBaseId: 'kb-1', - assertedWorkspaceId: WORKSPACE_ID, - name: 'guide.pdf', - contentType: 'application/pdf', - size: 1024, - metadata: { tag1: 'product' }, - }, - request: call.request, - }) - expect(await response.json()).toMatchObject({ - data: { session: { id: 'upload-1' }, uploadToken: 'token' }, - }) - }) -}) diff --git a/apps/sim/app/api/knowledge/[id]/documents/uploads/route.ts b/apps/sim/app/api/knowledge/[id]/documents/uploads/route.ts index 26a46296fe2..3d50830db0b 100644 --- a/apps/sim/app/api/knowledge/[id]/documents/uploads/route.ts +++ b/apps/sim/app/api/knowledge/[id]/documents/uploads/route.ts @@ -1,53 +1,39 @@ -import { type NextRequest, NextResponse } from 'next/server' import { createKnowledgeDocumentUploadContract } from '@/lib/api/contracts/knowledge/upload-sessions' -import { parseRequest } from '@/lib/api/server' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createKnowledgeDocumentUpload } from '@/lib/knowledge/application/upload-sessions' import { - knowledgeDocumentUploadErrorResponse, - requireKnowledgeDocumentUploadActor, -} from '@/app/api/knowledge/[id]/documents/uploads/utils' -import { toV2KnowledgeDocumentUpload } from '@/app/api/v2/knowledge/[id]/documents/uploads/utils' - -interface KnowledgeDocumentUploadsRouteParams { - params: Promise<{ id: string }> -} + defineInternalJsonRoute, + internalRateLimits, + internalSessionAuth, +} from '@/lib/api/server/routes' +import { toInternalKnowledgeDocumentUpload } from '@/lib/knowledge/api/internal-route' +import { internalKnowledgeErrorPolicies } from '@/lib/knowledge/api/route-policies' +import { knowledgeOperations } from '@/lib/knowledge/application/operations' +import { createKnowledgeDocumentUpload } from '@/lib/knowledge/application/upload-sessions' -export const POST = withRouteHandler( - async (request: NextRequest, context: KnowledgeDocumentUploadsRouteParams) => { - const actor = await requireKnowledgeDocumentUploadActor() - if (actor instanceof NextResponse) return actor - const parsed = await parseRequest(createKnowledgeDocumentUploadContract, request, context) - if (!parsed.success) return parsed.response - const { id: knowledgeBaseId } = parsed.data.params - const { workspaceId, name, contentType, size, ...metadata } = parsed.data.body - try { - const upload = await createKnowledgeDocumentUpload.execute({ - principal: { kind: 'session', userId: actor.id, sessionId: actor.sessionId }, - input: { - knowledgeBaseId, - assertedWorkspaceId: workspaceId, - name, - contentType, - size, - metadata, - }, - request, - }) - return NextResponse.json( - { - data: { - session: toV2KnowledgeDocumentUpload(upload, null), - uploadToken: upload.uploadToken, - transfer: upload.transfer, - }, - }, - { status: 201 } - ) - } catch (error) { - const classified = knowledgeDocumentUploadErrorResponse(error) - if (classified) return classified - throw error +export const POST = defineInternalJsonRoute({ + contract: createKnowledgeDocumentUploadContract, + auth: internalSessionAuth, + operation: knowledgeOperations.uploadCreate, + rateLimit: internalRateLimits.none({ + reason: 'Preserve existing internal upload-session creation behavior', + }), + errorPolicy: internalKnowledgeErrorPolicies.uploads, + mapInput: ({ params, body }) => { + const { workspaceId, name, contentType, size, ...metadata } = body + return { + knowledgeBaseId: params.id, + assertedWorkspaceId: workspaceId, + name, + contentType, + size, + metadata, } - } -) + }, + useCase: createKnowledgeDocumentUpload, + present: (upload) => ({ + data: { + session: toInternalKnowledgeDocumentUpload(upload, null), + uploadToken: upload.uploadToken, + transfer: upload.transfer, + }, + }), +}) diff --git a/apps/sim/app/api/knowledge/[id]/documents/uploads/utils.test.ts b/apps/sim/app/api/knowledge/[id]/documents/uploads/utils.test.ts deleted file mode 100644 index 80ae8df1f47..00000000000 --- a/apps/sim/app/api/knowledge/[id]/documents/uploads/utils.test.ts +++ /dev/null @@ -1,36 +0,0 @@ -/** - * @vitest-environment node - */ -import { beforeEach, describe, expect, it, vi } from 'vitest' - -const mocks = vi.hoisted(() => ({ getSession: vi.fn() })) - -vi.mock('@/lib/auth', () => ({ getSession: mocks.getSession })) - -import { requireKnowledgeDocumentUploadActor } from '@/app/api/knowledge/[id]/documents/uploads/utils' - -describe('knowledge-document upload session authentication', () => { - beforeEach(() => vi.clearAllMocks()) - - it('returns the authoritative session id with the authenticated user', async () => { - mocks.getSession.mockResolvedValue({ - user: { id: 'user-1', name: 'User', email: 'user@example.com' }, - session: { id: 'session-1' }, - }) - - await expect(requireKnowledgeDocumentUploadActor()).resolves.toEqual({ - id: 'user-1', - sessionId: 'session-1', - name: 'User', - email: 'user@example.com', - }) - }) - - it('fails fast when authenticated state has no session id', async () => { - mocks.getSession.mockResolvedValue({ user: { id: 'user-1' }, session: {} }) - - await expect(requireKnowledgeDocumentUploadActor()).rejects.toThrow( - 'Authenticated session is missing its session ID' - ) - }) -}) diff --git a/apps/sim/app/api/knowledge/[id]/documents/uploads/utils.ts b/apps/sim/app/api/knowledge/[id]/documents/uploads/utils.ts deleted file mode 100644 index 04aa96cc40a..00000000000 --- a/apps/sim/app/api/knowledge/[id]/documents/uploads/utils.ts +++ /dev/null @@ -1,39 +0,0 @@ -import { NextResponse } from 'next/server' -import { getSession } from '@/lib/auth' -import { KnowledgeUsageLimitExceededError } from '@/lib/knowledge/application/billing' -import { KnowledgeDocumentUnsupportedMediaTypeError } from '@/lib/knowledge/application/upload-sessions' -import { uploadSessionErrorResponse } from '@/app/api/files/uploads/utils' - -export interface KnowledgeDocumentUploadActor { - id: string - sessionId: string - name?: string | null - email?: string | null -} - -export async function requireKnowledgeDocumentUploadActor(): Promise< - KnowledgeDocumentUploadActor | NextResponse -> { - const session = await getSession() - if (!session?.user?.id) { - return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) - } - const sessionId = session.session?.id - if (!sessionId) throw new Error('Authenticated session is missing its session ID') - return { - id: session.user.id, - sessionId, - name: session.user.name, - email: session.user.email, - } -} - -export function knowledgeDocumentUploadErrorResponse(error: unknown): NextResponse | null { - if (error instanceof KnowledgeDocumentUnsupportedMediaTypeError) { - return NextResponse.json({ error: error.message }, { status: 415 }) - } - if (error instanceof KnowledgeUsageLimitExceededError) { - return NextResponse.json({ error: error.message }, { status: 402 }) - } - return uploadSessionErrorResponse(error) -} diff --git a/apps/sim/app/api/knowledge/[id]/documents/upsert/route.test.ts b/apps/sim/app/api/knowledge/[id]/documents/upsert/route.test.ts deleted file mode 100644 index ddd9c80cb7f..00000000000 --- a/apps/sim/app/api/knowledge/[id]/documents/upsert/route.test.ts +++ /dev/null @@ -1,114 +0,0 @@ -/** - * Tests for knowledge base document upsert API route - * - * @vitest-environment node - */ -import { - auditMock, - createMockRequest, - hybridAuthMockFns, - knowledgeApiUtilsMock, - resetDbChainMock, -} from '@sim/testing' -import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest' - -vi.mock('@/app/api/knowledge/utils', () => knowledgeApiUtilsMock) -vi.mock('@sim/audit', () => auditMock) - -vi.mock('@/lib/billing/calculations/usage-monitor', () => ({ - checkActorUsageLimits: vi.fn().mockResolvedValue({ isExceeded: false }), -})) - -vi.mock('@/lib/billing/core/billing-attribution', () => ({ - resolveBillingAttribution: vi.fn().mockResolvedValue({ - actorUserId: 'user-1', - workspaceId: 'ws-1', - billingEntity: { type: 'organization', id: 'org-1' }, - }), - checkAttributedUsageLimits: vi.fn().mockResolvedValue({ isExceeded: false }), -})) - -vi.mock('@/lib/knowledge/documents/service', () => ({ - createDocumentRecords: vi.fn(), - deleteDocument: vi.fn(), - getProcessingConfig: vi.fn().mockReturnValue({ maxConcurrentDocuments: 1, batchSize: 1 }), - processDocumentsWithQueue: vi.fn(), - KnowledgeBaseFileOwnershipError: class KnowledgeBaseFileOwnershipError extends Error {}, -})) - -import { createDocumentRecords, processDocumentsWithQueue } from '@/lib/knowledge/documents/service' -import { POST } from '@/app/api/knowledge/[id]/documents/upsert/route' -import { checkKnowledgeBaseWriteAccess } from '@/app/api/knowledge/utils' - -describe('POST /api/knowledge/[id]/documents/upsert', () => { - const params = Promise.resolve({ id: 'kb-123' }) - - beforeEach(() => { - vi.clearAllMocks() - resetDbChainMock() - - hybridAuthMockFns.mockCheckSessionOrInternalAuth.mockResolvedValue({ - success: true, - userId: 'user-1', - authType: 'session', - userName: 'Test User', - userEmail: 'test@example.com', - }) - - vi.mocked(checkKnowledgeBaseWriteAccess).mockResolvedValue({ - hasAccess: true, - knowledgeBase: { id: 'kb-123', userId: 'user-1', workspaceId: 'ws-1', name: 'KB' }, - } as any) - - vi.mocked(createDocumentRecords).mockResolvedValue([ - { documentId: 'doc-new', filename: 'note.txt' }, - ] as any) - vi.mocked(processDocumentsWithQueue).mockResolvedValue(undefined as any) - }) - - afterAll(() => { - resetDbChainMock() - }) - - const baseBody = { - filename: 'note.txt', - fileSize: 11, - mimeType: 'text/plain', - } - - it('accepts a data: URI', async () => { - const req = createMockRequest('POST', { - ...baseBody, - fileUrl: 'data:text/plain;base64,SGVsbG8gd29ybGQ=', - }) - const res = await POST(req, { params }) - expect(res.status).toBe(200) - expect(createDocumentRecords).toHaveBeenCalled() - }) - - it('accepts an https URL', async () => { - const req = createMockRequest('POST', { - ...baseBody, - fileUrl: 'https://example.com/note.txt', - }) - const res = await POST(req, { params }) - expect(res.status).toBe(200) - expect(createDocumentRecords).toHaveBeenCalled() - }) - - it.each([ - ['absolute local path', '/etc/passwd'], - ['app config path', '/app/.env'], - ['file:// URL', 'file:///etc/passwd'], - ['relative serve path', '/api/files/serve/kb/foo.pdf'], - ['ftp URL', 'ftp://example.com/file.pdf'], - ['parent traversal', '../../etc/passwd'], - ['windows path', 'C:\\Windows\\System32\\config\\SAM'], - ])('rejects %s with 400 and never invokes the pipeline', async (_label, fileUrl) => { - const req = createMockRequest('POST', { ...baseBody, fileUrl }) - const res = await POST(req, { params }) - expect(res.status).toBe(400) - expect(createDocumentRecords).not.toHaveBeenCalled() - expect(processDocumentsWithQueue).not.toHaveBeenCalled() - }) -}) diff --git a/apps/sim/app/api/knowledge/[id]/documents/upsert/route.ts b/apps/sim/app/api/knowledge/[id]/documents/upsert/route.ts index 8792517dd4e..2da21e78cd9 100644 --- a/apps/sim/app/api/knowledge/[id]/documents/upsert/route.ts +++ b/apps/sim/app/api/knowledge/[id]/documents/upsert/route.ts @@ -1,308 +1,92 @@ -import { AuditAction, AuditResourceType, recordAudit } from '@sim/audit' -import { db } from '@sim/db' -import { document } from '@sim/db/schema' -import { createLogger } from '@sim/logger' -import { authorizeWorkflowByWorkspacePermission } from '@sim/platform-authz/workflow' -import { getErrorMessage, toError } from '@sim/utils/errors' -import { generateId } from '@sim/utils/id' -import { and, eq, isNull } from 'drizzle-orm' -import { type NextRequest, NextResponse } from 'next/server' import { upsertKnowledgeDocumentContract } from '@/lib/api/contracts/knowledge' -import { parseRequest } from '@/lib/api/server' -import { AuthType, checkSessionOrInternalAuth } from '@/lib/auth/hybrid' -import { checkActorUsageLimits } from '@/lib/billing/calculations/usage-monitor' +import { defineInternalJsonRoute, internalRateLimits } from '@/lib/api/server/routes' +import { OrchestrationError } from '@/lib/core/orchestration/types' import { - checkAttributedUsageLimits, - requireBillingAttributionHeader, - resolveBillingAttribution, -} from '@/lib/billing/core/billing-attribution' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' + internalKnowledgeAnalytics, + internalKnowledgeAuthType, + resolveInternalKnowledgeBillingAttribution, +} from '@/lib/knowledge/api/internal-route' import { - createDocumentRecords, - deleteDocument, - getProcessingConfig, - KnowledgeBaseFileOwnershipError, - processDocumentsWithQueue, -} from '@/lib/knowledge/documents/service' + internalKnowledgeErrorPolicies, + internalKnowledgeSessionOrExecutorAuth, +} from '@/lib/knowledge/api/route-policies' +import { upsertKnowledgeDocument } from '@/lib/knowledge/application/documents' +import { knowledgeOperations } from '@/lib/knowledge/application/operations' import { - createKnowledgeProvenanceResponse, + finalizeKnowledgeProvenanceResponse, resolveKnowledgeDocumentWriteSecretProvenance, } from '@/app/api/knowledge/secret-provenance' -import { checkKnowledgeBaseWriteAccess } from '@/app/api/knowledge/utils' -const logger = createLogger('DocumentUpsertAPI') - -export const POST = withRouteHandler( - async (req: NextRequest, context: { params: Promise<{ id: string }> }) => { - const requestId = generateId().slice(0, 8) - const { id: knowledgeBaseId } = await context.params - - try { - const parsed = await parseRequest(upsertKnowledgeDocumentContract, req, context) - if (!parsed.success) return parsed.response - const validatedData = parsed.data.body - - logger.info(`[${requestId}] Knowledge base document upsert request`, { - knowledgeBaseId, - hasDocumentId: !!validatedData.documentId, - mimeType: validatedData.mimeType, - fileSize: validatedData.fileSize, - }) - - const auth = await checkSessionOrInternalAuth(req, { requireWorkflowId: false }) - if (!auth.success || !auth.userId) { - logger.warn(`[${requestId}] Authentication failed: ${auth.error || 'Unauthorized'}`) - return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) - } - const userId = auth.userId - - if (validatedData.workflowId) { - const authorization = await authorizeWorkflowByWorkspacePermission({ - workflowId: validatedData.workflowId, - userId, - action: 'write', - }) - if (!authorization.allowed) { - return NextResponse.json( - { error: authorization.message || 'Access denied' }, - { status: authorization.status } - ) - } - } - - const accessCheck = await checkKnowledgeBaseWriteAccess(knowledgeBaseId, userId) - - if (!accessCheck.hasAccess) { - if ('notFound' in accessCheck && accessCheck.notFound) { - logger.warn(`[${requestId}] Knowledge base not found: ${knowledgeBaseId}`) - return NextResponse.json({ error: 'Knowledge base not found' }, { status: 404 }) - } - logger.warn( - `[${requestId}] User ${userId} attempted to upsert document in unauthorized knowledge base ${knowledgeBaseId}` - ) - return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) - } - - /** - * Gate the workspace payer and uploader before mutation so an over-limit - * upsert cannot delete an indexed document. Workspace-less legacy KBs - * retain account-only enforcement. - */ - const kbWorkspaceId = accessCheck.knowledgeBase?.workspaceId - const billingAttribution = kbWorkspaceId - ? auth.authType === AuthType.INTERNAL_JWT - ? requireBillingAttributionHeader(req.headers, { - actorUserId: userId, - workspaceId: kbWorkspaceId, - }) - : await resolveBillingAttribution({ - actorUserId: userId, - workspaceId: kbWorkspaceId, - }) - : undefined - const usage = billingAttribution - ? await checkAttributedUsageLimits(billingAttribution) - : await checkActorUsageLimits(userId) - if (usage.isExceeded) { - return NextResponse.json( - { - error: usage.message || 'Usage limit exceeded. Please upgrade your plan to continue.', - }, - { status: 402 } - ) - } - - const writeProvenance = resolveKnowledgeDocumentWriteSecretProvenance({ - request: req, - payload: validatedData, - authType: auth.authType, - userId, - ...(kbWorkspaceId ? { workspaceId: kbWorkspaceId } : {}), - documents: [validatedData], - }) - if (!writeProvenance.success) return writeProvenance.response - - let existingDocumentId: string | null = null - let isUpdate = false - - if (validatedData.documentId) { - const existingDoc = await db - .select({ id: document.id }) - .from(document) - .where( - and( - eq(document.id, validatedData.documentId), - eq(document.knowledgeBaseId, knowledgeBaseId), - isNull(document.deletedAt) - ) - ) - .limit(1) - - if (existingDoc.length > 0) { - existingDocumentId = existingDoc[0].id - } - } else { - const docsByFilename = await db - .select({ id: document.id }) - .from(document) - .where( - and( - eq(document.filename, validatedData.filename), - eq(document.knowledgeBaseId, knowledgeBaseId), - isNull(document.deletedAt) - ) - ) - .limit(1) - - if (docsByFilename.length > 0) { - existingDocumentId = docsByFilename[0].id - } - } - - if (existingDocumentId) { - isUpdate = true - logger.info( - `[${requestId}] Found existing document ${existingDocumentId}, creating replacement before deleting old` - ) - } - - const createdDocuments = await createDocumentRecords( - [ - { - filename: validatedData.filename, - fileUrl: validatedData.fileUrl, - fileSize: validatedData.fileSize, - mimeType: validatedData.mimeType, - ...(validatedData.documentTagsData && { - documentTagsData: validatedData.documentTagsData, - }), - }, - ], - knowledgeBaseId, - requestId, +export const POST = defineInternalJsonRoute({ + contract: upsertKnowledgeDocumentContract, + auth: internalKnowledgeSessionOrExecutorAuth, + operation: knowledgeOperations.uploadDocument, + rateLimit: internalRateLimits.none({ + reason: 'Preserve existing internal document-upsert behavior', + }), + errorPolicy: internalKnowledgeErrorPolicies.upsert, + parseOptions: { maxBodyBytes: 2 * 1024 * 1024 }, + mapInput: ({ params, body }, { principal, request }) => ({ + knowledgeBaseId: params.id, + documentId: body.documentId, + filename: body.filename, + fileUrl: body.fileUrl, + fileSize: body.fileSize, + mimeType: body.mimeType, + documentTagsData: body.documentTagsData, + processingOptions: body.processingOptions, + resolveBillingAttribution: (workspaceId: string) => + resolveInternalKnowledgeBillingAttribution(request, principal, workspaceId), + resolveSecretProvenances: ({ + userId, + workspaceId, + }: { + userId: string + workspaceId: string + }) => { + const resolved = resolveKnowledgeDocumentWriteSecretProvenance({ + request, + payload: body, + authType: internalKnowledgeAuthType(principal), userId, - writeProvenance.provenances - ) - - const firstDocument = createdDocuments[0] - if (!firstDocument) { - logger.error(`[${requestId}] createDocumentRecords returned empty array unexpectedly`) - return NextResponse.json({ error: 'Failed to create document record' }, { status: 500 }) - } - - if (existingDocumentId) { - try { - await deleteDocument(existingDocumentId, requestId) - } catch (deleteError) { - logger.error( - `[${requestId}] Failed to delete old document ${existingDocumentId}, rolling back new record`, - { errorType: toError(deleteError).name } - ) - await deleteDocument(firstDocument.documentId, requestId).catch(() => {}) - return NextResponse.json( - { error: 'Failed to replace existing document' }, - { status: 500 } - ) - } - } - - processDocumentsWithQueue( - createdDocuments, - knowledgeBaseId, - validatedData.processingOptions ?? {}, - requestId, - billingAttribution - ).catch((error: unknown) => { - logger.error(`[${requestId}] Critical error in document processing pipeline`, { - errorType: toError(error).name, - }) + workspaceId, + documents: [body], }) - - try { - const { PlatformEvents } = await import('@/lib/core/telemetry') - PlatformEvents.knowledgeBaseDocumentsUploaded({ - knowledgeBaseId, - documentsCount: 1, - uploadType: 'single', - recipe: validatedData.processingOptions?.recipe, - }) - } catch (_e) { - // Silently fail + if (!resolved.success) { + throw new OrchestrationError('validation', 'Invalid knowledge secret provenance') } - - recordAudit({ - workspaceId: accessCheck.knowledgeBase?.workspaceId ?? null, - actorId: userId, - actorName: auth.userName, - actorEmail: auth.userEmail, - action: isUpdate ? AuditAction.DOCUMENT_UPDATED : AuditAction.DOCUMENT_UPLOADED, - resourceType: AuditResourceType.DOCUMENT, - resourceId: knowledgeBaseId, - resourceName: validatedData.filename, - description: isUpdate - ? `Upserted (replaced) document "${validatedData.filename}" in knowledge base "${knowledgeBaseId}"` - : `Upserted (created) document "${validatedData.filename}" in knowledge base "${knowledgeBaseId}"`, - metadata: { - knowledgeBaseName: accessCheck.knowledgeBase?.name, - fileName: validatedData.filename, - fileType: validatedData.mimeType, - fileSize: validatedData.fileSize, - previousDocumentId: existingDocumentId, - isUpdate, - }, - request: req, - }) - - return createKnowledgeProvenanceResponse({ - request: req, - authType: auth.authType, - userId, - ...(kbWorkspaceId ? { workspaceId: kbWorkspaceId } : {}), - provenances: - writeProvenance.provenances?.flatMap((provenance) => [ - provenance.filename, - ...provenance.tags.map((tag) => tag.provenance), - ]) ?? [], - body: { - success: true, - data: { - documentsCreated: [ - { - documentId: firstDocument.documentId, - filename: firstDocument.filename, - status: 'pending', - }, - ], - isUpdate, - previousDocumentId: existingDocumentId, - processingMethod: 'background', - processingConfig: { - maxConcurrentDocuments: getProcessingConfig().maxConcurrentDocuments, - batchSize: getProcessingConfig().batchSize, - }, - }, + return resolved.provenances + }, + }), + useCase: upsertKnowledgeDocument, + onSuccess: internalKnowledgeAnalytics.documentUpserted, + present: ({ document, isUpdate, previousDocumentId, processingConfig }) => ({ + success: true as const, + data: { + documentsCreated: [ + { + documentId: document.documentId, + filename: document.filename, + status: 'pending' as const, }, - }) - } catch (error) { - logger.error(`[${requestId}] Error upserting document`, { - errorType: toError(error).name, - }) - - if (error instanceof KnowledgeBaseFileOwnershipError) { - return NextResponse.json( - { error: 'File URL does not reference a file owned by this knowledge base' }, - { status: 403 } - ) - } - const errorMessage = getErrorMessage(error, 'Failed to upsert document') - const isStorageLimitError = - errorMessage.includes('Storage limit exceeded') || errorMessage.includes('storage limit') - const isMissingKnowledgeBase = errorMessage === 'Knowledge base not found' - - return NextResponse.json( - { error: errorMessage }, - { status: isMissingKnowledgeBase ? 404 : isStorageLimitError ? 413 : 500 } - ) - } - } -) + ], + isUpdate, + previousDocumentId, + processingMethod: 'background' as const, + processingConfig, + }, + }), + finalizeResponse: ({ request, principal, result, body }) => + finalizeKnowledgeProvenanceResponse({ + request, + authType: internalKnowledgeAuthType(principal), + userId: result.userId, + workspaceId: result.workspaceId, + body, + provenances: + result.secretProvenances?.flatMap((provenance) => [ + provenance.filename, + ...provenance.tags.map((tag) => tag.provenance), + ]) ?? [], + }), +}) diff --git a/apps/sim/app/api/knowledge/[id]/next-available-slot/route.ts b/apps/sim/app/api/knowledge/[id]/next-available-slot/route.ts index 6e46b7bb18f..ac872c35417 100644 --- a/apps/sim/app/api/knowledge/[id]/next-available-slot/route.ts +++ b/apps/sim/app/api/knowledge/[id]/next-available-slot/route.ts @@ -1,77 +1,19 @@ -import { createLogger } from '@sim/logger' -import { generateId } from '@sim/utils/id' -import { type NextRequest, NextResponse } from 'next/server' import { nextAvailableSlotContract } from '@/lib/api/contracts/knowledge' -import { parseRequest } from '@/lib/api/server' -import { getSession } from '@/lib/auth' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { getNextAvailableSlot, getTagDefinitions } from '@/lib/knowledge/tags/service' -import { checkKnowledgeBaseAccess } from '@/app/api/knowledge/utils' - -const logger = createLogger('NextAvailableSlotAPI') - -// GET /api/knowledge/[id]/next-available-slot - Get the next available tag slot for a knowledge base and field type -export const GET = withRouteHandler( - async (req: NextRequest, context: { params: Promise<{ id: string }> }) => { - const requestId = generateId().slice(0, 8) - const parsed = await parseRequest(nextAvailableSlotContract, req, context) - if (!parsed.success) { - return NextResponse.json({ error: 'fieldType parameter is required' }, { status: 400 }) - } - const { id: knowledgeBaseId } = parsed.data.params - const { fieldType } = parsed.data.query - - try { - logger.info( - `[${requestId}] Getting next available slot for knowledge base ${knowledgeBaseId}, fieldType: ${fieldType}` - ) - - const session = await getSession() - if (!session?.user?.id) { - return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) - } - - const accessCheck = await checkKnowledgeBaseAccess(knowledgeBaseId, session.user.id) - if (!accessCheck.hasAccess) { - return NextResponse.json( - { error: accessCheck.notFound ? 'Not found' : 'Forbidden' }, - { status: accessCheck.notFound ? 404 : 403 } - ) - } - - // Get existing definitions once and reuse - const existingDefinitions = await getTagDefinitions(knowledgeBaseId) - const usedSlots = existingDefinitions - .filter((def) => def.fieldType === fieldType) - .map((def) => def.tagSlot) - - // Create a map for efficient lookup and pass to avoid redundant query - const existingBySlot = new Map(existingDefinitions.map((def) => [def.tagSlot as string, def])) - const nextAvailableSlot = await getNextAvailableSlot( - knowledgeBaseId, - fieldType, - existingBySlot - ) - - logger.info( - `[${requestId}] Next available slot for fieldType ${fieldType}: ${nextAvailableSlot}` - ) - - const result = { - nextAvailableSlot, - fieldType, - usedSlots, - totalSlots: 7, - availableSlots: nextAvailableSlot ? 7 - usedSlots.length : 0, - } - - return NextResponse.json({ - success: true, - data: result, - }) - } catch (error) { - logger.error(`[${requestId}] Error getting next available slot`, error) - return NextResponse.json({ error: 'Failed to get next available slot' }, { status: 500 }) - } - } -) +import { defineInternalJsonRoute, internalRateLimits } from '@/lib/api/server/routes' +import { + internalKnowledgeErrorPolicies, + internalKnowledgeSessionOrExecutorAuth, +} from '@/lib/knowledge/api/route-policies' +import { knowledgeOperations } from '@/lib/knowledge/application/operations' +import { readNextKnowledgeTagSlot } from '@/lib/knowledge/application/tags' + +export const GET = defineInternalJsonRoute({ + contract: nextAvailableSlotContract, + auth: internalKnowledgeSessionOrExecutorAuth, + operation: knowledgeOperations.readNextTagSlot, + rateLimit: internalRateLimits.none({ reason: 'Preserve existing internal tag-slot behavior' }), + errorPolicy: internalKnowledgeErrorPolicies.tags, + mapInput: ({ params, query }) => ({ knowledgeBaseId: params.id, fieldType: query.fieldType }), + useCase: readNextKnowledgeTagSlot, + present: (data) => ({ success: true as const, data }), +}) diff --git a/apps/sim/app/api/knowledge/[id]/restore/route.ts b/apps/sim/app/api/knowledge/[id]/restore/route.ts index a5ed8b85808..fa65567869e 100644 --- a/apps/sim/app/api/knowledge/[id]/restore/route.ts +++ b/apps/sim/app/api/knowledge/[id]/restore/route.ts @@ -1,76 +1,23 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' import { restoreKnowledgeBaseContract } from '@/lib/api/contracts/knowledge' -import { parseRequest } from '@/lib/api/server' -import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid' import { - messageForOrchestrationError, - statusForOrchestrationError, -} from '@/lib/core/orchestration/types' -import { generateRequestId } from '@/lib/core/utils/request' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { - getRestorableKnowledgeBase, - performRestoreKnowledgeBase, -} from '@/lib/knowledge/orchestration' -import { getUserEntityPermissions } from '@/lib/workspaces/permissions/utils' - -const logger = createLogger('RestoreKnowledgeBaseAPI') - -export const POST = withRouteHandler( - async (request: NextRequest, context: { params: Promise<{ id: string }> }) => { - const requestId = generateRequestId() - const parsed = await parseRequest(restoreKnowledgeBaseContract, request, context) - if (!parsed.success) return parsed.response - const { id } = parsed.data.params - - try { - const auth = await checkSessionOrInternalAuth(request, { requireWorkflowId: false }) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) - } - - const kb = await getRestorableKnowledgeBase(id) - - if (!kb) { - return NextResponse.json({ error: 'Knowledge base not found' }, { status: 404 }) - } - - if (kb.workspaceId) { - const permission = await getUserEntityPermissions(auth.userId, 'workspace', kb.workspaceId) - if (permission !== 'admin' && permission !== 'write') { - return NextResponse.json({ error: 'Insufficient permissions' }, { status: 403 }) - } - } else if (kb.userId !== auth.userId) { - return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) - } - - const result = await performRestoreKnowledgeBase({ - knowledgeBaseId: id, - userId: auth.userId, - actorName: auth.userName, - actorEmail: auth.userEmail, - source: 'ui', - requestId, - request, - }) - if (!result.success) { - return NextResponse.json( - { error: messageForOrchestrationError(result, 'Failed to restore knowledge base') }, - { status: statusForOrchestrationError(result.errorCode) } - ) - } - - logger.info(`[${requestId}] Restored knowledge base ${id}`) - - return NextResponse.json({ success: true }) - } catch (error) { - logger.error(`[${requestId}] Error restoring knowledge base ${id}`, error) - return NextResponse.json( - { error: getErrorMessage(error, 'Internal server error') }, - { status: 500 } - ) - } - } -) + defineInternalJsonRoute, + internalJsonPresenters, + internalRateLimits, + internalSessionAuth, +} from '@/lib/api/server/routes' +import { internalKnowledgeErrorPolicies } from '@/lib/knowledge/api/route-policies' +import { restoreInternalKnowledgeBase } from '@/lib/knowledge/application/knowledge-bases' +import { knowledgeSessionOperations } from '@/lib/knowledge/application/operations' + +export const POST = defineInternalJsonRoute({ + contract: restoreKnowledgeBaseContract, + auth: internalSessionAuth, + operation: knowledgeSessionOperations.restore, + rateLimit: internalRateLimits.none({ + reason: 'Preserve existing internal knowledge base restore behavior', + }), + errorPolicy: internalKnowledgeErrorPolicies.restore, + mapInput: ({ params }) => ({ knowledgeBaseId: params.id }), + useCase: restoreInternalKnowledgeBase, + present: internalJsonPresenters.successFrom('success'), +}) diff --git a/apps/sim/app/api/knowledge/[id]/route.test.ts b/apps/sim/app/api/knowledge/[id]/route.test.ts deleted file mode 100644 index 882a1df6f0d..00000000000 --- a/apps/sim/app/api/knowledge/[id]/route.test.ts +++ /dev/null @@ -1,454 +0,0 @@ -/** - * Tests for knowledge base by ID API route - * - * @vitest-environment node - */ -import { - auditMock, - authMockFns, - createMockRequest, - knowledgeApiUtilsMock, - resetDbChainMock, -} from '@sim/testing' -import { afterAll, afterEach, beforeEach, describe, expect, it, vi } from 'vitest' - -vi.mock('@sim/audit', () => auditMock) - -vi.mock('@/lib/knowledge/service', async (importOriginal) => { - const actual = await importOriginal() - return { - ...actual, - getKnowledgeBaseById: vi.fn(), - updateKnowledgeBase: vi.fn(), - deleteKnowledgeBase: vi.fn(), - KnowledgeBasePermissionError: actual.KnowledgeBasePermissionError, - } -}) - -vi.mock('@/app/api/knowledge/utils', () => knowledgeApiUtilsMock) - -import { - deleteKnowledgeBase, - getKnowledgeBaseById, - KnowledgeBasePermissionError, - updateKnowledgeBase, -} from '@/lib/knowledge/service' -import { DELETE, GET, PUT } from '@/app/api/knowledge/[id]/route' -import { checkKnowledgeBaseAccess, checkKnowledgeBaseWriteAccess } from '@/app/api/knowledge/utils' - -describe('Knowledge Base By ID API Route', () => { - const mockKnowledgeBase = { - id: 'kb-123', - userId: 'user-123', - name: 'Test Knowledge Base', - description: 'Test description', - tokenCount: 100, - embeddingModel: 'text-embedding-3-small', - embeddingDimension: 1536, - chunkingConfig: { maxSize: 1024, minSize: 100, overlap: 200 }, - createdAt: new Date(), - updatedAt: new Date(), - workspaceId: null, - deletedAt: null, - } - - const resetMocks = () => { - vi.clearAllMocks() - resetDbChainMock() - } - - beforeEach(() => { - vi.clearAllMocks() - resetDbChainMock() - - vi.stubGlobal('crypto', { - randomUUID: vi.fn().mockReturnValue('mock-uuid-1234-5678'), - }) - }) - - afterEach(() => { - vi.clearAllMocks() - }) - - afterAll(() => { - resetDbChainMock() - }) - - describe('GET /api/knowledge/[id]', () => { - const mockParams = Promise.resolve({ id: 'kb-123' }) - - it('should retrieve knowledge base successfully for authenticated user', async () => { - authMockFns.mockGetSession.mockResolvedValue({ - user: { id: 'user-123', email: 'test@example.com' }, - }) - - vi.mocked(checkKnowledgeBaseAccess).mockResolvedValueOnce({ - hasAccess: true, - knowledgeBase: { id: 'kb-123', userId: 'user-123' }, - }) - - vi.mocked(getKnowledgeBaseById).mockResolvedValueOnce(mockKnowledgeBase) - - const req = createMockRequest('GET') - const response = await GET(req, { params: mockParams }) - const data = await response.json() - - expect(response.status).toBe(200) - expect(data.success).toBe(true) - expect(data.data.id).toBe('kb-123') - expect(data.data.name).toBe('Test Knowledge Base') - expect(checkKnowledgeBaseAccess).toHaveBeenCalledWith('kb-123', 'user-123') - expect(getKnowledgeBaseById).toHaveBeenCalledWith('kb-123') - }) - - it('should return unauthorized for unauthenticated user', async () => { - authMockFns.mockGetSession.mockResolvedValue(null) - - const req = createMockRequest('GET') - const response = await GET(req, { params: mockParams }) - const data = await response.json() - - expect(response.status).toBe(401) - expect(data.error).toBe('Unauthorized') - }) - - it('should return not found for non-existent knowledge base', async () => { - authMockFns.mockGetSession.mockResolvedValue({ - user: { id: 'user-123', email: 'test@example.com' }, - }) - - vi.mocked(checkKnowledgeBaseAccess).mockResolvedValueOnce({ - hasAccess: false, - notFound: true, - }) - - const req = createMockRequest('GET') - const response = await GET(req, { params: mockParams }) - const data = await response.json() - - expect(response.status).toBe(404) - expect(data.error).toBe('Knowledge base not found') - }) - - it('should return unauthorized for knowledge base owned by different user', async () => { - authMockFns.mockGetSession.mockResolvedValue({ - user: { id: 'user-123', email: 'test@example.com' }, - }) - - vi.mocked(checkKnowledgeBaseAccess).mockResolvedValueOnce({ - hasAccess: false, - notFound: false, - }) - - const req = createMockRequest('GET') - const response = await GET(req, { params: mockParams }) - const data = await response.json() - - expect(response.status).toBe(401) - expect(data.error).toBe('Unauthorized') - }) - - it('should return not found when service returns null', async () => { - authMockFns.mockGetSession.mockResolvedValue({ - user: { id: 'user-123', email: 'test@example.com' }, - }) - - vi.mocked(checkKnowledgeBaseAccess).mockResolvedValueOnce({ - hasAccess: true, - knowledgeBase: { id: 'kb-123', userId: 'user-123' }, - }) - - vi.mocked(getKnowledgeBaseById).mockResolvedValueOnce(null) - - const req = createMockRequest('GET') - const response = await GET(req, { params: mockParams }) - const data = await response.json() - - expect(response.status).toBe(404) - expect(data.error).toBe('Knowledge base not found') - }) - - it('should handle database errors', async () => { - authMockFns.mockGetSession.mockResolvedValue({ - user: { id: 'user-123', email: 'test@example.com' }, - }) - - vi.mocked(checkKnowledgeBaseAccess).mockRejectedValueOnce(new Error('Database error')) - - const req = createMockRequest('GET') - const response = await GET(req, { params: mockParams }) - const data = await response.json() - - expect(response.status).toBe(500) - expect(data.error).toBe('Failed to fetch knowledge base') - }) - }) - - describe('PUT /api/knowledge/[id]', () => { - const mockParams = Promise.resolve({ id: 'kb-123' }) - const validUpdateData = { - name: 'Updated Knowledge Base', - description: 'Updated description', - } - - it('should update knowledge base successfully', async () => { - authMockFns.mockGetSession.mockResolvedValue({ - user: { id: 'user-123', email: 'test@example.com' }, - }) - - resetMocks() - - vi.mocked(checkKnowledgeBaseWriteAccess).mockResolvedValueOnce({ - hasAccess: true, - knowledgeBase: { id: 'kb-123', userId: 'user-123' }, - }) - - const updatedKnowledgeBase = { ...mockKnowledgeBase, ...validUpdateData } - vi.mocked(updateKnowledgeBase).mockResolvedValueOnce(updatedKnowledgeBase) - - const req = createMockRequest('PUT', validUpdateData) - const response = await PUT(req, { params: mockParams }) - const data = await response.json() - - expect(response.status).toBe(200) - expect(data.success).toBe(true) - expect(data.data.name).toBe('Updated Knowledge Base') - expect(checkKnowledgeBaseWriteAccess).toHaveBeenCalledWith('kb-123', 'user-123') - expect(updateKnowledgeBase).toHaveBeenCalledWith( - 'kb-123', - { - name: validUpdateData.name, - description: validUpdateData.description, - workspaceId: undefined, - chunkingConfig: undefined, - }, - expect.any(String), - { actorUserId: 'user-123' } - ) - }) - - it('returns 403 when service rejects a cross-workspace transfer', async () => { - authMockFns.mockGetSession.mockResolvedValue({ - user: { id: 'attacker', email: 'a@example.com' }, - }) - - resetMocks() - - vi.mocked(checkKnowledgeBaseWriteAccess).mockResolvedValueOnce({ - hasAccess: true, - knowledgeBase: { id: 'kb-123', userId: 'user-123', workspaceId: 'ws-current' }, - }) - - vi.mocked(updateKnowledgeBase).mockRejectedValueOnce( - new KnowledgeBasePermissionError('User does not have permission on the target workspace') - ) - - const req = createMockRequest('PUT', { workspaceId: 'ws-target' }) - const response = await PUT(req, { params: mockParams }) - const data = await response.json() - - expect(response.status).toBe(403) - expect(data.error).toBe('User does not have permission on the target workspace') - }) - - it('returns 403 when service rejects clearing workspaceId', async () => { - authMockFns.mockGetSession.mockResolvedValue({ - user: { id: 'user-123', email: 'test@example.com' }, - }) - - resetMocks() - - vi.mocked(checkKnowledgeBaseWriteAccess).mockResolvedValueOnce({ - hasAccess: true, - knowledgeBase: { id: 'kb-123', userId: 'user-123', workspaceId: 'ws-current' }, - }) - - vi.mocked(updateKnowledgeBase).mockRejectedValueOnce( - new KnowledgeBasePermissionError('Knowledge base workspace cannot be cleared') - ) - - const req = createMockRequest('PUT', { workspaceId: null }) - const response = await PUT(req, { params: mockParams }) - const data = await response.json() - - expect(response.status).toBe(403) - expect(data.error).toBe('Knowledge base workspace cannot be cleared') - }) - - it('should return unauthorized for unauthenticated user', async () => { - authMockFns.mockGetSession.mockResolvedValue(null) - - const req = createMockRequest('PUT', validUpdateData) - const response = await PUT(req, { params: mockParams }) - const data = await response.json() - - expect(response.status).toBe(401) - expect(data.error).toBe('Unauthorized') - }) - - it('should return not found for non-existent knowledge base', async () => { - authMockFns.mockGetSession.mockResolvedValue({ - user: { id: 'user-123', email: 'test@example.com' }, - }) - - resetMocks() - - vi.mocked(checkKnowledgeBaseWriteAccess).mockResolvedValueOnce({ - hasAccess: false, - notFound: true, - }) - - const req = createMockRequest('PUT', validUpdateData) - const response = await PUT(req, { params: mockParams }) - const data = await response.json() - - expect(response.status).toBe(404) - expect(data.error).toBe('Knowledge base not found') - }) - - it('should validate update data', async () => { - authMockFns.mockGetSession.mockResolvedValue({ - user: { id: 'user-123', email: 'test@example.com' }, - }) - - resetMocks() - - vi.mocked(checkKnowledgeBaseWriteAccess).mockResolvedValueOnce({ - hasAccess: true, - knowledgeBase: { id: 'kb-123', userId: 'user-123' }, - }) - - const invalidData = { - name: '', - } - - const req = createMockRequest('PUT', invalidData) - const response = await PUT(req, { params: mockParams }) - const data = await response.json() - - expect(response.status).toBe(400) - expect(data.error).toBe('Validation error') - expect(data.details).toBeDefined() - }) - - it('should handle database errors during update', async () => { - authMockFns.mockGetSession.mockResolvedValue({ - user: { id: 'user-123', email: 'test@example.com' }, - }) - - vi.mocked(checkKnowledgeBaseWriteAccess).mockResolvedValueOnce({ - hasAccess: true, - knowledgeBase: { id: 'kb-123', userId: 'user-123' }, - }) - - vi.mocked(updateKnowledgeBase).mockRejectedValueOnce(new Error('Database error')) - - const req = createMockRequest('PUT', validUpdateData) - const response = await PUT(req, { params: mockParams }) - const data = await response.json() - - expect(response.status).toBe(500) - expect(data.error).toBe('Failed to update knowledge base') - }) - }) - - describe('DELETE /api/knowledge/[id]', () => { - const mockParams = Promise.resolve({ id: 'kb-123' }) - - it('should delete knowledge base successfully', async () => { - authMockFns.mockGetSession.mockResolvedValue({ - user: { id: 'user-123', email: 'test@example.com' }, - }) - - resetMocks() - - vi.mocked(checkKnowledgeBaseWriteAccess).mockResolvedValueOnce({ - hasAccess: true, - knowledgeBase: { id: 'kb-123', userId: 'user-123' }, - }) - - vi.mocked(deleteKnowledgeBase).mockResolvedValueOnce(undefined) - - const req = createMockRequest('DELETE') - const response = await DELETE(req, { params: mockParams }) - const data = await response.json() - - expect(response.status).toBe(200) - expect(data.success).toBe(true) - expect(data.data.message).toBe('Knowledge base deleted successfully') - expect(checkKnowledgeBaseWriteAccess).toHaveBeenCalledWith('kb-123', 'user-123') - expect(deleteKnowledgeBase).toHaveBeenCalledWith('kb-123', expect.any(String)) - }) - - it('should return unauthorized for unauthenticated user', async () => { - authMockFns.mockGetSession.mockResolvedValue(null) - - const req = createMockRequest('DELETE') - const response = await DELETE(req, { params: mockParams }) - const data = await response.json() - - expect(response.status).toBe(401) - expect(data.error).toBe('Unauthorized') - }) - - it('should return not found for non-existent knowledge base', async () => { - authMockFns.mockGetSession.mockResolvedValue({ - user: { id: 'user-123', email: 'test@example.com' }, - }) - - resetMocks() - - vi.mocked(checkKnowledgeBaseWriteAccess).mockResolvedValueOnce({ - hasAccess: false, - notFound: true, - }) - - const req = createMockRequest('DELETE') - const response = await DELETE(req, { params: mockParams }) - const data = await response.json() - - expect(response.status).toBe(404) - expect(data.error).toBe('Knowledge base not found') - }) - - it('should return unauthorized for knowledge base owned by different user', async () => { - authMockFns.mockGetSession.mockResolvedValue({ - user: { id: 'user-123', email: 'test@example.com' }, - }) - - resetMocks() - - vi.mocked(checkKnowledgeBaseWriteAccess).mockResolvedValueOnce({ - hasAccess: false, - notFound: false, - }) - - const req = createMockRequest('DELETE') - const response = await DELETE(req, { params: mockParams }) - const data = await response.json() - - expect(response.status).toBe(401) - expect(data.error).toBe('Unauthorized') - }) - - it('should handle database errors during delete', async () => { - authMockFns.mockGetSession.mockResolvedValue({ - user: { id: 'user-123', email: 'test@example.com' }, - }) - - vi.mocked(checkKnowledgeBaseWriteAccess).mockResolvedValueOnce({ - hasAccess: true, - knowledgeBase: { id: 'kb-123', userId: 'user-123' }, - }) - - vi.mocked(deleteKnowledgeBase).mockRejectedValueOnce(new Error('Database error')) - - const req = createMockRequest('DELETE') - const response = await DELETE(req, { params: mockParams }) - const data = await response.json() - - expect(response.status).toBe(500) - expect(data.error).toBe('Failed to delete knowledge base') - }) - }) -}) diff --git a/apps/sim/app/api/knowledge/[id]/route.ts b/apps/sim/app/api/knowledge/[id]/route.ts index 3b91289af20..42e10e2ea08 100644 --- a/apps/sim/app/api/knowledge/[id]/route.ts +++ b/apps/sim/app/api/knowledge/[id]/route.ts @@ -1,174 +1,68 @@ -import { createLogger } from '@sim/logger' -import { type NextRequest, NextResponse } from 'next/server' -import { updateKnowledgeBaseContract } from '@/lib/api/contracts/knowledge' -import { parseRequest } from '@/lib/api/server' -import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid' import { - messageForOrchestrationError, - statusForOrchestrationError, -} from '@/lib/core/orchestration/types' -import { generateRequestId } from '@/lib/core/utils/request' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' + deleteKnowledgeBaseContract, + getKnowledgeBaseContract, + updateKnowledgeBaseContract, +} from '@/lib/api/contracts/knowledge' +import { validationErrorResponse } from '@/lib/api/server' import { - performDeleteKnowledgeBase, - performUpdateKnowledgeBase, -} from '@/lib/knowledge/orchestration' -import { getKnowledgeBaseById } from '@/lib/knowledge/service' -import { checkKnowledgeBaseAccess, checkKnowledgeBaseWriteAccess } from '@/app/api/knowledge/utils' - -const logger = createLogger('KnowledgeBaseByIdAPI') - -export const GET = withRouteHandler( - async (_request: NextRequest, { params }: { params: Promise<{ id: string }> }) => { - const requestId = generateRequestId() - const { id } = await params - - try { - const auth = await checkSessionOrInternalAuth(_request, { requireWorkflowId: false }) - if (!auth.success || !auth.userId) { - logger.warn(`[${requestId}] Unauthorized knowledge base access attempt`) - return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) - } - const userId = auth.userId - - const accessCheck = await checkKnowledgeBaseAccess(id, userId) - - if (!accessCheck.hasAccess) { - if ('notFound' in accessCheck && accessCheck.notFound) { - logger.warn(`[${requestId}] Knowledge base not found: ${id}`) - return NextResponse.json({ error: 'Knowledge base not found' }, { status: 404 }) - } - logger.warn( - `[${requestId}] User ${userId} attempted to access unauthorized knowledge base ${id}` - ) - return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) - } - - const knowledgeBaseData = await getKnowledgeBaseById(id) - - if (!knowledgeBaseData) { - return NextResponse.json({ error: 'Knowledge base not found' }, { status: 404 }) - } - - logger.info(`[${requestId}] Retrieved knowledge base: ${id} for user ${userId}`) - - return NextResponse.json({ - success: true, - data: knowledgeBaseData, - }) - } catch (error) { - logger.error(`[${requestId}] Error fetching knowledge base`, error) - return NextResponse.json({ error: 'Failed to fetch knowledge base' }, { status: 500 }) - } - } -) - -export const PUT = withRouteHandler( - async (req: NextRequest, context: { params: Promise<{ id: string }> }) => { - const requestId = generateRequestId() - const { id } = await context.params - - const auth = await checkSessionOrInternalAuth(req, { requireWorkflowId: false }) - if (!auth.success || !auth.userId) { - logger.warn(`[${requestId}] Unauthorized knowledge base update attempt`) - return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) - } - const userId = auth.userId - - const accessCheck = await checkKnowledgeBaseWriteAccess(id, userId) - - if (!accessCheck.hasAccess) { - if ('notFound' in accessCheck && accessCheck.notFound) { - logger.warn(`[${requestId}] Knowledge base not found: ${id}`) - return NextResponse.json({ error: 'Knowledge base not found' }, { status: 404 }) - } - logger.warn( - `[${requestId}] User ${userId} attempted to update unauthorized knowledge base ${id}` - ) - return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseRequest(updateKnowledgeBaseContract, req, context) - if (!parsed.success) return parsed.response - - const body = parsed.data.body - - const outcome = await performUpdateKnowledgeBase({ - knowledgeBaseId: id, - workspaceId: accessCheck.knowledgeBase.workspaceId ?? null, - userId, - actorName: auth.userName, - actorEmail: auth.userEmail, - source: 'ui', - updates: { - name: body.name, - description: body.description, - workspaceId: body.workspaceId, - folderId: body.folderId, - chunkingConfig: body.chunkingConfig, - }, - requestId, - request: req, - }) - if (!outcome.success) { - return NextResponse.json( - { error: messageForOrchestrationError(outcome, 'Failed to update knowledge base') }, - { status: statusForOrchestrationError(outcome.errorCode) } - ) - } - - return NextResponse.json({ success: true, data: outcome.knowledgeBase }) - } -) - -export const DELETE = withRouteHandler( - async (_request: NextRequest, { params }: { params: Promise<{ id: string }> }) => { - const requestId = generateRequestId() - const { id } = await params - - const auth = await checkSessionOrInternalAuth(_request, { requireWorkflowId: false }) - if (!auth.success || !auth.userId) { - logger.warn(`[${requestId}] Unauthorized knowledge base delete attempt`) - return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) - } - const userId = auth.userId - - const accessCheck = await checkKnowledgeBaseWriteAccess(id, userId) - - if (!accessCheck.hasAccess) { - if ('notFound' in accessCheck && accessCheck.notFound) { - logger.warn(`[${requestId}] Knowledge base not found: ${id}`) - return NextResponse.json({ error: 'Knowledge base not found' }, { status: 404 }) - } - logger.warn( - `[${requestId}] User ${userId} attempted to delete unauthorized knowledge base ${id}` - ) - return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) - } - - const outcome = await performDeleteKnowledgeBase({ - knowledgeBase: { - id, - name: accessCheck.knowledgeBase.name, - workspaceId: accessCheck.knowledgeBase.workspaceId ?? null, - }, - userId, - actorName: auth.userName, - actorEmail: auth.userEmail, - source: 'ui', - requestId, - request: _request, - }) - if (!outcome.success) { - return NextResponse.json( - { error: messageForOrchestrationError(outcome, 'Failed to delete knowledge base') }, - { status: statusForOrchestrationError(outcome.errorCode) } - ) - } - - return NextResponse.json({ - success: true, - data: { message: 'Knowledge base deleted successfully' }, - }) - } -) + defineInternalJsonRoute, + internalRateLimits, + internalSessionAuth, +} from '@/lib/api/server/routes' +import { internalKnowledgePresenters } from '@/lib/knowledge/api/internal-route' +import { internalKnowledgeErrorPolicies } from '@/lib/knowledge/api/route-policies' +import { + deleteInternalKnowledgeBase, + readInternalKnowledgeBase, + updateInternalKnowledgeBase, +} from '@/lib/knowledge/application/knowledge-bases' +import { knowledgeSessionOperations } from '@/lib/knowledge/application/operations' + +export const GET = defineInternalJsonRoute({ + contract: getKnowledgeBaseContract, + auth: internalSessionAuth, + operation: knowledgeSessionOperations.read, + rateLimit: internalRateLimits.none({ + reason: 'Preserve existing internal knowledge base detail behavior', + }), + errorPolicy: internalKnowledgeErrorPolicies.read, + mapInput: ({ params }) => ({ knowledgeBaseId: params.id }), + useCase: readInternalKnowledgeBase, + present: internalKnowledgePresenters.read, +}) + +export const PUT = defineInternalJsonRoute({ + contract: updateKnowledgeBaseContract, + auth: internalSessionAuth, + operation: knowledgeSessionOperations.update, + rateLimit: internalRateLimits.none({ + reason: 'Preserve existing internal knowledge base update behavior', + }), + errorPolicy: internalKnowledgeErrorPolicies.update, + parseOptions: { + validationErrorResponse: (error) => validationErrorResponse(error, 'Invalid request data'), + }, + mapInput: ({ params, body }) => ({ + knowledgeBaseId: params.id, + name: body.name, + description: body.description, + workspaceId: body.workspaceId, + folderId: body.folderId, + chunkingConfig: body.chunkingConfig, + }), + useCase: updateInternalKnowledgeBase, + present: internalKnowledgePresenters.read, +}) + +export const DELETE = defineInternalJsonRoute({ + contract: deleteKnowledgeBaseContract, + auth: internalSessionAuth, + operation: knowledgeSessionOperations.delete, + rateLimit: internalRateLimits.none({ + reason: 'Preserve existing internal knowledge base deletion behavior', + }), + errorPolicy: internalKnowledgeErrorPolicies.delete, + mapInput: ({ params }) => ({ knowledgeBaseId: params.id }), + useCase: deleteInternalKnowledgeBase, + present: internalKnowledgePresenters.deleted, +}) diff --git a/apps/sim/app/api/knowledge/[id]/tag-definitions/[tagId]/route.ts b/apps/sim/app/api/knowledge/[id]/tag-definitions/[tagId]/route.ts index 17772638f3e..0f40474f658 100644 --- a/apps/sim/app/api/knowledge/[id]/tag-definitions/[tagId]/route.ts +++ b/apps/sim/app/api/knowledge/[id]/tag-definitions/[tagId]/route.ts @@ -1,58 +1,28 @@ -import { createLogger } from '@sim/logger' -import { generateId } from '@sim/utils/id' -import { type NextRequest, NextResponse } from 'next/server' import { deleteTagDefinitionContract } from '@/lib/api/contracts/knowledge' -import { parseRequest } from '@/lib/api/server' -import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' +import { defineInternalJsonRoute, internalRateLimits } from '@/lib/api/server/routes' import { - deleteTagDefinition, - KnowledgeTagProvenanceConflictError, -} from '@/lib/knowledge/tags/service' -import { checkKnowledgeBaseWriteAccess } from '@/app/api/knowledge/utils' + internalKnowledgeErrorPolicies, + internalKnowledgeSessionOrExecutorAuth, +} from '@/lib/knowledge/api/route-policies' +import { knowledgeOperations } from '@/lib/knowledge/application/operations' +import { deleteKnowledgeTag } from '@/lib/knowledge/application/tags' export const dynamic = 'force-dynamic' -const logger = createLogger('TagDefinitionAPI') - -// DELETE /api/knowledge/[id]/tag-definitions/[tagId] - Delete a tag definition -export const DELETE = withRouteHandler( - async (req: NextRequest, context: { params: Promise<{ id: string; tagId: string }> }) => { - const requestId = generateId().slice(0, 8) - const parsed = await parseRequest(deleteTagDefinitionContract, req, context) - if (!parsed.success) return parsed.response - const { id: knowledgeBaseId, tagId } = parsed.data.params - - try { - logger.info( - `[${requestId}] Deleting tag definition ${tagId} from knowledge base ${knowledgeBaseId}` - ) - - const auth = await checkSessionOrInternalAuth(req, { requireWorkflowId: false }) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) - } - - const accessCheck = await checkKnowledgeBaseWriteAccess(knowledgeBaseId, auth.userId) - if (!accessCheck.hasAccess) { - return NextResponse.json( - { error: accessCheck.notFound ? 'Not found' : 'Forbidden' }, - { status: accessCheck.notFound ? 404 : 403 } - ) - } - - const deletedTag = await deleteTagDefinition(knowledgeBaseId, tagId, requestId) - - return NextResponse.json({ - success: true, - message: `Tag definition "${deletedTag.displayName}" deleted successfully`, - }) - } catch (error) { - if (error instanceof KnowledgeTagProvenanceConflictError) { - return NextResponse.json({ error: error.message }, { status: 409 }) - } - logger.error(`[${requestId}] Error deleting tag definition`, error) - return NextResponse.json({ error: 'Failed to delete tag definition' }, { status: 500 }) - } - } -) +export const DELETE = defineInternalJsonRoute({ + contract: deleteTagDefinitionContract, + auth: internalKnowledgeSessionOrExecutorAuth, + operation: knowledgeOperations.deleteTag, + rateLimit: internalRateLimits.none({ reason: 'Preserve existing internal tag-delete behavior' }), + errorPolicy: internalKnowledgeErrorPolicies.tags, + mapInput: ({ params }) => ({ + knowledgeBaseId: params.id, + tagDefinitionId: params.tagId, + source: 'ui' as const, + }), + useCase: deleteKnowledgeTag, + present: (deleted) => ({ + success: true as const, + message: `Tag definition "${deleted.displayName}" deleted successfully`, + }), +}) diff --git a/apps/sim/app/api/knowledge/[id]/tag-definitions/route.ts b/apps/sim/app/api/knowledge/[id]/tag-definitions/route.ts index 8d8b1cc41be..baed4d96f80 100644 --- a/apps/sim/app/api/knowledge/[id]/tag-definitions/route.ts +++ b/apps/sim/app/api/knowledge/[id]/tag-definitions/route.ts @@ -1,113 +1,48 @@ -import { createLogger } from '@sim/logger' -import { generateId } from '@sim/utils/id' -import { type NextRequest, NextResponse } from 'next/server' -import { createTagDefinitionContract } from '@/lib/api/contracts/knowledge' -import { parseRequest } from '@/lib/api/server' -import { AuthType, checkSessionOrInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { SUPPORTED_FIELD_TYPES } from '@/lib/knowledge/constants' -import { createTagDefinition, getTagDefinitions } from '@/lib/knowledge/tags/service' -import { checkKnowledgeBaseWriteAccess } from '@/app/api/knowledge/utils' +import { + createTagDefinitionContract, + listTagDefinitionsContract, +} from '@/lib/api/contracts/knowledge' +import { defineInternalJsonRoute, internalRateLimits } from '@/lib/api/server/routes' +import { toInternalKnowledgeTag } from '@/lib/knowledge/api/internal-route' +import { + internalKnowledgeErrorPolicies, + internalKnowledgeSessionOrExecutorAuth, +} from '@/lib/knowledge/api/route-policies' +import { knowledgeOperations } from '@/lib/knowledge/application/operations' +import { createKnowledgeTag, listKnowledgeTags } from '@/lib/knowledge/application/tags' export const dynamic = 'force-dynamic' -const logger = createLogger('KnowledgeBaseTagDefinitionsAPI') - -// GET /api/knowledge/[id]/tag-definitions - Get all tag definitions for a knowledge base -export const GET = withRouteHandler( - async (req: NextRequest, { params }: { params: Promise<{ id: string }> }) => { - const requestId = generateId().slice(0, 8) - const { id: knowledgeBaseId } = await params - - try { - logger.info(`[${requestId}] Getting tag definitions for knowledge base ${knowledgeBaseId}`) - - const auth = await checkSessionOrInternalAuth(req, { requireWorkflowId: false }) - if (!auth.success) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - // For session auth, verify KB access. Internal JWT is trusted. - if (auth.authType === AuthType.SESSION && auth.userId) { - const accessCheck = await checkKnowledgeBaseWriteAccess(knowledgeBaseId, auth.userId) - if (!accessCheck.hasAccess) { - return NextResponse.json( - { error: accessCheck.notFound ? 'Not found' : 'Forbidden' }, - { status: accessCheck.notFound ? 404 : 403 } - ) - } - } - - const tagDefinitions = await getTagDefinitions(knowledgeBaseId) - - logger.info( - `[${requestId}] Retrieved ${tagDefinitions.length} tag definitions (${auth.authType})` - ) - - return NextResponse.json({ - success: true, - data: tagDefinitions, - }) - } catch (error) { - logger.error(`[${requestId}] Error getting tag definitions`, error) - return NextResponse.json({ error: 'Failed to get tag definitions' }, { status: 500 }) - } - } -) - -// POST /api/knowledge/[id]/tag-definitions - Create a new tag definition -export const POST = withRouteHandler( - async (req: NextRequest, context: { params: Promise<{ id: string }> }) => { - const requestId = generateId().slice(0, 8) - const { id: knowledgeBaseId } = await context.params - - try { - logger.info(`[${requestId}] Creating tag definition for knowledge base ${knowledgeBaseId}`) - - const auth = await checkSessionOrInternalAuth(req, { requireWorkflowId: false }) - if (!auth.success) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - // For session auth, verify KB access. Internal JWT is trusted. - if (auth.authType === AuthType.SESSION && auth.userId) { - const accessCheck = await checkKnowledgeBaseWriteAccess(knowledgeBaseId, auth.userId) - if (!accessCheck.hasAccess) { - return NextResponse.json( - { error: accessCheck.notFound ? 'Not found' : 'Forbidden' }, - { status: accessCheck.notFound ? 404 : 403 } - ) - } - } - - const parsed = await parseRequest(createTagDefinitionContract, req, context) - if (!parsed.success) return parsed.response - - const validatedData = parsed.data.body - if (!(SUPPORTED_FIELD_TYPES as readonly string[]).includes(validatedData.fieldType)) { - return NextResponse.json( - { error: 'Invalid request data', details: 'Invalid field type' }, - { status: 400 } - ) - } - - const newTagDefinition = await createTagDefinition( - { - knowledgeBaseId, - tagSlot: validatedData.tagSlot, - displayName: validatedData.displayName, - fieldType: validatedData.fieldType, - }, - requestId - ) - - return NextResponse.json({ - success: true, - data: newTagDefinition, - }) - } catch (error) { - logger.error(`[${requestId}] Error creating tag definition`, error) - return NextResponse.json({ error: 'Failed to create tag definition' }, { status: 500 }) - } - } -) +export const GET = defineInternalJsonRoute({ + contract: listTagDefinitionsContract, + auth: internalKnowledgeSessionOrExecutorAuth, + operation: knowledgeOperations.listTags, + rateLimit: internalRateLimits.none({ reason: 'Preserve existing internal tag-list behavior' }), + errorPolicy: internalKnowledgeErrorPolicies.tags, + mapInput: ({ params }) => ({ knowledgeBaseId: params.id }), + useCase: listKnowledgeTags, + present: ({ tagDefinitions }) => ({ + success: true as const, + data: tagDefinitions.map(toInternalKnowledgeTag), + }), +}) + +export const POST = defineInternalJsonRoute({ + contract: createTagDefinitionContract, + auth: internalKnowledgeSessionOrExecutorAuth, + operation: knowledgeOperations.createTag, + rateLimit: internalRateLimits.none({ reason: 'Preserve existing internal tag-create behavior' }), + errorPolicy: internalKnowledgeErrorPolicies.tags, + mapInput: ({ params, body }) => ({ + knowledgeBaseId: params.id, + tagSlot: body.tagSlot, + displayName: body.displayName, + fieldType: body.fieldType, + source: 'ui' as const, + }), + useCase: createKnowledgeTag, + present: ({ tagDefinition }) => ({ + success: true as const, + data: toInternalKnowledgeTag(tagDefinition), + }), +}) diff --git a/apps/sim/app/api/knowledge/[id]/tag-usage/route.ts b/apps/sim/app/api/knowledge/[id]/tag-usage/route.ts index 7412ccf307a..b71e73fb548 100644 --- a/apps/sim/app/api/knowledge/[id]/tag-usage/route.ts +++ b/apps/sim/app/api/knowledge/[id]/tag-usage/route.ts @@ -1,56 +1,21 @@ -import { createLogger } from '@sim/logger' -import { generateId } from '@sim/utils/id' -import { type NextRequest, NextResponse } from 'next/server' import { getTagUsageContract } from '@/lib/api/contracts/knowledge' -import { parseRequest } from '@/lib/api/server' -import { getSession } from '@/lib/auth' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { getTagUsage } from '@/lib/knowledge/tags/service' -import { checkKnowledgeBaseAccess } from '@/app/api/knowledge/utils' +import { defineInternalJsonRoute, internalRateLimits } from '@/lib/api/server/routes' +import { + internalKnowledgeErrorPolicies, + internalKnowledgeSessionOrExecutorAuth, +} from '@/lib/knowledge/api/route-policies' +import { knowledgeOperations } from '@/lib/knowledge/application/operations' +import { readDetailedKnowledgeTagUsage } from '@/lib/knowledge/application/tags' export const dynamic = 'force-dynamic' -const logger = createLogger('TagUsageAPI') - -// GET /api/knowledge/[id]/tag-usage - Get usage statistics for all tag definitions -export const GET = withRouteHandler( - async (req: NextRequest, context: { params: Promise<{ id: string }> }) => { - const requestId = generateId().slice(0, 8) - const parsed = await parseRequest(getTagUsageContract, req, context) - if (!parsed.success) return parsed.response - const { id: knowledgeBaseId } = parsed.data.params - - try { - logger.info( - `[${requestId}] Getting tag usage statistics for knowledge base ${knowledgeBaseId}` - ) - - const session = await getSession() - if (!session?.user?.id) { - return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) - } - - const accessCheck = await checkKnowledgeBaseAccess(knowledgeBaseId, session.user.id) - if (!accessCheck.hasAccess) { - return NextResponse.json( - { error: accessCheck.notFound ? 'Not found' : 'Forbidden' }, - { status: accessCheck.notFound ? 404 : 403 } - ) - } - - const usageStats = await getTagUsage(knowledgeBaseId, requestId) - - logger.info( - `[${requestId}] Retrieved usage statistics for ${usageStats.length} tag definitions` - ) - - return NextResponse.json({ - success: true, - data: usageStats, - }) - } catch (error) { - logger.error(`[${requestId}] Error getting tag usage statistics`, error) - return NextResponse.json({ error: 'Failed to get tag usage statistics' }, { status: 500 }) - } - } -) +export const GET = defineInternalJsonRoute({ + contract: getTagUsageContract, + auth: internalKnowledgeSessionOrExecutorAuth, + operation: knowledgeOperations.readDetailedTagUsage, + rateLimit: internalRateLimits.none({ reason: 'Preserve existing internal tag-usage behavior' }), + errorPolicy: internalKnowledgeErrorPolicies.tags, + mapInput: ({ params }) => ({ knowledgeBaseId: params.id }), + useCase: readDetailedKnowledgeTagUsage, + present: ({ usage }) => ({ success: true as const, data: usage }), +}) diff --git a/apps/sim/app/api/knowledge/migrated-routes.test.ts b/apps/sim/app/api/knowledge/migrated-routes.test.ts new file mode 100644 index 00000000000..c25aa75d757 --- /dev/null +++ b/apps/sim/app/api/knowledge/migrated-routes.test.ts @@ -0,0 +1,552 @@ +/** + * @vitest-environment node + */ +import { authMockFns, createMockRequest } from '@sim/testing' +import { NextRequest } from 'next/server' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + useCase: (id: string, execute: ReturnType) => ({ operation: { id }, execute }), + listDocuments: vi.fn(), + createDocuments: vi.fn(), + bulkDocuments: vi.fn(), + readDocument: vi.fn(), + updateDocument: vi.fn(), + deleteDocument: vi.fn(), + upsertDocument: vi.fn(), + listConnectors: vi.fn(), + createConnector: vi.fn(), + readConnector: vi.fn(), + updateConnector: vi.fn(), + deleteConnector: vi.fn(), + syncConnector: vi.fn(), + listConnectorDocuments: vi.fn(), + updateConnectorDocuments: vi.fn(), + search: vi.fn(), + createUpload: vi.fn(), + issueParts: vi.fn(), + completeUpload: vi.fn(), + cancelUpload: vi.fn(), + persistedResponse: vi.fn(), + provenanceResponse: vi.fn(), + registryResponse: vi.fn(), + resolveDocumentProvenance: vi.fn(), + readKnowledgeBase: vi.fn(), + updateKnowledgeBase: vi.fn(), + deleteKnowledgeBase: vi.fn(), + restoreKnowledgeBase: vi.fn(), + platformUpload: vi.fn(), + capture: vi.fn(), +})) + +vi.mock('@/lib/knowledge/application/documents', () => ({ + listKnowledgeDocuments: mocks.useCase('knowledge.documents.list', mocks.listDocuments), + createKnowledgeDocuments: mocks.useCase('knowledge.documents.upload', mocks.createDocuments), + bulkUpdateKnowledgeDocuments: mocks.useCase('knowledge.documents.bulk', mocks.bulkDocuments), + readKnowledgeDocument: mocks.useCase('knowledge.documents.read', mocks.readDocument), + updateKnowledgeDocument: mocks.useCase('knowledge.documents.update', mocks.updateDocument), + deleteKnowledgeDocument: mocks.useCase('knowledge.documents.delete', mocks.deleteDocument), + upsertKnowledgeDocument: mocks.useCase('knowledge.documents.upload', mocks.upsertDocument), +})) + +vi.mock('@/lib/knowledge/application/knowledge-bases', () => ({ + readInternalKnowledgeBase: mocks.useCase('knowledge.session.read', mocks.readKnowledgeBase), + updateInternalKnowledgeBase: mocks.useCase('knowledge.session.update', mocks.updateKnowledgeBase), + deleteInternalKnowledgeBase: mocks.useCase('knowledge.session.delete', mocks.deleteKnowledgeBase), + restoreInternalKnowledgeBase: mocks.useCase( + 'knowledge.session.restore', + mocks.restoreKnowledgeBase + ), +})) + +vi.mock('@/lib/knowledge/application/connectors', () => ({ + listKnowledgeConnectors: mocks.useCase('knowledge.connectors.list', mocks.listConnectors), + createKnowledgeConnector: mocks.useCase('knowledge.connectors.create', mocks.createConnector), + readKnowledgeConnector: mocks.useCase('knowledge.connectors.read', mocks.readConnector), + updateKnowledgeConnector: mocks.useCase('knowledge.connectors.update', mocks.updateConnector), + deleteKnowledgeConnector: mocks.useCase('knowledge.connectors.delete', mocks.deleteConnector), + syncKnowledgeConnector: mocks.useCase('knowledge.connectors.sync', mocks.syncConnector), + listKnowledgeConnectorDocuments: mocks.useCase( + 'knowledge.connectors.documents.list', + mocks.listConnectorDocuments + ), + updateKnowledgeConnectorDocuments: mocks.useCase( + 'knowledge.connectors.documents.update', + mocks.updateConnectorDocuments + ), +})) + +vi.mock('@/lib/knowledge/application/search', () => ({ + KnowledgeSearchProvenanceUnavailableError: class extends Error {}, + searchKnowledge: mocks.useCase('knowledge.search', mocks.search), +})) + +vi.mock('@/lib/knowledge/application/upload-sessions', () => ({ + KnowledgeDocumentUnsupportedMediaTypeError: class extends Error {}, + createKnowledgeDocumentUpload: mocks.useCase( + 'knowledge.documents.upload.create', + mocks.createUpload + ), + issueKnowledgeDocumentUploadParts: mocks.useCase( + 'knowledge.documents.upload.parts', + mocks.issueParts + ), + completeKnowledgeDocumentUpload: mocks.useCase( + 'knowledge.documents.upload.complete', + mocks.completeUpload + ), + cancelKnowledgeDocumentUpload: mocks.useCase( + 'knowledge.documents.upload.cancel', + mocks.cancelUpload + ), +})) + +vi.mock('@/app/api/knowledge/secret-provenance', () => ({ + finalizeKnowledgePersistedResponse: mocks.persistedResponse, + finalizeKnowledgeProvenanceResponse: mocks.provenanceResponse, + finalizeKnowledgeRegistryResponse: mocks.registryResponse, + resolveKnowledgeDocumentWriteSecretProvenance: mocks.resolveDocumentProvenance, +})) + +vi.mock('@/lib/core/telemetry', () => ({ + PlatformEvents: { knowledgeBaseDocumentsUploaded: mocks.platformUpload }, +})) + +vi.mock('@/lib/posthog/server', () => ({ captureServerEvent: mocks.capture })) + +import { + GET as listConnectorDocuments, + PATCH as updateConnectorDocuments, +} from '@/app/api/knowledge/[id]/connectors/[connectorId]/documents/route' +import { PUT as updateDocument } from '@/app/api/knowledge/[id]/documents/[documentId]/route' +import { + POST as createDocuments, + GET as listDocuments, +} from '@/app/api/knowledge/[id]/documents/route' +import { POST as completeUpload } from '@/app/api/knowledge/[id]/documents/uploads/[uploadId]/complete/route' +import { POST as upsertDocument } from '@/app/api/knowledge/[id]/documents/upsert/route' +import { POST as restoreKnowledgeBase } from '@/app/api/knowledge/[id]/restore/route' +import { + DELETE as deleteKnowledgeBase, + GET as readKnowledgeBase, + PUT as updateKnowledgeBase, +} from '@/app/api/knowledge/[id]/route' +import { POST as search } from '@/app/api/knowledge/search/route' + +const session = { + user: { id: 'user-1', email: 'user@example.com', name: 'User' }, + session: { id: 'session-1' }, +} + +const document = { + id: 'document-1', + knowledgeBaseId: 'knowledge-1', + filename: 'guide.pdf', + fileUrl: 'https://example.com/guide.pdf', + fileSize: 42, + mimeType: 'application/pdf', + chunkCount: 1, + tokenCount: 5, + characterCount: 20, + processingStatus: 'completed' as const, + enabled: true, + uploadedAt: new Date('2026-01-01T00:00:00Z'), +} + +const knowledgeBase = { + id: 'knowledge-1', + userId: 'user-1', + name: 'Docs', + description: null, + tokenCount: 0, + embeddingModel: 'text-embedding-3-small', + embeddingDimension: 1536, + chunkingConfig: { maxSize: 1024, minSize: 100, overlap: 200 }, + createdAt: new Date('2026-01-01T00:00:00Z'), + updatedAt: new Date('2026-01-01T00:00:00Z'), + deletedAt: null, + workspaceId: null, + folderId: null, + docCount: 0, + connectorTypes: [], +} + +describe('migrated internal Knowledge routes', () => { + beforeEach(() => { + vi.clearAllMocks() + authMockFns.mockGetSession.mockResolvedValue(session) + mocks.persistedResponse.mockResolvedValue({}) + mocks.provenanceResponse.mockResolvedValue({}) + mocks.registryResponse.mockReturnValue({}) + mocks.resolveDocumentProvenance.mockReturnValue({ success: true }) + }) + + it('authenticates before parsing malformed document JSON', async () => { + authMockFns.mockGetSession.mockResolvedValueOnce(null) + const request = new NextRequest('http://localhost/api/knowledge/knowledge-1/documents', { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: '{', + }) + + const response = await createDocuments(request, { + params: Promise.resolve({ id: 'knowledge-1' }), + }) + + expect(response.status).toBe(401) + expect(mocks.createDocuments).not.toHaveBeenCalled() + }) + + it('authenticates before parsing malformed knowledge base JSON', async () => { + authMockFns.mockGetSession.mockResolvedValueOnce(null) + const request = new NextRequest('http://localhost/api/knowledge/knowledge-1', { + method: 'PUT', + headers: { 'content-type': 'application/json' }, + body: '{', + }) + + const response = await updateKnowledgeBase(request, { + params: Promise.resolve({ id: 'knowledge-1' }), + }) + + expect(response.status).toBe(401) + expect(mocks.updateKnowledgeBase).not.toHaveBeenCalled() + }) + + it('preserves knowledge base detail, delete, and restore envelopes', async () => { + mocks.readKnowledgeBase.mockResolvedValue({ knowledgeBase }) + const readResponse = await readKnowledgeBase(createMockRequest('GET'), { + params: Promise.resolve({ id: 'knowledge-1' }), + }) + await expect(readResponse.json()).resolves.toEqual({ + success: true, + data: expect.objectContaining({ + id: 'knowledge-1', + workspaceId: null, + createdAt: '2026-01-01T00:00:00.000Z', + }), + }) + + mocks.deleteKnowledgeBase.mockResolvedValue({ success: true }) + const deleteResponse = await deleteKnowledgeBase(createMockRequest('DELETE'), { + params: Promise.resolve({ id: 'knowledge-1' }), + }) + await expect(deleteResponse.json()).resolves.toEqual({ + success: true, + data: { message: 'Knowledge base deleted successfully' }, + }) + + mocks.restoreKnowledgeBase.mockResolvedValue({ success: true }) + const restoreResponse = await restoreKnowledgeBase(createMockRequest('POST'), { + params: Promise.resolve({ id: 'knowledge-1' }), + }) + await expect(restoreResponse.json()).resolves.toEqual({ success: true }) + }) + + it('renders unknown knowledge base failures safely', async () => { + mocks.readKnowledgeBase.mockRejectedValue(new Error('postgres password=secret')) + const response = await readKnowledgeBase(createMockRequest('GET'), { + params: Promise.resolve({ id: 'knowledge-1' }), + }) + + expect(response.status).toBe(500) + await expect(response.json()).resolves.toEqual({ error: 'Failed to fetch knowledge base' }) + }) + + it('preserves the document-list envelope and canonical application input', async () => { + mocks.listDocuments.mockResolvedValue({ + documents: [document], + pagination: { total: 1, limit: 50, offset: 0, hasMore: false }, + workspaceId: 'workspace-1', + }) + const request = createMockRequest( + 'GET', + undefined, + {}, + 'http://localhost/api/knowledge/knowledge-1/documents?enabledFilter=all' + ) + + const response = await listDocuments(request, { + params: Promise.resolve({ id: 'knowledge-1' }), + }) + + expect(mocks.listDocuments).toHaveBeenCalledWith({ + principal: { kind: 'session', userId: 'user-1', sessionId: 'session-1' }, + input: expect.objectContaining({ knowledgeBaseId: 'knowledge-1', enabledFilter: 'all' }), + request, + }) + await expect(response.json()).resolves.toEqual({ + success: true, + data: { + documents: [ + expect.objectContaining({ id: 'document-1', uploadedAt: '2026-01-01T00:00:00.000Z' }), + ], + pagination: { total: 1, limit: 50, offset: 0, hasMore: false }, + }, + }) + }) + + it('renders unknown document failures safely', async () => { + mocks.updateDocument.mockRejectedValue(new Error('postgres password=secret')) + const response = await updateDocument(createMockRequest('PUT', { filename: 'renamed.pdf' }), { + params: Promise.resolve({ id: 'knowledge-1', documentId: 'document-1' }), + }) + + expect(response.status).toBe(500) + await expect(response.json()).resolves.toEqual({ + error: 'Failed to process knowledge document request', + }) + }) + + it('keeps document upsert admission behind auth and preserves its response', async () => { + mocks.upsertDocument.mockResolvedValue({ + document: { documentId: 'document-2', filename: 'new.txt' }, + knowledgeBaseId: 'knowledge-1', + isUpdate: false, + previousDocumentId: null, + processingConfig: { maxConcurrentDocuments: 5, batchSize: 10 }, + workspaceId: 'workspace-1', + userId: 'user-1', + }) + const response = await upsertDocument( + createMockRequest('POST', { + filename: 'new.txt', + fileUrl: 'data:text/plain;base64,aGVsbG8=', + fileSize: 5, + mimeType: 'text/plain', + }), + { params: Promise.resolve({ id: 'knowledge-1' }) } + ) + + expect(response.status).toBe(200) + await expect(response.json()).resolves.toEqual({ + success: true, + data: expect.objectContaining({ + documentsCreated: [{ documentId: 'document-2', filename: 'new.txt', status: 'pending' }], + isUpdate: false, + previousDocumentId: null, + processingMethod: 'background', + }), + }) + expect(mocks.platformUpload).toHaveBeenCalledWith( + expect.objectContaining({ knowledgeBaseId: 'knowledge-1', documentsCount: 1 }) + ) + }) + + it('runs internal document analytics only after application success', async () => { + mocks.createDocuments.mockResolvedValue({ + kind: 'single', + data: document, + workspaceId: 'workspace-1', + userId: 'user-1', + }) + const request = createMockRequest('POST', { + bulk: false, + filename: document.filename, + fileUrl: document.fileUrl, + fileSize: document.fileSize, + mimeType: document.mimeType, + }) + + const response = await createDocuments(request, { + params: Promise.resolve({ id: 'knowledge-1' }), + }) + + expect(response.status).toBe(200) + expect(mocks.platformUpload).toHaveBeenCalledOnce() + expect(mocks.capture).toHaveBeenCalledOnce() + expect(mocks.createDocuments.mock.invocationCallOrder[0]).toBeLessThan( + mocks.platformUpload.mock.invocationCallOrder[0] + ) + }) + + it('rejects oversized document-create arrays at the contract boundary', async () => { + const response = await createDocuments( + createMockRequest('POST', { + bulk: true, + documents: Array.from({ length: 101 }, (_, index) => ({ + filename: `document-${index}.txt`, + fileUrl: `https://example.com/document-${index}.txt`, + fileSize: 1, + mimeType: 'text/plain', + })), + }), + { params: Promise.resolve({ id: 'knowledge-1' }) } + ) + + expect(response.status).toBe(400) + expect(mocks.createDocuments).not.toHaveBeenCalled() + }) + + it('preserves connector-document list and mutation envelopes', async () => { + mocks.listConnectorDocuments.mockResolvedValue({ + documents: [ + { + id: 'document-1', + filename: 'Guide', + externalId: null, + sourceUrl: null, + enabled: true, + userExcluded: false, + uploadedAt: new Date('2026-01-01T00:00:00Z'), + processingStatus: 'completed', + }, + ], + counts: { active: 1, excluded: 0 }, + }) + const params = Promise.resolve({ id: 'knowledge-1', connectorId: 'connector-1' }) + const listResponse = await listConnectorDocuments( + new NextRequest( + 'http://localhost/api/knowledge/knowledge-1/connectors/connector-1/documents?includeExcluded=true&limit=25&offset=50' + ), + { params } + ) + await expect(listResponse.json()).resolves.toEqual({ + success: true, + data: { + documents: [ + expect.objectContaining({ id: 'document-1', uploadedAt: '2026-01-01T00:00:00.000Z' }), + ], + counts: { active: 1, excluded: 0 }, + }, + }) + expect(mocks.listConnectorDocuments).toHaveBeenLastCalledWith( + expect.objectContaining({ + input: expect.objectContaining({ includeExcluded: true, limit: 25, offset: 50 }), + }) + ) + + const filteredListResponse = await listConnectorDocuments( + new NextRequest( + 'http://localhost/api/knowledge/knowledge-1/connectors/connector-1/documents?includeExcluded=false' + ), + { params: Promise.resolve({ id: 'knowledge-1', connectorId: 'connector-1' }) } + ) + expect(filteredListResponse.status).toBe(200) + expect(mocks.listConnectorDocuments).toHaveBeenLastCalledWith( + expect.objectContaining({ + input: expect.objectContaining({ includeExcluded: false }), + }) + ) + + mocks.updateConnectorDocuments.mockResolvedValue({ + operation: 'exclude', + count: 1, + documentIds: ['document-1'], + }) + const updateResponse = await updateConnectorDocuments( + createMockRequest('PATCH', { operation: 'exclude', documentIds: ['document-1'] }), + { params: Promise.resolve({ id: 'knowledge-1', connectorId: 'connector-1' }) } + ) + await expect(updateResponse.json()).resolves.toEqual({ + success: true, + data: { excludedCount: 1, documentIds: ['document-1'] }, + }) + }) + + it('rejects oversized connector-document mutations at the contract boundary', async () => { + const response = await updateConnectorDocuments( + createMockRequest('PATCH', { + operation: 'exclude', + documentIds: Array.from({ length: 101 }, (_, index) => `document-${index}`), + }), + { params: Promise.resolve({ id: 'knowledge-1', connectorId: 'connector-1' }) } + ) + + expect(response.status).toBe(400) + expect(mocks.updateConnectorDocuments).not.toHaveBeenCalled() + }) + + it('preserves search cost shape and sanitizes infrastructure errors', async () => { + const registry = {} + mocks.search.mockResolvedValue({ + results: [ + { + embeddingId: 'embedding-1', + documentId: 'document-1', + documentName: 'Guide', + sourceUrl: null, + content: 'hello', + chunkIndex: 0, + metadata: {}, + similarity: 0.9, + }, + ], + query: 'hello', + knowledgeBaseIds: ['knowledge-1'], + knowledgeBaseId: 'knowledge-1', + topK: 10, + totalResults: 1, + workspaceId: 'workspace-1', + userId: 'user-1', + resultSecretRegistry: registry, + cost: { + input: 0.1, + output: 0, + total: 0.1, + tokens: { prompt: 1, completion: 0, total: 1 }, + model: 'text-embedding-3-small', + pricing: { input: 0.1, output: 0 }, + }, + }) + const response = await search( + createMockRequest('POST', { + knowledgeBaseIds: ['knowledge-1'], + query: 'hello', + }) + ) + const body = await response.json() + expect(body.data.results[0]).not.toHaveProperty('embeddingId') + expect(body.data.cost).toEqual(expect.objectContaining({ total: 0.1 })) + + mocks.search.mockRejectedValueOnce(new Error('database host secret.internal')) + const failure = await search( + createMockRequest('POST', { knowledgeBaseIds: ['knowledge-1'], query: 'hello' }) + ) + expect(failure.status).toBe(500) + await expect(failure.json()).resolves.toEqual({ error: 'Failed to perform vector search' }) + }) + + it('runs upload analytics only after a newly-created completion', async () => { + const completed = { + session: { + id: 'upload-1', + knowledgeBaseId: 'knowledge-1', + workspaceId: 'workspace-1', + status: 'completed', + uploadToken: 'token', + objectKey: 'key', + fileName: 'guide.pdf', + contentType: 'application/pdf', + fileSize: 42, + expiresAt: new Date('2026-01-02T00:00:00Z'), + error: null, + createdAt: new Date('2026-01-01T00:00:00Z'), + updatedAt: new Date('2026-01-01T00:00:00Z'), + }, + value: { + created: true, + document: { + ...document, + documentId: document.id, + }, + }, + knowledgeBaseId: 'knowledge-1', + workspaceId: 'workspace-1', + } + mocks.completeUpload.mockResolvedValue(completed) + const response = await completeUpload( + createMockRequest( + 'POST', + undefined, + { 'upload-token': 'token' }, + 'http://localhost/api/knowledge/knowledge-1/documents/uploads/upload-1/complete?workspaceId=workspace-1' + ), + { params: Promise.resolve({ id: 'knowledge-1', uploadId: 'upload-1' }) } + ) + + expect(response.status).toBe(200) + expect(mocks.capture).toHaveBeenCalledOnce() + expect(mocks.platformUpload).toHaveBeenCalledOnce() + }) +}) diff --git a/apps/sim/app/api/knowledge/route.test.ts b/apps/sim/app/api/knowledge/route.test.ts index 3c8f8083b79..011d7822057 100644 --- a/apps/sim/app/api/knowledge/route.test.ts +++ b/apps/sim/app/api/knowledge/route.test.ts @@ -1,211 +1,227 @@ /** - * Tests for knowledge base API route - * * @vitest-environment node */ -import { - auditMock, - authMockFns, - createMockRequest, - dbChainMockFns, - permissionsMock, - permissionsMockFns, - resetDbChainMock, -} from '@sim/testing' -import { afterAll, afterEach, beforeEach, describe, expect, it, vi } from 'vitest' - -vi.mock('@sim/audit', () => auditMock) - -vi.mock('@/lib/workspaces/permissions/utils', () => permissionsMock) - +import { authMockFns, createMockRequest } from '@sim/testing' +import { NextRequest } from 'next/server' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + list: vi.fn(), + create: vi.fn(), + platformCreated: vi.fn(), + capture: vi.fn(), +})) + +vi.mock('@/lib/knowledge/application/knowledge-bases', () => ({ + listInternalKnowledgeBases: { + operation: { id: 'knowledge.session.list' }, + execute: mocks.list, + }, + createKnowledgeBase: { + operation: { id: 'knowledge.create' }, + execute: mocks.create, + }, +})) + +vi.mock('@/lib/core/telemetry', () => ({ + PlatformEvents: { knowledgeBaseCreated: mocks.platformCreated }, +})) + +vi.mock('@/lib/posthog/server', () => ({ captureServerEvent: mocks.capture })) + +import { OrchestrationError } from '@/lib/core/orchestration/types' import { GET, POST } from '@/app/api/knowledge/route' -describe('Knowledge Base API Route', () => { +const session = { + user: { id: 'user-123', email: 'test@example.com', name: 'Test User' }, + session: { id: 'session-123' }, +} + +const knowledgeBase = { + id: 'knowledge-1', + userId: 'user-123', + name: 'Test Knowledge Base', + description: 'Test description', + tokenCount: 0, + embeddingModel: 'text-embedding-3-small', + embeddingDimension: 1536, + chunkingConfig: { maxSize: 1024, minSize: 100, overlap: 200 }, + createdAt: new Date('2026-01-01T00:00:00Z'), + updatedAt: new Date('2026-01-02T00:00:00Z'), + deletedAt: null, + workspaceId: 'workspace-1', + folderId: null, + docCount: 0, + connectorTypes: [], +} + +const expectedKnowledgeBase = { + ...knowledgeBase, + createdAt: '2026-01-01T00:00:00.000Z', + updatedAt: '2026-01-02T00:00:00.000Z', +} + +describe('/api/knowledge internal route composition', () => { beforeEach(() => { vi.clearAllMocks() - resetDbChainMock() - - permissionsMockFns.mockGetUserEntityPermissions.mockResolvedValue('admin') + authMockFns.mockGetSession.mockResolvedValue(session) + mocks.list.mockResolvedValue({ knowledgeBases: [knowledgeBase] }) + mocks.create.mockResolvedValue({ knowledgeBase, folderPath: '/' }) + }) - vi.stubGlobal('crypto', { - randomUUID: vi.fn().mockReturnValue('mock-uuid-1234-5678'), + it('authenticates before parsing malformed JSON', async () => { + authMockFns.mockGetSession.mockResolvedValueOnce(null) + const request = new NextRequest('http://localhost/api/knowledge', { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: '{', }) - }) - afterEach(() => { - vi.clearAllMocks() - }) + const response = await POST(request) - afterAll(() => { - resetDbChainMock() + expect(response.status).toBe(401) + await expect(response.json()).resolves.toEqual({ error: 'Unauthorized' }) + expect(mocks.create).not.toHaveBeenCalled() }) - describe('GET /api/knowledge', () => { - it('should return unauthorized for unauthenticated user', async () => { - authMockFns.mockGetSession.mockResolvedValue(null) + it('preserves the legacy personal listing envelope without inventing a workspace', async () => { + const request = createMockRequest( + 'GET', + undefined, + {}, + 'http://localhost/api/knowledge?scope=all' + ) - const req = createMockRequest('GET') - const response = await GET(req) - const data = await response.json() + const response = await GET(request) - expect(response.status).toBe(401) - expect(data.error).toBe('Unauthorized') + expect(mocks.list).toHaveBeenCalledWith({ + principal: { kind: 'session', userId: 'user-123', sessionId: 'session-123' }, + input: { workspaceId: undefined, scope: 'all' }, + request, }) - - it('should handle database errors', async () => { - authMockFns.mockGetSession.mockResolvedValue({ - user: { id: 'user-123', email: 'test@example.com' }, - }) - dbChainMockFns.orderBy.mockRejectedValueOnce(new Error('Database error')) - - const req = createMockRequest('GET') - const response = await GET(req) - const data = await response.json() - - expect(response.status).toBe(500) - expect(data.error).toBe('Failed to fetch knowledge bases') + expect(response.status).toBe(200) + await expect(response.json()).resolves.toEqual({ + success: true, + data: [expectedKnowledgeBase], }) }) - describe('POST /api/knowledge', () => { - const validKnowledgeBaseData = { - name: 'Test Knowledge Base', - description: 'Test description', - workspaceId: 'test-workspace-id', - chunkingConfig: { - maxSize: 1024, - minSize: 100, - overlap: 200, - }, - } - - it('should create knowledge base successfully', async () => { - authMockFns.mockGetSession.mockResolvedValue({ - user: { id: 'user-123', email: 'test@example.com' }, - }) + it('preserves workspace-scoped archived listing input and envelope', async () => { + const request = createMockRequest( + 'GET', + undefined, + {}, + 'http://localhost/api/knowledge?workspaceId=workspace-1&scope=archived' + ) - const req = createMockRequest('POST', validKnowledgeBaseData) - const response = await POST(req) - const data = await response.json() + const response = await GET(request) - expect(response.status).toBe(200) - expect(data.success).toBe(true) - expect(data.data.name).toBe(validKnowledgeBaseData.name) - expect(data.data.description).toBe(validKnowledgeBaseData.description) - expect(dbChainMockFns.insert).toHaveBeenCalled() + expect(mocks.list).toHaveBeenCalledWith({ + principal: { kind: 'session', userId: 'user-123', sessionId: 'session-123' }, + input: { workspaceId: 'workspace-1', scope: 'archived' }, + request, }) - - it('should return unauthorized for unauthenticated user', async () => { - authMockFns.mockGetSession.mockResolvedValue(null) - - const req = createMockRequest('POST', validKnowledgeBaseData) - const response = await POST(req) - const data = await response.json() - - expect(response.status).toBe(401) - expect(data.error).toBe('Unauthorized') + await expect(response.json()).resolves.toEqual({ + success: true, + data: [expectedKnowledgeBase], }) + }) - it('should validate required fields', async () => { - authMockFns.mockGetSession.mockResolvedValue({ - user: { id: 'user-123', email: 'test@example.com' }, - }) - - const req = createMockRequest('POST', { description: 'Missing name' }) - const response = await POST(req) - const data = await response.json() + it('preserves the legacy query validation envelope', async () => { + const response = await GET( + createMockRequest('GET', undefined, {}, 'http://localhost/api/knowledge?scope=invalid') + ) + const body = await response.json() - expect(response.status).toBe(400) - expect(data.error).toBe('Invalid request data') - expect(data.details).toBeDefined() + expect(response.status).toBe(400) + expect(body).toEqual({ + error: 'Invalid query parameters', + details: expect.any(Array), }) + expect(mocks.list).not.toHaveBeenCalled() + }) - it('should require workspaceId', async () => { - authMockFns.mockGetSession.mockResolvedValue({ - user: { id: 'user-123', email: 'test@example.com' }, - }) - - const req = createMockRequest('POST', { name: 'Test KB' }) - const response = await POST(req) - const data = await response.json() - - expect(response.status).toBe(400) - expect(data.error).toBe('Invalid request data') - expect(data.details).toBeDefined() + it('preserves the exact creation envelope and runs internal analytics after success', async () => { + const request = createMockRequest('POST', { + name: 'Test Knowledge Base', + description: 'Test description', + workspaceId: 'workspace-1', + folderId: null, }) - it('returns 403 when user lacks permission on target workspace', async () => { - authMockFns.mockGetSession.mockResolvedValue({ - user: { id: 'attacker', email: 'a@example.com' }, - }) - permissionsMockFns.mockGetUserEntityPermissions.mockResolvedValueOnce('read') - - const req = createMockRequest('POST', validKnowledgeBaseData) - const response = await POST(req) - const data = await response.json() - - expect(response.status).toBe(403) - expect(data.error).toBe( - 'User does not have permission to create knowledge bases in this workspace' - ) - expect(dbChainMockFns.insert).not.toHaveBeenCalled() + const response = await POST(request) + + expect(mocks.create).toHaveBeenCalledWith({ + principal: { kind: 'session', userId: 'user-123', sessionId: 'session-123' }, + input: { + workspaceId: 'workspace-1', + name: 'Test Knowledge Base', + description: 'Test description', + folderId: null, + chunkingConfig: { maxSize: 1024, minSize: 100, overlap: 200 }, + source: 'ui', + }, + request, }) - - it('should validate chunking config constraints', async () => { - authMockFns.mockGetSession.mockResolvedValue({ - user: { id: 'user-123', email: 'test@example.com' }, - }) - - const invalidData = { - name: 'Test KB', - workspaceId: 'test-workspace-id', - chunkingConfig: { - maxSize: 100, // 100 tokens = 400 characters - minSize: 500, // Invalid: minSize (500 chars) > maxSize (400 chars) - overlap: 50, - }, + expect(mocks.platformCreated).toHaveBeenCalledWith({ + knowledgeBaseId: 'knowledge-1', + name: 'Test Knowledge Base', + workspaceId: 'workspace-1', + }) + expect(mocks.capture).toHaveBeenCalledWith( + 'user-123', + 'knowledge_base_created', + { + knowledge_base_id: 'knowledge-1', + workspace_id: 'workspace-1', + name: 'Test Knowledge Base', + }, + { + groups: { workspace: 'workspace-1' }, + setOnce: { first_kb_created_at: expect.any(String) }, } + ) + expect(mocks.create.mock.invocationCallOrder[0]).toBeLessThan( + mocks.platformCreated.mock.invocationCallOrder[0] + ) + expect(response.status).toBe(200) + await expect(response.json()).resolves.toEqual({ + success: true, + data: expectedKnowledgeBase, + }) + }) - const req = createMockRequest('POST', invalidData) - const response = await POST(req) - const data = await response.json() + it('preserves the legacy body validation envelope', async () => { + const response = await POST(createMockRequest('POST', { description: 'Missing fields' })) + const body = await response.json() - expect(response.status).toBe(400) - expect(data.error).toBe('Invalid request data') - }) + expect(response.status).toBe(400) + expect(body).toEqual({ error: 'Invalid request data', details: expect.any(Array) }) + expect(mocks.create).not.toHaveBeenCalled() + }) - it('should use default values for optional fields', async () => { - authMockFns.mockGetSession.mockResolvedValue({ - user: { id: 'user-123', email: 'test@example.com' }, - }) + it('projects typed application errors without running analytics', async () => { + mocks.create.mockRejectedValueOnce(new OrchestrationError('conflict', 'Already exists')) - const minimalData = { name: 'Test KB', workspaceId: 'test-workspace-id' } - const req = createMockRequest('POST', minimalData) - const response = await POST(req) - const data = await response.json() - - expect(response.status).toBe(200) - expect(data.data.embeddingModel).toBe('text-embedding-3-small') - expect(data.data.embeddingDimension).toBe(1536) - expect(data.data.chunkingConfig).toEqual({ - maxSize: 1024, - minSize: 100, - overlap: 200, + const response = await POST( + createMockRequest('POST', { + name: 'Test Knowledge Base', + workspaceId: 'workspace-1', }) - }) + ) - it('should handle database errors during creation', async () => { - authMockFns.mockGetSession.mockResolvedValue({ - user: { id: 'user-123', email: 'test@example.com' }, - }) - dbChainMockFns.values.mockRejectedValueOnce(new Error('Database error')) + expect(response.status).toBe(409) + await expect(response.json()).resolves.toEqual({ error: 'Already exists' }) + expect(mocks.platformCreated).not.toHaveBeenCalled() + expect(mocks.capture).not.toHaveBeenCalled() + }) - const req = createMockRequest('POST', validKnowledgeBaseData) - const response = await POST(req) - const data = await response.json() + it('returns a safe list error for unknown infrastructure failures', async () => { + mocks.list.mockRejectedValueOnce(new Error('database DSN secret')) - expect(response.status).toBe(500) - expect(data.error).toBe('Failed to create knowledge base') - }) + const response = await GET(createMockRequest('GET')) + + expect(response.status).toBe(500) + await expect(response.json()).resolves.toEqual({ error: 'Failed to fetch knowledge bases' }) }) }) diff --git a/apps/sim/app/api/knowledge/route.ts b/apps/sim/app/api/knowledge/route.ts index 09178f9dff1..40ef69cb20f 100644 --- a/apps/sim/app/api/knowledge/route.ts +++ b/apps/sim/app/api/knowledge/route.ts @@ -1,107 +1,63 @@ -import { createLogger } from '@sim/logger' -import { type NextRequest, NextResponse } from 'next/server' import { createKnowledgeBaseContract, - listKnowledgeBasesQuerySchema, + listKnowledgeBasesContract, } from '@/lib/api/contracts/knowledge' -import { parseRequest } from '@/lib/api/server' -import { getSession } from '@/lib/auth' +import { validationErrorResponse } from '@/lib/api/server' import { - messageForOrchestrationError, - statusForOrchestrationError, -} from '@/lib/core/orchestration/types' -import { generateRequestId } from '@/lib/core/utils/request' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { performCreateKnowledgeBase } from '@/lib/knowledge/orchestration' -import { getKnowledgeBases, type KnowledgeBaseScope } from '@/lib/knowledge/service' - -const logger = createLogger('KnowledgeBaseAPI') - -export const GET = withRouteHandler(async (req: NextRequest) => { - const requestId = generateRequestId() - - try { - const session = await getSession() - if (!session?.user?.id) { - logger.warn(`[${requestId}] Unauthorized knowledge base access attempt`) - return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) - } - - const { searchParams } = new URL(req.url) - const query = listKnowledgeBasesQuerySchema.safeParse({ - workspaceId: searchParams.get('workspaceId') ?? undefined, - scope: searchParams.get('scope') ?? undefined, - }) - if (!query.success) { - return NextResponse.json( - { error: 'Invalid query parameters', details: query.error.issues }, - { status: 400 } - ) - } - const { workspaceId, scope } = query.data - - const knowledgeBasesWithCounts = await getKnowledgeBases( - session.user.id, - workspaceId, - scope as KnowledgeBaseScope - ) - - return NextResponse.json({ - success: true, - data: knowledgeBasesWithCounts, - }) - } catch (error) { - logger.error(`[${requestId}] Error fetching knowledge bases`, error) - return NextResponse.json({ error: 'Failed to fetch knowledge bases' }, { status: 500 }) - } + defineInternalJsonRoute, + internalRateLimits, + internalSessionAuth, +} from '@/lib/api/server/routes' +import { + internalKnowledgeAnalytics, + internalKnowledgePresenters, +} from '@/lib/knowledge/api/internal-route' +import { internalKnowledgeErrorPolicies } from '@/lib/knowledge/api/route-policies' +import { + createKnowledgeBase, + listInternalKnowledgeBases, +} from '@/lib/knowledge/application/knowledge-bases' +import { + knowledgeOperations, + knowledgeSessionOperations, +} from '@/lib/knowledge/application/operations' + +export const GET = defineInternalJsonRoute({ + contract: listKnowledgeBasesContract, + auth: internalSessionAuth, + operation: knowledgeSessionOperations.list, + rateLimit: internalRateLimits.none({ + reason: 'Preserve existing internal knowledge base listing behavior', + }), + errorPolicy: internalKnowledgeErrorPolicies.list, + parseOptions: { + validationErrorResponse: (error) => validationErrorResponse(error, 'Invalid query parameters'), + }, + mapInput: ({ query }) => ({ workspaceId: query.workspaceId, scope: query.scope }), + useCase: listInternalKnowledgeBases, + present: internalKnowledgePresenters.list, }) -export const POST = withRouteHandler(async (req: NextRequest) => { - const requestId = generateRequestId() - - const session = await getSession() - if (!session?.user?.id) { - logger.warn(`[${requestId}] Unauthorized knowledge base creation attempt`) - return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseRequest( - createKnowledgeBaseContract, - req, - {}, - { - validationErrorResponse: (error) => { - logger.warn(`[${requestId}] Invalid knowledge base data`, { errors: error.issues }) - return NextResponse.json( - { error: 'Invalid request data', details: error.issues }, - { status: 400 } - ) - }, - } - ) - if (!parsed.success) return parsed.response - - const body = parsed.data.body - - const outcome = await performCreateKnowledgeBase({ - userId: session.user.id, - actorName: session.user.name, - actorEmail: session.user.email, - source: 'ui', +export const POST = defineInternalJsonRoute({ + contract: createKnowledgeBaseContract, + auth: internalSessionAuth, + operation: knowledgeOperations.create, + rateLimit: internalRateLimits.none({ + reason: 'Preserve existing internal knowledge base creation behavior', + }), + errorPolicy: internalKnowledgeErrorPolicies.create, + parseOptions: { + validationErrorResponse: (error) => validationErrorResponse(error, 'Invalid request data'), + }, + mapInput: ({ body }) => ({ workspaceId: body.workspaceId, name: body.name, description: body.description, folderId: body.folderId, chunkingConfig: body.chunkingConfig, - requestId, - request: req, - }) - if (!outcome.success) { - return NextResponse.json( - { error: messageForOrchestrationError(outcome, 'Failed to create knowledge base') }, - { status: statusForOrchestrationError(outcome.errorCode) } - ) - } - - return NextResponse.json({ success: true, data: outcome.knowledgeBase }) + source: 'ui', + }), + useCase: createKnowledgeBase, + onSuccess: internalKnowledgeAnalytics.created, + present: internalKnowledgePresenters.create, }) diff --git a/apps/sim/app/api/knowledge/search/route.test.ts b/apps/sim/app/api/knowledge/search/route.test.ts deleted file mode 100644 index 07526264200..00000000000 --- a/apps/sim/app/api/knowledge/search/route.test.ts +++ /dev/null @@ -1,1278 +0,0 @@ -/** - * Tests for knowledge search API route - * Focuses on route-specific functionality: authentication, validation, API contract, error handling - * Search logic is tested in utils.test.ts - * - * @vitest-environment node - */ -import { - createMockRequest, - dbChainMockFns, - hybridAuthMockFns, - knowledgeApiUtilsMock, - knowledgeApiUtilsMockFns, - resetDbChainMock, - resetEnvMock, - setEnv, - workflowAuthzMockFns, - workflowsUtilsMock, -} from '@sim/testing' -import { afterAll, afterEach, beforeEach, describe, expect, it, vi } from 'vitest' - -const { - mockGetDocumentTagDefinitions, - mockExecuteKnowledgeSearch, - mockGenerateSearchEmbedding, - mockImportKnowledgeSearchResultSecretProvenance, -} = vi.hoisted(() => ({ - mockGetDocumentTagDefinitions: vi.fn(), - mockExecuteKnowledgeSearch: vi.fn(), - mockGenerateSearchEmbedding: vi.fn(), - mockImportKnowledgeSearchResultSecretProvenance: vi.fn(), -})) - -const mockCheckKnowledgeBaseAccess = knowledgeApiUtilsMockFns.mockCheckKnowledgeBaseAccess - -vi.mock('@/lib/workflows/utils', () => workflowsUtilsMock) - -vi.mock('@/lib/documents/utils', () => ({ - retryWithExponentialBackoff: vi.fn().mockImplementation((fn) => fn()), -})) - -vi.mock('@/lib/tokenization/estimators', () => ({ - estimateTokenCount: vi.fn().mockReturnValue({ count: 521 }), -})) - -vi.mock('@/providers/utils', () => ({ - isFunctionToolCall: (toolCall: unknown) => - typeof toolCall === 'object' && - toolCall !== null && - 'function' in toolCall && - (toolCall as { function?: unknown }).function != null, - calculateCost: vi.fn().mockReturnValue({ - input: 0.00001042, - output: 0, - total: 0.00001042, - pricing: { - input: 0.02, - output: 0, - updatedAt: '2025-07-10', - }, - }), -})) - -vi.mock('@/app/api/knowledge/utils', () => knowledgeApiUtilsMock) - -vi.mock('@/lib/knowledge/tags/service', () => ({ - getDocumentTagDefinitions: mockGetDocumentTagDefinitions, -})) - -vi.mock('@/lib/knowledge/secret-provenance', () => ({ - importKnowledgeSearchResultSecretProvenance: mockImportKnowledgeSearchResultSecretProvenance, -})) - -vi.mock('@/lib/knowledge/search/queries', () => ({ - executeKnowledgeSearch: mockExecuteKnowledgeSearch, - generateSearchEmbedding: mockGenerateSearchEmbedding, - APIError: class APIError extends Error { - public status: number - constructor(message: string, status: number) { - super(message) - this.name = 'APIError' - this.status = status - } - }, -})) - -import { PRIVATE_MODEL_INPUT_PROVENANCE_HEADER } from '@/lib/execution/model-input-provenance' -import { - RESOLVED_SECRET_PROVENANCE_FIELD, - RESOLVED_SECRET_PROVENANCE_METADATA_V1, -} from '@/lib/execution/private-tool-metadata' -import { estimateTokenCount } from '@/lib/tokenization/estimators' -import { POST } from '@/app/api/knowledge/search/route' -import { calculateCost } from '@/providers/utils' - -describe('Knowledge Search API Route', () => { - const mockGetUserId = vi.fn() - const mockFetch = vi.fn() - - const mockEmbedding = [0.1, 0.2, 0.3, 0.4, 0.5] - const mockSearchResults = [ - { - id: 'chunk-1', - content: 'This is a test chunk', - documentId: 'doc-1', - chunkIndex: 0, - metadata: { title: 'Test Document' }, - distance: 0.2, - }, - { - id: 'chunk-2', - content: 'Another test chunk', - documentId: 'doc-2', - chunkIndex: 1, - metadata: { title: 'Another Document' }, - distance: 0.3, - }, - ] - - beforeEach(() => { - vi.clearAllMocks() - resetDbChainMock() - setEnv({ OPENAI_API_KEY: 'test-api-key' }) - - mockExecuteKnowledgeSearch.mockClear() - mockGenerateSearchEmbedding - .mockClear() - .mockResolvedValue({ embedding: [0.1, 0.2, 0.3, 0.4, 0.5], isBYOK: false }) - mockImportKnowledgeSearchResultSecretProvenance.mockClear().mockResolvedValue({ - imported: true, - documentMetadata: { - doc1: { - filename: 'Document 1', - sourceUrl: null, - provenance: { status: 'known', entries: [] }, - }, - doc2: { - filename: 'Document 2', - sourceUrl: null, - provenance: { status: 'known', entries: [] }, - }, - }, - }) - mockGetDocumentTagDefinitions.mockClear() - hybridAuthMockFns.mockCheckSessionOrInternalAuth.mockClear().mockResolvedValue({ - success: true, - userId: 'user-123', - authType: 'session', - }) - workflowAuthzMockFns.mockAuthorizeWorkflowByWorkspacePermission.mockClear().mockResolvedValue({ - allowed: true, - status: 200, - }) - - vi.stubGlobal('crypto', { - randomUUID: vi.fn().mockReturnValue('mock-uuid-1234-5678'), - }) - - vi.stubGlobal('fetch', mockFetch) - }) - - afterEach(() => { - vi.clearAllMocks() - }) - - afterAll(() => { - resetDbChainMock() - resetEnvMock() - }) - - describe('POST /api/knowledge/search', () => { - const validSearchData = { - knowledgeBaseIds: 'kb-123', - query: 'test search query', - topK: 10, - } - - const mockKnowledgeBases = [ - { - id: 'kb-123', - userId: 'user-123', - name: 'Test KB', - deletedAt: null, - }, - ] - - it('should perform search successfully with single knowledge base', async () => { - mockGetUserId.mockResolvedValue('user-123') - - mockCheckKnowledgeBaseAccess.mockResolvedValue({ - hasAccess: true, - knowledgeBase: { - id: 'kb-123', - userId: 'user-123', - name: 'Test KB', - deletedAt: null, - }, - }) - - dbChainMockFns.limit.mockResolvedValue([]) - - mockExecuteKnowledgeSearch.mockResolvedValue(mockSearchResults) - - mockFetch.mockResolvedValue({ - ok: true, - json: () => - Promise.resolve({ - data: [{ embedding: mockEmbedding }], - }), - }) - - const req = createMockRequest('POST', validSearchData) - const response = await POST(req) - const data = await response.json() - - expect(response.status).toBe(200) - expect(data.success).toBe(true) - expect(data.data.results).toHaveLength(2) - expect(data.data.results[0].similarity).toBe(0.8) // 1 - 0.2 - expect(data.data.query).toBe(validSearchData.query) - expect(data.data.knowledgeBaseIds).toEqual(['kb-123']) - expect(mockExecuteKnowledgeSearch).toHaveBeenCalledWith({ - knowledgeBaseIds: ['kb-123'], - topK: 10, - searchMode: 'vector', - query: validSearchData.query, - queryVector: JSON.stringify(mockEmbedding), - structuredFilters: undefined, - }) - }) - - it('should forward the hybrid searchMode opt-in to the retrieval layer', async () => { - mockGetUserId.mockResolvedValue('user-123') - - mockCheckKnowledgeBaseAccess.mockResolvedValue({ - hasAccess: true, - knowledgeBase: { - id: 'kb-123', - userId: 'user-123', - name: 'Test KB', - deletedAt: null, - }, - }) - - dbChainMockFns.limit.mockResolvedValue([]) - - mockExecuteKnowledgeSearch.mockResolvedValue(mockSearchResults) - - mockFetch.mockResolvedValue({ - ok: true, - json: () => - Promise.resolve({ - data: [{ embedding: mockEmbedding }], - }), - }) - - const req = createMockRequest('POST', { ...validSearchData, searchMode: 'hybrid' }) - const response = await POST(req) - - expect(response.status).toBe(200) - expect(mockExecuteKnowledgeSearch).toHaveBeenCalledWith( - expect.objectContaining({ searchMode: 'hybrid' }) - ) - }) - - it('should perform search successfully with multiple knowledge bases', async () => { - const multiKbData = { - ...validSearchData, - knowledgeBaseIds: ['kb-123', 'kb-456'], - } - - const multiKbs = [ - ...mockKnowledgeBases, - { id: 'kb-456', userId: 'user-123', name: 'Test KB 2', deletedAt: null }, - ] - - mockGetUserId.mockResolvedValue('user-123') - - mockCheckKnowledgeBaseAccess - .mockResolvedValueOnce({ hasAccess: true, knowledgeBase: multiKbs[0] }) - .mockResolvedValueOnce({ hasAccess: true, knowledgeBase: multiKbs[1] }) - - dbChainMockFns.limit.mockResolvedValue([]) - - mockExecuteKnowledgeSearch.mockResolvedValue(mockSearchResults) - - mockFetch.mockResolvedValue({ - ok: true, - json: () => - Promise.resolve({ - data: [{ embedding: mockEmbedding }], - }), - }) - - const req = createMockRequest('POST', multiKbData) - const response = await POST(req) - const data = await response.json() - - expect(response.status).toBe(200) - expect(data.success).toBe(true) - expect(data.data.knowledgeBaseIds).toEqual(['kb-123', 'kb-456']) - expect(mockExecuteKnowledgeSearch).toHaveBeenCalledWith({ - knowledgeBaseIds: ['kb-123', 'kb-456'], - topK: 10, - searchMode: 'vector', - query: multiKbData.query, - queryVector: JSON.stringify(mockEmbedding), - structuredFilters: undefined, - }) - }) - - it('should handle workflow-based authentication', async () => { - const workflowData = { - ...validSearchData, - workflowId: 'workflow-123', - } - - mockGetUserId.mockResolvedValue('user-123') - - mockCheckKnowledgeBaseAccess.mockResolvedValue({ - hasAccess: true, - knowledgeBase: { - id: 'kb-123', - userId: 'user-123', - name: 'Test KB', - deletedAt: null, - }, - }) - - dbChainMockFns.limit.mockResolvedValue([]) - - mockExecuteKnowledgeSearch.mockResolvedValue(mockSearchResults) - - mockFetch.mockResolvedValue({ - ok: true, - json: () => - Promise.resolve({ - data: [{ embedding: mockEmbedding }], - }), - }) - - const req = createMockRequest('POST', workflowData) - const response = await POST(req) - const data = await response.json() - - expect(response.status).toBe(200) - expect(data.success).toBe(true) - expect(workflowAuthzMockFns.mockAuthorizeWorkflowByWorkspacePermission).toHaveBeenCalledWith({ - workflowId: 'workflow-123', - userId: 'user-123', - action: 'read', - }) - }) - - it('fails before embedding work when an internal workspace request omits attribution', async () => { - hybridAuthMockFns.mockCheckSessionOrInternalAuth.mockResolvedValueOnce({ - success: true, - userId: 'user-123', - authType: 'internal_jwt', - }) - mockCheckKnowledgeBaseAccess.mockResolvedValue({ - hasAccess: true, - knowledgeBase: { - id: 'kb-123', - userId: 'user-123', - workspaceId: 'workspace-123', - embeddingModel: 'text-embedding-3-small', - }, - }) - - const req = createMockRequest('POST', { - ...validSearchData, - skipUsageBilling: true, - }) - const response = await POST(req) - - expect(response.status).toBe(500) - expect(mockGenerateSearchEmbedding).not.toHaveBeenCalled() - }) - - it('uses the immutable header for an internal unmetered workspace search', async () => { - hybridAuthMockFns.mockCheckSessionOrInternalAuth.mockResolvedValueOnce({ - success: true, - userId: 'user-123', - authType: 'internal_jwt', - }) - mockCheckKnowledgeBaseAccess.mockResolvedValue({ - hasAccess: true, - knowledgeBase: { - id: 'kb-123', - userId: 'user-123', - workspaceId: 'workspace-123', - embeddingModel: 'text-embedding-3-small', - }, - }) - mockExecuteKnowledgeSearch.mockResolvedValue(mockSearchResults) - const attribution = encodeURIComponent( - JSON.stringify({ - actorUserId: 'user-123', - workspaceId: 'workspace-123', - organizationId: 'organization-123', - billedAccountUserId: 'owner-123', - billingEntity: { type: 'organization', id: 'organization-123' }, - billingPeriod: { - start: '2026-07-01T00:00:00.000Z', - end: '2026-08-01T00:00:00.000Z', - }, - payerSubscription: null, - }) - ) - - const req = createMockRequest( - 'POST', - { - ...validSearchData, - skipUsageBilling: true, - [RESOLVED_SECRET_PROVENANCE_FIELD]: { version: 1, complete: true, entries: [] }, - }, - { - 'x-sim-billing-attribution': attribution, - [PRIVATE_MODEL_INPUT_PROVENANCE_HEADER]: RESOLVED_SECRET_PROVENANCE_METADATA_V1, - } - ) - const response = await POST(req) - - expect(response.status).toBe(200) - expect(mockGenerateSearchEmbedding).toHaveBeenCalledOnce() - }) - - it.concurrent('should return unauthorized for unauthenticated request', async () => { - hybridAuthMockFns.mockCheckSessionOrInternalAuth.mockResolvedValueOnce({ - success: false, - error: 'Unauthorized', - }) - - const req = createMockRequest('POST', validSearchData) - const response = await POST(req) - const data = await response.json() - - expect(response.status).toBe(401) - expect(data.error).toBe('Unauthorized') - }) - - it.concurrent('should return not found for workflow that does not exist', async () => { - const workflowData = { - ...validSearchData, - workflowId: 'nonexistent-workflow', - } - - workflowAuthzMockFns.mockAuthorizeWorkflowByWorkspacePermission.mockResolvedValueOnce({ - allowed: false, - status: 404, - message: 'Workflow not found', - }) - - const req = createMockRequest('POST', workflowData) - const response = await POST(req) - const data = await response.json() - - expect(response.status).toBe(404) - expect(data.error).toBe('Workflow not found') - }) - - it('should return not found for non-existent knowledge base', async () => { - mockGetUserId.mockResolvedValue('user-123') - - mockCheckKnowledgeBaseAccess.mockResolvedValue({ - hasAccess: false, - notFound: true, - }) - - const req = createMockRequest('POST', validSearchData) - const response = await POST(req) - const data = await response.json() - - expect(response.status).toBe(404) - expect(data.error).toBe('Knowledge base not found or access denied') - }) - - it('should return not found for some missing knowledge bases', async () => { - const multiKbData = { - ...validSearchData, - knowledgeBaseIds: ['kb-123', 'kb-missing'], - } - - mockGetUserId.mockResolvedValue('user-123') - - mockCheckKnowledgeBaseAccess - .mockResolvedValueOnce({ hasAccess: true, knowledgeBase: mockKnowledgeBases[0] }) - .mockResolvedValueOnce({ hasAccess: false, notFound: true }) - - const req = createMockRequest('POST', multiKbData) - const response = await POST(req) - const data = await response.json() - - expect(response.status).toBe(404) - expect(data.error).toBe('Knowledge bases not found or access denied: kb-missing') - }) - - it.concurrent('should validate search parameters', async () => { - const invalidData = { - knowledgeBaseIds: '', // Empty string - query: '', // Empty query - topK: 150, // Too high - } - - const req = createMockRequest('POST', invalidData) - const response = await POST(req) - const data = await response.json() - - expect(response.status).toBe(400) - expect(data.error).toBe('Validation error') - expect(data.details).toBeDefined() - }) - - it('should use default topK value when not provided', async () => { - const dataWithoutTopK = { - knowledgeBaseIds: 'kb-123', - query: 'test search query', - } - - mockGetUserId.mockResolvedValue('user-123') - - mockCheckKnowledgeBaseAccess.mockResolvedValue({ - hasAccess: true, - knowledgeBase: { - id: 'kb-123', - userId: 'user-123', - name: 'Test KB', - deletedAt: null, - embeddingModel: 'text-embedding-3-small', - }, - }) - - dbChainMockFns.limit.mockResolvedValueOnce(mockSearchResults) // Search results - - mockFetch.mockResolvedValue({ - ok: true, - json: () => - Promise.resolve({ - data: [{ embedding: mockEmbedding }], - }), - }) - - const req = createMockRequest('POST', dataWithoutTopK) - const response = await POST(req) - const data = await response.json() - - expect(response.status).toBe(200) - expect(data.data.topK).toBe(10) // Default value - }) - - it.concurrent('should handle OpenAI API errors', async () => { - mockGetUserId.mockResolvedValue('user-123') - dbChainMockFns.limit.mockResolvedValueOnce(mockKnowledgeBases) - - mockGenerateSearchEmbedding.mockRejectedValueOnce( - new Error('OpenAI API error: 401 Unauthorized - Invalid API key') - ) - - const req = createMockRequest('POST', validSearchData) - const response = await POST(req) - const data = await response.json() - - expect(response.status).toBe(500) - expect(data.error).toBe('Failed to perform vector search') - }) - - it.concurrent('should handle missing OpenAI API key', async () => { - mockGetUserId.mockResolvedValue('user-123') - dbChainMockFns.limit.mockResolvedValueOnce(mockKnowledgeBases) - - mockGenerateSearchEmbedding.mockRejectedValueOnce(new Error('OPENAI_API_KEY not configured')) - - const req = createMockRequest('POST', validSearchData) - const response = await POST(req) - const data = await response.json() - - expect(response.status).toBe(500) - expect(data.error).toBe('Failed to perform vector search') - }) - - it.concurrent('should handle database errors during search', async () => { - mockGetUserId.mockResolvedValue('user-123') - dbChainMockFns.limit.mockResolvedValueOnce(mockKnowledgeBases) - - mockExecuteKnowledgeSearch.mockRejectedValueOnce(new Error('Database error')) - - const req = createMockRequest('POST', validSearchData) - const response = await POST(req) - const data = await response.json() - - expect(response.status).toBe(500) - expect(data.error).toBe('Failed to perform vector search') - }) - - it.concurrent('should handle invalid OpenAI response format', async () => { - mockGetUserId.mockResolvedValue('user-123') - dbChainMockFns.limit.mockResolvedValueOnce(mockKnowledgeBases) - - mockGenerateSearchEmbedding.mockRejectedValueOnce( - new Error('Invalid response format from OpenAI embeddings API') - ) - - const req = createMockRequest('POST', validSearchData) - const response = await POST(req) - const data = await response.json() - - expect(response.status).toBe(500) - expect(data.error).toBe('Failed to perform vector search') - }) - - describe('Cost tracking', () => { - it.concurrent('should include cost information in successful search response', async () => { - mockGetUserId.mockResolvedValue('user-123') - - mockCheckKnowledgeBaseAccess.mockResolvedValue({ - hasAccess: true, - knowledgeBase: { - id: 'kb-123', - userId: 'user-123', - name: 'Test KB', - deletedAt: null, - embeddingModel: 'text-embedding-3-small', - }, - }) - - dbChainMockFns.limit.mockResolvedValueOnce(mockSearchResults) - - mockFetch.mockResolvedValue({ - ok: true, - json: () => - Promise.resolve({ - data: [{ embedding: mockEmbedding }], - }), - }) - - const req = createMockRequest('POST', validSearchData) - const response = await POST(req) - const data = await response.json() - - expect(response.status).toBe(200) - expect(data.success).toBe(true) - - expect(data.data.cost).toBeDefined() - expect(data.data.cost.input).toBe(0.00001042) - expect(data.data.cost.output).toBe(0) - expect(data.data.cost.total).toBe(0.00001042) - expect(data.data.cost.tokens).toEqual({ - prompt: 521, - completion: 0, - total: 521, - }) - expect(data.data.cost.model).toBe('text-embedding-3-small') - expect(data.data.cost.pricing).toEqual({ - input: 0.02, - output: 0, - updatedAt: '2025-07-10', - }) - }) - - it('should call cost calculation functions with correct parameters', async () => { - mockGetUserId.mockResolvedValue('user-123') - - mockCheckKnowledgeBaseAccess.mockResolvedValue({ - hasAccess: true, - knowledgeBase: { - id: 'kb-123', - userId: 'user-123', - name: 'Test KB', - deletedAt: null, - embeddingModel: 'text-embedding-3-small', - }, - }) - - dbChainMockFns.limit.mockResolvedValueOnce(mockSearchResults) - - mockFetch.mockResolvedValue({ - ok: true, - json: () => - Promise.resolve({ - data: [{ embedding: mockEmbedding }], - }), - }) - - const req = createMockRequest('POST', validSearchData) - await POST(req) - - expect(estimateTokenCount).toHaveBeenCalledWith('test search query', 'openai') - - expect(calculateCost).toHaveBeenCalledWith('text-embedding-3-small', 521, 0, false) - }) - - it('should handle cost calculation with different query lengths', async () => { - vi.mocked(estimateTokenCount).mockReturnValue({ - count: 1042, - confidence: 'high', - provider: 'openai', - method: 'precise', - }) - vi.mocked(calculateCost).mockReturnValue({ - input: 0.00002084, - output: 0, - total: 0.00002084, - pricing: { - input: 0.02, - output: 0, - updatedAt: '2025-07-10', - }, - }) - - const longQueryData = { - ...validSearchData, - query: - 'This is a much longer search query with many more tokens to test cost calculation accuracy', - } - - mockGetUserId.mockResolvedValue('user-123') - - mockCheckKnowledgeBaseAccess.mockResolvedValue({ - hasAccess: true, - knowledgeBase: { - id: 'kb-123', - userId: 'user-123', - name: 'Test KB', - deletedAt: null, - embeddingModel: 'text-embedding-3-small', - }, - }) - - dbChainMockFns.limit.mockResolvedValueOnce(mockSearchResults) - - mockFetch.mockResolvedValue({ - ok: true, - json: () => - Promise.resolve({ - data: [{ embedding: mockEmbedding }], - }), - }) - - const req = createMockRequest('POST', longQueryData) - const response = await POST(req) - const data = await response.json() - - expect(response.status).toBe(200) - expect(data.data.cost.input).toBe(0.00002084) - expect(data.data.cost.tokens.prompt).toBe(1042) - expect(calculateCost).toHaveBeenCalledWith('text-embedding-3-small', 1042, 0, false) - }) - }) - }) - - describe('Optional Query Search', () => { - const mockTagDefinitions = [ - { tagSlot: 'tag1', displayName: 'category', fieldType: 'text' }, - { tagSlot: 'tag2', displayName: 'priority', fieldType: 'text' }, - ] - - const mockTaggedResults = [ - { - id: 'chunk-1', - content: 'Tagged content 1', - documentId: 'doc-1', - chunkIndex: 0, - tag1: 'api', - tag2: 'high', - distance: 0, - knowledgeBaseId: 'kb-123', - }, - { - id: 'chunk-2', - content: 'Tagged content 2', - documentId: 'doc-2', - chunkIndex: 1, - tag1: 'docs', - tag2: 'medium', - distance: 0, - knowledgeBaseId: 'kb-123', - }, - ] - - it('should perform tag-only search without query', async () => { - const tagOnlyData = { - knowledgeBaseIds: 'kb-123', - tagFilters: [{ tagName: 'category', value: 'api', fieldType: 'text', operator: 'eq' }], - topK: 10, - } - - mockGetUserId.mockResolvedValue('user-123') - mockCheckKnowledgeBaseAccess.mockResolvedValue({ - hasAccess: true, - knowledgeBase: { - id: 'kb-123', - userId: 'user-123', - name: 'Test KB', - deletedAt: null, - embeddingModel: 'text-embedding-3-small', - }, - }) - - mockGetDocumentTagDefinitions.mockResolvedValue(mockTagDefinitions) - - dbChainMockFns.limit.mockResolvedValueOnce(mockTagDefinitions) - - mockExecuteKnowledgeSearch.mockResolvedValue(mockTaggedResults) - - const req = createMockRequest('POST', tagOnlyData) - const response = await POST(req) - const data = await response.json() - - expect(response.status).toBe(200) - expect(data.success).toBe(true) - expect(data.data.results).toHaveLength(2) - expect(data.data.results[0].similarity).toBe(1) // Perfect similarity for tag-only - expect(data.data.query).toBe('') // Empty query - expect(data.data.cost).toBeUndefined() // No cost for tag-only search - expect(mockGenerateSearchEmbedding).not.toHaveBeenCalled() // No embedding API call - expect(mockExecuteKnowledgeSearch).toHaveBeenCalledWith({ - knowledgeBaseIds: ['kb-123'], - topK: 10, - searchMode: 'vector', - structuredFilters: [ - { tagSlot: 'tag1', fieldType: 'text', operator: 'eq', value: 'api', valueTo: undefined }, - ], - }) - }) - - it('should perform query + tag combination search', async () => { - const combinedData = { - knowledgeBaseIds: 'kb-123', - query: 'test search', - tagFilters: [{ tagName: 'category', value: 'api', fieldType: 'text', operator: 'eq' }], - topK: 10, - } - - mockGetUserId.mockResolvedValue('user-123') - mockCheckKnowledgeBaseAccess.mockResolvedValue({ - hasAccess: true, - knowledgeBase: { - id: 'kb-123', - userId: 'user-123', - name: 'Test KB', - deletedAt: null, - embeddingModel: 'text-embedding-3-small', - }, - }) - - mockGetDocumentTagDefinitions.mockResolvedValue(mockTagDefinitions) - - dbChainMockFns.limit.mockResolvedValueOnce(mockTagDefinitions) - - mockExecuteKnowledgeSearch.mockResolvedValue(mockSearchResults) - - mockFetch.mockResolvedValue({ - ok: true, - json: () => - Promise.resolve({ - data: [{ embedding: mockEmbedding }], - }), - }) - - const req = createMockRequest('POST', combinedData) - const response = await POST(req) - const data = await response.json() - - expect(response.status).toBe(200) - expect(data.success).toBe(true) - expect(data.data.results).toHaveLength(2) - expect(data.data.query).toBe('test search') - expect(data.data.cost).toBeDefined() // Cost included for vector search - expect(mockGenerateSearchEmbedding).toHaveBeenCalled() // Embedding API called - expect(mockExecuteKnowledgeSearch).toHaveBeenCalledWith({ - knowledgeBaseIds: ['kb-123'], - topK: 10, - searchMode: 'vector', - query: 'test search', - queryVector: JSON.stringify(mockEmbedding), - structuredFilters: [ - { tagSlot: 'tag1', fieldType: 'text', operator: 'eq', value: 'api', valueTo: undefined }, - ], - }) - }) - - it('should validate that either query or filters are provided', async () => { - const emptyData = { - knowledgeBaseIds: 'kb-123', - topK: 10, - } - - const req = createMockRequest('POST', emptyData) - const response = await POST(req) - const data = await response.json() - - expect(response.status).toBe(400) - expect(data.error).toBe('Validation error') - expect(data.details).toEqual( - expect.arrayContaining([ - expect.objectContaining({ - message: - 'Please provide either a search query or tag filters to search your knowledge base', - }), - ]) - ) - }) - - it('should validate that empty query with empty filters fails', async () => { - const emptyFiltersData = { - knowledgeBaseIds: 'kb-123', - query: '', - filters: {}, - topK: 10, - } - - const req = createMockRequest('POST', emptyFiltersData) - const response = await POST(req) - const data = await response.json() - - expect(response.status).toBe(400) - expect(data.error).toBe('Validation error') - }) - - it('should handle empty tag values gracefully', async () => { - const emptyTagValueData = { - knowledgeBaseIds: 'kb-123', - query: '', - topK: 10, - } - - const req = createMockRequest('POST', emptyTagValueData) - const response = await POST(req) - const data = await response.json() - - expect(response.status).toBe(400) - expect(data.error).toBe('Validation error') - expect(data.details).toEqual( - expect.arrayContaining([ - expect.objectContaining({ - message: - 'Please provide either a search query or tag filters to search your knowledge base', - }), - ]) - ) - }) - - it('should handle null values from frontend gracefully', async () => { - const nullValuesData = { - knowledgeBaseIds: 'kb-123', - topK: null, - query: null, - filters: null, - } - - const req = createMockRequest('POST', nullValuesData) - const response = await POST(req) - const data = await response.json() - - expect(response.status).toBe(400) - expect(data.error).toBe('Validation error') - expect(data.details).toEqual( - expect.arrayContaining([ - expect.objectContaining({ - message: - 'Please provide either a search query or tag filters to search your knowledge base', - }), - ]) - ) - }) - - it('should perform query-only search (existing behavior)', async () => { - const queryOnlyData = { - knowledgeBaseIds: 'kb-123', - query: 'test search query', - topK: 10, - } - - mockGetUserId.mockResolvedValue('user-123') - mockCheckKnowledgeBaseAccess.mockResolvedValue({ - hasAccess: true, - knowledgeBase: { - id: 'kb-123', - userId: 'user-123', - name: 'Test KB', - deletedAt: null, - embeddingModel: 'text-embedding-3-small', - }, - }) - - dbChainMockFns.limit.mockResolvedValueOnce(mockSearchResults) - - mockFetch.mockResolvedValue({ - ok: true, - json: () => - Promise.resolve({ - data: [{ embedding: mockEmbedding }], - }), - }) - - const req = createMockRequest('POST', queryOnlyData) - const response = await POST(req) - const data = await response.json() - - expect(response.status).toBe(200) - expect(data.success).toBe(true) - expect(data.data.results).toHaveLength(2) - expect(data.data.query).toBe('test search query') - expect(data.data.cost).toBeDefined() // Cost included for vector search - expect(mockGenerateSearchEmbedding).toHaveBeenCalled() // Embedding API called - }) - - it('should handle tag-only search with multiple knowledge bases', async () => { - const multiKbTagData = { - knowledgeBaseIds: ['kb-123', 'kb-456'], - tagFilters: [ - { tagName: 'category', value: 'docs', fieldType: 'text', operator: 'eq' }, - { tagName: 'priority', value: 'high', fieldType: 'text', operator: 'eq' }, - ], - topK: 10, - } - - mockGetUserId.mockResolvedValue('user-123') - mockCheckKnowledgeBaseAccess - .mockResolvedValueOnce({ - hasAccess: true, - knowledgeBase: { - id: 'kb-123', - userId: 'user-123', - name: 'Test KB', - deletedAt: null, - embeddingModel: 'text-embedding-3-small', - }, - }) - .mockResolvedValueOnce({ - hasAccess: true, - knowledgeBase: { - id: 'kb-456', - userId: 'user-123', - name: 'Test KB 2', - embeddingModel: 'text-embedding-3-small', - }, - }) - - mockGetDocumentTagDefinitions.mockResolvedValue(mockTagDefinitions) - - mockExecuteKnowledgeSearch.mockResolvedValue(mockTaggedResults) - - dbChainMockFns.limit.mockResolvedValueOnce(mockTagDefinitions) - - const req = createMockRequest('POST', multiKbTagData) - const response = await POST(req) - const data = await response.json() - - expect(response.status).toBe(200) - expect(data.success).toBe(true) - expect(data.data.knowledgeBaseIds).toEqual(['kb-123', 'kb-456']) - expect(mockGenerateSearchEmbedding).not.toHaveBeenCalled() // No embedding for tag-only - }) - }) - - describe('Deleted document filtering', () => { - it('should exclude results from deleted documents in vector search', async () => { - mockGetUserId.mockResolvedValue('user-123') - - mockCheckKnowledgeBaseAccess.mockResolvedValue({ - hasAccess: true, - knowledgeBase: { - id: 'kb-123', - userId: 'user-123', - name: 'Test KB', - deletedAt: null, - }, - }) - - mockExecuteKnowledgeSearch.mockResolvedValue([ - { - id: 'chunk-1', - content: 'Content from active document', - documentId: 'doc-active', - chunkIndex: 0, - tag1: null, - tag2: null, - tag3: null, - tag4: null, - tag5: null, - tag6: null, - tag7: null, - distance: 0.2, - knowledgeBaseId: 'kb-123', - }, - ]) - - mockGenerateSearchEmbedding.mockResolvedValue({ embedding: [0.1, 0.2, 0.3], isBYOK: false }) - mockImportKnowledgeSearchResultSecretProvenance.mockResolvedValue({ - imported: true, - documentMetadata: { - 'doc-active': { - filename: 'Active Document.pdf', - sourceUrl: 'https://example.atlassian.net/wiki/spaces/DOCS/pages/12345', - provenance: { status: 'known', entries: [] }, - }, - }, - }) - - const mockTagDefs = { - select: vi.fn().mockReturnThis(), - from: vi.fn().mockReturnThis(), - where: vi.fn().mockResolvedValue([]), - } - dbChainMockFns.select.mockReturnValueOnce(mockTagDefs) - - const req = createMockRequest('POST', { - knowledgeBaseIds: ['kb-123'], - query: 'test query', - topK: 10, - }) - - const response = await POST(req) - const data = await response.json() - - expect(response.status).toBe(200) - expect(data.success).toBe(true) - expect(data.data.results).toHaveLength(1) - expect(data.data.results[0].documentId).toBe('doc-active') - expect(data.data.results[0].documentName).toBe('Active Document.pdf') - expect(data.data.results[0].sourceUrl).toBe( - 'https://example.atlassian.net/wiki/spaces/DOCS/pages/12345' - ) - }) - - it('should exclude results from deleted documents in tag search', async () => { - mockGetUserId.mockResolvedValue('user-123') - - mockCheckKnowledgeBaseAccess.mockResolvedValue({ - hasAccess: true, - knowledgeBase: { - id: 'kb-123', - userId: 'user-123', - name: 'Test KB', - deletedAt: null, - }, - }) - - mockGetDocumentTagDefinitions.mockResolvedValue([ - { tagSlot: 'tag1', displayName: 'tag1', fieldType: 'text' }, - ]) - - mockExecuteKnowledgeSearch.mockResolvedValue([ - { - id: 'chunk-2', - content: 'Content from active document with tag', - documentId: 'doc-active-tagged', - chunkIndex: 0, - tag1: 'api', - tag2: null, - tag3: null, - tag4: null, - tag5: null, - tag6: null, - tag7: null, - distance: 0, - knowledgeBaseId: 'kb-123', - }, - ]) - - mockImportKnowledgeSearchResultSecretProvenance.mockResolvedValue({ - imported: true, - documentMetadata: { - 'doc-active-tagged': { - filename: 'Active Tagged Document.pdf', - sourceUrl: null, - tag1: 'api', - provenance: { status: 'known', entries: [] }, - }, - }, - }) - - const mockTagDefs = { - select: vi.fn().mockReturnThis(), - from: vi.fn().mockReturnThis(), - where: vi - .fn() - .mockResolvedValue([{ tagSlot: 'tag1', displayName: 'tag1', fieldType: 'text' }]), - } - dbChainMockFns.select.mockReturnValueOnce(mockTagDefs) - - const req = createMockRequest('POST', { - knowledgeBaseIds: ['kb-123'], - tagFilters: [{ tagName: 'tag1', value: 'api', fieldType: 'text', operator: 'eq' }], - topK: 10, - }) - - const response = await POST(req) - const data = await response.json() - - expect(response.status).toBe(200) - expect(data.success).toBe(true) - expect(data.data.results).toHaveLength(1) - expect(data.data.results[0].documentId).toBe('doc-active-tagged') - expect(data.data.results[0].documentName).toBe('Active Tagged Document.pdf') - expect(data.data.results[0].metadata).toEqual({ tag1: 'api' }) - }) - - it('should exclude results from deleted documents in combined tag+vector search', async () => { - mockGetUserId.mockResolvedValue('user-123') - - mockCheckKnowledgeBaseAccess.mockResolvedValue({ - hasAccess: true, - knowledgeBase: { - id: 'kb-123', - userId: 'user-123', - name: 'Test KB', - deletedAt: null, - }, - }) - - mockGetDocumentTagDefinitions.mockResolvedValue([ - { tagSlot: 'tag1', displayName: 'tag1', fieldType: 'text' }, - ]) - - mockExecuteKnowledgeSearch.mockResolvedValue([ - { - id: 'chunk-3', - content: 'Relevant content from active document', - documentId: 'doc-active-combined', - chunkIndex: 0, - tag1: 'guide', - tag2: null, - tag3: null, - tag4: null, - tag5: null, - tag6: null, - tag7: null, - distance: 0.15, - knowledgeBaseId: 'kb-123', - }, - ]) - - mockGenerateSearchEmbedding.mockResolvedValue({ embedding: [0.1, 0.2, 0.3], isBYOK: false }) - mockImportKnowledgeSearchResultSecretProvenance.mockResolvedValue({ - imported: true, - documentMetadata: { - 'doc-active-combined': { - filename: 'Active Combined Search.pdf', - sourceUrl: null, - tag1: 'guide', - provenance: { status: 'known', entries: [] }, - }, - }, - }) - - const mockTagDefs = { - select: vi.fn().mockReturnThis(), - from: vi.fn().mockReturnThis(), - where: vi - .fn() - .mockResolvedValue([{ tagSlot: 'tag1', displayName: 'tag1', fieldType: 'text' }]), - } - dbChainMockFns.select.mockReturnValueOnce(mockTagDefs) - - const req = createMockRequest('POST', { - knowledgeBaseIds: ['kb-123'], - query: 'relevant content', - tagFilters: [{ tagName: 'tag1', value: 'guide', fieldType: 'text', operator: 'eq' }], - topK: 10, - }) - - const response = await POST(req) - const data = await response.json() - - expect(response.status).toBe(200) - expect(data.success).toBe(true) - expect(data.data.results).toHaveLength(1) - expect(data.data.results[0].documentId).toBe('doc-active-combined') - expect(data.data.results[0].documentName).toBe('Active Combined Search.pdf') - expect(data.data.results[0].metadata).toEqual({ tag1: 'guide' }) - expect(data.data.results[0].similarity).toBe(0.85) // 1 - 0.15 distance - }) - }) -}) diff --git a/apps/sim/app/api/knowledge/search/route.ts b/apps/sim/app/api/knowledge/search/route.ts index e44bb92571e..c9a99ffa3de 100644 --- a/apps/sim/app/api/knowledge/search/route.ts +++ b/apps/sim/app/api/knowledge/search/route.ts @@ -1,659 +1,84 @@ -import { createLogger } from '@sim/logger' -import { authorizeWorkflowByWorkspacePermission } from '@sim/platform-authz/workflow' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { knowledgeSearchBodySchema } from '@/lib/api/contracts/knowledge' -import { parseJsonBody, validationErrorResponse } from '@/lib/api/server' -import { AuthType, checkSessionOrInternalAuth } from '@/lib/auth/hybrid' -import { checkActorUsageLimits } from '@/lib/billing/calculations/usage-monitor' +import { internalKnowledgeSearchContract } from '@/lib/api/contracts/knowledge' +import { defineInternalJsonRoute, internalRateLimits } from '@/lib/api/server/routes' +import { OrchestrationError } from '@/lib/core/orchestration/types' import { - checkAttributedUsageLimits, - requireBillingAttributionHeader, - resolveBillingAttribution, - toBillingContext, -} from '@/lib/billing/core/billing-attribution' + internalKnowledgeAuthType, + resolveInternalKnowledgeBillingAttribution, +} from '@/lib/knowledge/api/internal-route' import { - checkAndBillOverageThreshold, - checkAndBillPayerOverageThreshold, -} from '@/lib/billing/threshold-billing' -import { PlatformEvents } from '@/lib/core/telemetry' -import { generateRequestId } from '@/lib/core/utils/request' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { importDurableSecretProvenance } from '@/lib/execution/durable-secret-provenance' -import { ALL_TAG_SLOTS } from '@/lib/knowledge/constants' -import { getEmbeddingModelInfo } from '@/lib/knowledge/embedding-models' -import { - prepareKnowledgeModelInputProvenance, - runWithKnowledgeModelInputProvenance, -} from '@/lib/knowledge/model-input-provenance' -import { rerank } from '@/lib/knowledge/reranker' -import { - executeKnowledgeSearch, - generateSearchEmbedding, - type SearchResult, -} from '@/lib/knowledge/search/queries' -import { importKnowledgeSearchResultSecretProvenance } from '@/lib/knowledge/secret-provenance' -import { getDocumentTagDefinitions } from '@/lib/knowledge/tags/service' -import { buildUndefinedTagsError, validateTagValue } from '@/lib/knowledge/tags/utils' -import type { StructuredFilter } from '@/lib/knowledge/types' -import { estimateTokenCount } from '@/lib/tokenization/estimators' -import { createKnowledgeRegistryResponse } from '@/app/api/knowledge/secret-provenance' -import { checkKnowledgeBaseAccess, type KnowledgeBaseAccessResult } from '@/app/api/knowledge/utils' -import { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry' -import { getRerankModelPricing } from '@/providers/models' -import { calculateCost } from '@/providers/utils' - -const logger = createLogger('VectorSearchAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - try { - const parsedBody = await parseJsonBody(request) - if (!parsedBody.success) return parsedBody.response - const body = parsedBody.data as Record - const { workflowId, skipUsageBilling, ...searchParams } = body - - const auth = await checkSessionOrInternalAuth(request, { requireWorkflowId: false }) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) - } - const userId = auth.userId - - // Only the internal workflow tool may suppress route metering (it rolls the - // cost into the executor's usage instead). Session/API-key callers cannot set - // skipUsageBilling to dodge their own embedding/reranker charge. - const shouldMeter = !(skipUsageBilling === true && auth.authType === AuthType.INTERNAL_JWT) - - if (workflowId) { - const authorization = await authorizeWorkflowByWorkspacePermission({ - workflowId: workflowId as string, - userId, - action: 'read', - }) - if (!authorization.allowed) { - return NextResponse.json( - { error: authorization.message || 'Access denied' }, - { status: authorization.status } - ) - } - } - - const validation = knowledgeSearchBodySchema.safeParse(searchParams) - if (!validation.success) return validationErrorResponse(validation.error) - const validatedData = validation.data - - const knowledgeBaseIds = Array.isArray(validatedData.knowledgeBaseIds) - ? validatedData.knowledgeBaseIds - : [validatedData.knowledgeBaseIds] - - const accessChecks = await Promise.all( - knowledgeBaseIds.map((kbId) => checkKnowledgeBaseAccess(kbId, userId)) - ) - const accessibleKbIds: string[] = knowledgeBaseIds.filter( - (_, idx) => accessChecks[idx]?.hasAccess - ) - - let structuredFilters: StructuredFilter[] = [] - - if (validatedData.tagFilters && accessibleKbIds.length > 0) { - const kbTagDefs = await Promise.all( - accessibleKbIds.map(async (kbId) => ({ - kbId, - tagDefs: await getDocumentTagDefinitions(kbId), - })) - ) - - const displayNameToTagDef: Record = {} - for (const { kbId, tagDefs } of kbTagDefs) { - const perKbMap = new Map( - tagDefs.map((def) => [ - def.displayName, - { tagSlot: def.tagSlot, fieldType: def.fieldType }, - ]) - ) - - for (const filter of validatedData.tagFilters) { - const current = perKbMap.get(filter.tagName) - if (!current) { - if (accessibleKbIds.length > 1) { - return NextResponse.json( - { - error: `Tag "${filter.tagName}" does not exist in all selected knowledge bases. Search those knowledge bases separately.`, - }, - { status: 400 } - ) - } - continue - } - - const existing = displayNameToTagDef[filter.tagName] - if ( - existing && - (existing.tagSlot !== current.tagSlot || existing.fieldType !== current.fieldType) - ) { - return NextResponse.json( - { - error: `Tag "${filter.tagName}" is not mapped consistently across the selected knowledge bases. Search those knowledge bases separately.`, - }, - { status: 400 } - ) - } - - displayNameToTagDef[filter.tagName] = current - } - - logger.debug(`[${requestId}] Loaded tag definitions for KB ${kbId}`, { - tagCount: tagDefs.length, - }) - } - - const undefinedTags: string[] = [] - const typeErrors: string[] = [] - - for (const filter of validatedData.tagFilters) { - const tagDef = displayNameToTagDef[filter.tagName] - - if (!tagDef) { - undefinedTags.push(filter.tagName) - continue - } - - const validationError = validateTagValue( - filter.tagName, - String(filter.value), - tagDef.fieldType - ) - if (validationError) { - typeErrors.push(validationError) - } - } - - if (undefinedTags.length > 0 || typeErrors.length > 0) { - const errorParts: string[] = [] - - if (undefinedTags.length > 0) { - errorParts.push(buildUndefinedTagsError(undefinedTags)) - } - - if (typeErrors.length > 0) { - errorParts.push(...typeErrors) - } - - return NextResponse.json({ error: errorParts.join('\n') }, { status: 400 }) - } - - structuredFilters = validatedData.tagFilters.map((filter) => { - const tagDef = displayNameToTagDef[filter.tagName]! - const tagSlot = tagDef.tagSlot - const fieldType = tagDef.fieldType - - logger.debug( - `[${requestId}] Structured filter: ${filter.tagName} -> ${tagSlot} (${fieldType}) ${filter.operator}` - ) - - return { - tagSlot, - fieldType, - operator: filter.operator, - value: filter.value, - valueTo: filter.valueTo, - } - }) - } - - if (accessibleKbIds.length === 0) { - return NextResponse.json( - { error: 'Knowledge base not found or access denied' }, - { status: 404 } - ) - } - - const accessibleKbs = accessChecks - .filter((ac): ac is KnowledgeBaseAccessResult => Boolean(ac?.hasAccess)) - .map((ac) => ac.knowledgeBase) - const useReranker = validatedData.rerankerEnabled && Boolean(validatedData.query?.trim()) - const rerankerModel = useReranker ? validatedData.rerankerModel : null - - const hasQuery = validatedData.query && validatedData.query.trim().length > 0 - const workspaceIds = new Set(accessibleKbs.map((kb) => kb.workspaceId ?? null)) - if (hasQuery && workspaceIds.size > 1) { - return NextResponse.json( - { error: 'Selected knowledge bases must belong to the same workspace' }, - { status: 400 } - ) - } - const workspaceId = accessibleKbs[0]?.workspaceId - - if (workflowId) { - const authorization = await authorizeWorkflowByWorkspacePermission({ - workflowId: workflowId as string, - userId, - action: 'read', - }) - const workflowWorkspaceId = authorization.workflow?.workspaceId ?? null - if ( - workflowWorkspaceId && - accessChecks.some( - (accessCheck) => - accessCheck?.hasAccess && accessCheck.knowledgeBase?.workspaceId !== workflowWorkspaceId - ) - ) { - return NextResponse.json( - { error: 'Knowledge base does not belong to the workflow workspace' }, - { status: 400 } - ) - } - } - - const billingAttribution = - hasQuery && workspaceId - ? auth.authType === AuthType.INTERNAL_JWT - ? requireBillingAttributionHeader(request.headers, { - actorUserId: userId, - workspaceId, - }) - : shouldMeter - ? await resolveBillingAttribution({ - actorUserId: userId, - workspaceId, - }) - : undefined - : undefined - const embeddingModels = Array.from(new Set(accessibleKbs.map((kb) => kb.embeddingModel))) - if (hasQuery && embeddingModels.length > 1) { - return NextResponse.json( - { - error: - 'Selected knowledge bases use different embedding models and cannot be searched together. Search them separately.', - }, - { status: 400 } - ) - } - const queryEmbeddingModel = embeddingModels[0] - - const inaccessibleKbIds = knowledgeBaseIds.filter((id) => !accessibleKbIds.includes(id)) - - if (inaccessibleKbIds.length > 0) { - return NextResponse.json( - { error: `Knowledge bases not found or access denied: ${inaccessibleKbIds.join(', ')}` }, - { status: 404 } - ) - } - - /** - * Gate the workspace payer and actor before hosted embedding cost. Internal - * workflow tools were gated during preprocessing, and tag-only search is free. - */ - if (shouldMeter && hasQuery) { - const usage = billingAttribution - ? await checkAttributedUsageLimits(billingAttribution) - : await checkActorUsageLimits(userId) - if (usage.isExceeded) { - return NextResponse.json( - { error: usage.message || 'Usage limit exceeded. Please upgrade your plan to continue.' }, - { status: 402 } - ) - } - } - - const modelInputProvenance = await prepareKnowledgeModelInputProvenance({ - headers: request.headers, - payload: body, - isInternalRequest: auth.authType === AuthType.INTERNAL_JWT, + internalKnowledgeErrorPolicies, + internalKnowledgeSessionOrExecutorAuth, +} from '@/lib/knowledge/api/route-policies' +import { knowledgeOperations } from '@/lib/knowledge/application/operations' +import { searchKnowledge } from '@/lib/knowledge/application/search' +import { prepareKnowledgeModelInputProvenance } from '@/lib/knowledge/model-input-provenance' +import { finalizeKnowledgeRegistryResponse } from '@/app/api/knowledge/secret-provenance' + +export const POST = defineInternalJsonRoute({ + contract: internalKnowledgeSearchContract, + auth: internalKnowledgeSessionOrExecutorAuth, + operation: knowledgeOperations.search, + rateLimit: internalRateLimits.none({ + reason: 'Preserve existing internal Knowledge-search behavior', + }), + errorPolicy: internalKnowledgeErrorPolicies.search, + parseOptions: { maxBodyBytes: 2 * 1024 * 1024 }, + mapInput: ({ body }, { principal, request }) => ({ + knowledgeBaseIds: Array.isArray(body.knowledgeBaseIds) + ? body.knowledgeBaseIds + : [body.knowledgeBaseIds], + query: body.query, + topK: body.topK, + tagFilters: body.tagFilters, + searchMode: body.searchMode, + rerankerEnabled: body.rerankerEnabled, + rerankerModel: body.rerankerModel, + rerankerInputCount: body.rerankerInputCount, + rerankerApiKey: body.rerankerApiKey, + skipUsageBilling: body.skipUsageBilling, + resolveBillingAttribution: (workspaceId: string) => + resolveInternalKnowledgeBillingAttribution(request, principal, workspaceId), + prepareModelInputProvenance: async ({ userId, - workspaceId: workspaceId ?? undefined, - modelInput: validatedData.query, - }) - if (!modelInputProvenance.success) { - return NextResponse.json( - { error: modelInputProvenance.error }, - { status: modelInputProvenance.status } - ) - } - - const queryEmbeddingPromise = hasQuery - ? runWithKnowledgeModelInputProvenance(modelInputProvenance.registry, () => - generateSearchEmbedding(validatedData.query!, queryEmbeddingModel, workspaceId) - ) - : Promise.resolve(null) - - let results: SearchResult[] - - const hasFilters = structuredFilters && structuredFilters.length > 0 - - /** Oversample vector results when reranking so the reranker has more to choose from. - * Cap at 100 to bound Cohere request cost (1 search unit = ≤100 docs). When the caller - * supplies `rerankerInputCount`, honor it but never let it drop below `topK` - * (which would defeat the purpose) or exceed 100 (which would split into >1 search units). */ - const rawInputCount = validatedData.rerankerInputCount - if (useReranker && rawInputCount !== undefined && rawInputCount < validatedData.topK) { - logger.warn( - `[${requestId}] rerankerInputCount (${rawInputCount}) is below topK (${validatedData.topK}); raising to topK` - ) - } - const candidateTopK = useReranker - ? rawInputCount !== undefined - ? Math.min(100, Math.max(validatedData.topK, rawInputCount)) - : Math.min(100, validatedData.topK * 4) - : validatedData.topK - - if (!hasQuery && hasFilters) { - results = await executeKnowledgeSearch({ - knowledgeBaseIds: accessibleKbIds, - topK: validatedData.topK, - searchMode: validatedData.searchMode, - structuredFilters, - }) - } else if (hasQuery) { - logger.debug( - `[${requestId}] Executing ${validatedData.searchMode} search`, - hasFilters ? { filterCount: structuredFilters?.length ?? 0 } : undefined - ) - const queryVector = JSON.stringify((await queryEmbeddingPromise)?.embedding ?? null) - - results = await executeKnowledgeSearch({ - knowledgeBaseIds: accessibleKbIds, - topK: candidateTopK, - searchMode: validatedData.searchMode, - query: validatedData.query, - queryVector, - structuredFilters: hasFilters ? structuredFilters : undefined, - }) - } else { - return NextResponse.json( - { - error: - 'Please provide either a search query or tag filters to search your knowledge base', - }, - { status: 400 } - ) - } - - const resultSecretRegistry = - modelInputProvenance.registry ?? - new ResolvedSecretTraceRegistry([], { + workspaceId, + }: { + userId: string + workspaceId: string + }) => { + const prepared = await prepareKnowledgeModelInputProvenance({ + headers: request.headers, + payload: body, + isInternalRequest: principal.kind === 'delegated', userId, - ...(workspaceId ? { workspaceId } : {}), + workspaceId, + modelInput: body.query, }) - const resultProvenanceSnapshot = await importKnowledgeSearchResultSecretProvenance({ - registry: resultSecretRegistry, - results, - }) - if (!resultProvenanceSnapshot.imported) { - resultSecretRegistry.markIncomplete() - if (useReranker) { - return NextResponse.json( - { error: 'Knowledge result secret provenance is unavailable' }, - { status: 422 } - ) - } - } - - /** Optional Cohere rerank pass on top of vector results. - * `rerankBilled` = Cohere was successfully called (even with 0 results) and we owe the search unit. */ - const rerankedScores = new Map() - let rerankBilled = false - let rerankIsBYOK = false - if (useReranker && rerankerModel && results.length > 0) { - const candidateCount = results.length - try { - const { results: ranked, isBYOK } = await runWithKnowledgeModelInputProvenance( - resultSecretRegistry, - () => - rerank( - validatedData.query!, - results.map((r) => ({ id: r.id, text: r.content })), - { - model: rerankerModel, - topN: validatedData.topK, - workspaceId, - apiKey: validatedData.rerankerApiKey, - } - ) - ) - rerankBilled = true - rerankIsBYOK = isBYOK - if (ranked.length === 0) { - logger.warn( - `[${requestId}] Reranker returned 0 results; falling back to vector ordering`, - { model: rerankerModel, candidateCount } - ) - results = results.slice(0, validatedData.topK) - } else { - const idToResult = new Map(results.map((r) => [r.id, r])) - results = ranked - .map((r) => idToResult.get(r.item.id)) - .filter((r): r is SearchResult => Boolean(r)) - for (const r of ranked) rerankedScores.set(r.item.id, r.relevanceScore) - logger.info(`[${requestId}] Reranked ${candidateCount} → ${results.length} results`, { - model: rerankerModel, - }) - } - } catch (error) { - if (resultSecretRegistry.isPermanentlyIncomplete()) throw error - logger.warn(`[${requestId}] Reranker failed; falling back to vector ordering`, { - error: getErrorMessage(error, 'Unknown error'), - model: rerankerModel, - candidateCount, - workspaceId, - }) - results = results.slice(0, validatedData.topK) - } - } else if (useReranker) { - results = results.slice(0, validatedData.topK) - } - - let cost = null - let tokenCount = null - if (hasQuery) { - try { - tokenCount = estimateTokenCount( - validatedData.query!, - getEmbeddingModelInfo(queryEmbeddingModel).tokenizerProvider - ) - // BYOK query embeddings incur no Sim cost, so don't bill (or roll up) them. - const queryEmbeddingResult = await queryEmbeddingPromise - if (!queryEmbeddingResult?.isBYOK) { - cost = calculateCost(queryEmbeddingModel, tokenCount.count, 0, false) - } - } catch (error) { - logger.warn(`[${requestId}] Failed to calculate cost for search query`, { - error: getErrorMessage(error, 'Unknown error'), - }) - } - } - - /** Add Cohere rerank cost (1 search unit per successful call, since we cap candidates ≤100). - * Bill on every successful API response — Cohere charges even when 0 results are returned. */ - let rerankerCost = 0 - if (rerankBilled && rerankerModel && !rerankIsBYOK) { - const pricing = getRerankModelPricing(rerankerModel) - if (pricing) { - rerankerCost = pricing.perSearchUnit - if (cost) { - cost = { - ...cost, - input: cost.input + rerankerCost, - total: cost.total + rerankerCost, - } - } else { - cost = { - input: rerankerCost, - output: 0, - total: rerankerCost, - pricing: { input: 0, output: 0, updatedAt: pricing.updatedAt }, - } - } - } else { - logger.warn(`[${requestId}] No pricing entry for rerank model ${rerankerModel}`) - } - } - - // Record query-embedding + reranker cost for standalone callers (UI, copilot, - // guardrail RAG). The workflow tool sets skipUsageBilling and rolls the cost - // up via the executor instead, so this never double-bills; BYOK already - // resolved to 0 above. - if (shouldMeter && cost && cost.total > 0) { - const { recordUsage } = await import('@/lib/billing/core/usage-log') - try { - await recordUsage({ - userId, - workspaceId: workspaceId ?? undefined, - ...(billingAttribution ? toBillingContext(billingAttribution) : {}), - entries: [ - { - category: 'model', - source: 'knowledge-base', - description: queryEmbeddingModel, - cost: cost.total, - sourceReference: `kb-search:${requestId}`, - }, - ], - }) - if (billingAttribution) { - await checkAndBillPayerOverageThreshold(billingAttribution.billingEntity) - } else { - await checkAndBillOverageThreshold(userId) - } - } catch (billingError) { - logger.error(`[${requestId}] Failed to record KB search usage`, { error: billingError }) - } - } - - const tagDefsResults = await Promise.all( - accessibleKbIds.map(async (kbId) => { - try { - const tagDefs = await getDocumentTagDefinitions(kbId) - const map: Record = {} - tagDefs.forEach((def) => { - map[def.tagSlot] = def.displayName - }) - return { kbId, map } - } catch (error) { - logger.warn(`[${requestId}] Failed to fetch tag definitions for display mapping:`, error) - return { kbId, map: {} as Record } - } - }) - ) - const tagDefinitionsMap: Record> = {} - tagDefsResults.forEach(({ kbId, map }) => { - tagDefinitionsMap[kbId] = map - }) - - const documentMetadataMap = resultProvenanceSnapshot.documentMetadata - - try { - PlatformEvents.knowledgeBaseSearched({ - knowledgeBaseId: accessibleKbIds[0], - resultsCount: results.length, - workspaceId: workspaceId || undefined, - }) - } catch { - // Telemetry should not fail the operation - } - - const renderedResults = results.map((result) => { - const kbTagMap = tagDefinitionsMap[result.knowledgeBaseId] || {} - logger.debug( - `[${requestId}] Result KB: ${result.knowledgeBaseId}, available mappings:`, - kbTagMap - ) - - const tags: Record = {} - const docMeta = documentMetadataMap[result.documentId] - ALL_TAG_SLOTS.forEach((slot) => { - const tagValue = slot.startsWith('tag') - ? docMeta?.[ - slot as keyof Pick< - typeof docMeta, - 'tag1' | 'tag2' | 'tag3' | 'tag4' | 'tag5' | 'tag6' | 'tag7' - > - ] - : result[slot] - if (tagValue !== null && tagValue !== undefined) { - const displayName = kbTagMap[slot] || slot - logger.debug(`[${requestId}] Mapping ${slot} -> "${displayName}"`) - tags[displayName] = tagValue - } - }) - - const rerankerScore = rerankedScores.get(result.id) - return { - documentId: result.documentId, - documentName: docMeta?.filename || undefined, - sourceUrl: docMeta?.sourceUrl ?? null, - content: result.content, - chunkIndex: result.chunkIndex, - metadata: tags, - similarity: hasQuery ? 1 - result.distance : 1, - ...(rerankerScore !== undefined && { rerankerScore }), - } - }) - - for (const [documentId, metadata] of Object.entries(documentMetadataMap)) { - const renderedMetadata = renderedResults - .filter((result) => result.documentId === documentId) - .map((result) => ({ - documentName: result.documentName, - sourceUrl: result.sourceUrl, - metadata: result.metadata, - })) - if ( - renderedMetadata.length > 0 && - !(await importDurableSecretProvenance( - resultSecretRegistry, - metadata.provenance, - renderedMetadata - )) - ) { - resultSecretRegistry.markIncomplete() - } - } - - const responseBody = { - success: true, - data: { - results: renderedResults, - query: validatedData.query || '', - knowledgeBaseIds: accessibleKbIds, - knowledgeBaseId: accessibleKbIds[0], - topK: validatedData.topK, - totalResults: results.length, - ...(cost - ? { - cost: { - input: cost.input, - output: cost.output, - total: cost.total, - tokens: { - prompt: tokenCount?.count ?? 0, - completion: 0, - total: tokenCount?.count ?? 0, - }, - model: queryEmbeddingModel, - pricing: cost.pricing, - ...(rerankBilled && !rerankIsBYOK - ? { rerankerCost, rerankerModel, rerankerSearchUnits: 1 } - : {}), - }, - } - : {}), - }, - } - return createKnowledgeRegistryResponse({ + if (!prepared.success) throw new OrchestrationError('validation', prepared.error) + return prepared.registry + }, + }), + useCase: searchKnowledge, + present: (result) => ({ + success: true as const, + data: { + results: result.results.map(({ embeddingId: _embeddingId, ...item }) => item), + query: result.query, + knowledgeBaseIds: result.knowledgeBaseIds, + knowledgeBaseId: result.knowledgeBaseId, + topK: result.topK, + totalResults: result.totalResults, + ...(result.cost ? { cost: result.cost } : {}), + }, + }), + finalizeResponse: ({ request, principal, result, body }) => { + if (!result.resultSecretRegistry) { + throw new Error('Internal Knowledge search did not produce a provenance registry') + } + return finalizeKnowledgeRegistryResponse({ request, - authType: auth.authType, - body: responseBody, - registry: resultSecretRegistry, + authType: internalKnowledgeAuthType(principal), + body, + registry: result.resultSecretRegistry, }) - } catch (error) { - return NextResponse.json( - { - error: 'Failed to perform vector search', - message: getErrorMessage(error, 'Unknown error'), - }, - { status: 500 } - ) - } + }, }) diff --git a/apps/sim/app/api/knowledge/secret-provenance.ts b/apps/sim/app/api/knowledge/secret-provenance.ts index 45d45206b4d..10ba3b6fe7c 100644 --- a/apps/sim/app/api/knowledge/secret-provenance.ts +++ b/apps/sim/app/api/knowledge/secret-provenance.ts @@ -1,5 +1,7 @@ import { type NextRequest, NextResponse } from 'next/server' +import type { InternalJsonResponseFinalization } from '@/lib/api/server/routes/internal-json-route' import { AuthType, type AuthTypeValue } from '@/lib/auth/hybrid' +import { OrchestrationError } from '@/lib/core/orchestration/types' import { createDurableSecretProvenanceRegistry, type DurableSecretProvenance, @@ -12,6 +14,7 @@ import { } from '@/lib/execution/model-input-provenance' import { negotiatePrivateToolMetadataResponse, + RESOLVED_SECRET_PROVENANCE_FIELD, RESOLVED_SECRET_PROVENANCE_METADATA_V1, serializePrivateToolMetadataResponseEnvelope, } from '@/lib/execution/private-tool-metadata' @@ -33,6 +36,21 @@ function invalidKnowledgeProvenanceResponse(): NextResponse { return NextResponse.json({ error: 'Invalid knowledge secret provenance' }, { status: 400 }) } +function rejectInvalidKnowledgeProvenance(): never { + throw new OrchestrationError('validation', 'Invalid knowledge secret provenance') +} + +function finalizeKnowledgeMetadataEnvelope( + envelope: ReturnType +): InternalJsonResponseFinalization { + return { + bodyFields: { + [RESOLVED_SECRET_PROVENANCE_FIELD]: envelope.body[RESOLVED_SECRET_PROVENANCE_FIELD], + }, + headers: envelope.headers, + } +} + type KnowledgeWriteProvenanceResolution = | { success: true; provenances?: DurableSecretProvenance[] } | { success: false; response: NextResponse } @@ -134,23 +152,23 @@ export function resolveKnowledgeDocumentWriteSecretProvenance(options: { return { success: true, provenances } } -/** Adds private provenance for a raw Knowledge response without changing its functional shape. */ -export async function createKnowledgeProvenanceResponse(options: { +/** Finalizes private provenance after the functional Knowledge response passes its contract. */ +export async function finalizeKnowledgeProvenanceResponse(options: { request: NextRequest authType: AuthTypeValue | undefined userId: string workspaceId?: string body: Record provenances: readonly DurableSecretProvenance[] -}): Promise { +}): Promise { const { request } = options const negotiation = negotiatePrivateToolMetadataResponse( request.headers, RESOLVED_SECRET_PROVENANCE_METADATA_V1, options.authType === AuthType.INTERNAL_JWT ) - if (negotiation.status === 'not-requested') return NextResponse.json(options.body) - if (negotiation.status === 'rejected') return invalidKnowledgeProvenanceResponse() + if (negotiation.status === 'not-requested') return {} + if (negotiation.status === 'rejected') rejectInvalidKnowledgeProvenance() const registry = new ResolvedSecretTraceRegistry([], { userId: options.userId, ...(options.workspaceId ? { workspaceId: options.workspaceId } : {}), @@ -171,34 +189,34 @@ export async function createKnowledgeProvenanceResponse(options: { RESOLVED_SECRET_PROVENANCE_METADATA_V1, registry.exportCommittedProvenanceForValue(options.body) ) - return NextResponse.json(envelope.body, { headers: envelope.headers }) + return finalizeKnowledgeMetadataEnvelope(envelope) } /** Serializes an already-populated request registry as private response metadata. */ -export function createKnowledgeRegistryResponse(options: { +export function finalizeKnowledgeRegistryResponse(options: { request: NextRequest authType: AuthTypeValue | undefined body: Record registry: ResolvedSecretTraceRegistry -}): NextResponse { +}): InternalJsonResponseFinalization { const { request } = options const negotiation = negotiatePrivateToolMetadataResponse( request.headers, RESOLVED_SECRET_PROVENANCE_METADATA_V1, options.authType === AuthType.INTERNAL_JWT ) - if (negotiation.status === 'not-requested') return NextResponse.json(options.body) - if (negotiation.status === 'rejected') return invalidKnowledgeProvenanceResponse() + if (negotiation.status === 'not-requested') return {} + if (negotiation.status === 'rejected') rejectInvalidKnowledgeProvenance() const envelope = serializePrivateToolMetadataResponseEnvelope( options.body, RESOLVED_SECRET_PROVENANCE_METADATA_V1, options.registry.exportCommittedProvenanceForValue(options.body) ) - return NextResponse.json(envelope.body, { headers: envelope.headers }) + return finalizeKnowledgeMetadataEnvelope(envelope) } /** Emits private response provenance for a bounded exact snapshot of persisted KB rows. */ -export async function createKnowledgePersistedResponse(options: { +export async function finalizeKnowledgePersistedResponse(options: { request: NextRequest authType: AuthTypeValue | undefined userId: string @@ -215,15 +233,15 @@ export async function createKnowledgePersistedResponse(options: { content: string value: unknown }[] -}): Promise { +}): Promise { const { request } = options const negotiation = negotiatePrivateToolMetadataResponse( request.headers, RESOLVED_SECRET_PROVENANCE_METADATA_V1, options.authType === AuthType.INTERNAL_JWT ) - if (negotiation.status === 'not-requested') return NextResponse.json(options.body) - if (negotiation.status === 'rejected') return invalidKnowledgeProvenanceResponse() + if (negotiation.status === 'not-requested') return {} + if (negotiation.status === 'rejected') rejectInvalidKnowledgeProvenance() const registry = new ResolvedSecretTraceRegistry([], { userId: options.userId, @@ -234,7 +252,7 @@ export async function createKnowledgePersistedResponse(options: { documents: options.documents, chunks: options.chunks, }) - return createKnowledgeRegistryResponse({ + return finalizeKnowledgeRegistryResponse({ request: options.request, authType: options.authType, body: options.body, diff --git a/apps/sim/app/api/providers/route.test.ts b/apps/sim/app/api/providers/route.test.ts index 34558934d2e..05f809a8184 100644 --- a/apps/sim/app/api/providers/route.test.ts +++ b/apps/sim/app/api/providers/route.test.ts @@ -49,7 +49,7 @@ vi.mock('@/providers/model-input-provenance', () => ({ collectProviderModelInputProvenanceValues: mockCollectProviderModelInputProvenanceValues, })) -vi.mock('@/app/api/auth/oauth/utils', () => ({ +vi.mock('@/lib/oauth/credential-service', () => ({ getServiceAccountToken: vi.fn(), refreshTokenIfNeeded: vi.fn(), resolveOAuthAccountId: vi.fn(), diff --git a/apps/sim/app/api/providers/route.ts b/apps/sim/app/api/providers/route.ts index bc335f9ceaa..a5292978d3b 100644 --- a/apps/sim/app/api/providers/route.ts +++ b/apps/sim/app/api/providers/route.ts @@ -17,12 +17,12 @@ import { prepareCopilotEnvironmentContext } from '@/lib/copilot/environment-cont import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { inspectModelInputProvenanceRequest } from '@/lib/execution/model-input-provenance' -import { checkWorkspaceAccess } from '@/lib/workspaces/permissions/utils' import { getServiceAccountToken, refreshTokenIfNeeded, resolveOAuthAccountId, -} from '@/app/api/auth/oauth/utils' +} from '@/lib/oauth/credential-service' +import { checkWorkspaceAccess } from '@/lib/workspaces/permissions/utils' import { assertPermissionsAllowed, IntegrationNotAllowedError, diff --git a/apps/sim/app/api/tools/airtable/bases/route.ts b/apps/sim/app/api/tools/airtable/bases/route.ts index 20b3b3459d7..a309daa1f08 100644 --- a/apps/sim/app/api/tools/airtable/bases/route.ts +++ b/apps/sim/app/api/tools/airtable/bases/route.ts @@ -5,7 +5,7 @@ import { parseRequest } from '@/lib/api/server' import { authorizeCredentialUse } from '@/lib/auth/credential-access' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { refreshAccessTokenIfNeeded } from '@/app/api/auth/oauth/utils' +import { refreshAccessTokenIfNeeded } from '@/lib/oauth/credential-service' const logger = createLogger('AirtableBasesAPI') diff --git a/apps/sim/app/api/tools/airtable/tables/route.ts b/apps/sim/app/api/tools/airtable/tables/route.ts index 5d08b698747..3f4ba3c9739 100644 --- a/apps/sim/app/api/tools/airtable/tables/route.ts +++ b/apps/sim/app/api/tools/airtable/tables/route.ts @@ -6,7 +6,7 @@ import { authorizeCredentialUse } from '@/lib/auth/credential-access' import { validateAirtableId } from '@/lib/core/security/input-validation' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { refreshAccessTokenIfNeeded } from '@/app/api/auth/oauth/utils' +import { refreshAccessTokenIfNeeded } from '@/lib/oauth/credential-service' const logger = createLogger('AirtableTablesAPI') diff --git a/apps/sim/app/api/tools/asana/workspaces/route.ts b/apps/sim/app/api/tools/asana/workspaces/route.ts index 0f71376c6df..0aaec253c95 100644 --- a/apps/sim/app/api/tools/asana/workspaces/route.ts +++ b/apps/sim/app/api/tools/asana/workspaces/route.ts @@ -5,7 +5,7 @@ import { parseRequest } from '@/lib/api/server' import { authorizeCredentialUse } from '@/lib/auth/credential-access' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { refreshAccessTokenIfNeeded } from '@/app/api/auth/oauth/utils' +import { refreshAccessTokenIfNeeded } from '@/lib/oauth/credential-service' const logger = createLogger('AsanaWorkspacesAPI') diff --git a/apps/sim/app/api/tools/attio/lists/route.ts b/apps/sim/app/api/tools/attio/lists/route.ts index ea30238d737..310784e6fb4 100644 --- a/apps/sim/app/api/tools/attio/lists/route.ts +++ b/apps/sim/app/api/tools/attio/lists/route.ts @@ -5,7 +5,7 @@ import { getValidationErrorMessage, parseRequest } from '@/lib/api/server' import { authorizeCredentialUse } from '@/lib/auth/credential-access' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { refreshAccessTokenIfNeeded } from '@/app/api/auth/oauth/utils' +import { refreshAccessTokenIfNeeded } from '@/lib/oauth/credential-service' const logger = createLogger('AttioListsAPI') diff --git a/apps/sim/app/api/tools/attio/objects/route.ts b/apps/sim/app/api/tools/attio/objects/route.ts index 38cc19d4bbc..de0e7820f91 100644 --- a/apps/sim/app/api/tools/attio/objects/route.ts +++ b/apps/sim/app/api/tools/attio/objects/route.ts @@ -5,7 +5,7 @@ import { getValidationErrorMessage, parseRequest } from '@/lib/api/server' import { authorizeCredentialUse } from '@/lib/auth/credential-access' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { refreshAccessTokenIfNeeded } from '@/app/api/auth/oauth/utils' +import { refreshAccessTokenIfNeeded } from '@/lib/oauth/credential-service' const logger = createLogger('AttioObjectsAPI') diff --git a/apps/sim/app/api/tools/calcom/event-types/route.ts b/apps/sim/app/api/tools/calcom/event-types/route.ts index a9ab63da8e4..0bb4a9cd8d0 100644 --- a/apps/sim/app/api/tools/calcom/event-types/route.ts +++ b/apps/sim/app/api/tools/calcom/event-types/route.ts @@ -5,7 +5,7 @@ import { parseRequest } from '@/lib/api/server' import { authorizeCredentialUse } from '@/lib/auth/credential-access' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { refreshAccessTokenIfNeeded } from '@/app/api/auth/oauth/utils' +import { refreshAccessTokenIfNeeded } from '@/lib/oauth/credential-service' const logger = createLogger('CalcomEventTypesAPI') diff --git a/apps/sim/app/api/tools/calcom/schedules/route.ts b/apps/sim/app/api/tools/calcom/schedules/route.ts index 15b6e1dfc6e..8ccb0700017 100644 --- a/apps/sim/app/api/tools/calcom/schedules/route.ts +++ b/apps/sim/app/api/tools/calcom/schedules/route.ts @@ -5,7 +5,7 @@ import { parseRequest } from '@/lib/api/server' import { authorizeCredentialUse } from '@/lib/auth/credential-access' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { refreshAccessTokenIfNeeded } from '@/app/api/auth/oauth/utils' +import { refreshAccessTokenIfNeeded } from '@/lib/oauth/credential-service' const logger = createLogger('CalcomSchedulesAPI') diff --git a/apps/sim/app/api/tools/clickup/folders/route.ts b/apps/sim/app/api/tools/clickup/folders/route.ts index 6613be9189e..fc9063ce7a6 100644 --- a/apps/sim/app/api/tools/clickup/folders/route.ts +++ b/apps/sim/app/api/tools/clickup/folders/route.ts @@ -6,7 +6,7 @@ import { authorizeCredentialUse } from '@/lib/auth/credential-access' import { validateAlphanumericId } from '@/lib/core/security/input-validation' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { refreshAccessTokenIfNeeded } from '@/app/api/auth/oauth/utils' +import { refreshAccessTokenIfNeeded } from '@/lib/oauth/credential-service' import { CLICKUP_API_BASE_URL, clickupAuthorizationHeader } from '@/tools/clickup/shared' const logger = createLogger('ClickUpFoldersAPI') diff --git a/apps/sim/app/api/tools/clickup/lists/route.ts b/apps/sim/app/api/tools/clickup/lists/route.ts index a07712b36e4..f77d7153b2b 100644 --- a/apps/sim/app/api/tools/clickup/lists/route.ts +++ b/apps/sim/app/api/tools/clickup/lists/route.ts @@ -6,7 +6,7 @@ import { authorizeCredentialUse } from '@/lib/auth/credential-access' import { validateAlphanumericId } from '@/lib/core/security/input-validation' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { refreshAccessTokenIfNeeded } from '@/app/api/auth/oauth/utils' +import { refreshAccessTokenIfNeeded } from '@/lib/oauth/credential-service' import { CLICKUP_API_BASE_URL, clickupAuthorizationHeader } from '@/tools/clickup/shared' const logger = createLogger('ClickUpListsAPI') diff --git a/apps/sim/app/api/tools/clickup/spaces/route.ts b/apps/sim/app/api/tools/clickup/spaces/route.ts index 197e5245ca4..2f8d6ca5932 100644 --- a/apps/sim/app/api/tools/clickup/spaces/route.ts +++ b/apps/sim/app/api/tools/clickup/spaces/route.ts @@ -6,7 +6,7 @@ import { authorizeCredentialUse } from '@/lib/auth/credential-access' import { validateAlphanumericId } from '@/lib/core/security/input-validation' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { refreshAccessTokenIfNeeded } from '@/app/api/auth/oauth/utils' +import { refreshAccessTokenIfNeeded } from '@/lib/oauth/credential-service' import { CLICKUP_API_BASE_URL, clickupAuthorizationHeader } from '@/tools/clickup/shared' const logger = createLogger('ClickUpSpacesAPI') diff --git a/apps/sim/app/api/tools/clickup/workspaces/route.ts b/apps/sim/app/api/tools/clickup/workspaces/route.ts index 2ff80e7f2dd..9c2990a18f7 100644 --- a/apps/sim/app/api/tools/clickup/workspaces/route.ts +++ b/apps/sim/app/api/tools/clickup/workspaces/route.ts @@ -5,7 +5,7 @@ import { parseRequest } from '@/lib/api/server' import { authorizeCredentialUse } from '@/lib/auth/credential-access' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { refreshAccessTokenIfNeeded } from '@/app/api/auth/oauth/utils' +import { refreshAccessTokenIfNeeded } from '@/lib/oauth/credential-service' import { CLICKUP_API_BASE_URL, clickupAuthorizationHeader } from '@/tools/clickup/shared' const logger = createLogger('ClickUpWorkspacesAPI') diff --git a/apps/sim/app/api/tools/confluence/selector-spaces/route.ts b/apps/sim/app/api/tools/confluence/selector-spaces/route.ts index 4ad5c0f2629..ca7431294c5 100644 --- a/apps/sim/app/api/tools/confluence/selector-spaces/route.ts +++ b/apps/sim/app/api/tools/confluence/selector-spaces/route.ts @@ -6,12 +6,12 @@ import { authorizeCredentialUse } from '@/lib/auth/credential-access' import { validateJiraCloudId } from '@/lib/core/security/input-validation' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { ATLASSIAN_SERVICE_ACCOUNT_PROVIDER_ID } from '@/lib/oauth/types' import { getAtlassianServiceAccountSecret, refreshAccessTokenIfNeeded, resolveOAuthAccountId, -} from '@/app/api/auth/oauth/utils' +} from '@/lib/oauth/credential-service' +import { ATLASSIAN_SERVICE_ACCOUNT_PROVIDER_ID } from '@/lib/oauth/types' import { getConfluenceCloudId } from '@/tools/confluence/utils' import { parseAtlassianErrorMessage } from '@/tools/jira/utils' diff --git a/apps/sim/app/api/tools/drive/file/route.ts b/apps/sim/app/api/tools/drive/file/route.ts index 85af8e72bc7..69e569564cf 100644 --- a/apps/sim/app/api/tools/drive/file/route.ts +++ b/apps/sim/app/api/tools/drive/file/route.ts @@ -7,8 +7,11 @@ import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid' import { validateAlphanumericId } from '@/lib/core/security/input-validation' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' +import { + refreshAccessTokenIfNeeded, + ServiceAccountTokenError, +} from '@/lib/oauth/credential-service' import { getScopesForService } from '@/lib/oauth/utils' -import { refreshAccessTokenIfNeeded, ServiceAccountTokenError } from '@/app/api/auth/oauth/utils' export const dynamic = 'force-dynamic' const logger = createLogger('GoogleDriveFileAPI') diff --git a/apps/sim/app/api/tools/drive/files/route.ts b/apps/sim/app/api/tools/drive/files/route.ts index 4c38334b14d..c3bdd3c6911 100644 --- a/apps/sim/app/api/tools/drive/files/route.ts +++ b/apps/sim/app/api/tools/drive/files/route.ts @@ -7,9 +7,12 @@ import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid' import { validateAlphanumericId } from '@/lib/core/security/input-validation' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' +import { + refreshAccessTokenIfNeeded, + ServiceAccountTokenError, +} from '@/lib/oauth/credential-service' import { drainGooglePagedList, GooglePageError } from '@/lib/oauth/google-pagination' import { getScopesForService } from '@/lib/oauth/utils' -import { refreshAccessTokenIfNeeded, ServiceAccountTokenError } from '@/app/api/auth/oauth/utils' export const dynamic = 'force-dynamic' const logger = createLogger('GoogleDriveFilesAPI') diff --git a/apps/sim/app/api/tools/gmail/label/route.ts b/apps/sim/app/api/tools/gmail/label/route.ts index 75cd890a522..f1abd52383c 100644 --- a/apps/sim/app/api/tools/gmail/label/route.ts +++ b/apps/sim/app/api/tools/gmail/label/route.ts @@ -6,8 +6,11 @@ import { authorizeCredentialUse } from '@/lib/auth/credential-access' import { validateAlphanumericId } from '@/lib/core/security/input-validation' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' +import { + refreshAccessTokenIfNeeded, + ServiceAccountTokenError, +} from '@/lib/oauth/credential-service' import { getScopesForService } from '@/lib/oauth/utils' -import { refreshAccessTokenIfNeeded, ServiceAccountTokenError } from '@/app/api/auth/oauth/utils' export const dynamic = 'force-dynamic' diff --git a/apps/sim/app/api/tools/gmail/labels/route.ts b/apps/sim/app/api/tools/gmail/labels/route.ts index 3b05cf12a9e..d531ec0a61d 100644 --- a/apps/sim/app/api/tools/gmail/labels/route.ts +++ b/apps/sim/app/api/tools/gmail/labels/route.ts @@ -6,12 +6,12 @@ import { authorizeCredentialUse } from '@/lib/auth/credential-access' import { validateAlphanumericId } from '@/lib/core/security/input-validation' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { getScopesForService } from '@/lib/oauth/utils' import { getServiceAccountToken, refreshAccessTokenIfNeeded, ServiceAccountTokenError, -} from '@/app/api/auth/oauth/utils' +} from '@/lib/oauth/credential-service' +import { getScopesForService } from '@/lib/oauth/utils' export const dynamic = 'force-dynamic' const logger = createLogger('GmailLabelsAPI') diff --git a/apps/sim/app/api/tools/google_bigquery/datasets/route.ts b/apps/sim/app/api/tools/google_bigquery/datasets/route.ts index 695a371e170..db98979b24e 100644 --- a/apps/sim/app/api/tools/google_bigquery/datasets/route.ts +++ b/apps/sim/app/api/tools/google_bigquery/datasets/route.ts @@ -5,9 +5,12 @@ import { getValidationErrorMessage, parseRequest } from '@/lib/api/server' import { authorizeCredentialUse } from '@/lib/auth/credential-access' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' +import { + refreshAccessTokenIfNeeded, + ServiceAccountTokenError, +} from '@/lib/oauth/credential-service' import { drainGooglePagedList, GooglePageError } from '@/lib/oauth/google-pagination' import { getScopesForService } from '@/lib/oauth/utils' -import { refreshAccessTokenIfNeeded, ServiceAccountTokenError } from '@/app/api/auth/oauth/utils' const logger = createLogger('GoogleBigQueryDatasetsAPI') diff --git a/apps/sim/app/api/tools/google_bigquery/tables/route.ts b/apps/sim/app/api/tools/google_bigquery/tables/route.ts index af013790595..8cb630b944e 100644 --- a/apps/sim/app/api/tools/google_bigquery/tables/route.ts +++ b/apps/sim/app/api/tools/google_bigquery/tables/route.ts @@ -5,9 +5,12 @@ import { parseRequest } from '@/lib/api/server' import { authorizeCredentialUse } from '@/lib/auth/credential-access' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' +import { + refreshAccessTokenIfNeeded, + ServiceAccountTokenError, +} from '@/lib/oauth/credential-service' import { drainGooglePagedList, GooglePageError } from '@/lib/oauth/google-pagination' import { getScopesForService } from '@/lib/oauth/utils' -import { refreshAccessTokenIfNeeded, ServiceAccountTokenError } from '@/app/api/auth/oauth/utils' const logger = createLogger('GoogleBigQueryTablesAPI') diff --git a/apps/sim/app/api/tools/google_calendar/calendars/route.ts b/apps/sim/app/api/tools/google_calendar/calendars/route.ts index 752cc72b229..0102f8c3b78 100644 --- a/apps/sim/app/api/tools/google_calendar/calendars/route.ts +++ b/apps/sim/app/api/tools/google_calendar/calendars/route.ts @@ -5,9 +5,12 @@ import { parseRequest } from '@/lib/api/server' import { authorizeCredentialUse } from '@/lib/auth/credential-access' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' +import { + refreshAccessTokenIfNeeded, + ServiceAccountTokenError, +} from '@/lib/oauth/credential-service' import { drainGooglePagedList, GooglePageError } from '@/lib/oauth/google-pagination' import { getScopesForService } from '@/lib/oauth/utils' -import { refreshAccessTokenIfNeeded, ServiceAccountTokenError } from '@/app/api/auth/oauth/utils' export const dynamic = 'force-dynamic' const logger = createLogger('GoogleCalendarAPI') diff --git a/apps/sim/app/api/tools/google_sheets/sheets/route.ts b/apps/sim/app/api/tools/google_sheets/sheets/route.ts index 951c31f67e8..18fca36e377 100644 --- a/apps/sim/app/api/tools/google_sheets/sheets/route.ts +++ b/apps/sim/app/api/tools/google_sheets/sheets/route.ts @@ -6,8 +6,11 @@ import { authorizeCredentialUse } from '@/lib/auth/credential-access' import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' +import { + refreshAccessTokenIfNeeded, + ServiceAccountTokenError, +} from '@/lib/oauth/credential-service' import { getScopesForService } from '@/lib/oauth/utils' -import { refreshAccessTokenIfNeeded, ServiceAccountTokenError } from '@/app/api/auth/oauth/utils' export const dynamic = 'force-dynamic' diff --git a/apps/sim/app/api/tools/google_tasks/task-lists/route.ts b/apps/sim/app/api/tools/google_tasks/task-lists/route.ts index 80c5a99f598..6b6dff75db7 100644 --- a/apps/sim/app/api/tools/google_tasks/task-lists/route.ts +++ b/apps/sim/app/api/tools/google_tasks/task-lists/route.ts @@ -5,9 +5,12 @@ import { parseRequest } from '@/lib/api/server' import { authorizeCredentialUse } from '@/lib/auth/credential-access' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' +import { + refreshAccessTokenIfNeeded, + ServiceAccountTokenError, +} from '@/lib/oauth/credential-service' import { drainGooglePagedList, GooglePageError } from '@/lib/oauth/google-pagination' import { getScopesForService } from '@/lib/oauth/utils' -import { refreshAccessTokenIfNeeded, ServiceAccountTokenError } from '@/app/api/auth/oauth/utils' const logger = createLogger('GoogleTasksTaskListsAPI') diff --git a/apps/sim/app/api/tools/hubspot/lists/route.ts b/apps/sim/app/api/tools/hubspot/lists/route.ts index ab7cf55230e..171474f48c5 100644 --- a/apps/sim/app/api/tools/hubspot/lists/route.ts +++ b/apps/sim/app/api/tools/hubspot/lists/route.ts @@ -6,7 +6,7 @@ import { authorizeCredentialUse } from '@/lib/auth/credential-access' import { validateAlphanumericId } from '@/lib/core/security/input-validation' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { refreshAccessTokenIfNeeded } from '@/app/api/auth/oauth/utils' +import { refreshAccessTokenIfNeeded } from '@/lib/oauth/credential-service' export const dynamic = 'force-dynamic' diff --git a/apps/sim/app/api/tools/hubspot/owners/route.ts b/apps/sim/app/api/tools/hubspot/owners/route.ts index be34256def9..43f0b3576b6 100644 --- a/apps/sim/app/api/tools/hubspot/owners/route.ts +++ b/apps/sim/app/api/tools/hubspot/owners/route.ts @@ -6,7 +6,7 @@ import { authorizeCredentialUse } from '@/lib/auth/credential-access' import { validateAlphanumericId } from '@/lib/core/security/input-validation' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { refreshAccessTokenIfNeeded } from '@/app/api/auth/oauth/utils' +import { refreshAccessTokenIfNeeded } from '@/lib/oauth/credential-service' export const dynamic = 'force-dynamic' diff --git a/apps/sim/app/api/tools/hubspot/pipelines/route.ts b/apps/sim/app/api/tools/hubspot/pipelines/route.ts index fd9643bed3a..c7fd92edaa1 100644 --- a/apps/sim/app/api/tools/hubspot/pipelines/route.ts +++ b/apps/sim/app/api/tools/hubspot/pipelines/route.ts @@ -6,7 +6,7 @@ import { authorizeCredentialUse } from '@/lib/auth/credential-access' import { validateAlphanumericId } from '@/lib/core/security/input-validation' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { refreshAccessTokenIfNeeded } from '@/app/api/auth/oauth/utils' +import { refreshAccessTokenIfNeeded } from '@/lib/oauth/credential-service' export const dynamic = 'force-dynamic' diff --git a/apps/sim/app/api/tools/hubspot/properties/route.ts b/apps/sim/app/api/tools/hubspot/properties/route.ts index e52185455fc..3d52b9b0d8f 100644 --- a/apps/sim/app/api/tools/hubspot/properties/route.ts +++ b/apps/sim/app/api/tools/hubspot/properties/route.ts @@ -6,7 +6,7 @@ import { authorizeCredentialUse } from '@/lib/auth/credential-access' import { validateAlphanumericId } from '@/lib/core/security/input-validation' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { refreshAccessTokenIfNeeded } from '@/app/api/auth/oauth/utils' +import { refreshAccessTokenIfNeeded } from '@/lib/oauth/credential-service' export const dynamic = 'force-dynamic' diff --git a/apps/sim/app/api/tools/jsm/selector-requesttypes/route.ts b/apps/sim/app/api/tools/jsm/selector-requesttypes/route.ts index b23b4b7c7a7..705dc2be2e0 100644 --- a/apps/sim/app/api/tools/jsm/selector-requesttypes/route.ts +++ b/apps/sim/app/api/tools/jsm/selector-requesttypes/route.ts @@ -6,7 +6,7 @@ import { authorizeCredentialUse } from '@/lib/auth/credential-access' import { validateAlphanumericId, validateJiraCloudId } from '@/lib/core/security/input-validation' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { refreshAccessTokenIfNeeded } from '@/app/api/auth/oauth/utils' +import { refreshAccessTokenIfNeeded } from '@/lib/oauth/credential-service' import { getJiraCloudId, parseAtlassianErrorMessage } from '@/tools/jira/utils' import { getJsmApiBaseUrl, getJsmHeaders } from '@/tools/jsm/utils' diff --git a/apps/sim/app/api/tools/jsm/selector-servicedesks/route.ts b/apps/sim/app/api/tools/jsm/selector-servicedesks/route.ts index 786483630dd..9dc55ea0a83 100644 --- a/apps/sim/app/api/tools/jsm/selector-servicedesks/route.ts +++ b/apps/sim/app/api/tools/jsm/selector-servicedesks/route.ts @@ -6,7 +6,7 @@ import { authorizeCredentialUse } from '@/lib/auth/credential-access' import { validateJiraCloudId } from '@/lib/core/security/input-validation' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { refreshAccessTokenIfNeeded } from '@/app/api/auth/oauth/utils' +import { refreshAccessTokenIfNeeded } from '@/lib/oauth/credential-service' import { getJiraCloudId, parseAtlassianErrorMessage } from '@/tools/jira/utils' import { getJsmApiBaseUrl, getJsmHeaders } from '@/tools/jsm/utils' diff --git a/apps/sim/app/api/tools/linear/projects/route.ts b/apps/sim/app/api/tools/linear/projects/route.ts index c549654b80a..360453b615a 100644 --- a/apps/sim/app/api/tools/linear/projects/route.ts +++ b/apps/sim/app/api/tools/linear/projects/route.ts @@ -7,7 +7,7 @@ import { parseRequest } from '@/lib/api/server' import { authorizeCredentialUse } from '@/lib/auth/credential-access' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { refreshAccessTokenIfNeeded } from '@/app/api/auth/oauth/utils' +import { refreshAccessTokenIfNeeded } from '@/lib/oauth/credential-service' export const dynamic = 'force-dynamic' diff --git a/apps/sim/app/api/tools/linear/teams/route.ts b/apps/sim/app/api/tools/linear/teams/route.ts index a03ccaea4d6..5d788bc2b07 100644 --- a/apps/sim/app/api/tools/linear/teams/route.ts +++ b/apps/sim/app/api/tools/linear/teams/route.ts @@ -7,7 +7,7 @@ import { parseRequest } from '@/lib/api/server' import { authorizeCredentialUse } from '@/lib/auth/credential-access' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { refreshAccessTokenIfNeeded } from '@/app/api/auth/oauth/utils' +import { refreshAccessTokenIfNeeded } from '@/lib/oauth/credential-service' export const dynamic = 'force-dynamic' diff --git a/apps/sim/app/api/tools/managed-agent/list/route.ts b/apps/sim/app/api/tools/managed-agent/list/route.ts index 1776f198793..97a1ca6fa47 100644 --- a/apps/sim/app/api/tools/managed-agent/list/route.ts +++ b/apps/sim/app/api/tools/managed-agent/list/route.ts @@ -12,8 +12,8 @@ import { authorizeCredentialUse } from '@/lib/auth/credential-access' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { CLAUDE_PLATFORM_SERVICE_ACCOUNT_PROVIDER_ID } from '@/lib/credentials/token-service-accounts/descriptors' import { AGENT_MEMORY_BETA, managedAgentsList } from '@/lib/managed-agents/session-client' +import { resolveOAuthAccountId, resolveServiceAccountToken } from '@/lib/oauth/credential-service' import { captureServerEvent } from '@/lib/posthog/server' -import { resolveOAuthAccountId, resolveServiceAccountToken } from '@/app/api/auth/oauth/utils' export const dynamic = 'force-dynamic' diff --git a/apps/sim/app/api/tools/microsoft-teams/channels/route.ts b/apps/sim/app/api/tools/microsoft-teams/channels/route.ts index c8bd6ddcb57..27070fb419a 100644 --- a/apps/sim/app/api/tools/microsoft-teams/channels/route.ts +++ b/apps/sim/app/api/tools/microsoft-teams/channels/route.ts @@ -6,7 +6,7 @@ import { parseRequest } from '@/lib/api/server' import { authorizeCredentialUse } from '@/lib/auth/credential-access' import { validateMicrosoftGraphId } from '@/lib/core/security/input-validation' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { refreshAccessTokenIfNeeded } from '@/app/api/auth/oauth/utils' +import { refreshAccessTokenIfNeeded } from '@/lib/oauth/credential-service' import { assertGraphNextPageUrl, getGraphNextPageUrl } from '@/tools/sharepoint/utils' export const dynamic = 'force-dynamic' diff --git a/apps/sim/app/api/tools/microsoft-teams/chats/route.ts b/apps/sim/app/api/tools/microsoft-teams/chats/route.ts index d709bcd62e6..afc5fc7668d 100644 --- a/apps/sim/app/api/tools/microsoft-teams/chats/route.ts +++ b/apps/sim/app/api/tools/microsoft-teams/chats/route.ts @@ -6,7 +6,7 @@ import { parseRequest } from '@/lib/api/server' import { authorizeCredentialUse } from '@/lib/auth/credential-access' import { validateMicrosoftGraphId } from '@/lib/core/security/input-validation' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { refreshAccessTokenIfNeeded } from '@/app/api/auth/oauth/utils' +import { refreshAccessTokenIfNeeded } from '@/lib/oauth/credential-service' import { assertGraphNextPageUrl, getGraphNextPageUrl } from '@/tools/sharepoint/utils' export const dynamic = 'force-dynamic' diff --git a/apps/sim/app/api/tools/microsoft-teams/teams/route.ts b/apps/sim/app/api/tools/microsoft-teams/teams/route.ts index 990bfd282d2..7206e85ab24 100644 --- a/apps/sim/app/api/tools/microsoft-teams/teams/route.ts +++ b/apps/sim/app/api/tools/microsoft-teams/teams/route.ts @@ -5,7 +5,7 @@ import { microsoftTeamsSelectorContract } from '@/lib/api/contracts/selectors/mi import { parseRequest } from '@/lib/api/server' import { authorizeCredentialUse } from '@/lib/auth/credential-access' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { refreshAccessTokenIfNeeded } from '@/app/api/auth/oauth/utils' +import { refreshAccessTokenIfNeeded } from '@/lib/oauth/credential-service' import { assertGraphNextPageUrl, getGraphNextPageUrl } from '@/tools/sharepoint/utils' export const dynamic = 'force-dynamic' diff --git a/apps/sim/app/api/tools/microsoft_excel/drives/route.ts b/apps/sim/app/api/tools/microsoft_excel/drives/route.ts index 97d921a5ea4..99dbd4d9f09 100644 --- a/apps/sim/app/api/tools/microsoft_excel/drives/route.ts +++ b/apps/sim/app/api/tools/microsoft_excel/drives/route.ts @@ -6,7 +6,7 @@ import { authorizeCredentialUse } from '@/lib/auth/credential-access' import { validatePathSegment, validateSharePointSiteId } from '@/lib/core/security/input-validation' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { refreshAccessTokenIfNeeded } from '@/app/api/auth/oauth/utils' +import { refreshAccessTokenIfNeeded } from '@/lib/oauth/credential-service' import { extractGraphError, GRAPH_ID_PATTERN } from '@/tools/microsoft_excel/utils' import { assertGraphNextPageUrl, getGraphNextPageUrl } from '@/tools/sharepoint/utils' diff --git a/apps/sim/app/api/tools/microsoft_excel/sheets/route.ts b/apps/sim/app/api/tools/microsoft_excel/sheets/route.ts index f08f968734c..bd6ff64e8df 100644 --- a/apps/sim/app/api/tools/microsoft_excel/sheets/route.ts +++ b/apps/sim/app/api/tools/microsoft_excel/sheets/route.ts @@ -6,7 +6,7 @@ import { parseRequest } from '@/lib/api/server' import { authorizeCredentialUse } from '@/lib/auth/credential-access' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { refreshAccessTokenIfNeeded } from '@/app/api/auth/oauth/utils' +import { refreshAccessTokenIfNeeded } from '@/lib/oauth/credential-service' import { extractGraphError, getItemBasePath } from '@/tools/microsoft_excel/utils' export const dynamic = 'force-dynamic' diff --git a/apps/sim/app/api/tools/microsoft_planner/plans/route.ts b/apps/sim/app/api/tools/microsoft_planner/plans/route.ts index 604f7c85b34..bee66b58459 100644 --- a/apps/sim/app/api/tools/microsoft_planner/plans/route.ts +++ b/apps/sim/app/api/tools/microsoft_planner/plans/route.ts @@ -5,7 +5,7 @@ import { parseRequest } from '@/lib/api/server' import { authorizeCredentialUse } from '@/lib/auth/credential-access' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { refreshAccessTokenIfNeeded } from '@/app/api/auth/oauth/utils' +import { refreshAccessTokenIfNeeded } from '@/lib/oauth/credential-service' import { assertGraphNextPageUrl, getGraphNextPageUrl } from '@/tools/sharepoint/utils' const logger = createLogger('MicrosoftPlannerPlansAPI') diff --git a/apps/sim/app/api/tools/microsoft_planner/tasks/route.ts b/apps/sim/app/api/tools/microsoft_planner/tasks/route.ts index b9b764089bd..e04a9bc2e3f 100644 --- a/apps/sim/app/api/tools/microsoft_planner/tasks/route.ts +++ b/apps/sim/app/api/tools/microsoft_planner/tasks/route.ts @@ -6,7 +6,7 @@ import { authorizeCredentialUse } from '@/lib/auth/credential-access' import { validateMicrosoftGraphId } from '@/lib/core/security/input-validation' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { refreshAccessTokenIfNeeded } from '@/app/api/auth/oauth/utils' +import { refreshAccessTokenIfNeeded } from '@/lib/oauth/credential-service' import type { PlannerTask } from '@/tools/microsoft_planner/types' import { assertGraphNextPageUrl, getGraphNextPageUrl } from '@/tools/sharepoint/utils' diff --git a/apps/sim/app/api/tools/monday/boards/route.ts b/apps/sim/app/api/tools/monday/boards/route.ts index e5d3dc5fadc..a3de9b41989 100644 --- a/apps/sim/app/api/tools/monday/boards/route.ts +++ b/apps/sim/app/api/tools/monday/boards/route.ts @@ -5,7 +5,7 @@ import { parseRequest } from '@/lib/api/server' import { authorizeCredentialUse } from '@/lib/auth/credential-access' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { refreshAccessTokenIfNeeded } from '@/app/api/auth/oauth/utils' +import { refreshAccessTokenIfNeeded } from '@/lib/oauth/credential-service' export const dynamic = 'force-dynamic' diff --git a/apps/sim/app/api/tools/monday/groups/route.ts b/apps/sim/app/api/tools/monday/groups/route.ts index 49021443e64..3492f448564 100644 --- a/apps/sim/app/api/tools/monday/groups/route.ts +++ b/apps/sim/app/api/tools/monday/groups/route.ts @@ -6,7 +6,7 @@ import { authorizeCredentialUse } from '@/lib/auth/credential-access' import { validateMondayNumericId } from '@/lib/core/security/input-validation' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { refreshAccessTokenIfNeeded } from '@/app/api/auth/oauth/utils' +import { refreshAccessTokenIfNeeded } from '@/lib/oauth/credential-service' export const dynamic = 'force-dynamic' diff --git a/apps/sim/app/api/tools/notion/databases/route.ts b/apps/sim/app/api/tools/notion/databases/route.ts index c3f844495d9..966ac49fb51 100644 --- a/apps/sim/app/api/tools/notion/databases/route.ts +++ b/apps/sim/app/api/tools/notion/databases/route.ts @@ -6,7 +6,7 @@ import { parseRequest } from '@/lib/api/server' import { authorizeCredentialUse } from '@/lib/auth/credential-access' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { refreshAccessTokenIfNeeded } from '@/app/api/auth/oauth/utils' +import { refreshAccessTokenIfNeeded } from '@/lib/oauth/credential-service' import { extractTitleFromItem } from '@/tools/notion/utils' const logger = createLogger('NotionDatabasesAPI') diff --git a/apps/sim/app/api/tools/notion/pages/route.ts b/apps/sim/app/api/tools/notion/pages/route.ts index e48eadf8a41..4a0f486e495 100644 --- a/apps/sim/app/api/tools/notion/pages/route.ts +++ b/apps/sim/app/api/tools/notion/pages/route.ts @@ -6,7 +6,7 @@ import { parseRequest } from '@/lib/api/server' import { authorizeCredentialUse } from '@/lib/auth/credential-access' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { refreshAccessTokenIfNeeded } from '@/app/api/auth/oauth/utils' +import { refreshAccessTokenIfNeeded } from '@/lib/oauth/credential-service' import { extractTitleFromItem } from '@/tools/notion/utils' const logger = createLogger('NotionPagesAPI') diff --git a/apps/sim/app/api/tools/onedrive/files/route.ts b/apps/sim/app/api/tools/onedrive/files/route.ts index 5bf2a580ac0..992ecd16875 100644 --- a/apps/sim/app/api/tools/onedrive/files/route.ts +++ b/apps/sim/app/api/tools/onedrive/files/route.ts @@ -6,7 +6,7 @@ import { getValidationErrorMessage } from '@/lib/api/server' import { authorizeCredentialUse } from '@/lib/auth/credential-access' import { validateMicrosoftGraphId } from '@/lib/core/security/input-validation' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { refreshAccessTokenIfNeeded } from '@/app/api/auth/oauth/utils' +import { refreshAccessTokenIfNeeded } from '@/lib/oauth/credential-service' import type { MicrosoftGraphDriveItem } from '@/tools/onedrive/types' import { assertGraphNextPageUrl, getGraphNextPageUrl } from '@/tools/sharepoint/utils' diff --git a/apps/sim/app/api/tools/onedrive/folder/route.ts b/apps/sim/app/api/tools/onedrive/folder/route.ts index 17ff0b02d8e..df3d192ad9c 100644 --- a/apps/sim/app/api/tools/onedrive/folder/route.ts +++ b/apps/sim/app/api/tools/onedrive/folder/route.ts @@ -6,7 +6,7 @@ import { getValidationErrorMessage } from '@/lib/api/server' import { authorizeCredentialUse } from '@/lib/auth/credential-access' import { validateMicrosoftGraphId } from '@/lib/core/security/input-validation' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { refreshAccessTokenIfNeeded } from '@/app/api/auth/oauth/utils' +import { refreshAccessTokenIfNeeded } from '@/lib/oauth/credential-service' export const dynamic = 'force-dynamic' diff --git a/apps/sim/app/api/tools/onedrive/folders/route.ts b/apps/sim/app/api/tools/onedrive/folders/route.ts index bcfd9273c2e..2ce737a57fe 100644 --- a/apps/sim/app/api/tools/onedrive/folders/route.ts +++ b/apps/sim/app/api/tools/onedrive/folders/route.ts @@ -6,7 +6,7 @@ import { getValidationErrorMessage } from '@/lib/api/server' import { authorizeCredentialUse } from '@/lib/auth/credential-access' import { validateMicrosoftGraphId } from '@/lib/core/security/input-validation' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { refreshAccessTokenIfNeeded } from '@/app/api/auth/oauth/utils' +import { refreshAccessTokenIfNeeded } from '@/lib/oauth/credential-service' import type { MicrosoftGraphDriveItem } from '@/tools/onedrive/types' import { assertGraphNextPageUrl, getGraphNextPageUrl } from '@/tools/sharepoint/utils' diff --git a/apps/sim/app/api/tools/outlook/calendars/route.ts b/apps/sim/app/api/tools/outlook/calendars/route.ts index 2c6a68c70b1..4227222c9e0 100644 --- a/apps/sim/app/api/tools/outlook/calendars/route.ts +++ b/apps/sim/app/api/tools/outlook/calendars/route.ts @@ -7,7 +7,7 @@ import { authorizeCredentialUse } from '@/lib/auth/credential-access' import { validateAlphanumericId } from '@/lib/core/security/input-validation' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { refreshAccessTokenIfNeeded } from '@/app/api/auth/oauth/utils' +import { refreshAccessTokenIfNeeded } from '@/lib/oauth/credential-service' import { assertGraphNextPageUrl, getGraphNextPageUrl } from '@/tools/sharepoint/utils' export const dynamic = 'force-dynamic' diff --git a/apps/sim/app/api/tools/outlook/folders/route.ts b/apps/sim/app/api/tools/outlook/folders/route.ts index 8ae9d3e9e21..1a6721b703b 100644 --- a/apps/sim/app/api/tools/outlook/folders/route.ts +++ b/apps/sim/app/api/tools/outlook/folders/route.ts @@ -7,7 +7,7 @@ import { authorizeCredentialUse } from '@/lib/auth/credential-access' import { validateAlphanumericId } from '@/lib/core/security/input-validation' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { refreshAccessTokenIfNeeded } from '@/app/api/auth/oauth/utils' +import { refreshAccessTokenIfNeeded } from '@/lib/oauth/credential-service' import { assertGraphNextPageUrl, getGraphNextPageUrl } from '@/tools/sharepoint/utils' export const dynamic = 'force-dynamic' diff --git a/apps/sim/app/api/tools/pipedrive/pipelines/route.ts b/apps/sim/app/api/tools/pipedrive/pipelines/route.ts index 16381901228..be03aa5bde7 100644 --- a/apps/sim/app/api/tools/pipedrive/pipelines/route.ts +++ b/apps/sim/app/api/tools/pipedrive/pipelines/route.ts @@ -5,7 +5,7 @@ import { parseRequest } from '@/lib/api/server' import { authorizeCredentialUse } from '@/lib/auth/credential-access' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { resolveCredentialAccessToken } from '@/app/api/auth/oauth/utils' +import { resolveCredentialAccessToken } from '@/lib/oauth/credential-service' import { getPipedriveAuthHeaders } from '@/tools/pipedrive/utils' const logger = createLogger('PipedrivePipelinesAPI') diff --git a/apps/sim/app/api/tools/sharepoint/lists/route.ts b/apps/sim/app/api/tools/sharepoint/lists/route.ts index a3970a6f043..43dc8ae95c9 100644 --- a/apps/sim/app/api/tools/sharepoint/lists/route.ts +++ b/apps/sim/app/api/tools/sharepoint/lists/route.ts @@ -6,7 +6,7 @@ import { authorizeCredentialUse } from '@/lib/auth/credential-access' import { validateSharePointSiteId } from '@/lib/core/security/input-validation' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { refreshAccessTokenIfNeeded } from '@/app/api/auth/oauth/utils' +import { refreshAccessTokenIfNeeded } from '@/lib/oauth/credential-service' import { assertGraphNextPageUrl, getGraphNextPageUrl } from '@/tools/sharepoint/utils' export const dynamic = 'force-dynamic' diff --git a/apps/sim/app/api/tools/sharepoint/site/route.ts b/apps/sim/app/api/tools/sharepoint/site/route.ts index ce7bcefcbc9..4dc2b508917 100644 --- a/apps/sim/app/api/tools/sharepoint/site/route.ts +++ b/apps/sim/app/api/tools/sharepoint/site/route.ts @@ -6,7 +6,7 @@ import { getValidationErrorMessage } from '@/lib/api/server' import { authorizeCredentialUse } from '@/lib/auth/credential-access' import { validateMicrosoftGraphId } from '@/lib/core/security/input-validation' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { refreshAccessTokenIfNeeded } from '@/app/api/auth/oauth/utils' +import { refreshAccessTokenIfNeeded } from '@/lib/oauth/credential-service' export const dynamic = 'force-dynamic' diff --git a/apps/sim/app/api/tools/sharepoint/sites/route.ts b/apps/sim/app/api/tools/sharepoint/sites/route.ts index fc8db948c7d..64bdd3684d5 100644 --- a/apps/sim/app/api/tools/sharepoint/sites/route.ts +++ b/apps/sim/app/api/tools/sharepoint/sites/route.ts @@ -5,7 +5,7 @@ import { parseRequest } from '@/lib/api/server' import { authorizeCredentialUse } from '@/lib/auth/credential-access' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { refreshAccessTokenIfNeeded } from '@/app/api/auth/oauth/utils' +import { refreshAccessTokenIfNeeded } from '@/lib/oauth/credential-service' import type { SharepointSite } from '@/tools/sharepoint/types' import { assertGraphNextPageUrl, getGraphNextPageUrl } from '@/tools/sharepoint/utils' diff --git a/apps/sim/app/api/tools/slack/channels/route.ts b/apps/sim/app/api/tools/slack/channels/route.ts index feb5a7b5153..3648f70e4fd 100644 --- a/apps/sim/app/api/tools/slack/channels/route.ts +++ b/apps/sim/app/api/tools/slack/channels/route.ts @@ -9,7 +9,7 @@ import { authorizeCredentialUse } from '@/lib/auth/credential-access' import { validateAlphanumericId } from '@/lib/core/security/input-validation' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { refreshAccessTokenIfNeeded } from '@/app/api/auth/oauth/utils' +import { refreshAccessTokenIfNeeded } from '@/lib/oauth/credential-service' export const dynamic = 'force-dynamic' diff --git a/apps/sim/app/api/tools/slack/users/route.ts b/apps/sim/app/api/tools/slack/users/route.ts index cb1e69569f4..6d0ae67e31a 100644 --- a/apps/sim/app/api/tools/slack/users/route.ts +++ b/apps/sim/app/api/tools/slack/users/route.ts @@ -6,7 +6,7 @@ import { authorizeCredentialUse } from '@/lib/auth/credential-access' import { validateAlphanumericId } from '@/lib/core/security/input-validation' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { refreshAccessTokenIfNeeded } from '@/app/api/auth/oauth/utils' +import { refreshAccessTokenIfNeeded } from '@/lib/oauth/credential-service' export const dynamic = 'force-dynamic' diff --git a/apps/sim/app/api/tools/trello/boards/route.ts b/apps/sim/app/api/tools/trello/boards/route.ts index e4ca2f42461..8ce07d0500f 100644 --- a/apps/sim/app/api/tools/trello/boards/route.ts +++ b/apps/sim/app/api/tools/trello/boards/route.ts @@ -5,7 +5,7 @@ import { parseRequest } from '@/lib/api/server' import { authorizeCredentialUse } from '@/lib/auth/credential-access' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { refreshAccessTokenIfNeeded } from '@/app/api/auth/oauth/utils' +import { refreshAccessTokenIfNeeded } from '@/lib/oauth/credential-service' const logger = createLogger('TrelloBoardsAPI') diff --git a/apps/sim/app/api/tools/wealthbox/item/route.ts b/apps/sim/app/api/tools/wealthbox/item/route.ts index 066cfb6fcbe..da8ad62b91f 100644 --- a/apps/sim/app/api/tools/wealthbox/item/route.ts +++ b/apps/sim/app/api/tools/wealthbox/item/route.ts @@ -6,7 +6,7 @@ import { authorizeCredentialUse } from '@/lib/auth/credential-access' import { validatePathSegment } from '@/lib/core/security/input-validation' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { refreshAccessTokenIfNeeded } from '@/app/api/auth/oauth/utils' +import { refreshAccessTokenIfNeeded } from '@/lib/oauth/credential-service' export const dynamic = 'force-dynamic' diff --git a/apps/sim/app/api/tools/wealthbox/items/route.ts b/apps/sim/app/api/tools/wealthbox/items/route.ts index a10c4672eb3..98223382e94 100644 --- a/apps/sim/app/api/tools/wealthbox/items/route.ts +++ b/apps/sim/app/api/tools/wealthbox/items/route.ts @@ -6,7 +6,7 @@ import { authorizeCredentialUse } from '@/lib/auth/credential-access' import { validatePathSegment } from '@/lib/core/security/input-validation' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { refreshAccessTokenIfNeeded } from '@/app/api/auth/oauth/utils' +import { refreshAccessTokenIfNeeded } from '@/lib/oauth/credential-service' export const dynamic = 'force-dynamic' diff --git a/apps/sim/app/api/tools/webflow/collections/route.ts b/apps/sim/app/api/tools/webflow/collections/route.ts index 4df1bceaeca..4fd56b91f58 100644 --- a/apps/sim/app/api/tools/webflow/collections/route.ts +++ b/apps/sim/app/api/tools/webflow/collections/route.ts @@ -6,7 +6,7 @@ import { authorizeCredentialUse } from '@/lib/auth/credential-access' import { validateAlphanumericId } from '@/lib/core/security/input-validation' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { refreshAccessTokenIfNeeded } from '@/app/api/auth/oauth/utils' +import { refreshAccessTokenIfNeeded } from '@/lib/oauth/credential-service' const logger = createLogger('WebflowCollectionsAPI') diff --git a/apps/sim/app/api/tools/webflow/items/route.ts b/apps/sim/app/api/tools/webflow/items/route.ts index 4feb0f40417..3a363c5eacc 100644 --- a/apps/sim/app/api/tools/webflow/items/route.ts +++ b/apps/sim/app/api/tools/webflow/items/route.ts @@ -6,7 +6,7 @@ import { authorizeCredentialUse } from '@/lib/auth/credential-access' import { validateAlphanumericId } from '@/lib/core/security/input-validation' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { refreshAccessTokenIfNeeded } from '@/app/api/auth/oauth/utils' +import { refreshAccessTokenIfNeeded } from '@/lib/oauth/credential-service' const logger = createLogger('WebflowItemsAPI') diff --git a/apps/sim/app/api/tools/webflow/sites/route.ts b/apps/sim/app/api/tools/webflow/sites/route.ts index 89073c6eeb5..4aab5b999b2 100644 --- a/apps/sim/app/api/tools/webflow/sites/route.ts +++ b/apps/sim/app/api/tools/webflow/sites/route.ts @@ -6,7 +6,7 @@ import { authorizeCredentialUse } from '@/lib/auth/credential-access' import { validateAlphanumericId } from '@/lib/core/security/input-validation' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { refreshAccessTokenIfNeeded } from '@/app/api/auth/oauth/utils' +import { refreshAccessTokenIfNeeded } from '@/lib/oauth/credential-service' const logger = createLogger('WebflowSitesAPI') diff --git a/apps/sim/app/api/tools/zoho_desk/selector-credential.ts b/apps/sim/app/api/tools/zoho_desk/selector-credential.ts index dbffa284a14..9d59d67ffdd 100644 --- a/apps/sim/app/api/tools/zoho_desk/selector-credential.ts +++ b/apps/sim/app/api/tools/zoho_desk/selector-credential.ts @@ -4,7 +4,7 @@ import { createLogger } from '@sim/logger' import { eq } from 'drizzle-orm' import { type NextRequest, NextResponse } from 'next/server' import { authorizeCredentialUse } from '@/lib/auth/credential-access' -import { resolveCredentialAccessToken, resolveOAuthAccountId } from '@/app/api/auth/oauth/utils' +import { resolveCredentialAccessToken, resolveOAuthAccountId } from '@/lib/oauth/credential-service' import { extractZohoDeskBaseFromScope } from '@/tools/zoho_desk/host-allowlist' import { getZohoDeskApiBase } from '@/tools/zoho_desk/utils' diff --git a/apps/sim/app/api/tools/zoom/meetings/route.ts b/apps/sim/app/api/tools/zoom/meetings/route.ts index 53e78f408ca..48ba158659e 100644 --- a/apps/sim/app/api/tools/zoom/meetings/route.ts +++ b/apps/sim/app/api/tools/zoom/meetings/route.ts @@ -5,7 +5,7 @@ import { parseRequest } from '@/lib/api/server' import { authorizeCredentialUse } from '@/lib/auth/credential-access' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { refreshAccessTokenIfNeeded } from '@/app/api/auth/oauth/utils' +import { refreshAccessTokenIfNeeded } from '@/lib/oauth/credential-service' const logger = createLogger('ZoomMeetingsAPI') diff --git a/apps/sim/app/api/v2/knowledge/[id]/documents/route.test.ts b/apps/sim/app/api/v2/knowledge/[id]/documents/route.test.ts index cbb5e0d88d2..401d0f63f9b 100644 --- a/apps/sim/app/api/v2/knowledge/[id]/documents/route.test.ts +++ b/apps/sim/app/api/v2/knowledge/[id]/documents/route.test.ts @@ -1,13 +1,18 @@ /** * @vitest-environment node */ +import { + V2_OPERATION_RATE_LIMIT_ALLOWED, + V2_PREAUTH_RATE_LIMIT_ALLOWED, + v2ApiKeyAuthModuleMock, + v2GateModuleMock, + v2RateLimiterModuleMock, + v2RouteMocks, +} from '@sim/testing' import { NextRequest } from 'next/server' import { beforeEach, describe, expect, it, vi } from 'vitest' const { - mockAuthenticate, - mockCheckPreAuth, - mockCheckRateLimit, mockAdmitUpload, mockUploadDocument, mockReadFormData, @@ -15,10 +20,8 @@ const { mockUploadWorkspaceFile, mockPlatformUploaded, mockCapture, + mockIsPayloadSizeLimitError, } = vi.hoisted(() => ({ - mockAuthenticate: vi.fn(), - mockCheckPreAuth: vi.fn(), - mockCheckRateLimit: vi.fn(), mockAdmitUpload: vi.fn(), mockUploadDocument: vi.fn(), mockReadFormData: vi.fn(), @@ -26,27 +29,12 @@ const { mockUploadWorkspaceFile: vi.fn(), mockPlatformUploaded: vi.fn(), mockCapture: vi.fn(), + mockIsPayloadSizeLimitError: vi.fn(), })) -vi.mock('@/lib/api/server/routes/v2-api-key-auth', () => ({ - authenticateV2ApiKey: mockAuthenticate, - V2ApiKeyUnauthenticatedError: class V2ApiKeyUnauthenticatedError extends Error {}, -})) - -vi.mock('@/lib/core/rate-limiter', () => ({ - getRateLimit: () => ({ maxTokens: 100, refillRate: 100, refillIntervalMs: 60_000 }), - RateLimiter: class RateLimiter { - checkRateLimitDirect(...args: unknown[]) { - return mockCheckPreAuth(...args) - } - - checkRateLimitDirectOrThrow(...args: unknown[]) { - return mockCheckRateLimit(...args) - } - }, -})) - -vi.mock('@/app/api/v2/lib/gate', () => ({ v2ApiGateError: vi.fn().mockResolvedValue(null) })) +vi.mock('@/lib/api/server/routes/v2-api-key-auth', () => v2ApiKeyAuthModuleMock) +vi.mock('@/lib/core/rate-limiter', () => v2RateLimiterModuleMock) +vi.mock('@/app/api/v2/lib/gate', () => v2GateModuleMock) vi.mock('@/lib/knowledge/application/documents', () => ({ listKnowledgeDocuments: { @@ -64,7 +52,8 @@ vi.mock('@/lib/knowledge/application/documents', () => ({ })) vi.mock('@/lib/core/utils/stream-limits', () => ({ - isPayloadSizeLimitError: () => false, + MAX_MULTIPART_OVERHEAD_BYTES: 1024 * 1024, + isPayloadSizeLimitError: mockIsPayloadSizeLimitError, readFormDataWithLimit: mockReadFormData, readFileToBufferWithLimit: mockReadFile, })) @@ -79,16 +68,13 @@ vi.mock('@/lib/core/telemetry', () => ({ vi.mock('@/lib/posthog/server', () => ({ captureServerEvent: mockCapture })) +import { OrchestrationError } from '@/lib/core/orchestration/types' import { KnowledgeUsageLimitExceededError } from '@/lib/knowledge/application/billing' +import { MAX_KNOWLEDGE_DOCUMENT_FILE_SIZE } from '@/lib/uploads/shared/types' +import { validateFileType } from '@/lib/uploads/utils/validation' import { POST } from '@/app/api/v2/knowledge/[id]/documents/route' const WORKSPACE_ID = 'workspace-1' -const RATE_LIMIT_OK = { - allowed: true, - remaining: 99, - resetAt: new Date('2024-01-01T01:00:00Z'), - retryAfterMs: 0, -} const PRINCIPAL = { kind: 'personal_api_key', userId: 'user-1', keyId: 'key-1' } as const function buildRequest() { @@ -101,9 +87,11 @@ function buildRequest() { describe('POST /api/v2/knowledge/[id]/documents', () => { beforeEach(() => { vi.clearAllMocks() - mockCheckPreAuth.mockResolvedValue(RATE_LIMIT_OK) - mockCheckRateLimit.mockResolvedValue(RATE_LIMIT_OK) - mockAuthenticate.mockResolvedValue({ + v2RouteMocks.preauthRate.mockResolvedValue(V2_PREAUTH_RATE_LIMIT_ALLOWED) + v2RouteMocks.operationRate.mockResolvedValue(V2_OPERATION_RATE_LIMIT_ALLOWED) + v2RouteMocks.gate.mockResolvedValue(null) + mockIsPayloadSizeLimitError.mockReturnValue(false) + v2RouteMocks.authenticate.mockResolvedValue({ principal: PRINCIPAL, rolloutUserId: 'user-1', rateLimitSubjectIds: ['api-key:key-1', 'user:user-1'], @@ -184,6 +172,16 @@ describe('POST /api/v2/knowledge/[id]/documents', () => { expect.objectContaining({ knowledge_base_id: 'kb-1' }), expect.any(Object) ) + expect(mockReadFormData).toHaveBeenCalledWith(request, { + maxBytes: MAX_KNOWLEDGE_DOCUMENT_FILE_SIZE + 1024 * 1024, + label: 'knowledge document upload body', + }) + expect(mockReadFile).toHaveBeenCalledWith(expect.any(File), { + maxBytes: MAX_KNOWLEDGE_DOCUMENT_FILE_SIZE, + label: 'knowledge document file', + }) + expect(response.headers.get('cache-control')).toBe('private, no-store') + expect(response.headers.get('x-ratelimit-limit')).toBe('100') }) it('maps usage admission to the v2 error before multipart buffering', async () => { @@ -201,7 +199,7 @@ describe('POST /api/v2/knowledge/[id]/documents', () => { }) it('does not create human analytics for a workspace key', async () => { - mockAuthenticate.mockResolvedValue({ + v2RouteMocks.authenticate.mockResolvedValue({ principal: { kind: 'workspace_api_key', workspaceId: WORKSPACE_ID, keyId: 'key-2' }, rolloutUserId: 'billing-owner', rateLimitSubjectIds: ['api-key:key-2', `workspace:${WORKSPACE_ID}`], @@ -215,4 +213,109 @@ describe('POST /api/v2/knowledge/[id]/documents', () => { expect(mockPlatformUploaded).toHaveBeenCalledOnce() expect(mockCapture).not.toHaveBeenCalled() }) + + it('preserves the malformed multipart envelope without transferring storage', async () => { + mockReadFormData.mockRejectedValueOnce(new Error('multipart boundary missing')) + + const response = await POST(buildRequest(), { params: Promise.resolve({ id: 'kb-1' }) }) + + expect(response.status).toBe(400) + expect(await response.json()).toEqual({ + error: { code: 'BAD_REQUEST', message: 'Request body must be valid multipart form data' }, + }) + expect(mockUploadWorkspaceFile).not.toHaveBeenCalled() + expect(mockUploadDocument).not.toHaveBeenCalled() + expect(mockPlatformUploaded).not.toHaveBeenCalled() + }) + + it('preserves bounded multipart rejection and stops before storage transfer', async () => { + const error = new Error('knowledge document upload body exceeds maximum size') + mockReadFormData.mockRejectedValueOnce(error) + mockIsPayloadSizeLimitError.mockImplementation((candidate: unknown) => candidate === error) + + const response = await POST(buildRequest(), { params: Promise.resolve({ id: 'kb-1' }) }) + + expect(response.status).toBe(413) + expect(await response.json()).toEqual({ + error: { code: 'PAYLOAD_TOO_LARGE', message: error.message }, + }) + expect(mockUploadWorkspaceFile).not.toHaveBeenCalled() + expect(mockUploadDocument).not.toHaveBeenCalled() + }) + + it('requires a file form field before storage transfer', async () => { + mockReadFormData.mockResolvedValueOnce(new FormData()) + + const response = await POST(buildRequest(), { params: Promise.resolve({ id: 'kb-1' }) }) + + expect(response.status).toBe(400) + expect(await response.json()).toEqual({ + error: { code: 'BAD_REQUEST', message: 'file form field is required' }, + }) + expect(mockUploadWorkspaceFile).not.toHaveBeenCalled() + }) + + it('preserves the exact file-size rejection before reading file bytes', async () => { + const formData = new FormData() + const file = new File(['x'], 'large.txt', { type: 'text/plain' }) + Object.defineProperty(file, 'size', { value: MAX_KNOWLEDGE_DOCUMENT_FILE_SIZE + 1 }) + formData.set('file', file) + mockReadFormData.mockResolvedValueOnce(formData) + + const response = await POST(buildRequest(), { params: Promise.resolve({ id: 'kb-1' }) }) + + expect(response.status).toBe(413) + expect(await response.json()).toEqual({ + error: { code: 'PAYLOAD_TOO_LARGE', message: 'File size exceeds 100MB limit (100.00MB)' }, + }) + expect(mockReadFile).not.toHaveBeenCalled() + expect(mockUploadWorkspaceFile).not.toHaveBeenCalled() + }) + + it('preserves unsupported file-type validation before reading file bytes', async () => { + const formData = new FormData() + formData.set('file', new File(['x'], 'malware.exe', { type: 'application/octet-stream' })) + mockReadFormData.mockResolvedValueOnce(formData) + const expectedMessage = validateFileType('malware.exe', 'application/octet-stream')?.message + if (!expectedMessage) throw new Error('Expected unsupported file type validation to fail') + + const response = await POST(buildRequest(), { params: Promise.resolve({ id: 'kb-1' }) }) + + expect(response.status).toBe(415) + expect(await response.json()).toEqual({ + error: { code: 'UNSUPPORTED_MEDIA_TYPE', message: expectedMessage }, + }) + expect(mockReadFile).not.toHaveBeenCalled() + expect(mockUploadWorkspaceFile).not.toHaveBeenCalled() + }) + + it('does not register or emit effects when storage transfer fails', async () => { + mockUploadWorkspaceFile.mockRejectedValueOnce(new Error('storage unavailable')) + + const response = await POST(buildRequest(), { params: Promise.resolve({ id: 'kb-1' }) }) + + expect(response.status).toBe(500) + expect(await response.json()).toEqual({ + error: { code: 'INTERNAL_ERROR', message: 'Internal server error' }, + }) + expect(mockUploadDocument).not.toHaveBeenCalled() + expect(mockPlatformUploaded).not.toHaveBeenCalled() + expect(mockCapture).not.toHaveBeenCalled() + }) + + it('preserves application authorization errors after storage transfer', async () => { + mockUploadDocument.mockRejectedValueOnce( + new OrchestrationError('forbidden', 'Insufficient workspace permissions') + ) + + const response = await POST(buildRequest(), { params: Promise.resolve({ id: 'kb-1' }) }) + + expect(response.status).toBe(403) + expect(await response.json()).toEqual({ + error: { code: 'FORBIDDEN', message: 'Insufficient workspace permissions' }, + }) + expect(mockUploadWorkspaceFile).toHaveBeenCalledOnce() + expect(mockPlatformUploaded).not.toHaveBeenCalled() + expect(mockCapture).not.toHaveBeenCalled() + }) }) diff --git a/apps/sim/app/api/v2/knowledge/[id]/documents/route.ts b/apps/sim/app/api/v2/knowledge/[id]/documents/route.ts index 539ffd677a1..a8b04e82093 100644 --- a/apps/sim/app/api/v2/knowledge/[id]/documents/route.ts +++ b/apps/sim/app/api/v2/knowledge/[id]/documents/route.ts @@ -1,47 +1,41 @@ -import type { NextRequest } from 'next/server' -import { NextResponse } from 'next/server' import { type V2KnowledgeDocumentSummary, v2ListKnowledgeDocumentsContract, v2UploadKnowledgeDocumentContract, } from '@/lib/api/contracts/v2/knowledge' -import { parseRequest } from '@/lib/api/server' import { + defineV2BodyLifecycleRoute, defineV2JsonRoute, v2ApiKeyAuth, - v2OrchestrationErrorPolicy, v2RateLimits, } from '@/lib/api/server/routes' -import type { JsonRouteContext } from '@/lib/api/server/routes/types' -import { admitV2Request, V2RouteInfrastructureError } from '@/lib/api/server/routes/v2-json-route' import { OrchestrationError } from '@/lib/core/orchestration/types' import { PlatformEvents } from '@/lib/core/telemetry' import { isPayloadSizeLimitError, + MAX_MULTIPART_OVERHEAD_BYTES, readFileToBufferWithLimit, readFormDataWithLimit, } from '@/lib/core/utils/stream-limits' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { v2KnowledgeErrorPolicies } from '@/lib/knowledge/api/route-policies' -import { KnowledgeUsageLimitExceededError } from '@/lib/knowledge/application/billing' import { admitKnowledgeDocumentUpload, listKnowledgeDocuments, uploadKnowledgeDocument, } from '@/lib/knowledge/application/documents' import { knowledgeOperations } from '@/lib/knowledge/application/operations' +import { KnowledgeDocumentUnsupportedMediaTypeError } from '@/lib/knowledge/application/upload-sessions' import { captureServerEvent } from '@/lib/posthog/server' import { uploadWorkspaceFile } from '@/lib/uploads/contexts/workspace' import { MAX_KNOWLEDGE_DOCUMENT_FILE_SIZE } from '@/lib/uploads/shared/types' import { validateFileType } from '@/lib/uploads/utils/validation' import { serializeDate } from '@/app/api/v1/knowledge/utils' -import { decodeCursor, encodeCursor, v2Error, v2ValidationError } from '@/app/api/v2/lib/response' +import { decodeCursor, encodeCursor } from '@/app/api/v2/lib/response' export const dynamic = 'force-dynamic' export const revalidate = 0 const MAX_FILE_SIZE = MAX_KNOWLEDGE_DOCUMENT_FILE_SIZE -const MAX_MULTIPART_OVERHEAD_BYTES = 1024 * 1024 function toV2DocumentSummary(document: { id: string @@ -107,146 +101,98 @@ export const GET = defineV2JsonRoute({ }) /** POST /api/v2/knowledge/[id]/documents — Upload a document to a knowledge base. */ -export const POST = withRouteHandler( - async (request: NextRequest, context) => { - if (request.method !== v2UploadKnowledgeDocumentContract.method) { - throw new Error( - `Route received ${request.method} for ${v2UploadKnowledgeDocumentContract.method} contract ${v2UploadKnowledgeDocumentContract.path}` - ) - } - - const routeAdmission = await admitV2Request( - request, - knowledgeOperations.uploadDocument, - v2ApiKeyAuth, - v2RateLimits.publicApi - ) - if (!routeAdmission.success) return routeAdmission.response - - const parsed = await parseRequest(v2UploadKnowledgeDocumentContract, request, context ?? {}, { - validationErrorResponse: v2ValidationError, - }) - if (!parsed.success) return parsed.response - - const { principal } = routeAdmission.auth - const { id: knowledgeBaseId } = parsed.data.params - const { workspaceId } = parsed.data.query - +export const POST = defineV2BodyLifecycleRoute({ + contract: v2UploadKnowledgeDocumentContract, + auth: v2ApiKeyAuth, + operation: knowledgeOperations.uploadDocument, + rateLimit: v2RateLimits.publicApi, + errorPolicy: v2KnowledgeErrorPolicies.documentUpload, + admission: { + mapInput: ({ params, query }) => ({ + knowledgeBaseId: params.id, + assertedWorkspaceId: query.workspaceId, + }), + useCase: admitKnowledgeDocumentUpload, + }, + async readBody({ request }) { + let formData: FormData try { - const uploadAdmission = await admitKnowledgeDocumentUpload.execute({ - principal, - input: { knowledgeBaseId, assertedWorkspaceId: workspaceId }, - request, + formData = await readFormDataWithLimit(request, { + maxBytes: MAX_FILE_SIZE + MAX_MULTIPART_OVERHEAD_BYTES, + label: 'knowledge document upload body', }) + } catch (error) { + if (isPayloadSizeLimitError(error)) throw error + throw new OrchestrationError('validation', 'Request body must be valid multipart form data') + } - let formData: FormData - try { - formData = await readFormDataWithLimit(request, { - maxBytes: MAX_FILE_SIZE + MAX_MULTIPART_OVERHEAD_BYTES, - label: 'knowledge document upload body', - }) - } catch (error) { - if (isPayloadSizeLimitError(error)) { - return v2Error('PAYLOAD_TOO_LARGE', error.message) - } - return v2Error('BAD_REQUEST', 'Request body must be valid multipart form data') - } - - const rawFile = formData.get('file') - const file = rawFile instanceof File ? rawFile : null - if (!file) return v2Error('BAD_REQUEST', 'file form field is required') - - if (file.size > MAX_FILE_SIZE) { - return v2Error( - 'PAYLOAD_TOO_LARGE', - `File size exceeds 100MB limit (${(file.size / (1024 * 1024)).toFixed(2)}MB)` - ) - } - - const fileTypeError = validateFileType(file.name, file.type || '') - if (fileTypeError) { - return v2Error('UNSUPPORTED_MEDIA_TYPE', fileTypeError.message) - } - - const buffer = await readFileToBufferWithLimit(file, { - maxBytes: MAX_FILE_SIZE, - label: 'knowledge document file', - }) - const contentType = file.type || 'application/octet-stream' - const uploadedFile = await uploadWorkspaceFile( - uploadAdmission.workspaceId, - uploadAdmission.storageActorUserId, - buffer, - file.name, - contentType + const rawFile = formData.get('file') + if (!(rawFile instanceof File)) { + throw new OrchestrationError('validation', 'file form field is required') + } + if (rawFile.size > MAX_FILE_SIZE) { + throw new OrchestrationError( + 'payload_too_large', + `File size exceeds 100MB limit (${(rawFile.size / (1024 * 1024)).toFixed(2)}MB)` ) - - const result = await uploadKnowledgeDocument.execute({ - principal, - input: { - knowledgeBaseId, - assertedWorkspaceId: workspaceId, - document: { - filename: file.name, - fileUrl: uploadedFile.url, - fileSize: file.size, - mimeType: contentType, - }, - startProcessing: true, - usageAdmission: 'pre_admitted', - source: 'api', + } + const contentType = rawFile.type || 'application/octet-stream' + const fileTypeError = validateFileType(rawFile.name, rawFile.type || '') + if (fileTypeError) { + throw new KnowledgeDocumentUnsupportedMediaTypeError(fileTypeError.message) + } + const buffer = await readFileToBufferWithLimit(rawFile, { + maxBytes: MAX_FILE_SIZE, + label: 'knowledge document file', + }) + return { file: rawFile, buffer, contentType } + }, + transfer: ({ admission, body }) => + uploadWorkspaceFile( + admission.workspaceId, + admission.storageActorUserId, + body.buffer, + body.file.name, + body.contentType + ), + mapInput: ({ parsed, body, transfer }) => ({ + knowledgeBaseId: parsed.params.id, + assertedWorkspaceId: parsed.query.workspaceId, + document: { + filename: body.file.name, + fileUrl: transfer.url, + fileSize: body.file.size, + mimeType: body.contentType, + }, + startProcessing: true, + usageAdmission: 'pre_admitted' as const, + source: 'api' as const, + }), + useCase: uploadKnowledgeDocument, + present: (result) => ({ data: { document: toV2DocumentSummary(result.document) } }), + onSuccess: ({ principal, admission, result }) => { + PlatformEvents.knowledgeBaseDocumentsUploaded({ + knowledgeBaseId: result.document.knowledgeBaseId, + documentsCount: 1, + uploadType: 'single', + mimeType: result.document.mimeType, + fileSize: result.document.fileSize, + }) + if (principal.kind === 'personal_api_key') { + captureServerEvent( + principal.userId, + 'knowledge_base_document_uploaded', + { + knowledge_base_id: result.document.knowledgeBaseId, + workspace_id: admission.workspaceId, + document_count: 1, + upload_type: 'single', }, - request, - }) - - PlatformEvents.knowledgeBaseDocumentsUploaded({ - knowledgeBaseId, - documentsCount: 1, - uploadType: 'single', - mimeType: contentType, - fileSize: file.size, - }) - if (principal.kind === 'personal_api_key') { - captureServerEvent( - principal.userId, - 'knowledge_base_document_uploaded', - { - knowledge_base_id: knowledgeBaseId, - workspace_id: workspaceId, - document_count: 1, - upload_type: 'single', - }, - { - groups: { workspace: workspaceId }, - setOnce: { first_document_uploaded_at: new Date().toISOString() }, - } - ) - } - - const document = toV2DocumentSummary(result.document) - const body = v2UploadKnowledgeDocumentContract.response.schema.parse({ - data: { document }, - }) - return NextResponse.json(body, { - status: 201, - headers: { 'Cache-Control': 'private, no-store' }, - }) - } catch (error) { - if (error instanceof KnowledgeUsageLimitExceededError) { - return v2Error('USAGE_LIMIT_EXCEEDED', error.message) - } - if (isPayloadSizeLimitError(error)) { - return v2Error('PAYLOAD_TOO_LARGE', error.message) - } - const response = v2OrchestrationErrorPolicy.render(error) - if (response) return response - throw error + { + groups: { workspace: admission.workspaceId }, + setOnce: { first_document_uploaded_at: new Date().toISOString() }, + } + ) } }, - { - unhandledErrorResponse: ({ error }) => - error instanceof V2RouteInfrastructureError - ? v2Error('SERVICE_UNAVAILABLE', 'Service temporarily unavailable') - : v2Error('INTERNAL_ERROR', 'Internal server error'), - } -) +}) diff --git a/apps/sim/app/api/v2/knowledge/search/route.test.ts b/apps/sim/app/api/v2/knowledge/search/route.test.ts index 462518ba38f..6a307e41b96 100644 --- a/apps/sim/app/api/v2/knowledge/search/route.test.ts +++ b/apps/sim/app/api/v2/knowledge/search/route.test.ts @@ -1,56 +1,39 @@ /** * @vitest-environment node */ +import { + V2_OPERATION_RATE_LIMIT_ALLOWED, + V2_PREAUTH_RATE_LIMIT_ALLOWED, + v2ApiKeyAuthModuleMock, + v2GateModuleMock, + v2RateLimiterModuleMock, + v2RouteMocks, +} from '@sim/testing' import { NextRequest } from 'next/server' import { beforeEach, describe, expect, it, vi } from 'vitest' -const { mockAuthenticate, mockCheckPreAuth, mockCheckRateLimit, mockSearch } = vi.hoisted(() => ({ - mockAuthenticate: vi.fn(), - mockCheckPreAuth: vi.fn(), - mockCheckRateLimit: vi.fn(), +const { mockSearch } = vi.hoisted(() => ({ mockSearch: vi.fn(), })) -vi.mock('@/lib/api/server/routes/v2-api-key-auth', () => ({ - authenticateV2ApiKey: mockAuthenticate, - V2ApiKeyUnauthenticatedError: class V2ApiKeyUnauthenticatedError extends Error {}, -})) - -vi.mock('@/lib/core/rate-limiter', () => ({ - getRateLimit: () => ({ maxTokens: 100, refillRate: 100, refillIntervalMs: 60_000 }), - RateLimiter: class RateLimiter { - checkRateLimitDirect(...args: unknown[]) { - return mockCheckPreAuth(...args) - } - - checkRateLimitDirectOrThrow(...args: unknown[]) { - return mockCheckRateLimit(...args) - } - }, -})) - -vi.mock('@/app/api/v2/lib/gate', () => ({ v2ApiGateError: vi.fn().mockResolvedValue(null) })) +vi.mock('@/lib/api/server/routes/v2-api-key-auth', () => v2ApiKeyAuthModuleMock) +vi.mock('@/lib/core/rate-limiter', () => v2RateLimiterModuleMock) +vi.mock('@/app/api/v2/lib/gate', () => v2GateModuleMock) vi.mock('@/lib/knowledge/application/search', () => ({ searchKnowledge: { operation: { id: 'knowledge.search' }, execute: mockSearch }, })) +import { DEFAULT_MAX_JSON_BODY_BYTES } from '@/lib/api/server/validation' import { KnowledgeUsageLimitExceededError } from '@/lib/knowledge/application/billing' import { POST } from '@/app/api/v2/knowledge/search/route' const WORKSPACE_ID = 'workspace-1' const PRINCIPAL = { kind: 'workspace_api_key', workspaceId: WORKSPACE_ID, keyId: 'key-1' } as const -const RATE_LIMIT_OK = { - allowed: true, - remaining: 99, - resetAt: new Date('2024-01-01T01:00:00Z'), - retryAfterMs: 0, -} - -function buildRequest(body: string) { +function buildRequest(body: string, headers: Record = {}) { return new NextRequest('http://localhost/api/v2/knowledge/search', { method: 'POST', - headers: { 'content-type': 'application/json', 'x-api-key': 'secret' }, + headers: { 'content-type': 'application/json', 'x-api-key': 'secret', ...headers }, body, }) } @@ -58,9 +41,10 @@ function buildRequest(body: string) { describe('POST /api/v2/knowledge/search', () => { beforeEach(() => { vi.clearAllMocks() - mockCheckPreAuth.mockResolvedValue(RATE_LIMIT_OK) - mockCheckRateLimit.mockResolvedValue(RATE_LIMIT_OK) - mockAuthenticate.mockResolvedValue({ + v2RouteMocks.preauthRate.mockResolvedValue(V2_PREAUTH_RATE_LIMIT_ALLOWED) + v2RouteMocks.operationRate.mockResolvedValue(V2_OPERATION_RATE_LIMIT_ALLOWED) + v2RouteMocks.gate.mockResolvedValue(null) + v2RouteMocks.authenticate.mockResolvedValue({ principal: PRINCIPAL, rolloutUserId: 'billing-owner', rateLimitSubjectIds: ['api-key:key-1', `workspace:${WORKSPACE_ID}`], @@ -113,13 +97,15 @@ describe('POST /api/v2/knowledge/search', () => { expect(await response.json()).toEqual({ data: expect.objectContaining({ knowledgeBaseIds: ['kb-1'], totalResults: 1 }), }) + expect(response.headers.get('cache-control')).toBe('private, no-store') + expect(response.headers.get('x-ratelimit-limit')).toBe('100') }) it('authenticates before rejecting malformed JSON', async () => { const response = await POST(buildRequest('{')) expect(response.status).toBe(400) - expect(mockAuthenticate).toHaveBeenCalledOnce() + expect(v2RouteMocks.authenticate).toHaveBeenCalledOnce() expect(mockSearch).not.toHaveBeenCalled() }) @@ -142,4 +128,37 @@ describe('POST /api/v2/knowledge/search', () => { error: { code: 'USAGE_LIMIT_EXCEEDED', message: 'Upgrade required' }, }) }) + + it('preserves the bounded JSON rejection before application execution', async () => { + const response = await POST( + buildRequest('{}', { 'content-length': String(DEFAULT_MAX_JSON_BODY_BYTES + 1) }) + ) + + expect(response.status).toBe(413) + expect(await response.json()).toEqual({ + error: `Request body exceeds the maximum allowed size of ${DEFAULT_MAX_JSON_BODY_BYTES} bytes`, + }) + expect(mockSearch).not.toHaveBeenCalled() + expect(response.headers.get('x-ratelimit-limit')).toBe('100') + }) + + it('does not expose application infrastructure failures', async () => { + mockSearch.mockRejectedValueOnce(new Error('database host is private-db')) + + const response = await POST( + buildRequest( + JSON.stringify({ + workspaceId: WORKSPACE_ID, + knowledgeBaseIds: ['kb-1'], + query: 'hello', + topK: 10, + }) + ) + ) + + expect(response.status).toBe(500) + expect(await response.json()).toEqual({ + error: { code: 'INTERNAL_ERROR', message: 'Internal server error' }, + }) + }) }) diff --git a/apps/sim/app/api/v2/knowledge/search/route.ts b/apps/sim/app/api/v2/knowledge/search/route.ts index 3aa8d98f158..07ba633ed00 100644 --- a/apps/sim/app/api/v2/knowledge/search/route.ts +++ b/apps/sim/app/api/v2/knowledge/search/route.ts @@ -1,74 +1,32 @@ -import type { NextRequest } from 'next/server' -import { NextResponse } from 'next/server' import { v2SearchKnowledgeContract } from '@/lib/api/contracts/v2/knowledge' -import { parseRequest } from '@/lib/api/server' -import { v2ApiKeyAuth, v2OrchestrationErrorPolicy, v2RateLimits } from '@/lib/api/server/routes' -import type { JsonRouteContext } from '@/lib/api/server/routes/types' -import { admitV2Request, V2RouteInfrastructureError } from '@/lib/api/server/routes/v2-json-route' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { KnowledgeUsageLimitExceededError } from '@/lib/knowledge/application/billing' +import { defineV2JsonRoute, v2ApiKeyAuth, v2RateLimits } from '@/lib/api/server/routes' +import { v2KnowledgeErrorPolicies } from '@/lib/knowledge/api/route-policies' import { knowledgeOperations } from '@/lib/knowledge/application/operations' import { searchKnowledge } from '@/lib/knowledge/application/search' -import { v2Error, v2ValidationError } from '@/app/api/v2/lib/response' +import { v2Error } from '@/app/api/v2/lib/response' export const dynamic = 'force-dynamic' export const revalidate = 0 /** POST /api/v2/knowledge/search — Vector / tag search across knowledge bases. */ -export const POST = withRouteHandler( - async (request: NextRequest, context) => { - if (request.method !== v2SearchKnowledgeContract.method) { - throw new Error( - `Route received ${request.method} for ${v2SearchKnowledgeContract.method} contract ${v2SearchKnowledgeContract.path}` - ) - } - - const admission = await admitV2Request( - request, - knowledgeOperations.search, - v2ApiKeyAuth, - v2RateLimits.publicApi - ) - if (!admission.success) return admission.response - - const parsed = await parseRequest(v2SearchKnowledgeContract, request, context ?? {}, { - validationErrorResponse: v2ValidationError, - invalidJsonResponse: () => v2Error('BAD_REQUEST', 'Request body must be valid JSON'), - }) - if (!parsed.success) return parsed.response - - const { body } = parsed.data - try { - const result = await searchKnowledge.execute({ - principal: admission.auth.principal, - input: { - workspaceId: body.workspaceId, - knowledgeBaseIds: Array.isArray(body.knowledgeBaseIds) - ? body.knowledgeBaseIds - : [body.knowledgeBaseIds], - query: body.query, - topK: body.topK, - tagFilters: body.tagFilters, - }, - request, - }) - const responseBody = v2SearchKnowledgeContract.response.schema.parse({ data: result }) - return NextResponse.json(responseBody, { - headers: { 'Cache-Control': 'private, no-store' }, - }) - } catch (error) { - if (error instanceof KnowledgeUsageLimitExceededError) { - return v2Error('USAGE_LIMIT_EXCEEDED', error.message) - } - const response = v2OrchestrationErrorPolicy.render(error) - if (response) return response - throw error - } +export const POST = defineV2JsonRoute({ + contract: v2SearchKnowledgeContract, + auth: v2ApiKeyAuth, + operation: knowledgeOperations.search, + rateLimit: v2RateLimits.publicApi, + errorPolicy: v2KnowledgeErrorPolicies.usage, + parseOptions: { + invalidJsonResponse: () => v2Error('BAD_REQUEST', 'Request body must be valid JSON'), }, - { - unhandledErrorResponse: ({ error }) => - error instanceof V2RouteInfrastructureError - ? v2Error('SERVICE_UNAVAILABLE', 'Service temporarily unavailable') - : v2Error('INTERNAL_ERROR', 'Internal server error'), - } -) + mapInput: ({ body }) => ({ + workspaceId: body.workspaceId, + knowledgeBaseIds: Array.isArray(body.knowledgeBaseIds) + ? body.knowledgeBaseIds + : [body.knowledgeBaseIds], + query: body.query, + topK: body.topK, + tagFilters: body.tagFilters, + }), + useCase: searchKnowledge, + present: (result) => ({ data: result }), +}) diff --git a/apps/sim/app/api/webhooks/slack/custom/[credentialId]/route.test.ts b/apps/sim/app/api/webhooks/slack/custom/[credentialId]/route.test.ts index ae51cee4d19..b1d182cdbd1 100644 --- a/apps/sim/app/api/webhooks/slack/custom/[credentialId]/route.test.ts +++ b/apps/sim/app/api/webhooks/slack/custom/[credentialId]/route.test.ts @@ -24,7 +24,7 @@ vi.mock('@/lib/core/admission/gate', () => ({ admissionRejectedResponse: () => new Response(null, { status: 503 }), })) -vi.mock('@/app/api/auth/oauth/utils', () => ({ +vi.mock('@/lib/oauth/credential-service', () => ({ getSlackBotCredential: mockGetSlackBotCredential, })) diff --git a/apps/sim/app/api/webhooks/slack/custom/[credentialId]/route.ts b/apps/sim/app/api/webhooks/slack/custom/[credentialId]/route.ts index 2267179417d..9925aca7f50 100644 --- a/apps/sim/app/api/webhooks/slack/custom/[credentialId]/route.ts +++ b/apps/sim/app/api/webhooks/slack/custom/[credentialId]/route.ts @@ -3,10 +3,10 @@ import { type NextRequest, NextResponse } from 'next/server' import { admissionRejectedResponse, tryAdmit } from '@/lib/core/admission/gate' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' +import { getSlackBotCredential } from '@/lib/oauth/credential-service' import { findWebhooksByRoutingKey, parseWebhookBody } from '@/lib/webhooks/processor' import { handleSlackChallenge, verifySlackRequestSignature } from '@/lib/webhooks/providers/slack' import { dispatchSlackWebhooks } from '@/lib/webhooks/slack-dispatch' -import { getSlackBotCredential } from '@/app/api/auth/oauth/utils' const logger = createLogger('SlackCustomBotWebhookAPI') diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/edit-connector-modal/edit-connector-modal.tsx b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/edit-connector-modal/edit-connector-modal.tsx index 2bdf1e118c9..d23554278a9 100644 --- a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/edit-connector-modal/edit-connector-modal.tsx +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/edit-connector-modal/edit-connector-modal.tsx @@ -402,19 +402,27 @@ interface DocumentsTabProps { function DocumentsTab({ knowledgeBaseId, connectorId }: DocumentsTabProps) { const [filter, setFilter] = useState<'active' | 'excluded'>('active') - const { data, isLoading } = useConnectorDocuments(knowledgeBaseId, connectorId, { - includeExcluded: true, - }) + const { data, isLoading, hasNextPage, isFetchingNextPage, fetchNextPage } = useConnectorDocuments( + knowledgeBaseId, + connectorId, + { + includeExcluded: true, + } + ) const { mutate: excludeDoc, isPending: isExcluding } = useExcludeConnectorDocument() const { mutate: restoreDoc, isPending: isRestoring } = useRestoreConnectorDocument() const documents = useMemo(() => { - if (!data?.documents) return [] - return data.documents.filter((d) => (filter === 'excluded' ? d.userExcluded : !d.userExcluded)) - }, [data?.documents, filter]) + const loadedDocuments = data?.pages.flatMap((page) => page.documents) ?? [] + return loadedDocuments.filter((document) => + filter === 'excluded' ? document.userExcluded : !document.userExcluded + ) + }, [data?.pages, filter]) - const counts = data?.counts ?? { active: 0, excluded: 0 } + const counts = data?.pages[0]?.counts ?? { active: 0, excluded: 0 } + const visibleDocumentCount = filter === 'excluded' ? counts.excluded : counts.active + const hasMoreVisibleDocuments = Boolean(hasNextPage && documents.length < visibleDocumentCount) if (isLoading) { return ( @@ -435,7 +443,7 @@ function DocumentsTab({ knowledgeBaseId, connectorId }: DocumentsTabProps) {
- {documents.length === 0 ? ( + {visibleDocumentCount === 0 ? (

{filter === 'excluded' ? 'No excluded documents' : 'No documents yet'}

@@ -488,6 +496,17 @@ function DocumentsTab({ knowledgeBaseId, connectorId }: DocumentsTabProps) {
))} + {hasMoreVisibleDocuments && ( + + )} )} diff --git a/apps/sim/background/webhook-execution.test.ts b/apps/sim/background/webhook-execution.test.ts index 17953ae6feb..29baecab743 100644 --- a/apps/sim/background/webhook-execution.test.ts +++ b/apps/sim/background/webhook-execution.test.ts @@ -115,7 +115,7 @@ vi.mock('@/lib/webhooks/attachment-processor', () => ({ WebhookAttachmentProcessor: class {}, })) -vi.mock('@/app/api/auth/oauth/utils', () => ({ +vi.mock('@/lib/oauth/credential-service', () => ({ resolveOAuthAccountId: vi.fn(), })) diff --git a/apps/sim/background/webhook-execution.ts b/apps/sim/background/webhook-execution.ts index bb3ba290468..dc7782615cf 100644 --- a/apps/sim/background/webhook-execution.ts +++ b/apps/sim/background/webhook-execution.ts @@ -31,6 +31,7 @@ import { import { preprocessExecution } from '@/lib/execution/preprocessing' import { LoggingSession } from '@/lib/logs/execution/logging-session' import { buildTraceSpans } from '@/lib/logs/execution/trace-spans/trace-spans' +import { resolveOAuthAccountId } from '@/lib/oauth/credential-service' import { type WebhookAttachment, WebhookAttachmentProcessor, @@ -49,7 +50,6 @@ import { loadDeployedWorkflowState, loadWorkflowDeploymentVersionState, } from '@/lib/workflows/persistence/utils' -import { resolveOAuthAccountId } from '@/app/api/auth/oauth/utils' import { WEBHOOK_EXECUTION_CONCURRENCY_LIMIT } from '@/background/concurrency-limits' import { getBlock } from '@/blocks' import { ExecutionSnapshot } from '@/executor/execution/snapshot' diff --git a/apps/sim/executor/handlers/evaluator/evaluator-handler.test.ts b/apps/sim/executor/handlers/evaluator/evaluator-handler.test.ts index 1483c5d9fe0..bd04d3f07b8 100644 --- a/apps/sim/executor/handlers/evaluator/evaluator-handler.test.ts +++ b/apps/sim/executor/handlers/evaluator/evaluator-handler.test.ts @@ -8,7 +8,7 @@ const { mockResolveAutoModel } = vi.hoisted(() => ({ mockResolveAutoModel: vi.fn(), })) -vi.mock('@/app/api/auth/oauth/utils', () => authOAuthUtilsMock) +vi.mock('@/lib/oauth/credential-service', () => authOAuthUtilsMock) vi.mock('@/lib/credentials/access', () => ({ getCredentialActorContext: vi.fn().mockResolvedValue({ diff --git a/apps/sim/executor/handlers/router/router-handler.test.ts b/apps/sim/executor/handlers/router/router-handler.test.ts index db605abaa1a..1e256a9024a 100644 --- a/apps/sim/executor/handlers/router/router-handler.test.ts +++ b/apps/sim/executor/handlers/router/router-handler.test.ts @@ -8,7 +8,7 @@ const { mockResolveAutoModel } = vi.hoisted(() => ({ mockResolveAutoModel: vi.fn(), })) -vi.mock('@/app/api/auth/oauth/utils', () => authOAuthUtilsMock) +vi.mock('@/lib/oauth/credential-service', () => authOAuthUtilsMock) vi.mock('@/lib/credentials/access', () => ({ getCredentialActorContext: vi.fn().mockResolvedValue({ diff --git a/apps/sim/executor/utils/vertex-credential.test.ts b/apps/sim/executor/utils/vertex-credential.test.ts index 9776fac39d2..242da2ad413 100644 --- a/apps/sim/executor/utils/vertex-credential.test.ts +++ b/apps/sim/executor/utils/vertex-credential.test.ts @@ -13,7 +13,7 @@ const { mockGetCredentialActorContext, mockGetServiceAccountToken, mockRefreshTo vi.mock('@/lib/credentials/access', () => ({ getCredentialActorContext: mockGetCredentialActorContext, })) -vi.mock('@/app/api/auth/oauth/utils', () => ({ +vi.mock('@/lib/oauth/credential-service', () => ({ getServiceAccountToken: mockGetServiceAccountToken, refreshTokenIfNeeded: mockRefreshTokenIfNeeded, })) diff --git a/apps/sim/executor/utils/vertex-credential.ts b/apps/sim/executor/utils/vertex-credential.ts index 9da58ffe40e..33f37c33ba6 100644 --- a/apps/sim/executor/utils/vertex-credential.ts +++ b/apps/sim/executor/utils/vertex-credential.ts @@ -3,7 +3,7 @@ import { account } from '@sim/db/schema' import { createLogger } from '@sim/logger' import { eq } from 'drizzle-orm' import { getCredentialActorContext } from '@/lib/credentials/access' -import { getServiceAccountToken, refreshTokenIfNeeded } from '@/app/api/auth/oauth/utils' +import { getServiceAccountToken, refreshTokenIfNeeded } from '@/lib/oauth/credential-service' const logger = createLogger('VertexCredential') diff --git a/apps/sim/hooks/queries/kb/connectors.test.ts b/apps/sim/hooks/queries/kb/connectors.test.ts new file mode 100644 index 00000000000..14d7ead56ea --- /dev/null +++ b/apps/sim/hooks/queries/kb/connectors.test.ts @@ -0,0 +1,89 @@ +/** + * @vitest-environment node + */ + +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + requestJson: vi.fn(), + useInfiniteQuery: vi.fn(), +})) + +vi.mock('@tanstack/react-query', () => ({ + keepPreviousData: Symbol('keepPreviousData'), + useInfiniteQuery: mocks.useInfiniteQuery, + useMutation: vi.fn(), + useQuery: vi.fn(), + useQueryClient: vi.fn(() => ({ invalidateQueries: vi.fn() })), +})) + +vi.mock('@/lib/api/client/request', () => ({ + requestJson: mocks.requestJson, +})) + +import { listKnowledgeConnectorDocumentsContract } from '@/lib/api/contracts/knowledge' +import { MAX_KNOWLEDGE_CONNECTOR_DOCUMENT_PAGE_SIZE } from '@/lib/knowledge/constants' +import { useConnectorDocuments } from '@/hooks/queries/kb/connectors' + +interface ConnectorDocumentsPage { + documents: Array<{ id: string }> + counts: { active: number; excluded: number } +} + +interface ConnectorDocumentsQueryOptions { + initialPageParam: number + queryFn: (context: { signal: AbortSignal; pageParam: number }) => Promise + getNextPageParam: ( + lastPage: ConnectorDocumentsPage, + pages: ConnectorDocumentsPage[] + ) => number | undefined +} + +describe('useConnectorDocuments', () => { + beforeEach(() => { + vi.clearAllMocks() + }) + + it('requests bounded pages and advances until the authoritative total is loaded', async () => { + const firstPage = { + documents: [{ id: 'document-1' }, { id: 'document-2' }], + counts: { active: 2, excluded: 1 }, + } + const finalPage = { + documents: [{ id: 'document-3' }], + counts: firstPage.counts, + } + mocks.requestJson.mockResolvedValue({ data: firstPage }) + + useConnectorDocuments('knowledge-1', 'connector-1', { includeExcluded: true }) + + const options = mocks.useInfiniteQuery.mock.calls[0]?.[0] as ConnectorDocumentsQueryOptions + const signal = new AbortController().signal + await options.queryFn({ signal, pageParam: 200 }) + + expect(mocks.requestJson).toHaveBeenCalledWith(listKnowledgeConnectorDocumentsContract, { + params: { id: 'knowledge-1', connectorId: 'connector-1' }, + query: { + includeExcluded: true, + limit: MAX_KNOWLEDGE_CONNECTOR_DOCUMENT_PAGE_SIZE, + offset: 200, + }, + signal, + }) + expect(options.initialPageParam).toBe(0) + expect(options.getNextPageParam(firstPage, [firstPage])).toBe(2) + expect(options.getNextPageParam(finalPage, [firstPage, finalPage])).toBeUndefined() + }) + + it('does not page toward excluded documents when they were not requested', () => { + const activePage = { + documents: [{ id: 'document-1' }, { id: 'document-2' }], + counts: { active: 2, excluded: 10 }, + } + + useConnectorDocuments('knowledge-1', 'connector-1') + + const options = mocks.useInfiniteQuery.mock.calls[0]?.[0] as ConnectorDocumentsQueryOptions + expect(options.getNextPageParam(activePage, [activePage])).toBeUndefined() + }) +}) diff --git a/apps/sim/hooks/queries/kb/connectors.ts b/apps/sim/hooks/queries/kb/connectors.ts index fb31f7a90ef..fe4d6d21004 100644 --- a/apps/sim/hooks/queries/kb/connectors.ts +++ b/apps/sim/hooks/queries/kb/connectors.ts @@ -1,5 +1,11 @@ import { createLogger } from '@sim/logger' -import { keepPreviousData, useMutation, useQuery, useQueryClient } from '@tanstack/react-query' +import { + keepPreviousData, + useInfiniteQuery, + useMutation, + useQuery, + useQueryClient, +} from '@tanstack/react-query' import { requestJson } from '@/lib/api/client/request' import { type ConnectorData, @@ -15,6 +21,7 @@ import { triggerKnowledgeConnectorSyncContract, updateKnowledgeConnectorContract, } from '@/lib/api/contracts/knowledge' +import { MAX_KNOWLEDGE_CONNECTOR_DOCUMENT_PAGE_SIZE } from '@/lib/knowledge/constants' import { knowledgeKeys } from '@/hooks/queries/utils/knowledge-keys' const logger = createLogger('KnowledgeConnectorQueries') @@ -245,11 +252,16 @@ async function fetchConnectorDocuments( knowledgeBaseId: string, connectorId: string, includeExcluded: boolean, + offset: number, signal?: AbortSignal ): Promise { const result = await requestJson(listKnowledgeConnectorDocumentsContract, { params: { id: knowledgeBaseId, connectorId }, - query: { includeExcluded }, + query: { + includeExcluded, + limit: MAX_KNOWLEDGE_CONNECTOR_DOCUMENT_PAGE_SIZE, + offset, + }, signal, }) @@ -261,18 +273,24 @@ export function useConnectorDocuments( connectorId?: string, options?: { includeExcluded?: boolean } ) { - return useQuery({ - queryKey: [ - ...connectorDocumentKeys.list(knowledgeBaseId, connectorId), - options?.includeExcluded ?? false, - ], - queryFn: ({ signal }) => + const includeExcluded = options?.includeExcluded ?? false + return useInfiniteQuery({ + queryKey: [...connectorDocumentKeys.list(knowledgeBaseId, connectorId), includeExcluded], + queryFn: ({ signal, pageParam }) => fetchConnectorDocuments( knowledgeBaseId as string, connectorId as string, - options?.includeExcluded ?? false, + includeExcluded, + pageParam, signal ), + initialPageParam: 0, + getNextPageParam: (lastPage, pages) => { + const loadedCount = pages.reduce((total, page) => total + page.documents.length, 0) + const totalCount = lastPage.counts.active + (includeExcluded ? lastPage.counts.excluded : 0) + if (lastPage.documents.length === 0 || loadedCount >= totalCount) return undefined + return loadedCount + }, enabled: Boolean(knowledgeBaseId && connectorId), staleTime: CONNECTOR_DOCUMENT_LIST_STALE_TIME, placeholderData: keepPreviousData, diff --git a/apps/sim/hooks/queries/kb/knowledge.ts b/apps/sim/hooks/queries/kb/knowledge.ts index 118b81ba5e5..0dd4606b44f 100644 --- a/apps/sim/hooks/queries/kb/knowledge.ts +++ b/apps/sim/hooks/queries/kb/knowledge.ts @@ -39,6 +39,7 @@ import { saveDocumentTagDefinitionsContract, type TagDefinitionData, type TagUsageData, + type UpdateKnowledgeDocumentResponseData, updateKnowledgeBaseContract, updateKnowledgeChunkContract, updateKnowledgeDocumentContract, @@ -490,7 +491,7 @@ async function updateDocument({ knowledgeBaseId, documentId, updates, -}: UpdateDocumentParams): Promise { +}: UpdateDocumentParams): Promise { const result = await requestJson(updateKnowledgeDocumentContract, { params: { id: knowledgeBaseId, documentId }, body: updates, @@ -1000,6 +1001,7 @@ async function deleteDocumentTagDefinitions({ }: DeleteDocumentTagDefinitionsParams): Promise { await requestJson(deleteDocumentTagDefinitionsContract, { params: { id: knowledgeBaseId, documentId }, + query: {}, }) } diff --git a/apps/sim/lib/api/contracts/knowledge/connectors.ts b/apps/sim/lib/api/contracts/knowledge/connectors.ts index cef3d8718d1..88b90907d52 100644 --- a/apps/sim/lib/api/contracts/knowledge/connectors.ts +++ b/apps/sim/lib/api/contracts/knowledge/connectors.ts @@ -6,6 +6,11 @@ import { } from '@/lib/api/contracts/knowledge/shared' import { booleanQueryFlagSchema } from '@/lib/api/contracts/primitives' import { defineRouteContract } from '@/lib/api/contracts/types' +import { + DEFAULT_KNOWLEDGE_CONNECTOR_DOCUMENT_PAGE_SIZE, + MAX_KNOWLEDGE_CONNECTOR_DOCUMENT_MUTATION_ITEMS, + MAX_KNOWLEDGE_CONNECTOR_DOCUMENT_PAGE_SIZE, +} from '@/lib/knowledge/constants' export const createConnectorBodySchema = z.object({ connectorType: z.string().min(1), @@ -27,12 +32,23 @@ export const deleteConnectorQuerySchema = z.object({ }) export const connectorDocumentsQuerySchema = z.object({ - includeExcluded: z.boolean().optional(), + includeExcluded: booleanQueryFlagSchema.optional(), + limit: z.coerce + .number() + .int() + .min(1) + .max(MAX_KNOWLEDGE_CONNECTOR_DOCUMENT_PAGE_SIZE) + .optional() + .default(DEFAULT_KNOWLEDGE_CONNECTOR_DOCUMENT_PAGE_SIZE), + offset: z.coerce.number().int().min(0).optional().default(0), }) export const connectorDocumentsPatchBodySchema = z.object({ operation: z.enum(['restore', 'exclude']), - documentIds: z.array(z.string()).min(1), + documentIds: z + .array(z.string().min(1)) + .min(1) + .max(MAX_KNOWLEDGE_CONNECTOR_DOCUMENT_MUTATION_ITEMS), }) export const connectorDataSchema = z @@ -117,6 +133,7 @@ export const createKnowledgeConnectorContract = defineRouteContract({ response: { mode: 'json', schema: successResponseSchema(connectorDataSchema), + status: 201, }, }) diff --git a/apps/sim/lib/api/contracts/knowledge/documents.ts b/apps/sim/lib/api/contracts/knowledge/documents.ts index e3f54a9d111..e2b9fd37691 100644 --- a/apps/sim/lib/api/contracts/knowledge/documents.ts +++ b/apps/sim/lib/api/contracts/knowledge/documents.ts @@ -15,7 +15,7 @@ import { import { privateSecretProvenanceBundleSchema } from '@/lib/api/contracts/primitives' import { defineRouteContract } from '@/lib/api/contracts/types' import { PRIVATE_SECRET_PROVENANCE_FIELD } from '@/lib/execution/private-tool-metadata' -import { getFieldTypeForSlot } from '@/lib/knowledge/constants' +import { getFieldTypeForSlot, MAX_KNOWLEDGE_DOCUMENTS_PER_CREATE } from '@/lib/knowledge/constants' import { getOperatorsForFieldType, isValidFilterValue } from '@/lib/knowledge/filters/types' export const documentTagFilterSchema = z @@ -129,7 +129,13 @@ export const createDocumentBodySchema = z.object({ }) export const bulkCreateDocumentsBodySchema = z.object({ - documents: z.array(createDocumentBodySchema), + documents: z + .array(createDocumentBodySchema) + .min(1, 'At least one document is required') + .max( + MAX_KNOWLEDGE_DOCUMENTS_PER_CREATE, + `At most ${MAX_KNOWLEDGE_DOCUMENTS_PER_CREATE} documents may be created at once` + ), processingOptions: z .object({ recipe: z.string().optional(), @@ -338,9 +344,17 @@ export const updateKnowledgeDocumentContract = defineRouteContract({ body: updateDocumentBodySchema, response: { mode: 'json', - schema: successResponseSchema(documentDataSchema), + schema: successResponseSchema( + z.union([ + documentDataSchema, + z.object({ documentId: z.string(), status: z.string(), message: z.string() }), + ]) + ), }, }) +export type UpdateKnowledgeDocumentResponseData = z.output< + typeof updateKnowledgeDocumentContract.response.schema +>['data'] export const updateKnowledgeDocumentTagsContract = defineRouteContract({ method: 'PUT', @@ -381,6 +395,23 @@ export const upsertKnowledgeDocumentContract = defineRouteContract({ body: upsertDocumentBodySchema, response: { mode: 'json', - schema: successResponseSchema(documentDataSchema), + schema: successResponseSchema( + z.object({ + documentsCreated: z.array( + z.object({ + documentId: z.string(), + filename: z.string(), + status: z.literal('pending'), + }) + ), + isUpdate: z.boolean(), + previousDocumentId: z.string().nullable(), + processingMethod: z.literal('background'), + processingConfig: z.object({ + maxConcurrentDocuments: z.number(), + batchSize: z.number(), + }), + }) + ), }, }) diff --git a/apps/sim/lib/api/contracts/knowledge/search.ts b/apps/sim/lib/api/contracts/knowledge/search.ts index 3550311f21d..cd5653285e7 100644 --- a/apps/sim/lib/api/contracts/knowledge/search.ts +++ b/apps/sim/lib/api/contracts/knowledge/search.ts @@ -1,4 +1,5 @@ import { z } from 'zod' +import { defineRouteContract } from '@/lib/api/contracts/types' import { DEFAULT_RERANKER_MODEL, rerankerModelSchema } from '@/lib/knowledge/reranker-models' export const knowledgeSearchTagFilterSchema = z.object({ @@ -85,3 +86,63 @@ export const knowledgeSearchBodySchema = z } ) export type KnowledgeSearchBody = z.output + +export const internalKnowledgeSearchBodySchema = z.intersection( + knowledgeSearchBodySchema, + z.object({ + workflowId: z.string().optional(), + skipUsageBilling: z.boolean().optional(), + }) +) + +export const internalKnowledgeSearchResultSchema = z.object({ + documentId: z.string(), + documentName: z.string().nullable(), + sourceUrl: z.string().nullable(), + content: z.string(), + chunkIndex: z.number(), + metadata: z.record(z.string(), z.unknown()), + similarity: z.number(), + rerankerScore: z.number().optional(), +}) + +export const internalKnowledgeSearchContract = defineRouteContract({ + method: 'POST', + path: '/api/knowledge/search', + body: internalKnowledgeSearchBodySchema, + response: { + mode: 'json', + schema: z.object({ + success: z.literal(true), + data: z.object({ + results: z.array(internalKnowledgeSearchResultSchema), + query: z.string(), + knowledgeBaseIds: z.array(z.string()), + knowledgeBaseId: z.string(), + topK: z.number(), + totalResults: z.number(), + cost: z + .object({ + input: z.number(), + output: z.number(), + total: z.number(), + tokens: z.object({ + prompt: z.number(), + completion: z.number(), + total: z.number(), + }), + model: z.string(), + pricing: z.object({ + input: z.number(), + output: z.number(), + updatedAt: z.string().optional(), + }), + rerankerCost: z.number().optional(), + rerankerModel: z.string().optional(), + rerankerSearchUnits: z.number().optional(), + }) + .optional(), + }), + }), + }, +}) diff --git a/apps/sim/lib/api/contracts/knowledge/tags.ts b/apps/sim/lib/api/contracts/knowledge/tags.ts index c44cc9424d3..730202ae997 100644 --- a/apps/sim/lib/api/contracts/knowledge/tags.ts +++ b/apps/sim/lib/api/contracts/knowledge/tags.ts @@ -144,6 +144,7 @@ export const deleteDocumentTagDefinitionsContract = defineRouteContract({ method: 'DELETE', path: '/api/knowledge/[id]/documents/[documentId]/tag-definitions', params: knowledgeDocumentParamsSchema, + query: deleteDocumentTagDefinitionsQuerySchema, response: { mode: 'json', schema: z.object({ success: z.literal(true) }).passthrough(), diff --git a/apps/sim/lib/api/contracts/knowledge/upload-sessions.ts b/apps/sim/lib/api/contracts/knowledge/upload-sessions.ts index b37ad76428b..f90cf22422d 100644 --- a/apps/sim/lib/api/contracts/knowledge/upload-sessions.ts +++ b/apps/sim/lib/api/contracts/knowledge/upload-sessions.ts @@ -1,26 +1,134 @@ -import { defineRouteContract } from '@/lib/api/contracts/types' -import { - v2CreateKnowledgeDocumentUploadBodySchema, - v2CreateKnowledgeDocumentUploadDataSchema, - v2KnowledgeDocumentUploadParamsSchema, - v2KnowledgeDocumentUploadSchema, - v2UploadKnowledgeDocumentQuerySchema, -} from '@/lib/api/contracts/v2/knowledge' -import { v2DataResponse } from '@/lib/api/contracts/v2/shared' +import { z } from 'zod' +import { documentDataSchema } from '@/lib/api/contracts/knowledge/documents' import { - v2PartUrlsBodySchema, - v2PartUrlsDataSchema, - v2UploadTokenHeadersSchema, -} from '@/lib/api/contracts/v2/uploads' + knowledgeBaseParamsSchema, + nullableWireDateSchema, +} from '@/lib/api/contracts/knowledge/shared' +import { workspaceIdSchema } from '@/lib/api/contracts/primitives' +import { defineRouteContract } from '@/lib/api/contracts/types' +import { knowledgeDocumentUploadMetadataSchema } from '@/lib/knowledge/upload-metadata' +import { MAX_KNOWLEDGE_DOCUMENT_FILE_SIZE } from '@/lib/uploads/shared/types' + +const knowledgeDocumentUploadStatusSchema = z.enum([ + 'uploading', + 'completing', + 'finalizing', + 'completed', + 'failed', + 'aborting', + 'aborted', + 'expired', +]) + +const knowledgeDocumentUploadTokenHeadersSchema = z.object({ + 'upload-token': z.string().min(1, 'upload-token header is required'), +}) + +const knowledgeDocumentUploadTransferSchema = z.discriminatedUnion('method', [ + z + .object({ + method: z.literal('put'), + url: z.string().url(), + headers: z.record(z.string(), z.string()), + }) + .strict(), + z + .object({ + method: z.literal('multipart'), + partSize: z.number().int().positive(), + partCount: z.number().int().positive().max(640), + }) + .strict(), +]) +export type KnowledgeDocumentUploadTransfer = z.output + +const knowledgeDocumentUploadPartUrlsBodySchema = z + .object({ + partNumbers: z.array(z.number().int().min(1)).min(1).max(100), + }) + .strict() + +const knowledgeDocumentUploadPartUrlSchema = z.object({ + partNumber: z.number().int().min(1), + url: z.string().url(), + headers: z.record(z.string(), z.string()), + expiresAt: z.string().datetime(), +}) +export type KnowledgeDocumentUploadPartUrl = z.output + +const knowledgeDocumentUploadPartUrlsDataSchema = z.object({ + parts: z.array(knowledgeDocumentUploadPartUrlSchema).max(100), +}) + +const knowledgeDocumentSummarySchema = documentDataSchema + .pick({ + id: true, + knowledgeBaseId: true, + filename: true, + fileSize: true, + mimeType: true, + processingStatus: true, + chunkCount: true, + tokenCount: true, + characterCount: true, + enabled: true, + }) + .extend({ createdAt: nullableWireDateSchema }) +export type KnowledgeDocumentUploadSummary = z.output + +const knowledgeDocumentUploadParamsSchema = knowledgeBaseParamsSchema.extend({ + uploadId: z.string().min(1, 'uploadId is required'), +}) + +const uploadKnowledgeDocumentQuerySchema = z.object({ workspaceId: workspaceIdSchema }) + +export const knowledgeDocumentUploadMetadataBodySchema = z + .object({ ...knowledgeDocumentUploadMetadataSchema.shape }) + .strict() +export type KnowledgeDocumentUploadMetadataBody = z.input< + typeof knowledgeDocumentUploadMetadataBodySchema +> + +const createKnowledgeDocumentUploadBodySchema = z + .object({ + workspaceId: workspaceIdSchema, + name: z.string().trim().min(1, 'name is required').max(255, 'name is too long'), + contentType: z.string().trim().min(1, 'contentType is required').max(255), + size: z.number().int().min(1).max(MAX_KNOWLEDGE_DOCUMENT_FILE_SIZE), + ...knowledgeDocumentUploadMetadataBodySchema.shape, + }) + .strict() + +const knowledgeDocumentUploadSchema = z.object({ + id: z.string(), + knowledgeBaseId: z.string(), + status: knowledgeDocumentUploadStatusSchema, + name: z.string(), + contentType: z.string(), + size: z.number().int().positive(), + expiresAt: z.string().datetime(), + error: z.string().nullable(), + document: knowledgeDocumentSummarySchema.nullable(), +}) + +const createKnowledgeDocumentUploadDataSchema = z + .object({ + session: knowledgeDocumentUploadSchema, + uploadToken: z.string().min(1), + transfer: knowledgeDocumentUploadTransferSchema, + }) + .strict() + +const dataResponse = (schema: T) => z.object({ data: schema }) export const createKnowledgeDocumentUploadContract = defineRouteContract({ method: 'POST', path: '/api/knowledge/[id]/documents/uploads', - params: v2KnowledgeDocumentUploadParamsSchema.omit({ uploadId: true }), - body: v2CreateKnowledgeDocumentUploadBodySchema, + params: knowledgeDocumentUploadParamsSchema.omit({ uploadId: true }), + body: createKnowledgeDocumentUploadBodySchema, response: { mode: 'json', - schema: v2DataResponse(v2CreateKnowledgeDocumentUploadDataSchema), + schema: dataResponse(createKnowledgeDocumentUploadDataSchema), status: 201, }, }) @@ -28,27 +136,27 @@ export const createKnowledgeDocumentUploadContract = defineRouteContract({ export const abortKnowledgeDocumentUploadContract = defineRouteContract({ method: 'DELETE', path: '/api/knowledge/[id]/documents/uploads/[uploadId]', - params: v2KnowledgeDocumentUploadParamsSchema, - query: v2UploadKnowledgeDocumentQuerySchema, - headers: v2UploadTokenHeadersSchema, - response: { mode: 'json', schema: v2DataResponse(v2KnowledgeDocumentUploadSchema) }, + params: knowledgeDocumentUploadParamsSchema, + query: uploadKnowledgeDocumentQuerySchema, + headers: knowledgeDocumentUploadTokenHeadersSchema, + response: { mode: 'json', schema: dataResponse(knowledgeDocumentUploadSchema) }, }) export const createKnowledgeDocumentUploadPartUrlsContract = defineRouteContract({ method: 'POST', path: '/api/knowledge/[id]/documents/uploads/[uploadId]/parts', - params: v2KnowledgeDocumentUploadParamsSchema, - query: v2UploadKnowledgeDocumentQuerySchema, - headers: v2UploadTokenHeadersSchema, - body: v2PartUrlsBodySchema, - response: { mode: 'json', schema: v2DataResponse(v2PartUrlsDataSchema) }, + params: knowledgeDocumentUploadParamsSchema, + query: uploadKnowledgeDocumentQuerySchema, + headers: knowledgeDocumentUploadTokenHeadersSchema, + body: knowledgeDocumentUploadPartUrlsBodySchema, + response: { mode: 'json', schema: dataResponse(knowledgeDocumentUploadPartUrlsDataSchema) }, }) export const completeKnowledgeDocumentUploadContract = defineRouteContract({ method: 'POST', path: '/api/knowledge/[id]/documents/uploads/[uploadId]/complete', - params: v2KnowledgeDocumentUploadParamsSchema, - query: v2UploadKnowledgeDocumentQuerySchema, - headers: v2UploadTokenHeadersSchema, - response: { mode: 'json', schema: v2DataResponse(v2KnowledgeDocumentUploadSchema) }, + params: knowledgeDocumentUploadParamsSchema, + query: uploadKnowledgeDocumentQuerySchema, + headers: knowledgeDocumentUploadTokenHeadersSchema, + response: { mode: 'json', schema: dataResponse(knowledgeDocumentUploadSchema) }, }) diff --git a/apps/sim/lib/api/contracts/v2/knowledge.ts b/apps/sim/lib/api/contracts/v2/knowledge.ts index 4ee31c3b933..d4901710217 100644 --- a/apps/sim/lib/api/contracts/v2/knowledge.ts +++ b/apps/sim/lib/api/contracts/v2/knowledge.ts @@ -34,6 +34,7 @@ import { v2UploadTokenHeadersSchema, v2UploadTransferSchema, } from '@/lib/api/contracts/v2/uploads' +import { knowledgeDocumentUploadMetadataSchema } from '@/lib/knowledge/upload-metadata' import { MAX_KNOWLEDGE_DOCUMENT_FILE_SIZE } from '@/lib/uploads/shared/types' /** @@ -171,28 +172,8 @@ export const v2KnowledgeDocumentUploadParamsSchema = knowledgeBaseParamsSchema.e }) export type V2KnowledgeDocumentUploadParams = z.output -const knowledgeDocumentUploadTagSchema = z - .string() - .max(1000, 'Knowledge document tag values cannot exceed 1000 characters') - .optional() - export const v2KnowledgeDocumentUploadMetadataSchema = z - .object({ - tag1: knowledgeDocumentUploadTagSchema, - tag2: knowledgeDocumentUploadTagSchema, - tag3: knowledgeDocumentUploadTagSchema, - tag4: knowledgeDocumentUploadTagSchema, - tag5: knowledgeDocumentUploadTagSchema, - tag6: knowledgeDocumentUploadTagSchema, - tag7: knowledgeDocumentUploadTagSchema, - processingOptions: z - .object({ - recipe: z.string().max(255, 'recipe cannot exceed 255 characters').optional(), - lang: z.string().max(35, 'lang cannot exceed 35 characters').optional(), - }) - .strict() - .optional(), - }) + .object({ ...knowledgeDocumentUploadMetadataSchema.shape }) .strict() export type V2KnowledgeDocumentUploadMetadata = z.output< typeof v2KnowledgeDocumentUploadMetadataSchema diff --git a/apps/sim/lib/api/server/routes/index.ts b/apps/sim/lib/api/server/routes/index.ts index 22c753ca9a2..f75fe366a79 100644 --- a/apps/sim/lib/api/server/routes/index.ts +++ b/apps/sim/lib/api/server/routes/index.ts @@ -14,6 +14,7 @@ export { internalSessionAuth, } from '@/lib/api/server/routes/internal-json-route' export { defineV2BinaryRoute } from '@/lib/api/server/routes/v2-binary-route' +export { defineV2BodyLifecycleRoute } from '@/lib/api/server/routes/v2-body-lifecycle-route' export { admitV2Request, defineV2JsonRoute, diff --git a/apps/sim/lib/api/server/routes/internal-json-route.test.ts b/apps/sim/lib/api/server/routes/internal-json-route.test.ts index c295caec6d4..9449c25acd0 100644 --- a/apps/sim/lib/api/server/routes/internal-json-route.test.ts +++ b/apps/sim/lib/api/server/routes/internal-json-route.test.ts @@ -2,7 +2,7 @@ * @vitest-environment node */ import { NextRequest } from 'next/server' -import { describe, expect, it, vi } from 'vitest' +import { beforeEach, describe, expect, it, vi } from 'vitest' import { z } from 'zod' import { defineRouteContract } from '@/lib/api/contracts' import { @@ -32,6 +32,10 @@ const contract = defineRouteContract({ }) describe('defineInternalJsonRoute', () => { + beforeEach(() => { + vi.clearAllMocks() + }) + it('uses the use-case result directly when it already matches the contract', async () => { const handler = defineInternalJsonRoute({ contract, @@ -82,4 +86,173 @@ describe('defineInternalJsonRoute', () => { 'Internal error responses require a 4xx or 5xx status' ) }) + + it('orders auth, rate limiting, parsing, async mapping, and application execution', async () => { + const events: string[] = [] + const orderedContract = defineRouteContract({ + method: 'POST', + path: '/api/test/internal-json-route', + body: z.object({ value: z.string() }).transform((body) => { + events.push('parse') + return body + }), + response: { mode: 'json', schema: z.object({ value: z.string() }) }, + }) + const handler = defineInternalJsonRoute({ + contract: orderedContract, + auth: { + async authenticate() { + events.push('auth') + return { kind: 'session' as const, userId: 'user-1', sessionId: 'session-1' } + }, + }, + operation, + rateLimit: { + kind: 'none', + reason: 'Unit test', + async enforce() { + events.push('rate') + }, + }, + errorPolicy: internalPlainOrchestrationErrorPolicy, + async mapInput({ body }) { + events.push('map:start') + await Promise.resolve() + events.push('map:end') + return body.value + }, + useCase: { + operation, + async execute({ input }) { + events.push('use-case') + return { value: input } + }, + }, + }) + + const response = await handler( + new NextRequest('http://localhost/api/test/internal-json-route', { + method: 'POST', + body: JSON.stringify({ value: 'ok' }), + }) + ) + + expect(response.status).toBe(200) + expect(events).toEqual(['auth', 'rate', 'parse', 'map:start', 'map:end', 'use-case']) + }) + + it('projects async mapping errors before application execution', async () => { + const execute = vi.fn() + const handler = defineInternalJsonRoute({ + contract, + auth, + operation, + rateLimit: internalRateLimits.none({ reason: 'Unit test' }), + errorPolicy: internalPlainOrchestrationErrorPolicy, + async mapInput() { + await Promise.resolve() + throw new OrchestrationError('validation', 'Invalid mapped input') + }, + useCase: { operation, execute }, + }) + + const response = await handler(new NextRequest('http://localhost/api/test/internal-json-route')) + + expect(response.status).toBe(400) + await expect(response.json()).resolves.toEqual({ error: 'Invalid mapped input' }) + expect(execute).not.toHaveBeenCalled() + }) + + it('validates the contract response before invoking the finalizer', async () => { + const finalizeResponse = vi.fn() + const handler = defineInternalJsonRoute({ + contract, + auth, + operation, + rateLimit: internalRateLimits.none({ reason: 'Unit test' }), + errorPolicy: internalPlainOrchestrationErrorPolicy, + mapInput: () => undefined, + useCase: { + operation, + async execute() { + return { value: 42 } + }, + }, + present: ({ value }) => ({ value: value as unknown as string }), + finalizeResponse, + }) + + const response = await handler(new NextRequest('http://localhost/api/test/internal-json-route')) + + expect(response.status).toBe(500) + expect(finalizeResponse).not.toHaveBeenCalled() + }) + + it('projects finalizer failures through the declared error policy', async () => { + const handler = defineInternalJsonRoute({ + contract, + auth, + operation, + rateLimit: internalRateLimits.none({ reason: 'Unit test' }), + errorPolicy: internalPlainOrchestrationErrorPolicy, + mapInput: () => undefined, + useCase: { + operation, + async execute() { + return { value: 'ok' } + }, + }, + finalizeResponse() { + throw new OrchestrationError('conflict', 'Metadata conflict') + }, + }) + + const response = await handler(new NextRequest('http://localhost/api/test/internal-json-route')) + + expect(response.status).toBe(409) + await expect(response.json()).resolves.toEqual({ error: 'Metadata conflict' }) + }) + + it('appends finalizer metadata while preserving the success status and declared headers', async () => { + const createdContract = defineRouteContract({ + method: 'POST', + path: '/api/test/internal-json-route', + response: { + mode: 'json', + schema: z.object({ value: z.string() }), + status: 201, + }, + }) + const handler = defineInternalJsonRoute({ + contract: createdContract, + auth, + operation, + rateLimit: internalRateLimits.none({ reason: 'Unit test' }), + errorPolicy: internalPlainOrchestrationErrorPolicy, + mapInput: () => undefined, + useCase: { + operation, + async execute() { + return { value: 'ok' } + }, + }, + responseHeaders: () => ({ 'x-contract-header': 'preserved' }), + finalizeResponse: ({ body }) => ({ + bodyFields: { __privateMetadata: { value: body.value } }, + headers: { 'x-private-metadata': 'v1' }, + }), + }) + + const response = await handler( + new NextRequest('http://localhost/api/test/internal-json-route', { method: 'POST' }) + ) + + expect(response.status).toBe(201) + expect(response.headers.get('x-contract-header')).toBe('preserved') + expect(response.headers.get('x-private-metadata')).toBe('v1') + await expect(response.json()).resolves.toEqual({ + value: 'ok', + __privateMetadata: { value: 'ok' }, + }) + }) }) diff --git a/apps/sim/lib/api/server/routes/internal-json-route.ts b/apps/sim/lib/api/server/routes/internal-json-route.ts index 6563d5e7cdb..0330013ce95 100644 --- a/apps/sim/lib/api/server/routes/internal-json-route.ts +++ b/apps/sim/lib/api/server/routes/internal-json-route.ts @@ -179,6 +179,16 @@ export interface InternalAuthPolicy

{ ): Promise

} +export interface InternalJsonResponseFinalization { + bodyFields?: Readonly> + headers?: HeadersInit +} + +type InternalJsonParseOptions = Pick< + ParseRequestOptions, + 'maxBodyBytes' | 'validationErrorResponse' +> + type InternalJsonPresenter = [R] extends [ ContractJsonResponse, ] @@ -198,12 +208,12 @@ type InternalJsonRouteOptions< > = { contract: C operation: O - mapInput(input: ParsedRequest): I + mapInput(input: ParsedRequest, context: { principal: P; request: NextRequest }): I | Promise useCase: OperationUseCase, I, R> auth: InternalAuthPolicy

rateLimit: InternalRateLimitPolicy errorPolicy: InternalErrorPolicy - parseOptions?: Omit + parseOptions?: InternalJsonParseOptions beforeParse?(args: { request: NextRequest principal: P @@ -211,6 +221,13 @@ type InternalJsonRouteOptions< }): void | Promise onSuccess?(args: { principal: P; input: NoInfer; result: NoInfer }): void | Promise responseHeaders?(args: { principal: P; input: NoInfer; result: NoInfer }): HeadersInit + finalizeResponse?(args: { + request: NextRequest + principal: P + input: NoInfer + result: NoInfer + body: ContractJsonResponse + }): InternalJsonResponseFinalization | Promise } & InternalJsonPresenter function createJsonErrorResponse(descriptor: JsonErrorResponseDescriptor): NextResponse { @@ -220,6 +237,34 @@ function createJsonErrorResponse(descriptor: JsonErrorResponseDescriptor): NextR }) } +function appendFinalizedBodyFields( + body: unknown, + bodyFields?: Readonly> +): unknown { + if (!bodyFields || Object.keys(bodyFields).length === 0) return body + if (!body || typeof body !== 'object' || Array.isArray(body)) { + throw new Error('Internal JSON response metadata requires an object response body') + } + for (const key of Object.keys(bodyFields)) { + if (Object.hasOwn(body, key)) { + throw new Error(`Internal JSON response metadata cannot replace contract field "${key}"`) + } + } + return { ...body, ...bodyFields } +} + +function appendFinalizedHeaders(base: HeadersInit | undefined, additions?: HeadersInit): Headers { + const headers = new Headers(base) + if (!additions) return headers + new Headers(additions).forEach((value, key) => { + if (headers.has(key)) { + throw new Error(`Internal JSON response finalizer cannot replace header "${key}"`) + } + headers.set(key, value) + }) + return headers +} + export function defineInternalJsonRoute< C extends JsonApiRouteContract, O extends ApplicationOperation, @@ -271,7 +316,7 @@ export function defineInternalJsonRoute< if (!parsed.success) return parsed.response try { - const input = options.mapInput(parsed.data) + const input = await options.mapInput(parsed.data, { principal, request }) const result = await options.useCase.execute({ principal, input, @@ -283,11 +328,24 @@ export function defineInternalJsonRoute< if (responseSchema.mode !== 'json') { throw new Error('Internal JSON route response mode changed after initialization') } - const validatedBody = responseSchema.schema.parse(body) - return NextResponse.json(validatedBody, { - status: successStatus, - headers: options.responseHeaders?.({ principal, input, result }), - }) + const validatedBody = responseSchema.schema.parse(body) as ContractJsonResponse + const headers = options.responseHeaders?.({ principal, input, result }) + const finalization = options.finalizeResponse + ? await options.finalizeResponse({ + request, + principal, + input, + result, + body: validatedBody, + }) + : undefined + return NextResponse.json( + appendFinalizedBodyFields(validatedBody, finalization?.bodyFields), + { + status: successStatus, + headers: appendFinalizedHeaders(headers, finalization?.headers), + } + ) } catch (error) { const response = options.errorPolicy.project(error) if (response) return createJsonErrorResponse(response) diff --git a/apps/sim/lib/api/server/routes/v2-body-lifecycle-route.test.ts b/apps/sim/lib/api/server/routes/v2-body-lifecycle-route.test.ts new file mode 100644 index 00000000000..f0d940e8abb --- /dev/null +++ b/apps/sim/lib/api/server/routes/v2-body-lifecycle-route.test.ts @@ -0,0 +1,298 @@ +/** + * @vitest-environment node + */ + +import { + MockV2ApiKeyUnauthenticatedError, + V2_OPERATION_RATE_LIMIT_ALLOWED, + V2_PREAUTH_RATE_LIMIT_ALLOWED, + v2ApiKeyAuthModuleMock, + v2GateModuleMock, + v2RateLimiterModuleMock, + v2RouteMocks, +} from '@sim/testing' +import { NextRequest } from 'next/server' +import { beforeEach, describe, expect, it, vi } from 'vitest' +import { z } from 'zod' + +const mocks = vi.hoisted(() => ({ + order: [] as string[], +})) + +vi.mock('@/lib/api/server/routes/v2-api-key-auth', () => v2ApiKeyAuthModuleMock) +vi.mock('@/lib/core/rate-limiter', () => v2RateLimiterModuleMock) +vi.mock('@/app/api/v2/lib/gate', () => v2GateModuleMock) + +import { defineRouteContract } from '@/lib/api/contracts' +import { defineV2BodyLifecycleRoute } from '@/lib/api/server/routes/v2-body-lifecycle-route' +import { v2ApiKeyAuth, v2RateLimits } from '@/lib/api/server/routes/v2-json-route' +import { v2Error } from '@/app/api/v2/lib/response' + +const operation = { id: 'test.body_lifecycle' } as const +const contract = defineRouteContract({ + method: 'POST', + path: '/api/v2/body-lifecycle/[id]', + params: z.object({ id: z.string().min(1) }), + query: z.object({ workspaceId: z.string().min(1) }), + response: { + mode: 'json', + schema: z.object({ data: z.object({ id: z.string() }) }), + status: 201, + }, +}) + +class StageRejection extends Error {} + +type RejectableStage = 'admission' | 'body' | 'transfer' | 'application' | 'presenter' | 'effects' + +let rejectedStage: RejectableStage | null = null + +function rejectAt(stage: RejectableStage): void { + mocks.order.push(stage) + if (rejectedStage === stage) throw new StageRejection(`${stage} rejected`) +} + +function buildHandler() { + return defineV2BodyLifecycleRoute({ + contract, + auth: v2ApiKeyAuth, + operation, + rateLimit: v2RateLimits.publicApi, + errorPolicy: { + render(error) { + return error instanceof StageRejection ? v2Error('CONFLICT', error.message) : null + }, + }, + admission: { + mapInput: ({ params, query }) => { + mocks.order.push('contract') + return { id: params.id, workspaceId: query.workspaceId } + }, + useCase: { + operation, + async execute({ input }) { + rejectAt('admission') + return { canonicalWorkspaceId: input.workspaceId } + }, + }, + }, + async readBody() { + rejectAt('body') + return { bytes: Buffer.from('body') } + }, + async transfer({ admission }) { + rejectAt('transfer') + return { url: `stored://${admission.canonicalWorkspaceId}` } + }, + mapInput: ({ parsed, transfer }) => ({ id: parsed.params.id, url: transfer.url }), + useCase: { + operation, + async execute({ input }) { + rejectAt('application') + return input + }, + }, + present(result) { + rejectAt('presenter') + return { data: { id: result.id } } + }, + onSuccess() { + rejectAt('effects') + }, + }) +} + +function buildRequest() { + return new NextRequest('http://localhost/api/v2/body-lifecycle/item-1?workspaceId=workspace-1', { + method: 'POST', + headers: { 'x-api-key': 'secret' }, + body: 'unread-body', + }) +} + +function context(id = 'item-1') { + return { params: Promise.resolve({ id }) } +} + +describe('defineV2BodyLifecycleRoute', () => { + beforeEach(() => { + vi.clearAllMocks() + mocks.order.splice(0) + rejectedStage = null + v2RouteMocks.preauthRate.mockImplementation(async () => { + mocks.order.push('ip-limit') + return V2_PREAUTH_RATE_LIMIT_ALLOWED + }) + v2RouteMocks.operationRate.mockImplementation(async () => { + mocks.order.push('operation-limit') + return V2_OPERATION_RATE_LIMIT_ALLOWED + }) + v2RouteMocks.gate.mockImplementation(async () => { + mocks.order.push('rollout') + return null + }) + v2RouteMocks.authenticate.mockImplementation(async () => { + mocks.order.push('authenticate') + return { + principal: { kind: 'personal_api_key', userId: 'user-1', keyId: 'key-1' }, + rolloutUserId: 'user-1', + rateLimitSubjectIds: ['api-key:key-1', 'user:user-1'], + rateLimitSubscription: null, + keyType: 'personal', + } + }) + }) + + it('fails fast when a contract body would be read before staged admission', () => { + const bodyContract = defineRouteContract({ + method: 'POST', + path: '/api/v2/body-lifecycle', + body: z.object({ value: z.string() }), + response: { mode: 'json', schema: z.object({ data: z.object({ id: z.string() }) }) }, + }) + const useCase = { operation, execute: async () => ({ id: 'item-1' }) } + + expect(() => + defineV2BodyLifecycleRoute({ + contract: bodyContract, + auth: v2ApiKeyAuth, + operation, + rateLimit: v2RateLimits.publicApi, + errorPolicy: { render: () => null }, + admission: { mapInput: () => ({}), useCase }, + readBody: async () => Buffer.alloc(0), + transfer: async () => ({ url: 'stored://item-1' }), + mapInput: () => ({}), + useCase, + present: () => ({ data: { id: 'item-1' } }), + }) + ).toThrow('must omit its body schema so admission precedes body reads') + }) + + it('runs admission, bounded body work, registration, presentation, and effects in order', async () => { + const response = await buildHandler()(buildRequest(), context()) + + expect(response.status).toBe(201) + expect(await response.json()).toEqual({ data: { id: 'item-1' } }) + expect(mocks.order).toEqual([ + 'ip-limit', + 'authenticate', + 'rollout', + 'operation-limit', + 'operation-limit', + 'contract', + 'admission', + 'body', + 'transfer', + 'application', + 'presenter', + 'effects', + ]) + expect(response.headers.get('cache-control')).toBe('private, no-store') + expect(response.headers.get('x-ratelimit-limit')).toBe('100') + expect(response.headers.get('x-request-id')).toBeTruthy() + }) + + it('rejects at the IP abuse limit before authentication', async () => { + v2RouteMocks.preauthRate.mockImplementation(async () => { + mocks.order.push('ip-limit') + return { ...V2_PREAUTH_RATE_LIMIT_ALLOWED, allowed: false, remaining: 0 } + }) + + const response = await buildHandler()(buildRequest(), context()) + + expect(response.status).toBe(429) + expect(mocks.order).toEqual(['ip-limit']) + }) + + it('rejects unauthenticated requests before rollout and operation limiting', async () => { + v2RouteMocks.authenticate.mockImplementation(async () => { + mocks.order.push('authenticate') + throw new MockV2ApiKeyUnauthenticatedError('Authentication required') + }) + + const response = await buildHandler()(buildRequest(), context()) + + expect(response.status).toBe(401) + expect(mocks.order).toEqual(['ip-limit', 'authenticate']) + }) + + it('rejects rollout-gated requests before operation limiting', async () => { + v2RouteMocks.gate.mockImplementation(async () => { + mocks.order.push('rollout') + return v2Error('FORBIDDEN', 'V2 API access is not enabled') + }) + + const response = await buildHandler()(buildRequest(), context()) + + expect(response.status).toBe(403) + expect(mocks.order).toEqual(['ip-limit', 'authenticate', 'rollout']) + }) + + it('rejects operation-limited requests before contract or application admission', async () => { + v2RouteMocks.operationRate.mockImplementation(async () => { + mocks.order.push('operation-limit') + return { ...V2_OPERATION_RATE_LIMIT_ALLOWED, allowed: false, remaining: 0 } + }) + + const response = await buildHandler()(buildRequest(), context()) + + expect(response.status).toBe(429) + expect(mocks.order).toEqual([ + 'ip-limit', + 'authenticate', + 'rollout', + 'operation-limit', + 'operation-limit', + ]) + }) + + it('rejects invalid contract input before application admission or body reads', async () => { + const response = await buildHandler()(buildRequest(), context('')) + + expect(response.status).toBe(400) + expect(mocks.order).toEqual([ + 'ip-limit', + 'authenticate', + 'rollout', + 'operation-limit', + 'operation-limit', + ]) + }) + + it.each([ + 'admission', + 'body', + 'transfer', + 'application', + 'presenter', + 'effects', + ])('renders typed %s rejection without entering later phases', async (stage) => { + rejectedStage = stage + + const response = await buildHandler()(buildRequest(), context()) + + expect(response.status).toBe(409) + expect(await response.json()).toEqual({ + error: { code: 'CONFLICT', message: `${stage} rejected` }, + }) + expect(mocks.order.at(-1)).toBe(stage) + }) + + it.each(['authentication', 'rollout', 'rate_limit'] as const)( + 'maps %s infrastructure failures to service unavailable', + async (stage) => { + const failure = new Error(`${stage} unavailable`) + if (stage === 'authentication') v2RouteMocks.authenticate.mockRejectedValue(failure) + if (stage === 'rollout') v2RouteMocks.gate.mockRejectedValue(failure) + if (stage === 'rate_limit') v2RouteMocks.operationRate.mockRejectedValue(failure) + + const response = await buildHandler()(buildRequest(), context()) + + expect(response.status).toBe(503) + expect(await response.json()).toEqual({ + error: { code: 'SERVICE_UNAVAILABLE', message: 'Service temporarily unavailable' }, + }) + } + ) +}) diff --git a/apps/sim/lib/api/server/routes/v2-body-lifecycle-route.ts b/apps/sim/lib/api/server/routes/v2-body-lifecycle-route.ts new file mode 100644 index 00000000000..9e83ca0c155 --- /dev/null +++ b/apps/sim/lib/api/server/routes/v2-body-lifecycle-route.ts @@ -0,0 +1,174 @@ +import type { NextRequest } from 'next/server' +import { NextResponse } from 'next/server' +import type { ContractJsonResponse } from '@/lib/api/contracts' +import { requireJsonRouteDefinition } from '@/lib/api/server/routes/definition' +import type { + JsonApiRouteContract, + JsonNextRouteHandler, + JsonRouteContext, +} from '@/lib/api/server/routes/types' +import { + admitV2Request, + type V2ErrorPolicy, + type V2RateLimitPolicy, + V2RouteInfrastructureError, + type v2ApiKeyAuth, +} from '@/lib/api/server/routes/v2-json-route' +import { + type ParsedRequest, + type ParseRequestOptions, + parseRequest, +} from '@/lib/api/server/validation' +import type { ApplicationOperation, OperationUseCase } from '@/lib/core/application' +import { withRouteHandler } from '@/lib/core/utils/with-route-handler' +import { v2Error, v2ValidationError } from '@/app/api/v2/lib/response' + +interface V2BodyLifecycleAdmission< + O extends ApplicationOperation, + C extends JsonApiRouteContract, + I, + R, +> { + mapInput(input: ParsedRequest): I + useCase: OperationUseCase, I, R> +} + +interface V2BodyLifecycleContext { + request: NextRequest + principal: Awaited>['principal'] + parsed: ParsedRequest + admission: A +} + +interface V2BodyLifecycleTransferContext + extends V2BodyLifecycleContext { + body: B +} + +interface V2BodyLifecycleInputContext + extends V2BodyLifecycleTransferContext { + transfer: T +} + +interface V2BodyLifecycleSuccessContext + extends V2BodyLifecycleInputContext { + input: I + result: R +} + +interface V2BodyLifecycleRouteOptions< + C extends JsonApiRouteContract, + O extends ApplicationOperation, + AI, + A, + B, + T, + I, + R, +> { + contract: C + operation: O + auth: typeof v2ApiKeyAuth + rateLimit: V2RateLimitPolicy + errorPolicy: V2ErrorPolicy + parseOptions?: Omit + admission: V2BodyLifecycleAdmission + readBody(context: V2BodyLifecycleContext): Promise + transfer(context: V2BodyLifecycleTransferContext): Promise + mapInput(context: V2BodyLifecycleInputContext): I + useCase: OperationUseCase, I, R> + present(result: R): ContractJsonResponse | Promise> + onSuccess?( + context: V2BodyLifecycleSuccessContext, NoInfer> + ): void | Promise +} + +/** + * Defines a v2 route whose body must not be read until an application use case + * has completed cheap canonical admission. The contract intentionally omits a + * body schema because the bounded byte-plane reader owns that non-JSON payload. + */ +export function defineV2BodyLifecycleRoute< + C extends JsonApiRouteContract, + O extends ApplicationOperation, + AI, + A, + B, + T, + I, + R, +>(options: V2BodyLifecycleRouteOptions): JsonNextRouteHandler { + if (options.contract.body) { + throw new Error( + `${options.contract.method} ${options.contract.path} must omit its body schema so admission precedes body reads` + ) + } + const { successStatus } = requireJsonRouteDefinition( + options.contract, + options.operation, + options.useCase.operation + ) + requireJsonRouteDefinition( + options.contract, + options.operation, + options.admission.useCase.operation + ) + + const wrapped = withRouteHandler( + async (request, context) => { + if (request.method !== options.contract.method) { + throw new Error( + `Route received ${request.method} for ${options.contract.method} contract ${options.contract.path}` + ) + } + + const routeAdmission = await admitV2Request( + request, + options.operation, + options.auth, + options.rateLimit + ) + if (!routeAdmission.success) return routeAdmission.response + + const parsed = await parseRequest(options.contract, request, context ?? {}, { + ...options.parseOptions, + validationErrorResponse: v2ValidationError, + }) + if (!parsed.success) return parsed.response + + const { principal } = routeAdmission.auth + try { + const admissionInput = options.admission.mapInput(parsed.data) + const admission = await options.admission.useCase.execute({ + principal, + input: admissionInput, + request, + }) + const lifecycleContext = { request, principal, parsed: parsed.data, admission } + const body = await options.readBody(lifecycleContext) + const transfer = await options.transfer({ ...lifecycleContext, body }) + const inputContext = { ...lifecycleContext, body, transfer } + const input = options.mapInput(inputContext) + const result = await options.useCase.execute({ principal, input, request }) + const responseBody = options.contract.response.schema.parse(await options.present(result)) + await options.onSuccess?.({ ...inputContext, input, result }) + return NextResponse.json(responseBody, { + status: successStatus, + headers: { 'Cache-Control': 'private, no-store' }, + }) + } catch (error) { + const response = options.errorPolicy.render(error) + if (response) return response + throw error + } + }, + { + unhandledErrorResponse: ({ error }) => + error instanceof V2RouteInfrastructureError + ? v2Error('SERVICE_UNAVAILABLE', 'Service temporarily unavailable') + : v2Error('INTERNAL_ERROR', 'Internal server error'), + } + ) + + return async (request, context) => wrapped(request, context) +} diff --git a/apps/sim/lib/copilot/application/execute-knowledge-use-case.ts b/apps/sim/lib/copilot/application/execute-knowledge-use-case.ts index d88c5c940b3..f21b9b0c41a 100644 --- a/apps/sim/lib/copilot/application/execute-knowledge-use-case.ts +++ b/apps/sim/lib/copilot/application/execute-knowledge-use-case.ts @@ -5,6 +5,7 @@ import { COPILOT_APPLICATION_DELEGATION_TTL_MS, type CopilotExecutionContext, createCopilotApplicationPrincipal, + createTrustedCopilotPrincipal, requireTrustedCopilotExecutionContext, } from '@/lib/copilot/auth/application-delegation' import type { OperationUseCase } from '@/lib/core/application' @@ -16,6 +17,12 @@ import { export type CopilotKnowledgeDelegationContext = CopilotExecutionContext +export interface CopilotChatKnowledgeDelegationContext { + userId: string + workspaceId: string + chatId?: string +} + const knowledgeDelegation = { audience: knowledgeDelegationPolicy.audience, ttlMs: COPILOT_APPLICATION_DELEGATION_TTL_MS, @@ -29,6 +36,13 @@ const executeKnowledgeUseCase = createCopilotApplicationAdapter({ operations: knowledgeOperations, }) +/** Requires the immutable trusted workspace bound to a Copilot Knowledge execution. */ +export function requireCopilotKnowledgeWorkspaceId( + context: CopilotKnowledgeDelegationContext | undefined +): string { + return requireTrustedCopilotExecutionContext(context).workspaceId +} + /** Normalizes immutable Copilot execution identity into a knowledge delegation. */ export function resolveCopilotKnowledgePrincipal( context: CopilotKnowledgeDelegationContext | undefined @@ -39,6 +53,21 @@ export function resolveCopilotKnowledgePrincipal( ) } +/** Creates the trusted Knowledge principal used while resolving Copilot chat context. */ +export function createCopilotChatKnowledgePrincipal( + context: CopilotChatKnowledgeDelegationContext +): DelegatedPrincipal { + return createTrustedCopilotPrincipal( + { + userId: context.userId, + workspaceId: context.workspaceId, + delegationId: `copilot-chat:${context.chatId ?? context.workspaceId}`, + chatId: context.chatId, + }, + knowledgeDelegation + ) +} + /** Enters a registered knowledge application use case with trusted Copilot identity. */ export function executeCopilotKnowledgeUseCase( context: CopilotKnowledgeDelegationContext | undefined, diff --git a/apps/sim/lib/copilot/chat/process-contents.test.ts b/apps/sim/lib/copilot/chat/process-contents.test.ts index 0c1307fd4a5..96aaa155462 100644 --- a/apps/sim/lib/copilot/chat/process-contents.test.ts +++ b/apps/sim/lib/copilot/chat/process-contents.test.ts @@ -8,6 +8,7 @@ import { MAX_TABLE_SELECTION_CONTENT_LENGTH, MAX_TABLE_SELECTION_ROWS, } from '@/lib/copilot/chat/selection-context' +import { DelegatedWorkspaceAuthorizationError } from '@/lib/core/application' import type { ChatContext } from '@/stores/panel' const { @@ -20,6 +21,7 @@ const { readWorkspaceFileMetadata, getTableById, getRowsByIds, + readKnowledgeBase, getBlockVisibilityForCopilot, isIntegrationDeploymentAvailable, } = vi.hoisted(() => ({ @@ -32,6 +34,7 @@ const { readWorkspaceFileMetadata: vi.fn(), getTableById: vi.fn(), getRowsByIds: vi.fn(), + readKnowledgeBase: vi.fn(), getBlockVisibilityForCopilot: vi.fn(async () => null), isIntegrationDeploymentAvailable: vi.fn(() => true), })) @@ -50,6 +53,9 @@ vi.mock('@/lib/workspace-files/application/read-workspace-file-metadata', () => })) vi.mock('@/lib/table/service', () => ({ getTableById })) vi.mock('@/lib/table/rows/service', () => ({ getRowsByIds })) +vi.mock('@/lib/knowledge/application/knowledge-bases', () => ({ + readKnowledgeBase: { execute: readKnowledgeBase }, +})) /** * Overrides the global `@sim/db` mock: the logs-context tests below need @@ -58,6 +64,75 @@ vi.mock('@/lib/table/rows/service', () => ({ getRowsByIds })) import { processContextsServer } from './process-contents' +describe('processContextsServer - knowledge contexts', () => { + beforeEach(() => { + vi.clearAllMocks() + readKnowledgeBase.mockResolvedValue({ + knowledgeBase: { id: 'knowledge-1', name: 'Product docs' }, + }) + }) + + it('reads through the fixed application query with a trusted chat principal', async () => { + const result = await processContextsServer( + [{ kind: 'knowledge', knowledgeId: 'knowledge-1', label: 'Docs' } as ChatContext], + 'dual-workspace-user', + 'hello', + 'workspace-a', + 'chat-1' + ) + + expect(readKnowledgeBase).toHaveBeenCalledWith({ + principal: expect.objectContaining({ + kind: 'delegated', + serviceId: 'copilot', + subjectUserId: 'dual-workspace-user', + workspaceId: 'workspace-a', + audience: 'sim:knowledge', + }), + input: { + knowledgeBaseId: 'knowledge-1', + assertedWorkspaceId: 'workspace-a', + }, + }) + expect(result).toEqual([ + { + type: 'knowledge', + tag: '@Docs', + content: '', + path: 'knowledgebases/Product%20docs/meta.json', + }, + ]) + }) + + it('conceals a cross-workspace Knowledge target from Copilot context', async () => { + readKnowledgeBase.mockRejectedValueOnce(new DelegatedWorkspaceAuthorizationError()) + + await expect( + processContextsServer( + [{ kind: 'knowledge', knowledgeId: 'knowledge-b', label: 'Hidden' } as ChatContext], + 'dual-workspace-user', + 'hello', + 'workspace-a', + 'chat-1' + ) + ).resolves.toEqual([]) + }) + + it('conceals infrastructure details from Copilot context', async () => { + readKnowledgeBase.mockRejectedValueOnce(new Error('database host and password')) + + await expect( + processContextsServer( + [{ kind: 'knowledge', knowledgeId: 'knowledge-b', label: 'Hidden' } as ChatContext], + 'dual-workspace-user', + 'hello', + 'workspace-a', + 'chat-1' + ) + ).resolves.toEqual([]) + }) +}) + describe('processContextsServer - block contexts', () => { beforeEach(() => { vi.clearAllMocks() diff --git a/apps/sim/lib/copilot/chat/process-contents.ts b/apps/sim/lib/copilot/chat/process-contents.ts index 7dd0e3ac386..c3c108d09f2 100644 --- a/apps/sim/lib/copilot/chat/process-contents.ts +++ b/apps/sim/lib/copilot/chat/process-contents.ts @@ -1,11 +1,11 @@ -import { db, dbReplica } from '@sim/db' -import { knowledgeBase } from '@sim/db/schema' +import { db } from '@sim/db' import { createLogger } from '@sim/logger' import { authorizeWorkflowByWorkspacePermission, getActiveWorkflowRecord, } from '@sim/platform-authz/workflow' -import { and, eq, isNull } from 'drizzle-orm' +import { eq } from 'drizzle-orm' +import { createCopilotChatKnowledgePrincipal } from '@/lib/copilot/application/execute-knowledge-use-case' import { createCopilotChatFilePrincipal } from '@/lib/copilot/auth/file-delegation' import { getBlockVisibilityForCopilot } from '@/lib/copilot/block-visibility' import { @@ -27,6 +27,7 @@ import { import { EnvCapabilityConfigurationError } from '@/lib/core/config/env-capabilities' import { getAllowedIntegrationsFromEnv } from '@/lib/core/config/env-flags' import { isIntegrationDeploymentAvailableForVisibility } from '@/lib/integrations/availability.server' +import { readKnowledgeBase } from '@/lib/knowledge/application/knowledge-bases' import { toOverview } from '@/lib/logs/log-views' import type { TraceSpan } from '@/lib/logs/types' import { mcpService } from '@/lib/mcp/service' @@ -41,7 +42,6 @@ import { getWorkspaceFileFolderPath } from '@/lib/uploads/contexts/workspace/wor import { getSkillById } from '@/lib/workflows/skills/operations' import { listFolders } from '@/lib/workflows/utils' import { readWorkspaceFileMetadata } from '@/lib/workspace-files/application/read-workspace-file-metadata' -import { checkKnowledgeBaseAccess } from '@/app/api/knowledge/utils' import { getUserPermissionConfig } from '@/ee/access-control/utils/permission-check' import { escapeRegExp } from '@/executor/constants' import type { BrowserTextSelection, ChatContext, TerminalTextSelection } from '@/stores/panel' @@ -172,7 +172,8 @@ export async function processContextsServer( ctx.knowledgeId, userId, ctx.label ? `@${ctx.label}` : '@', - currentWorkspaceId + currentWorkspaceId, + chatId ) } if (ctx.kind === 'blocks' && ctx.blockIds?.length > 0) { @@ -559,33 +560,23 @@ async function processKnowledgeFromDb( knowledgeBaseId: string, userId: string | undefined, tag: string, - currentWorkspaceId?: string + currentWorkspaceId?: string, + chatId?: string ): Promise { try { - if (userId) { - const accessCheck = await checkKnowledgeBaseAccess(knowledgeBaseId, userId) - if (!accessCheck.hasAccess) { - return null - } - if (currentWorkspaceId && accessCheck.knowledgeBase?.workspaceId !== currentWorkspaceId) { - return null - } - } - - const conditions = [eq(knowledgeBase.id, knowledgeBaseId), isNull(knowledgeBase.deletedAt)] - if (currentWorkspaceId) { - conditions.push(eq(knowledgeBase.workspaceId, currentWorkspaceId)) - } - const kbRows = await dbReplica - .select({ - id: knowledgeBase.id, - name: knowledgeBase.name, - }) - .from(knowledgeBase) - .where(and(...conditions)) - .limit(1) - const kb = kbRows?.[0] - if (!kb) return null + if (!userId || !currentWorkspaceId) return null + const principal = createCopilotChatKnowledgePrincipal({ + userId, + workspaceId: currentWorkspaceId, + chatId, + }) + const { knowledgeBase: kb } = await readKnowledgeBase.execute({ + principal, + input: { + knowledgeBaseId, + assertedWorkspaceId: currentWorkspaceId, + }, + }) return { type: 'knowledge', @@ -836,7 +827,8 @@ export async function resolveActiveResourceContext( resourceId, userId, '@active_resource', - workspaceId + workspaceId, + chatId ) if (!ctx) return null return { diff --git a/apps/sim/lib/copilot/tool-executor/executor.test.ts b/apps/sim/lib/copilot/tool-executor/executor.test.ts index 30b5d8d4f17..2f241d2478b 100644 --- a/apps/sim/lib/copilot/tool-executor/executor.test.ts +++ b/apps/sim/lib/copilot/tool-executor/executor.test.ts @@ -138,7 +138,13 @@ describe('copilot tool executor fallback', () => { chatId: 'chat-1', enforceCredentialAccess: true, }), - }) + }), + { + internalExecutorDelegation: { + subjectUserId: 'user-1', + workflowId: 'workflow-1', + }, + } ) expect(result).toEqual({ success: true, output: { emails: [] } }) }) @@ -214,7 +220,13 @@ describe('copilot tool executor fallback', () => { query: 'hello', _context: expect.not.objectContaining({ resolvedSecretTraceRegistry: expect.anything() }), }), - { resolvedSecretTraceRegistry: registry } + { + resolvedSecretTraceRegistry: registry, + internalExecutorDelegation: { + subjectUserId: 'user-1', + workflowId: 'workflow-1', + }, + } ) const appParams = executeAppTool.mock.calls[0]?.[1] expect(JSON.stringify(appParams)).not.toContain('resolvedSecretTraceRegistry') @@ -274,7 +286,13 @@ describe('copilot tool executor fallback', () => { _context: expect.objectContaining({ copilotToolExecution: true, }), - }) + }), + { + internalExecutorDelegation: { + subjectUserId: 'user-1', + workflowId: 'workflow-1', + }, + } ) }) @@ -323,7 +341,13 @@ describe('copilot tool executor fallback', () => { 'function_execute', expect.objectContaining({ timeout: 10_000, - }) + }), + { + internalExecutorDelegation: { + subjectUserId: 'user-1', + workflowId: 'workflow-1', + }, + } ) }) @@ -347,7 +371,13 @@ describe('copilot tool executor fallback', () => { 'function_execute', expect.objectContaining({ timeout: 10_000, - }) + }), + { + internalExecutorDelegation: { + subjectUserId: 'user-1', + workflowId: 'workflow-1', + }, + } ) }) @@ -371,7 +401,13 @@ describe('copilot tool executor fallback', () => { 'function_execute', expect.objectContaining({ timeout: DEFAULT_EXECUTION_TIMEOUT_MS, - }) + }), + { + internalExecutorDelegation: { + subjectUserId: 'user-1', + workflowId: 'workflow-1', + }, + } ) }) }) diff --git a/apps/sim/lib/copilot/tool-executor/executor.ts b/apps/sim/lib/copilot/tool-executor/executor.ts index 6488b695f25..6184d682204 100644 --- a/apps/sim/lib/copilot/tool-executor/executor.ts +++ b/apps/sim/lib/copilot/tool-executor/executor.ts @@ -71,10 +71,22 @@ export async function executeTool( (isSimExecuted(toolId) || (isClientExecuted(toolId) && hasHandler(toolId))) if (!canUseRegisteredHandler) { const appParams = buildAppToolParams(normalizedParams, context) - return context.resolvedSecretTraceRegistry - ? executeAppTool(toolId, appParams, { - resolvedSecretTraceRegistry: context.resolvedSecretTraceRegistry, - }) + const options = { + ...(context.resolvedSecretTraceRegistry + ? { resolvedSecretTraceRegistry: context.resolvedSecretTraceRegistry } + : {}), + ...(context.workflowId + ? { + internalExecutorDelegation: { + subjectUserId: context.userId, + workflowId: context.workflowId, + ...(context.executionId ? { executionId: context.executionId } : {}), + }, + } + : {}), + } + return Object.keys(options).length > 0 + ? executeAppTool(toolId, appParams, options) : executeAppTool(toolId, appParams) } diff --git a/apps/sim/lib/copilot/tools/server/knowledge/knowledge-base.test.ts b/apps/sim/lib/copilot/tools/server/knowledge/knowledge-base.test.ts index 7e850b80dd5..711a9ca58eb 100644 --- a/apps/sim/lib/copilot/tools/server/knowledge/knowledge-base.test.ts +++ b/apps/sim/lib/copilot/tools/server/knowledge/knowledge-base.test.ts @@ -4,34 +4,81 @@ import { beforeEach, describe, expect, it, vi } from 'vitest' const { + mockAddWorkspaceFiles, + mockBulkDeleteKnowledgeBases, + mockBulkDeleteKnowledgeDocuments, mockCaptureServerEvent, mockCreateKnowledgeBase, - mockDeleteKnowledgeBase, - mockDeleteKnowledgeDocument, - mockGetBoundWorkspaceFileSecretProvenance, + mockDeleteKnowledgeConnector, + mockDeleteKnowledgeTag, mockKnowledgeBaseCreated, mockKnowledgeBaseDeleted, mockKnowledgeBaseDocumentsUploaded, mockReadKnowledgeBase, - mockResolveWorkspaceFileReference, + mockReadKnowledgeTagUsage, mockSearchKnowledge, + mockSyncKnowledgeConnector, mockUpdateKnowledgeBase, - mockUploadKnowledgeDocument, -} = vi.hoisted(() => ({ - mockCaptureServerEvent: vi.fn(), - mockCreateKnowledgeBase: vi.fn(), - mockDeleteKnowledgeBase: vi.fn(), - mockDeleteKnowledgeDocument: vi.fn(), - mockGetBoundWorkspaceFileSecretProvenance: vi.fn(), - mockKnowledgeBaseCreated: vi.fn(), - mockKnowledgeBaseDeleted: vi.fn(), - mockKnowledgeBaseDocumentsUploaded: vi.fn(), - mockReadKnowledgeBase: vi.fn(), - mockResolveWorkspaceFileReference: vi.fn(), - mockSearchKnowledge: vi.fn(), - mockUpdateKnowledgeBase: vi.fn(), - mockUploadKnowledgeDocument: vi.fn(), -})) + mockUpdateKnowledgeConnector, + mockUpdateKnowledgeDocument, + mockUpdateKnowledgeTag, + mockCreateKnowledgeConnector, + mockCreateKnowledgeTag, + mockListKnowledgeTags, + knowledgeOperations, +} = vi.hoisted(() => { + const defineOperation = (id: string, minimumRole: 'read' | 'write') => + Object.freeze({ + id, + minimumRole, + workspaceApiKey: 'deny' as const, + principalKinds: ['session', 'personal_api_key', 'delegated'] as const, + delegatedServices: ['copilot'] as const, + }) + + return { + mockAddWorkspaceFiles: vi.fn(), + mockBulkDeleteKnowledgeBases: vi.fn(), + mockBulkDeleteKnowledgeDocuments: vi.fn(), + mockCaptureServerEvent: vi.fn(), + mockCreateKnowledgeBase: vi.fn(), + mockDeleteKnowledgeConnector: vi.fn(), + mockDeleteKnowledgeTag: vi.fn(), + mockKnowledgeBaseCreated: vi.fn(), + mockKnowledgeBaseDeleted: vi.fn(), + mockKnowledgeBaseDocumentsUploaded: vi.fn(), + mockReadKnowledgeBase: vi.fn(), + mockReadKnowledgeTagUsage: vi.fn(), + mockSearchKnowledge: vi.fn(), + mockSyncKnowledgeConnector: vi.fn(), + mockUpdateKnowledgeBase: vi.fn(), + mockUpdateKnowledgeConnector: vi.fn(), + mockUpdateKnowledgeDocument: vi.fn(), + mockUpdateKnowledgeTag: vi.fn(), + mockCreateKnowledgeConnector: vi.fn(), + mockCreateKnowledgeTag: vi.fn(), + mockListKnowledgeTags: vi.fn(), + knowledgeOperations: { + addWorkspaceFiles: defineOperation('knowledge.documents.add_workspace_files', 'write'), + bulkDelete: defineOperation('knowledge.bulk_delete', 'write'), + bulkDeleteDocuments: defineOperation('knowledge.documents.bulk_delete', 'write'), + create: defineOperation('knowledge.create', 'write'), + createConnector: defineOperation('knowledge.connectors.create', 'write'), + createTag: defineOperation('knowledge.tags.create', 'write'), + deleteConnector: defineOperation('knowledge.connectors.delete', 'write'), + deleteTag: defineOperation('knowledge.tags.delete', 'write'), + listTags: defineOperation('knowledge.tags.list', 'read'), + read: defineOperation('knowledge.read', 'read'), + readTagUsage: defineOperation('knowledge.tags.read_usage', 'read'), + search: defineOperation('knowledge.search', 'read'), + syncConnector: defineOperation('knowledge.connectors.sync', 'write'), + update: defineOperation('knowledge.update', 'write'), + updateConnector: defineOperation('knowledge.connectors.update', 'write'), + updateDocument: defineOperation('knowledge.documents.update', 'write'), + updateTag: defineOperation('knowledge.tags.update', 'write'), + }, + } +}) vi.mock('@/lib/copilot/generated/tool-catalog-v1', () => ({ KnowledgeBase: { id: 'knowledge_base' }, @@ -44,49 +91,80 @@ vi.mock('@/lib/core/telemetry', () => ({ }, })) vi.mock('@/lib/posthog/server', () => ({ captureServerEvent: mockCaptureServerEvent })) +vi.mock('@/lib/knowledge/application/operations', () => ({ knowledgeOperations })) +vi.mock('@/lib/knowledge/application/add-workspace-files', () => ({ + addWorkspaceFilesToKnowledgeBase: { + operation: knowledgeOperations.addWorkspaceFiles, + execute: mockAddWorkspaceFiles, + }, +})) vi.mock('@/lib/knowledge/application/knowledge-bases', () => ({ - createKnowledgeBase: { execute: mockCreateKnowledgeBase }, - deleteKnowledgeBaseOperation: { execute: mockDeleteKnowledgeBase }, - readKnowledgeBase: { execute: mockReadKnowledgeBase }, - updateKnowledgeBaseOperation: { execute: mockUpdateKnowledgeBase }, + bulkDeleteKnowledgeBases: { + operation: knowledgeOperations.bulkDelete, + execute: mockBulkDeleteKnowledgeBases, + }, + createKnowledgeBase: { + operation: knowledgeOperations.create, + execute: mockCreateKnowledgeBase, + }, + readKnowledgeBase: { operation: knowledgeOperations.read, execute: mockReadKnowledgeBase }, + updateKnowledgeBaseOperation: { + operation: knowledgeOperations.update, + execute: mockUpdateKnowledgeBase, + }, })) vi.mock('@/lib/knowledge/application/documents', () => ({ - deleteKnowledgeDocument: { execute: mockDeleteKnowledgeDocument }, - uploadKnowledgeDocument: { execute: mockUploadKnowledgeDocument }, + bulkDeleteKnowledgeDocuments: { + operation: knowledgeOperations.bulkDeleteDocuments, + execute: mockBulkDeleteKnowledgeDocuments, + }, + updateKnowledgeDocument: { + operation: knowledgeOperations.updateDocument, + execute: mockUpdateKnowledgeDocument, + }, })) vi.mock('@/lib/knowledge/application/search', () => ({ - searchKnowledge: { execute: mockSearchKnowledge }, + searchKnowledge: { operation: knowledgeOperations.search, execute: mockSearchKnowledge }, })) -vi.mock('@/lib/knowledge/orchestration', () => ({ - performCreateKnowledgeConnector: vi.fn(), - performDeleteKnowledgeConnector: vi.fn(), - performSyncKnowledgeConnector: vi.fn(), - performUpdateKnowledgeConnector: vi.fn(), - performUpdateKnowledgeDocument: vi.fn(), -})) -vi.mock('@/lib/knowledge/tags/service', () => ({ - createTagDefinition: vi.fn(), - deleteTagDefinition: vi.fn(), - getDocumentTagDefinitions: vi.fn(), - getNextAvailableSlot: vi.fn(), - getTagDefinitionById: vi.fn(), - getTagUsageStats: vi.fn(), - updateTagDefinition: vi.fn(), -})) -vi.mock('@/lib/uploads', () => ({ - StorageService: { generatePresignedDownloadUrl: vi.fn().mockResolvedValue('https://file.test') }, -})) -vi.mock('@/lib/workspace-files/application/resolve-workspace-file-reference', () => ({ - resolveWorkspaceFileReference: mockResolveWorkspaceFileReference, -})) -vi.mock('@/lib/uploads/contexts/workspace/workspace-file-secret-provenance', () => ({ - getBoundWorkspaceFileSecretProvenance: mockGetBoundWorkspaceFileSecretProvenance, +vi.mock('@/lib/knowledge/application/connectors', () => ({ + createKnowledgeConnector: { + operation: knowledgeOperations.createConnector, + execute: mockCreateKnowledgeConnector, + }, + updateKnowledgeConnector: { + operation: knowledgeOperations.updateConnector, + execute: mockUpdateKnowledgeConnector, + }, + deleteKnowledgeConnector: { + operation: knowledgeOperations.deleteConnector, + execute: mockDeleteKnowledgeConnector, + }, + syncKnowledgeConnector: { + operation: knowledgeOperations.syncConnector, + execute: mockSyncKnowledgeConnector, + }, })) -vi.mock('@/app/api/auth/oauth/utils', () => ({ getCredential: vi.fn() })) -vi.mock('@/app/api/knowledge/utils', () => ({ - checkDocumentWriteAccess: vi.fn(), - checkKnowledgeBaseAccess: vi.fn(), - checkKnowledgeBaseWriteAccess: vi.fn(), +vi.mock('@/lib/knowledge/application/tags', () => ({ + createKnowledgeTag: { + operation: knowledgeOperations.createTag, + execute: mockCreateKnowledgeTag, + }, + deleteKnowledgeTag: { + operation: knowledgeOperations.deleteTag, + execute: mockDeleteKnowledgeTag, + }, + listKnowledgeTags: { + operation: knowledgeOperations.listTags, + execute: mockListKnowledgeTags, + }, + readKnowledgeTagUsage: { + operation: knowledgeOperations.readTagUsage, + execute: mockReadKnowledgeTagUsage, + }, + updateKnowledgeTag: { + operation: knowledgeOperations.updateTag, + execute: mockUpdateKnowledgeTag, + }, })) import type { ServerToolContext } from '@/lib/copilot/tools/server/base-tool' @@ -116,6 +194,22 @@ const CONTEXT = { copilotToolExecution: true, } satisfies ServerToolContext +const BILLED_CONTEXT = { + ...CONTEXT, + billingAttribution: { + actorUserId: 'external-admin', + workspaceId: 'workspace-paid', + billedAccountUserId: 'workspace-owner', + organizationId: null, + billingEntity: { type: 'user' as const, id: 'workspace-owner' }, + billingPeriod: { + start: '2026-08-01T00:00:00.000Z', + end: '2026-09-01T00:00:00.000Z', + }, + payerSubscription: null, + }, +} satisfies ServerToolContext + function expectDelegatedPrincipal(call: unknown): void { expect(call).toMatchObject({ principal: { @@ -136,15 +230,62 @@ describe('knowledge_base trusted application delegation', () => { mockReadKnowledgeBase.mockResolvedValue({ knowledgeBase: KNOWLEDGE_BASE, folderPath: '/' }) mockCreateKnowledgeBase.mockResolvedValue({ knowledgeBase: KNOWLEDGE_BASE, folderPath: '/' }) mockUpdateKnowledgeBase.mockResolvedValue({ knowledgeBase: KNOWLEDGE_BASE, folderPath: '/' }) - mockDeleteKnowledgeBase.mockResolvedValue({ id: KNOWLEDGE_BASE.id, name: KNOWLEDGE_BASE.name }) + mockBulkDeleteKnowledgeBases.mockResolvedValue({ + deleted: [{ id: KNOWLEDGE_BASE.id, name: KNOWLEDGE_BASE.name }], + notFound: [], + failed: [], + }) + mockBulkDeleteKnowledgeDocuments.mockResolvedValue({ + knowledgeBaseId: KNOWLEDGE_BASE.id, + deleted: ['document-1'], + failed: [], + deletedDocuments: [], + }) + mockAddWorkspaceFiles.mockResolvedValue({ + knowledgeBaseId: KNOWLEDGE_BASE.id, + knowledgeBaseName: KNOWLEDGE_BASE.name, + added: [ + { + documentId: 'document-1', + filename: 'report.pdf', + fileSize: 100, + mimeType: 'application/pdf', + }, + ], + failed: [], + }) mockSearchKnowledge.mockResolvedValue({ results: [], query: 'query', knowledgeBaseIds: [KNOWLEDGE_BASE.id], + knowledgeBases: [{ id: KNOWLEDGE_BASE.id, name: KNOWLEDGE_BASE.name }], topK: 5, totalResults: 0, }) - mockDeleteKnowledgeDocument.mockResolvedValue({ id: 'document-1', filename: 'doc.pdf' }) + mockCreateKnowledgeConnector.mockResolvedValue({ + connector: { + id: 'connector-1', + knowledgeBaseId: KNOWLEDGE_BASE.id, + connectorType: 'notion', + status: 'active', + syncIntervalMinutes: 1440, + }, + workspaceId: 'workspace-paid', + }) + mockDeleteKnowledgeConnector.mockResolvedValue({ + knowledgeBaseId: KNOWLEDGE_BASE.id, + workspaceId: 'workspace-paid', + connectorId: 'connector-1', + connectorType: 'notion', + documentsDeleted: 0, + documentsKept: 2, + }) + mockSyncKnowledgeConnector.mockResolvedValue({ + knowledgeBaseId: KNOWLEDGE_BASE.id, + workspaceId: 'workspace-paid', + connectorId: 'connector-1', + connectorType: 'notion', + }) }) it.each([ @@ -231,6 +372,7 @@ describe('knowledge_base trusted application delegation', () => { ], query: '{{KB_QUERY}}', knowledgeBaseIds: [KNOWLEDGE_BASE.id], + knowledgeBases: [{ id: KNOWLEDGE_BASE.id, name: KNOWLEDGE_BASE.name }], topK: 5, totalResults: 1, }) @@ -256,17 +398,19 @@ describe('knowledge_base trusted application delegation', () => { topK: 5, resultSecretRegistry: registry, }) + expect(mockReadKnowledgeBase).not.toHaveBeenCalled() }) - it('propagates search infrastructure failures', async () => { + it('returns a safe model result for search infrastructure failures', async () => { mockSearchKnowledge.mockRejectedValueOnce(new Error('database unavailable')) - await expect( - knowledgeBaseServerTool.execute( - { operation: 'query', args: { knowledgeBaseId: KNOWLEDGE_BASE.id, query: 'query' } }, - { ...CONTEXT, resolvedSecretTraceRegistry: new ResolvedSecretTraceRegistry() } - ) - ).rejects.toThrow('database unavailable') + const result = await knowledgeBaseServerTool.execute( + { operation: 'query', args: { knowledgeBaseId: KNOWLEDGE_BASE.id, query: 'query' } }, + { ...CONTEXT, resolvedSecretTraceRegistry: new ResolvedSecretTraceRegistry() } + ) + + expect(result).toEqual({ success: false, message: 'Failed to query knowledge base' }) + expect(result.message).not.toContain('database unavailable') }) it('updates through the semantic operation', async () => { @@ -286,7 +430,7 @@ describe('knowledge_base trusted application delegation', () => { }) }) - it('keeps the unexposed delete compatibility path on the shared delete operation', async () => { + it('delegates the unexposed delete compatibility path once to the bulk command', async () => { const result = await knowledgeBaseServerTool.execute( { operation: 'delete', args: { knowledgeBaseId: KNOWLEDGE_BASE.id } }, CONTEXT @@ -296,22 +440,23 @@ describe('knowledge_base trusted application delegation', () => { success: true, data: { deleted: [{ id: KNOWLEDGE_BASE.id, name: KNOWLEDGE_BASE.name }] }, }) - const call = mockDeleteKnowledgeBase.mock.calls[0][0] + const call = mockBulkDeleteKnowledgeBases.mock.calls[0][0] expectDelegatedPrincipal(call) - expect(call.input).toEqual({ - knowledgeBaseId: KNOWLEDGE_BASE.id, + expect(call.input).toMatchObject({ assertedWorkspaceId: 'workspace-paid', + knowledgeBaseIds: [KNOWLEDGE_BASE.id], source: 'agent', }) - expect(mockKnowledgeBaseDeleted).toHaveBeenCalledWith({ - knowledgeBaseId: KNOWLEDGE_BASE.id, - }) }) it('keeps classified delete failures in the batch result', async () => { - mockDeleteKnowledgeBase.mockRejectedValueOnce( - new OrchestrationError('conflict', 'Knowledge base is locked') - ) + mockBulkDeleteKnowledgeBases.mockResolvedValueOnce({ + deleted: [], + notFound: [], + failed: [ + { id: KNOWLEDGE_BASE.id, name: KNOWLEDGE_BASE.name, reason: 'Knowledge base is locked' }, + ], + }) const result = await knowledgeBaseServerTool.execute( { operation: 'delete', args: { knowledgeBaseId: KNOWLEDGE_BASE.id } }, @@ -330,9 +475,19 @@ describe('knowledge_base trusted application delegation', () => { }) it('delegates document deletion and retains partial batch results', async () => { - mockDeleteKnowledgeDocument.mockRejectedValueOnce( - new OrchestrationError('not_found', 'Document not found') - ) + mockBulkDeleteKnowledgeDocuments.mockResolvedValueOnce({ + knowledgeBaseId: KNOWLEDGE_BASE.id, + deleted: ['document-1'], + failed: ['missing'], + deletedDocuments: [ + { + id: 'document-1', + filename: 'guide.pdf', + fileSize: 42, + mimeType: 'application/pdf', + }, + ], + }) const result = await knowledgeBaseServerTool.execute( { @@ -346,7 +501,8 @@ describe('knowledge_base trusted application delegation', () => { success: true, data: { deleted: ['document-1'], failed: ['missing'] }, }) - expectDelegatedPrincipal(mockDeleteKnowledgeDocument.mock.calls[1][0]) + expectDelegatedPrincipal(mockBulkDeleteKnowledgeDocuments.mock.calls[0][0]) + expect(mockBulkDeleteKnowledgeDocuments).toHaveBeenCalledOnce() expect(mockCaptureServerEvent).toHaveBeenCalledWith( 'external-admin', 'knowledge_base_document_deleted', @@ -355,6 +511,149 @@ describe('knowledge_base trusted application delegation', () => { ) }) + it.each([ + [ + 'add_connector', + { + knowledgeBaseId: KNOWLEDGE_BASE.id, + connectorType: 'notion', + credentialId: 'credential-1', + }, + 'knowledge_base_connector_added', + ], + ['delete_connector', { connectorId: 'connector-1' }, 'knowledge_base_connector_removed'], + ['sync_connector', { connectorId: 'connector-1' }, 'knowledge_base_connector_synced'], + ])( + 'records %s product analytics only after application success', + async (operation, args, event) => { + const result = await knowledgeBaseServerTool.execute({ operation, args }, BILLED_CONTEXT) + + expect(result.success).toBe(true) + expect(mockCaptureServerEvent).toHaveBeenCalledWith( + 'external-admin', + event, + expect.objectContaining({ workspace_id: 'workspace-paid' }), + expect.any(Object) + ) + } + ) + + it('delegates document updates with only the trusted workspace assertion', async () => { + mockUpdateKnowledgeDocument.mockResolvedValueOnce({ document: {}, updatedFields: ['filename'] }) + + const result = await knowledgeBaseServerTool.execute( + { + operation: 'update_document', + args: { + knowledgeBaseId: KNOWLEDGE_BASE.id, + documentId: 'document-1', + filename: 'renamed.pdf', + }, + }, + CONTEXT + ) + + expect(result).toMatchObject({ success: true, data: { documentId: 'document-1' } }) + const call = mockUpdateKnowledgeDocument.mock.calls[0][0] + expectDelegatedPrincipal(call) + expect(call.input).toEqual({ + knowledgeBaseId: KNOWLEDGE_BASE.id, + documentId: 'document-1', + assertedWorkspaceId: 'workspace-paid', + filename: 'renamed.pdf', + source: 'agent', + }) + }) + + it('does not expose connector infrastructure errors to the model', async () => { + mockUpdateKnowledgeConnector.mockRejectedValueOnce(new Error('sql host=private-db')) + + const result = await knowledgeBaseServerTool.execute( + { + operation: 'update_connector', + args: { connectorId: 'connector-1', connectorStatus: 'paused' }, + }, + CONTEXT + ) + + expect(result).toEqual({ + success: false, + message: 'Failed to update connector', + }) + expect(result.message).not.toContain('private-db') + }) + + it.each([ + [ + 'update_document', + { + knowledgeBaseId: KNOWLEDGE_BASE.id, + documentId: 'document-1', + filename: 'renamed.pdf', + }, + mockUpdateKnowledgeDocument, + ], + [ + 'update_tag', + { + knowledgeBaseId: KNOWLEDGE_BASE.id, + tagDefinitionId: 'tag-1', + displayName: 'Renamed', + }, + mockUpdateKnowledgeTag, + ], + ])('does not expose %s infrastructure details to the model', async (operation, args, useCase) => { + useCase.mockRejectedValueOnce(new Error('database password=private')) + + const result = await knowledgeBaseServerTool.execute({ operation, args }, CONTEXT) + + expect(result.success).toBe(false) + expect(result.message).not.toContain('database') + expect(result.message).not.toContain('private') + }) + + it('preserves caller-actionable connector failure messages', async () => { + mockUpdateKnowledgeConnector.mockRejectedValueOnce( + new OrchestrationError('validation', 'At least one connector update is required') + ) + + const result = await knowledgeBaseServerTool.execute( + { + operation: 'update_connector', + args: { connectorId: 'connector-1', connectorStatus: 'paused' }, + }, + CONTEXT + ) + + expect(result).toEqual({ + success: false, + message: 'At least one connector update is required', + }) + }) + + it('preserves caller-actionable tag provenance conflicts', async () => { + mockDeleteKnowledgeTag.mockRejectedValueOnce( + new OrchestrationError( + 'conflict', + 'Tag definitions cannot be deleted while resolved-secret document provenance is present' + ) + ) + + const result = await knowledgeBaseServerTool.execute( + { + operation: 'delete_tag', + args: { knowledgeBaseId: KNOWLEDGE_BASE.id, tagDefinitionId: 'tag-1' }, + }, + CONTEXT + ) + + expect(result).toEqual({ + success: false, + message: + 'Failed to delete_tag knowledge base: Tag definitions cannot be deleted while resolved-secret document provenance is present', + }) + }) + it.each([ { operation: 'add_file', @@ -379,9 +678,9 @@ describe('knowledge_base trusted application delegation', () => { expect(result.success).toBe(false) expect(result.message).toContain('Maximum is 100') expect(mockReadKnowledgeBase).not.toHaveBeenCalled() - expect(mockDeleteKnowledgeBase).not.toHaveBeenCalled() - expect(mockDeleteKnowledgeDocument).not.toHaveBeenCalled() - expect(mockUploadKnowledgeDocument).not.toHaveBeenCalled() + expect(mockBulkDeleteKnowledgeBases).not.toHaveBeenCalled() + expect(mockBulkDeleteKnowledgeDocuments).not.toHaveBeenCalled() + expect(mockAddWorkspaceFiles).not.toHaveBeenCalled() } ) }) @@ -389,27 +688,22 @@ describe('knowledge_base trusted application delegation', () => { describe('knowledge_base add_file delegation', () => { beforeEach(() => { vi.clearAllMocks() - mockReadKnowledgeBase.mockResolvedValue({ knowledgeBase: KNOWLEDGE_BASE, folderPath: '/' }) - mockResolveWorkspaceFileReference.mockResolvedValue({ - id: 'file-1', - key: 'workspace/workspace-paid/report.pdf', - name: 'report.pdf', - size: 100, - type: 'application/pdf', - }) - mockGetBoundWorkspaceFileSecretProvenance.mockResolvedValue({ status: 'exact', entries: [] }) - mockUploadKnowledgeDocument.mockResolvedValue({ - created: true, - document: { - id: 'document-1', - filename: 'report.pdf', - fileSize: 100, - mimeType: 'application/pdf', - }, + mockAddWorkspaceFiles.mockResolvedValue({ + knowledgeBaseId: KNOWLEDGE_BASE.id, + knowledgeBaseName: KNOWLEDGE_BASE.name, + added: [ + { + documentId: 'document-1', + filename: 'report.pdf', + fileSize: 100, + mimeType: 'application/pdf', + }, + ], + failed: [], }) }) - it('preserves file resolution and performs current admission inside uploadKnowledgeDocument', async () => { + it('maps aliases and delegates the complete batch once to the application command', async () => { const result = await knowledgeBaseServerTool.execute( { operation: 'add_file', @@ -422,28 +716,32 @@ describe('knowledge_base add_file delegation', () => { success: true, data: { added: [{ documentId: 'document-1', filename: 'report.pdf' }] }, }) - expect(mockResolveWorkspaceFileReference).toHaveBeenCalledWith( - expect.objectContaining({ workspaceId: 'workspace-paid', reference: 'files/report.pdf' }) - ) - const call = mockUploadKnowledgeDocument.mock.calls[0][0] + const call = mockAddWorkspaceFiles.mock.calls[0][0] expectDelegatedPrincipal(call) expect(call.input).toMatchObject({ knowledgeBaseId: KNOWLEDGE_BASE.id, assertedWorkspaceId: 'workspace-paid', - startProcessing: true, + fileReferences: ['files/report.pdf'], source: 'agent', - document: { filename: 'report.pdf', fileSize: 100, mimeType: 'application/pdf' }, }) - expect(call.input).not.toHaveProperty('usageAdmission') + expect(mockAddWorkspaceFiles).toHaveBeenCalledOnce() expect(mockKnowledgeBaseDocumentsUploaded).toHaveBeenCalledWith( expect.objectContaining({ knowledgeBaseId: KNOWLEDGE_BASE.id, documentsCount: 1 }) ) + expect(mockCaptureServerEvent).toHaveBeenCalledWith( + 'external-admin', + 'knowledge_base_document_uploaded', + expect.objectContaining({ knowledge_base_id: KNOWLEDGE_BASE.id }), + expect.any(Object) + ) }) - it('rejects files carrying resolved-secret provenance before durable registration', async () => { - mockGetBoundWorkspaceFileSecretProvenance.mockResolvedValueOnce({ - status: 'exact', - entries: [{ name: 'API_KEY', encryptedValue: 'encrypted-secret' }], + it('preserves explicit partial failures returned by the application command', async () => { + mockAddWorkspaceFiles.mockResolvedValueOnce({ + knowledgeBaseId: KNOWLEDGE_BASE.id, + knowledgeBaseName: KNOWLEDGE_BASE.name, + added: [], + failed: ['files/report.pdf'], }) const result = await knowledgeBaseServerTool.execute( @@ -455,6 +753,47 @@ describe('knowledge_base add_file delegation', () => { ) expect(result.success).toBe(false) - expect(mockUploadKnowledgeDocument).not.toHaveBeenCalled() + expect(result).toMatchObject({ data: { added: [], failed: ['files/report.pdf'] } }) + }) + + it('does not expose add-file infrastructure failures to the model', async () => { + mockAddWorkspaceFiles.mockRejectedValueOnce(new Error('storage host=private-bucket')) + + const result = await knowledgeBaseServerTool.execute( + { + operation: 'add_file', + args: { knowledgeBaseId: KNOWLEDGE_BASE.id, filePaths: ['files/report.pdf'] }, + }, + CONTEXT + ) + + expect(result).toEqual({ success: false, message: 'Failed to add_file knowledge base' }) + expect(result.message).not.toContain('private-bucket') + }) + + it('rechecks cancellation after application composition before presenting a partial result', async () => { + const controller = new AbortController() + mockAddWorkspaceFiles.mockImplementationOnce(async () => { + controller.abort('user stopped') + return { + knowledgeBaseId: KNOWLEDGE_BASE.id, + knowledgeBaseName: KNOWLEDGE_BASE.name, + added: [{ documentId: 'document-1', filename: 'report.pdf' }], + failed: [], + cancelled: true, + } + }) + + await expect( + knowledgeBaseServerTool.execute( + { + operation: 'add_file', + args: { knowledgeBaseId: KNOWLEDGE_BASE.id, filePaths: ['files/report.pdf'] }, + }, + { ...CONTEXT, userStopSignal: controller.signal } + ) + ).rejects.toThrow('Request aborted before knowledge mutation could be applied') + + expect(mockAddWorkspaceFiles).toHaveBeenCalledOnce() }) }) diff --git a/apps/sim/lib/copilot/tools/server/knowledge/knowledge-base.ts b/apps/sim/lib/copilot/tools/server/knowledge/knowledge-base.ts index 924e77186d0..1f6de8b8938 100644 --- a/apps/sim/lib/copilot/tools/server/knowledge/knowledge-base.ts +++ b/apps/sim/lib/copilot/tools/server/knowledge/knowledge-base.ts @@ -1,20 +1,15 @@ -import { db } from '@sim/db' -import { knowledgeConnector } from '@sim/db/schema' import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' -import { generateId } from '@sim/utils/id' -import { filterUndefined } from '@sim/utils/object' import { truncate } from '@sim/utils/string' -import { and, eq, isNull } from 'drizzle-orm' import { assertBillingAttributionSnapshot, type BillingAttributionSnapshot, } from '@/lib/billing/core/billing-attribution' import { + executeCopilotKnowledgeUseCase, messageForCopilotKnowledgeError, - resolveCopilotKnowledgePrincipal, + requireCopilotKnowledgeWorkspaceId, } from '@/lib/copilot/application/execute-knowledge-use-case' -import { resolveCopilotFilePrincipal } from '@/lib/copilot/auth/file-delegation' import { KnowledgeBase } from '@/lib/copilot/generated/tool-catalog-v1' import { projectToolErrorMessageForCopilot } from '@/lib/copilot/request/tools/resolved-secret-result' import { @@ -23,54 +18,38 @@ import { type ServerToolContext, } from '@/lib/copilot/tools/server/base-tool' import { projectServerToolModelInput } from '@/lib/copilot/tools/server/model-input' -import { - asOrchestrationError, - messageForOrchestrationError, - type OrchestrationErrorCode, -} from '@/lib/core/orchestration/types' +import { asOrchestrationError } from '@/lib/core/orchestration/types' import { PlatformEvents } from '@/lib/core/telemetry' +import { addWorkspaceFilesToKnowledgeBase } from '@/lib/knowledge/application/add-workspace-files' +import { MAX_KNOWLEDGE_BATCH_ITEMS } from '@/lib/knowledge/application/batch-policy' import { KnowledgeUsageLimitExceededError } from '@/lib/knowledge/application/billing' import { - deleteKnowledgeDocument, - uploadKnowledgeDocument, + createKnowledgeConnector, + deleteKnowledgeConnector, + syncKnowledgeConnector, + updateKnowledgeConnector, +} from '@/lib/knowledge/application/connectors' +import { + bulkDeleteKnowledgeDocuments, + updateKnowledgeDocument, } from '@/lib/knowledge/application/documents' import { + bulkDeleteKnowledgeBases, createKnowledgeBase, - deleteKnowledgeBaseOperation, readKnowledgeBase, updateKnowledgeBaseOperation, } from '@/lib/knowledge/application/knowledge-bases' import { searchKnowledge } from '@/lib/knowledge/application/search' import { - performCreateKnowledgeConnector, - performDeleteKnowledgeConnector, - performSyncKnowledgeConnector, - performUpdateKnowledgeConnector, - performUpdateKnowledgeDocument, -} from '@/lib/knowledge/orchestration' -import { - createTagDefinition, - deleteTagDefinition, - getDocumentTagDefinitions, - getNextAvailableSlot, - getTagDefinitionById, - getTagUsageStats, - updateTagDefinition, -} from '@/lib/knowledge/tags/service' + createKnowledgeTag, + deleteKnowledgeTag, + listKnowledgeTags, + readKnowledgeTagUsage, + updateKnowledgeTag, +} from '@/lib/knowledge/application/tags' import { captureServerEvent } from '@/lib/posthog/server' -import { StorageService } from '@/lib/uploads' -import { getBoundWorkspaceFileSecretProvenance } from '@/lib/uploads/contexts/workspace/workspace-file-secret-provenance' -import { fileOperations } from '@/lib/workspace-files/application/operations' -import { resolveWorkspaceFileReference } from '@/lib/workspace-files/application/resolve-workspace-file-reference' -import { getCredential } from '@/app/api/auth/oauth/utils' -import { - checkDocumentWriteAccess, - checkKnowledgeBaseAccess, - checkKnowledgeBaseWriteAccess, -} from '@/app/api/knowledge/utils' const logger = createLogger('KnowledgeBaseServerTool') -const MAX_COPILOT_KNOWLEDGE_BATCH_SIZE = 100 function requireKnowledgeBillingAttribution( context: ServerToolContext, @@ -86,20 +65,7 @@ function requireKnowledgeBillingAttribution( return attribution } -/** - * The message the agent — and therefore the user — is shown for a failed - * operation. Mirrors `messageForOrchestrationError` on the HTTP surfaces: a - * classified failure is caller-fixable and safe to relay, an unclassified one - * carries whatever text the fault happened to have (a driver's failed SQL, say) - * and is replaced by the operation's own wording. - */ -function agentFacingError( - outcome: { error?: string; errorCode?: OrchestrationErrorCode }, - fallback: string -): string { - return messageForOrchestrationError(outcome, fallback) -} - +/** Records the existing Copilot product analytics after application success. */ function captureKnowledgeBaseCreated( userId: string, workspaceId: string, @@ -125,48 +91,133 @@ function captureKnowledgeBaseCreated( ) } -function captureKnowledgeDocumentUploaded( +function captureKnowledgeDocumentsUploaded( userId: string, workspaceId: string, knowledgeBaseId: string, - document: { mimeType: string; fileSize: number } + documents: readonly { mimeType: string; fileSize: number }[] +): void { + for (const document of documents) { + PlatformEvents.knowledgeBaseDocumentsUploaded({ + knowledgeBaseId, + documentsCount: 1, + uploadType: 'single', + mimeType: document.mimeType, + fileSize: document.fileSize, + }) + captureServerEvent( + userId, + 'knowledge_base_document_uploaded', + { + knowledge_base_id: knowledgeBaseId, + workspace_id: workspaceId, + document_count: 1, + upload_type: 'single', + }, + { + groups: { workspace: workspaceId }, + setOnce: { first_document_uploaded_at: new Date().toISOString() }, + } + ) + } +} + +function captureKnowledgeDocumentsDeleted( + userId: string, + workspaceId: string, + knowledgeBaseId: string, + count: number +): void { + for (let index = 0; index < count; index += 1) { + captureServerEvent( + userId, + 'knowledge_base_document_deleted', + { knowledge_base_id: knowledgeBaseId, workspace_id: workspaceId }, + { groups: { workspace: workspaceId } } + ) + } +} + +function captureKnowledgeConnectorAdded( + userId: string, + workspaceId: string, + knowledgeBaseId: string, + connectorType: string, + syncIntervalMinutes: number ): void { - PlatformEvents.knowledgeBaseDocumentsUploaded({ - knowledgeBaseId, - documentsCount: 1, - uploadType: 'single', - mimeType: document.mimeType, - fileSize: document.fileSize, - }) captureServerEvent( userId, - 'knowledge_base_document_uploaded', + 'knowledge_base_connector_added', { knowledge_base_id: knowledgeBaseId, workspace_id: workspaceId, - document_count: 1, - upload_type: 'single', + connector_type: connectorType, + sync_interval_minutes: syncIntervalMinutes, }, { groups: { workspace: workspaceId }, - setOnce: { first_document_uploaded_at: new Date().toISOString() }, + setOnce: { first_connector_added_at: new Date().toISOString() }, } ) } -function captureKnowledgeDocumentDeleted( +function captureKnowledgeConnectorRemoved( userId: string, workspaceId: string, - knowledgeBaseId: string + knowledgeBaseId: string, + connectorType: string, + documentsDeleted: number ): void { captureServerEvent( userId, - 'knowledge_base_document_deleted', - { knowledge_base_id: knowledgeBaseId, workspace_id: workspaceId }, + 'knowledge_base_connector_removed', + { + knowledge_base_id: knowledgeBaseId, + workspace_id: workspaceId, + connector_type: connectorType, + documents_deleted: documentsDeleted, + }, { groups: { workspace: workspaceId } } ) } +function captureKnowledgeConnectorSynced( + userId: string, + workspaceId: string, + knowledgeBaseId: string, + connectorType: string +): void { + captureServerEvent( + userId, + 'knowledge_base_connector_synced', + { + knowledge_base_id: knowledgeBaseId, + workspace_id: workspaceId, + connector_type: connectorType, + }, + { groups: { workspace: workspaceId } } + ) +} + +function applicationFailureFallback(operation: string): string | null { + switch (operation) { + case 'update_document': + return 'Failed to update document' + case 'create_tag': + return 'Failed to create tag' + case 'add_connector': + return 'Failed to add connector' + case 'update_connector': + return 'Failed to update connector' + case 'delete_connector': + return 'Failed to delete connector' + case 'sync_connector': + return 'Failed to sync connector' + default: + return null + } +} + type KnowledgeBaseArgs = { operation: string args?: Record @@ -188,26 +239,13 @@ export const knowledgeBaseServerTool: BaseServerTool { if (!context) throw new Error('Knowledge delegation requires a Copilot execution context') - const principal = resolveCopilotKnowledgePrincipal(context) const { operation, args = {} } = params - const workspaceId = principal.workspaceId + const workspaceId = requireCopilotKnowledgeWorkspaceId(context) const assertNotAborted = () => assertServerToolNotAborted( context, 'Request aborted before knowledge mutation could be applied.' ) - /** - * The acting agent, as every knowledge orchestration function expects it. - * `source: 'agent'` is what makes an agent-driven mutation distinguishable in - * the audit log — before these operations went through orchestration they - * were not recorded there at all. - */ - const actor = (requestId: string) => ({ - userId: context.userId as string, - source: 'agent' as const, - requestId, - }) - try { switch (operation) { case 'create': { @@ -226,16 +264,17 @@ export const knowledgeBaseServerTool: BaseServerTool MAX_COPILOT_KNOWLEDGE_BATCH_SIZE) { + if (fileRefs.length > MAX_KNOWLEDGE_BATCH_ITEMS) { return { success: false, - message: `Too many files (${fileRefs.length}). Maximum is ${MAX_COPILOT_KNOWLEDGE_BATCH_SIZE}.`, + message: `Too many files (${fileRefs.length}). Maximum is ${MAX_KNOWLEDGE_BATCH_ITEMS}.`, } } - const { knowledgeBase: targetKb } = await readKnowledgeBase.execute({ - principal, - input: { + assertNotAborted() + const outcome = await executeCopilotKnowledgeUseCase( + context, + addWorkspaceFilesToKnowledgeBase, + { knowledgeBaseId: args.knowledgeBaseId, assertedWorkspaceId: workspaceId, - }, - }) - - const added: Array<{ documentId: string; filename: string }> = [] - const failedFiles: string[] = [] - const filePrincipal = resolveCopilotFilePrincipal(context) - - for (const fileRef of fileRefs) { - let fileRecord - try { - fileRecord = await resolveWorkspaceFileReference({ - principal: filePrincipal, - operation: fileOperations.readContent, - workspaceId, - reference: fileRef, - }) - } catch (error) { - const classified = asOrchestrationError(error) - if (classified && classified.code !== 'internal') { - failedFiles.push(fileRef) - continue - } - throw error - } - - const fileProvenance = await getBoundWorkspaceFileSecretProvenance(workspaceId, { - fileId: fileRecord.id, - key: fileRecord.key, - context: 'workspace', - }) - if (fileProvenance.status !== 'exact' || fileProvenance.entries.length > 0) { - failedFiles.push(fileRef) - continue - } - - const presignedUrl = await StorageService.generatePresignedDownloadUrl( - fileRecord.key, - 'workspace', - 5 * 60 - ) - - assertNotAborted() - try { - const outcome = await uploadKnowledgeDocument.execute({ - principal, - input: { - knowledgeBaseId: args.knowledgeBaseId, - assertedWorkspaceId: workspaceId, - document: { - filename: fileRecord.name, - fileUrl: presignedUrl, - fileSize: fileRecord.size, - mimeType: fileRecord.type, - }, - startProcessing: true, - source: 'agent', - }, - }) - captureKnowledgeDocumentUploaded( - context.userId, - workspaceId, - args.knowledgeBaseId, - outcome.document - ) - added.push({ documentId: outcome.document.id, filename: fileRecord.name }) - } catch (error) { - if (error instanceof KnowledgeUsageLimitExceededError) { - return { success: false, message: error.message } - } - const classified = asOrchestrationError(error) - if (classified && classified.code !== 'internal') { - failedFiles.push(fileRef) - continue - } - throw error + fileReferences: fileRefs, + cancellationSignal: context.userStopSignal, + source: 'agent', } - } + ) + captureKnowledgeDocumentsUploaded( + context.userId, + workspaceId, + outcome.knowledgeBaseId, + outcome.added + ) + assertNotAborted() - const addedNames = added.map((a) => a.filename).join(', ') + const added = outcome.added.map(({ documentId, filename }) => ({ + documentId, + filename, + })) + const addedNames = added.map((item) => item.filename).join(', ') return { success: added.length > 0, message: added.length > 0 - ? `Added ${added.length} file(s) to "${targetKb.name}": ${addedNames}. Processing started.` + ? `Added ${added.length} file(s) to "${outcome.knowledgeBaseName}": ${addedNames}. Processing started.` : `No files could be added.`, data: { knowledgeBaseId: args.knowledgeBaseId, - knowledgeBaseName: targetKb.name, + knowledgeBaseName: outcome.knowledgeBaseName, added, - failed: failedFiles, + failed: outcome.failed, }, } } @@ -510,15 +484,16 @@ export const knowledgeBaseServerTool: BaseServerTool MAX_COPILOT_KNOWLEDGE_BATCH_SIZE) { + if (kbIds.length > MAX_KNOWLEDGE_BATCH_ITEMS) { return { success: false, - message: `Too many knowledge base IDs (${kbIds.length}). Maximum is ${MAX_COPILOT_KNOWLEDGE_BATCH_SIZE}.`, + message: `Too many knowledge base IDs (${kbIds.length}). Maximum is ${MAX_KNOWLEDGE_BATCH_ITEMS}.`, } } - const deleted: Array<{ id: string; name: string }> = [] - const notFound: string[] = [] - // A knowledge base that exists but could not be archived is neither - // deleted nor missing. Folding it into `notFound` told the user it was - // never there instead of why the delete failed. - const failed: Array<{ id: string; name: string; reason: string }> = [] - - for (const kbId of kbIds) { - let knowledgeBaseName = kbId - try { - const readResult = await readKnowledgeBase.execute({ - principal, - input: { knowledgeBaseId: kbId, assertedWorkspaceId: workspaceId }, - }) - knowledgeBaseName = readResult.knowledgeBase.name - assertNotAborted() - const deletedKnowledgeBase = await deleteKnowledgeBaseOperation.execute({ - principal, - input: { - knowledgeBaseId: kbId, - assertedWorkspaceId: workspaceId, - source: 'agent', - }, - }) - PlatformEvents.knowledgeBaseDeleted({ knowledgeBaseId: kbId }) - deleted.push(deletedKnowledgeBase) - } catch (error) { - const classified = asOrchestrationError(error) - if ( - classified?.code === 'not_found' || - classified?.code === 'forbidden' || - classified?.code === 'unauthorized' - ) { - notFound.push(kbId) - } else if (classified && classified.code !== 'internal') { - failed.push({ - id: kbId, - name: knowledgeBaseName, - reason: classified.message, - }) - } else { - throw error - } + assertNotAborted() + const { deleted, notFound, failed } = await executeCopilotKnowledgeUseCase( + context, + bulkDeleteKnowledgeBases, + { + assertedWorkspaceId: workspaceId, + knowledgeBaseIds: kbIds, + cancellationSignal: context.userStopSignal, + source: 'agent', } - } + ) + assertNotAborted() const deleteSummary = [ deleted.length > 0 ? `Deleted: ${deleted.map((d) => d.name).join(', ')}` : null, @@ -622,44 +564,34 @@ export const knowledgeBaseServerTool: BaseServerTool MAX_COPILOT_KNOWLEDGE_BATCH_SIZE) { + if (docIds.length > MAX_KNOWLEDGE_BATCH_ITEMS) { return { success: false, - message: `Too many document IDs (${docIds.length}). Maximum is ${MAX_COPILOT_KNOWLEDGE_BATCH_SIZE}.`, + message: `Too many document IDs (${docIds.length}). Maximum is ${MAX_KNOWLEDGE_BATCH_ITEMS}.`, } } - const deleted: string[] = [] - const failed: string[] = [] - - for (const docId of docIds) { - assertNotAborted() - try { - await deleteKnowledgeDocument.execute({ - principal, - input: { - knowledgeBaseId: args.knowledgeBaseId, - documentId: docId, - assertedWorkspaceId: workspaceId, - source: 'agent', - }, - }) - captureKnowledgeDocumentDeleted(context.userId, workspaceId, args.knowledgeBaseId) - deleted.push(docId) - } catch (error) { - const classified = asOrchestrationError(error) - if (classified && classified.code !== 'internal') { - failed.push(docId) - continue - } - throw error - } - } + assertNotAborted() + const { knowledgeBaseId, deleted, deletedDocuments, failed } = + await executeCopilotKnowledgeUseCase(context, bulkDeleteKnowledgeDocuments, { + knowledgeBaseId: args.knowledgeBaseId, + documentIds: docIds, + assertedWorkspaceId: workspaceId, + cancellationSignal: context.userStopSignal, + source: 'agent', + }) + captureKnowledgeDocumentsDeleted( + context.userId, + workspaceId, + knowledgeBaseId, + deletedDocuments.length + ) + assertNotAborted() return { success: deleted.length > 0, message: `Deleted ${deleted.length} document(s)${failed.length > 0 ? `, ${failed.length} failed` : ''}`, - data: { knowledgeBaseId: args.knowledgeBaseId, deleted, failed }, + data: { knowledgeBaseId, deleted, failed }, } } @@ -683,35 +615,14 @@ export const knowledgeBaseServerTool: BaseServerTool = { ...(args.sourceConfig ?? {}) } if (args.disabledTagIds?.length) { sourceConfig.disabledTagIds = args.disabledTagIds } - const requestId = generateId().slice(0, 8) assertNotAborted() - const outcome = await performCreateKnowledgeConnector({ - ...actor(requestId), - knowledgeBase: { - id: args.knowledgeBaseId, - name: writeAccess.knowledgeBase.name, - workspaceId: connectorWorkspaceId, - }, - connectorType: args.connectorType, - credentialId: args.credentialId, - apiKey: args.apiKey, - sourceConfig, - syncIntervalMinutes: args.syncIntervalMinutes ?? 1440, - resolveBillingAttribution: async () => billingAttribution, - resolveAccessToken: async (credentialId) => - (await getCredential(requestId, credentialId, context.userId as string)) - ?.accessToken ?? null, - }) - if (!outcome.success) { - return { success: false, message: agentFacingError(outcome, 'Failed to add connector') } - } - - const connector = outcome.connector + const { connector, workspaceId: canonicalWorkspaceId } = + await executeCopilotKnowledgeUseCase(context, createKnowledgeConnector, { + knowledgeBaseId: args.knowledgeBaseId, + assertedWorkspaceId: workspaceId, + connectorType: args.connectorType, + credentialId: args.credentialId, + apiKey: args.apiKey, + sourceConfig, + syncIntervalMinutes: args.syncIntervalMinutes ?? 1440, + resolveBillingAttribution: async (billingWorkspaceId) => + requireKnowledgeBillingAttribution(context, billingWorkspaceId), + source: 'agent', + }) + captureKnowledgeConnectorAdded( + context.userId, + canonicalWorkspaceId, + connector.knowledgeBaseId, + connector.connectorType, + connector.syncIntervalMinutes + ) return { success: true, message: `Connector "${args.connectorType}" added to knowledge base. Initial sync started.`, @@ -1039,7 +878,7 @@ export const knowledgeBaseServerTool: BaseServerTool billingAttribution, + assertedWorkspaceId: workspaceId, + resolveBillingAttribution: async (canonicalWorkspaceId) => + requireKnowledgeBillingAttribution(context, canonicalWorkspaceId), + source: 'agent', }) - if (!outcome.success) { - return { - success: false, - message: agentFacingError(outcome, 'Failed to sync connector'), - } - } + captureKnowledgeConnectorSynced( + context.userId, + outcome.workspaceId, + outcome.knowledgeBaseId, + outcome.connectorType + ) return { success: true, @@ -1221,40 +1004,39 @@ export const knowledgeBaseServerTool: BaseServerTool { - const rows = await db - .select({ knowledgeBaseId: knowledgeConnector.knowledgeBaseId }) - .from(knowledgeConnector) - .where( - and( - eq(knowledgeConnector.id, connectorId), - isNull(knowledgeConnector.archivedAt), - isNull(knowledgeConnector.deletedAt) - ) - ) - .limit(1) - - return rows[0]?.knowledgeBaseId ?? null -} diff --git a/apps/sim/lib/copilot/vfs/workspace-vfs.ts b/apps/sim/lib/copilot/vfs/workspace-vfs.ts index 1458b9249d4..60f9f033c6e 100644 --- a/apps/sim/lib/copilot/vfs/workspace-vfs.ts +++ b/apps/sim/lib/copilot/vfs/workspace-vfs.ts @@ -5,8 +5,6 @@ import { chat as chatTable, customTools as customToolsTable, folder as folderTable, - knowledgeBaseTagDefinitions, - knowledgeConnector, mcpServers as mcpServersTable, skill as skillTable, workflowDeploymentVersion, @@ -62,11 +60,7 @@ import { canonicalWorkspaceFilePath, encodeVfsPathSegments, } from '@/lib/copilot/vfs/path-utils' -import type { - DeploymentData, - KbTagDefinitionSummary, - VfsServiceAccountAuth, -} from '@/lib/copilot/vfs/serializers' +import type { DeploymentData, VfsServiceAccountAuth } from '@/lib/copilot/vfs/serializers' import { describeServiceAccountForOAuthProvider, serializeApiKeyIntegrations, @@ -117,9 +111,12 @@ import { isOAuthServiceDeploymentAvailable, } from '@/lib/integrations/availability.server' import { createIntegrationCredentialVisibility } from '@/lib/integrations/credential-visibility.server' +import { listKnowledgeConnectors } from '@/lib/knowledge/application/connectors' import { listKnowledgeDocuments } from '@/lib/knowledge/application/documents' -import { listKnowledgeBases } from '@/lib/knowledge/application/knowledge-bases' -import { getKnowledgeBases as getLegacyKnowledgeBases } from '@/lib/knowledge/service' +import { + listArchivedKnowledgeBases, + listKnowledgeBaseCatalog, +} from '@/lib/knowledge/application/knowledge-bases' import { validateMermaidSource } from '@/lib/mermaid/validate' import { isBlockTypeAccessControlExempt } from '@/lib/permission-groups/block-access' import { intersectIntegrationAllowlists } from '@/lib/permission-groups/integration-allowlist' @@ -851,7 +848,7 @@ export class WorkspaceVFS { this.files.set('WORKSPACE.md', buildWorkspaceMd(workspaceMdData)) this.files.set('WORKSPACE_CONTEXT.md', buildWorkspaceContextMd(workspaceMdData)) - await timed('recently_deleted', this.materializeRecentlyDeleted(workspaceId, userId)) + await timed('recently_deleted', this.materializeRecentlyDeleted(workspaceId)) // Per-viewer gating happens HERE, not in the shared builder: files // owned by blocks hidden for this viewer are skipped at stamp time. @@ -1064,6 +1061,13 @@ export class WorkspaceVFS { return this.filePrincipal } + private requireKnowledgePrincipal(): Principal { + if (!this.knowledgePrincipal) { + throw new Error('Workspace Knowledge reads require a trusted Copilot principal') + } + return this.knowledgePrincipal + } + /** * Renders a renderable doc (pptx/docx/pdf) record to a contact-sheet image and * returns it as a model readable JPEG attachment. Shared by the `/render` and @@ -1747,18 +1751,13 @@ export class WorkspaceVFS { private async materializeKnowledgeBases( workspaceId: string ): Promise { - if (!this.knowledgePrincipal) { - throw new Error('Workspace VFS knowledge materialization requires a trusted principal') - } - const { knowledgeBases } = await listKnowledgeBases.execute({ - principal: this.knowledgePrincipal, + const { knowledgeBases } = await listKnowledgeBaseCatalog.execute({ + principal: this.requireKnowledgePrincipal(), input: { workspaceId }, }) const kbs = knowledgeBases.map(({ knowledgeBase }) => knowledgeBase) - const tagDefinitionsByKb = await this.loadKbTagDefinitions(kbs.map((kb) => kb.id)) - - for (const kb of kbs) { + for (const { knowledgeBase: kb, tagDefinitions } of knowledgeBases) { const safeName = sanitizeName(kb.name) const prefix = `knowledgebases/${safeName}/` @@ -1775,7 +1774,11 @@ export class WorkspaceVFS { updatedAt: kb.updatedAt, documentCount: kb.docCount, connectorTypes: kb.connectorTypes, - tagDefinitions: tagDefinitionsByKb.get(kb.id), + tagDefinitions: tagDefinitions.map((definition) => ({ + tagName: definition.displayName, + tagSlot: definition.tagSlot, + fieldType: definition.fieldType, + })), }) ) @@ -1784,9 +1787,6 @@ export class WorkspaceVFS { // a read/glob, only when the artifact is read or grepped. if (kb.docCount > 0) { this.registerLazy(`${prefix}documents.json`, async () => { - if (!this.knowledgePrincipal) { - throw new Error('Workspace VFS knowledge document read requires a trusted principal') - } if (kb.docCount > MAX_VFS_KNOWLEDGE_DOCUMENTS) { throw new Error( `Knowledge base ${kb.id} has more than ${MAX_VFS_KNOWLEDGE_DOCUMENTS} documents; documents.json cannot be materialized` @@ -1797,7 +1797,7 @@ export class WorkspaceVFS { let offset = 0 while (true) { const page = await listKnowledgeDocuments.execute({ - principal: this.knowledgePrincipal, + principal: this.requireKnowledgePrincipal(), input: { knowledgeBaseId: kb.id, assertedWorkspaceId: workspaceId, @@ -1831,28 +1831,10 @@ export class WorkspaceVFS { if (kb.connectorTypes.length > 0) { this.registerLazy(`${prefix}connectors.json`, async () => { - const connectorRows = await db - .select({ - id: knowledgeConnector.id, - connectorType: knowledgeConnector.connectorType, - status: knowledgeConnector.status, - syncMode: knowledgeConnector.syncMode, - syncIntervalMinutes: knowledgeConnector.syncIntervalMinutes, - lastSyncAt: knowledgeConnector.lastSyncAt, - lastSyncError: knowledgeConnector.lastSyncError, - lastSyncDocCount: knowledgeConnector.lastSyncDocCount, - nextSyncAt: knowledgeConnector.nextSyncAt, - consecutiveFailures: knowledgeConnector.consecutiveFailures, - createdAt: knowledgeConnector.createdAt, - }) - .from(knowledgeConnector) - .where( - and( - eq(knowledgeConnector.knowledgeBaseId, kb.id), - isNull(knowledgeConnector.archivedAt), - isNull(knowledgeConnector.deletedAt) - ) - ) + const { connectors: connectorRows } = await listKnowledgeConnectors.execute({ + principal: this.requireKnowledgePrincipal(), + input: { knowledgeBaseId: kb.id, assertedWorkspaceId: workspaceId }, + }) return connectorRows.length > 0 ? serializeConnectors(connectorRows) : null }) } @@ -1866,61 +1848,6 @@ export class WorkspaceVFS { })) } - /** - * Load tag definitions for the given knowledge bases in a single query, grouped by - * KB id and ordered by tag slot. Surfaced inline in each KB's meta.json so the agent - * knows which tags exist (and their slot binding) when editing a knowledge-tag filter. - * - * @remarks - * Tag definitions are an optional enrichment, so a query failure degrades to a meta.json - * without them rather than rejecting. This materializer runs inside the top-level - * `Promise.all`, whose rejection would fail the entire workspace VFS build and leave the - * agent unable to read any file. - */ - private async loadKbTagDefinitions( - kbIds: string[] - ): Promise> { - const byKb = new Map() - if (kbIds.length === 0) return byKb - - let rows: Array<{ - knowledgeBaseId: string - tagSlot: string - displayName: string - fieldType: string - }> - try { - rows = await db - .select({ - knowledgeBaseId: knowledgeBaseTagDefinitions.knowledgeBaseId, - tagSlot: knowledgeBaseTagDefinitions.tagSlot, - displayName: knowledgeBaseTagDefinitions.displayName, - fieldType: knowledgeBaseTagDefinitions.fieldType, - }) - .from(knowledgeBaseTagDefinitions) - .where(inArray(knowledgeBaseTagDefinitions.knowledgeBaseId, kbIds)) - .orderBy(knowledgeBaseTagDefinitions.tagSlot) - } catch (err) { - logger.warn('Failed to load knowledge base tag definitions', { - error: toError(err).message, - }) - return byKb - } - - for (const row of rows) { - const entry = { - tagName: row.displayName, - tagSlot: row.tagSlot, - fieldType: row.fieldType, - } - const existing = byKb.get(row.knowledgeBaseId) - if (existing) existing.push(entry) - else byKb.set(row.knowledgeBaseId, [entry]) - } - - return byKb - } - /** * Materialize tables using the shared listTables function. * Returns a summary for WORKSPACE.md generation. @@ -2331,7 +2258,7 @@ export class WorkspaceVFS { return [] } } - private async materializeRecentlyDeleted(workspaceId: string, userId: string): Promise { + private async materializeRecentlyDeleted(workspaceId: string): Promise { try { const [ archivedWorkflows, @@ -2369,7 +2296,12 @@ export class WorkspaceVFS { input: { workspaceId, scope: 'archived' }, }) .then(({ folders }) => folders), - getLegacyKnowledgeBases(userId, workspaceId, 'archived'), + listArchivedKnowledgeBases + .execute({ + principal: this.requireKnowledgePrincipal(), + input: { workspaceId }, + }) + .then(({ knowledgeBases }) => knowledgeBases), ]) for (const wf of archivedWorkflows) { diff --git a/apps/sim/lib/credentials/service-account-secret.ts b/apps/sim/lib/credentials/service-account-secret.ts index d6b678d4a17..a2f6dfa8334 100644 --- a/apps/sim/lib/credentials/service-account-secret.ts +++ b/apps/sim/lib/credentials/service-account-secret.ts @@ -95,7 +95,7 @@ async function buildAtlassianServiceAccountSecret( ...(validation.emailAddress ? { label: validation.emailAddress } : {}), } // `atlassianAccountId` stays at the blob's top level: `getAtlassianServiceAccountSecret` - // in `app/api/auth/oauth/utils.ts` reads it there on every existing credential. + // in `lib/oauth/credential-service.ts` reads it there on every existing credential. const blob = JSON.stringify({ type: ATLASSIAN_SERVICE_ACCOUNT_SECRET_TYPE, apiToken, diff --git a/apps/sim/lib/guardrails/validate_hallucination.ts b/apps/sim/lib/guardrails/validate_hallucination.ts index d97cf94bf8a..f61b0c7ee13 100644 --- a/apps/sim/lib/guardrails/validate_hallucination.ts +++ b/apps/sim/lib/guardrails/validate_hallucination.ts @@ -20,7 +20,7 @@ import { RESOLVED_SECRET_PROVENANCE_FIELD, RESOLVED_SECRET_PROVENANCE_METADATA_V1, } from '@/lib/execution/private-tool-metadata' -import { refreshTokenIfNeeded } from '@/app/api/auth/oauth/utils' +import { refreshTokenIfNeeded } from '@/lib/oauth/credential-service' import { projectResolvedSecretModelContent } from '@/executor/utils/resolved-secret-content-projection' import type { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry' import { executeProviderRequest } from '@/providers' diff --git a/apps/sim/lib/knowledge/api/internal-route.ts b/apps/sim/lib/knowledge/api/internal-route.ts new file mode 100644 index 00000000000..b2a5e881b54 --- /dev/null +++ b/apps/sim/lib/knowledge/api/internal-route.ts @@ -0,0 +1,389 @@ +import { + type Principal, + requirePrincipalSubjectUserId, + type SessionPrincipal, +} from '@sim/auth/principal' +import type { NextRequest } from 'next/server' +import type { KnowledgeBaseData } from '@/lib/api/contracts/knowledge/base' +import { type ChunkData, chunkDataSchema } from '@/lib/api/contracts/knowledge/chunks' +import { + type ConnectorData, + type ConnectorDetailData, + connectorDataSchema, + connectorDetailDataSchema, +} from '@/lib/api/contracts/knowledge/connectors' +import { type DocumentData, documentDataSchema } from '@/lib/api/contracts/knowledge/documents' +import { type TagDefinitionData, tagDefinitionDataSchema } from '@/lib/api/contracts/knowledge/tags' +import { AuthType, type AuthTypeValue } from '@/lib/auth/hybrid' +import { + requireBillingAttributionHeader, + resolveBillingAttribution, +} from '@/lib/billing/core/billing-attribution' +import { PlatformEvents } from '@/lib/core/telemetry' +import type { + CreateKnowledgeBaseInput, + InternalKnowledgeBaseResult, + KnowledgeBaseResult, +} from '@/lib/knowledge/application/knowledge-bases' +import type { CreatedKnowledgeDocument } from '@/lib/knowledge/orchestration/documents' +import type { KnowledgeBaseWithCounts } from '@/lib/knowledge/types' +import { captureServerEvent } from '@/lib/posthog/server' +import type { UploadSessionRecord } from '@/lib/uploads/upload-session/service' + +export function internalKnowledgeActorUserId(principal: Principal): string { + return requirePrincipalSubjectUserId(principal) +} + +export function internalKnowledgeAuthType(principal: Principal): AuthTypeValue { + return principal.kind === 'delegated' ? AuthType.INTERNAL_JWT : AuthType.SESSION +} + +export async function resolveInternalKnowledgeBillingAttribution( + request: NextRequest, + principal: Principal, + workspaceId: string +) { + const actorUserId = internalKnowledgeActorUserId(principal) + return await (principal.kind === 'delegated' + ? requireBillingAttributionHeader(request.headers, { actorUserId, workspaceId }) + : resolveBillingAttribution({ actorUserId, workspaceId })) +} + +function serializeDate(date: Date | string): string { + return date instanceof Date ? date.toISOString() : date +} + +function serializeNullableDate(date: Date | string | null): string | null { + return date ? serializeDate(date) : null +} + +export function toInternalKnowledgeDocument< + T extends { + uploadedAt: Date | string + processingStartedAt?: Date | string | null + processingCompletedAt?: Date | string | null + date1?: Date | string | null + date2?: Date | string | null + }, +>(document: T): DocumentData { + return documentDataSchema.parse({ + ...document, + uploadedAt: serializeDate(document.uploadedAt), + processingStartedAt: serializeNullableDate(document.processingStartedAt ?? null), + processingCompletedAt: serializeNullableDate(document.processingCompletedAt ?? null), + date1: serializeNullableDate(document.date1 ?? null), + date2: serializeNullableDate(document.date2 ?? null), + }) +} + +export function toInternalKnowledgeChunk< + T extends { createdAt: Date | string; updatedAt: Date | string }, +>(chunk: T): ChunkData { + return chunkDataSchema.parse({ + ...chunk, + createdAt: serializeDate(chunk.createdAt), + updatedAt: serializeDate(chunk.updatedAt), + }) +} + +export function toInternalKnowledgeTag< + T extends { createdAt: Date | string; updatedAt: Date | string }, +>(tag: T): TagDefinitionData { + return tagDefinitionDataSchema.parse({ + ...tag, + createdAt: serializeDate(tag.createdAt), + updatedAt: serializeDate(tag.updatedAt), + }) +} + +export function toInternalKnowledgeConnector< + T extends { + sourceConfig: unknown + createdAt: Date | string + updatedAt: Date | string + lastSyncAt: Date | string | null + nextSyncAt: Date | string | null + }, +>(connector: T): ConnectorData { + return connectorDataSchema.parse({ + ...connector, + createdAt: serializeDate(connector.createdAt), + updatedAt: serializeDate(connector.updatedAt), + lastSyncAt: serializeNullableDate(connector.lastSyncAt), + nextSyncAt: serializeNullableDate(connector.nextSyncAt), + }) +} + +export function toInternalKnowledgeConnectorDetail< + T extends Parameters[0] & { + syncLogs: Array<{ + startedAt: Date | string + completedAt: Date | string | null + [key: string]: unknown + }> + }, +>(connector: T): ConnectorDetailData { + return connectorDetailDataSchema.parse({ + ...toInternalKnowledgeConnector(connector), + syncLogs: connector.syncLogs.map((log) => ({ + ...log, + startedAt: serializeDate(log.startedAt), + completedAt: serializeNullableDate(log.completedAt), + })), + }) +} + +export function toInternalKnowledgeDocumentUpload( + session: UploadSessionRecord, + document: CreatedKnowledgeDocument | null +) { + if (!session.knowledgeBaseId) { + throw new Error('Knowledge-document upload session is missing its knowledge base') + } + return { + id: session.id, + knowledgeBaseId: session.knowledgeBaseId, + status: session.status, + name: session.fileName, + contentType: session.contentType, + size: session.fileSize, + expiresAt: serializeDate(session.expiresAt), + error: session.error, + document: document + ? { + id: document.id, + knowledgeBaseId: document.knowledgeBaseId, + filename: document.filename, + fileSize: document.fileSize, + mimeType: document.mimeType, + processingStatus: document.processingStatus ?? 'pending', + chunkCount: document.chunkCount, + tokenCount: document.tokenCount, + characterCount: document.characterCount, + enabled: document.enabled, + createdAt: serializeNullableDate(document.uploadedAt), + } + : null, + } +} + +function toInternalKnowledgeBase(knowledgeBase: KnowledgeBaseWithCounts): KnowledgeBaseData { + return { + ...knowledgeBase, + chunkingConfig: { ...knowledgeBase.chunkingConfig }, + createdAt: serializeDate(knowledgeBase.createdAt), + updatedAt: serializeDate(knowledgeBase.updatedAt), + deletedAt: knowledgeBase.deletedAt ? serializeDate(knowledgeBase.deletedAt) : null, + } +} + +export const internalKnowledgePresenters = { + list({ knowledgeBases }: { knowledgeBases: KnowledgeBaseWithCounts[] }) { + return { success: true as const, data: knowledgeBases.map(toInternalKnowledgeBase) } + }, + create({ knowledgeBase }: KnowledgeBaseResult) { + return { success: true as const, data: toInternalKnowledgeBase(knowledgeBase) } + }, + read({ knowledgeBase }: InternalKnowledgeBaseResult) { + return { success: true as const, data: toInternalKnowledgeBase(knowledgeBase) } + }, + deleted() { + return { + success: true as const, + data: { message: 'Knowledge base deleted successfully' }, + } + }, +} as const + +export const internalKnowledgeAnalytics = { + created({ + principal, + result: { knowledgeBase }, + }: { + principal: SessionPrincipal + input: CreateKnowledgeBaseInput + result: KnowledgeBaseResult + }): void { + if (!knowledgeBase.workspaceId) { + throw new Error('Created knowledge base is missing its workspace analytics scope') + } + PlatformEvents.knowledgeBaseCreated({ + knowledgeBaseId: knowledgeBase.id, + name: knowledgeBase.name, + workspaceId: knowledgeBase.workspaceId, + }) + captureServerEvent( + principal.userId, + 'knowledge_base_created', + { + knowledge_base_id: knowledgeBase.id, + workspace_id: knowledgeBase.workspaceId, + name: knowledgeBase.name, + }, + { + groups: { workspace: knowledgeBase.workspaceId }, + setOnce: { first_kb_created_at: new Date().toISOString() }, + } + ) + }, + documentsUploaded({ + principal, + input, + result, + }: { + principal: Principal + input: { processingOptions?: { recipe?: string } } + result: + | { + kind: 'single' + workspaceId: string + data: { knowledgeBaseId: string; mimeType: string; fileSize: number } + } + | { kind: 'bulk'; workspaceId: string; data: { total: number }; knowledgeBaseId?: string } + }): void { + const userId = internalKnowledgeActorUserId(principal) + const documentCount = result.kind === 'bulk' ? result.data.total : 1 + const knowledgeBaseId = + result.kind === 'single' ? result.data.knowledgeBaseId : result.knowledgeBaseId + if (!knowledgeBaseId) { + throw new Error('Bulk document result is missing its knowledge base analytics scope') + } + PlatformEvents.knowledgeBaseDocumentsUploaded({ + knowledgeBaseId, + documentsCount: documentCount, + uploadType: result.kind, + ...(result.kind === 'single' + ? { mimeType: result.data.mimeType, fileSize: result.data.fileSize } + : { recipe: input.processingOptions?.recipe }), + }) + captureServerEvent( + userId, + 'knowledge_base_document_uploaded', + { + knowledge_base_id: knowledgeBaseId, + workspace_id: result.workspaceId, + document_count: documentCount, + upload_type: result.kind, + }, + { + groups: { workspace: result.workspaceId }, + setOnce: { first_document_uploaded_at: new Date().toISOString() }, + } + ) + }, + documentUpserted({ + input, + result, + }: { + principal: Principal + input: { processingOptions?: { recipe?: string } } + result: { knowledgeBaseId: string } + }): void { + PlatformEvents.knowledgeBaseDocumentsUploaded({ + knowledgeBaseId: result.knowledgeBaseId, + documentsCount: 1, + uploadType: 'single', + recipe: input.processingOptions?.recipe, + }) + }, + documentDeleted({ + principal, + result, + }: { + principal: Principal + result: { knowledgeBaseId: string; workspaceId?: string } + }): void { + const workspaceId = result.workspaceId + if (!workspaceId) throw new Error('Deleted document result is missing its workspace scope') + captureServerEvent( + internalKnowledgeActorUserId(principal), + 'knowledge_base_document_deleted', + { knowledge_base_id: result.knowledgeBaseId, workspace_id: workspaceId }, + { groups: { workspace: workspaceId } } + ) + }, + connectorAdded({ + principal, + result: { connector, workspaceId }, + }: { + principal: Principal + input: unknown + result: { + workspaceId: string + connector: { + knowledgeBaseId: string + connectorType: string + syncIntervalMinutes: number + } + } + }): void { + captureServerEvent( + internalKnowledgeActorUserId(principal), + 'knowledge_base_connector_added', + { + knowledge_base_id: connector.knowledgeBaseId, + workspace_id: workspaceId, + connector_type: connector.connectorType, + sync_interval_minutes: connector.syncIntervalMinutes, + }, + { + groups: { workspace: workspaceId }, + setOnce: { first_connector_added_at: new Date().toISOString() }, + } + ) + }, + connectorRemoved({ + principal, + result, + }: { + principal: Principal + input: unknown + result: { + knowledgeBaseId: string + connectorType: string + documentsDeleted: number + workspaceId?: string + } + }): void { + if (!result.workspaceId) { + throw new Error('Deleted connector result is missing its workspace analytics scope') + } + captureServerEvent( + internalKnowledgeActorUserId(principal), + 'knowledge_base_connector_removed', + { + knowledge_base_id: result.knowledgeBaseId, + workspace_id: result.workspaceId, + connector_type: result.connectorType, + documents_deleted: result.documentsDeleted, + }, + { groups: { workspace: result.workspaceId } } + ) + }, + connectorSynced({ + principal, + result, + }: { + principal: Principal + input: unknown + result: { + knowledgeBaseId: string + connectorType: string + workspaceId?: string + } + }): void { + if (!result.workspaceId) { + throw new Error('Synced connector result is missing its workspace analytics scope') + } + captureServerEvent( + internalKnowledgeActorUserId(principal), + 'knowledge_base_connector_synced', + { + knowledge_base_id: result.knowledgeBaseId, + workspace_id: result.workspaceId, + connector_type: result.connectorType, + }, + { groups: { workspace: result.workspaceId } } + ) + }, +} as const diff --git a/apps/sim/lib/knowledge/api/route-policies.test.ts b/apps/sim/lib/knowledge/api/route-policies.test.ts new file mode 100644 index 00000000000..e70bc59c2fe --- /dev/null +++ b/apps/sim/lib/knowledge/api/route-policies.test.ts @@ -0,0 +1,52 @@ +/** + * @vitest-environment node + */ + +import { describe, expect, it } from 'vitest' +import { + DelegatedWorkspaceAuthorizationError, + InsufficientWorkspacePermissionsError, + PersonalApiKeysDisabledError, + PrincipalKindAuthorizationError, + WorkspaceApiKeyAuthorizationError, +} from '@/lib/core/application' +import { OrchestrationError } from '@/lib/core/orchestration/types' +import { v2KnowledgeErrorPolicies } from '@/lib/knowledge/api/route-policies' + +describe('v2 knowledge error policies', () => { + it.each([ + new InsufficientWorkspacePermissionsError(), + new WorkspaceApiKeyAuthorizationError(), + new DelegatedWorkspaceAuthorizationError(), + new PrincipalKindAuthorizationError('workspace_api_key', 'knowledge.read'), + ])('conceals canonical resource authorization failures as absence', async (error) => { + const response = v2KnowledgeErrorPolicies.concealKnowledgeBaseAuthorization.render(error) + expect(response?.status).toBe(404) + expect(await response?.json()).toEqual({ + error: { code: 'NOT_FOUND', message: 'Knowledge base not found' }, + }) + }) + + it('preserves the personal-api-key policy failure as forbidden', async () => { + const response = v2KnowledgeErrorPolicies.concealKnowledgeBaseAuthorization.render( + new PersonalApiKeysDisabledError() + ) + expect(response?.status).toBe(403) + expect(await response?.json()).toEqual({ + error: { + code: 'FORBIDDEN', + message: 'Personal API keys are not allowed for this workspace', + }, + }) + }) + + it('does not conceal unrelated forbidden business errors', async () => { + const response = v2KnowledgeErrorPolicies.concealKnowledgeBaseAuthorization.render( + new OrchestrationError('forbidden', 'Knowledge base transition is forbidden') + ) + expect(response?.status).toBe(403) + expect(await response?.json()).toEqual({ + error: { code: 'FORBIDDEN', message: 'Knowledge base transition is forbidden' }, + }) + }) +}) diff --git a/apps/sim/lib/knowledge/api/route-policies.ts b/apps/sim/lib/knowledge/api/route-policies.ts index a832cbf32e5..83e7cc498b0 100644 --- a/apps/sim/lib/knowledge/api/route-policies.ts +++ b/apps/sim/lib/knowledge/api/route-policies.ts @@ -1,10 +1,97 @@ import { + createInternalSessionOrExecutorAuth, createV2ResourceConcealmentPolicy, + type InternalErrorPolicy, + internalErrorResponse, + internalPlainOrchestrationErrorPolicy, + type V2ErrorPolicy, v2OrchestrationErrorPolicy, } from '@/lib/api/server/routes' +import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' +import { KNOWLEDGE_DELEGATION_AUDIENCE } from '@/lib/knowledge/application/authorization' +import { KnowledgeUsageLimitExceededError } from '@/lib/knowledge/application/billing' +import { KnowledgeSearchProvenanceUnavailableError } from '@/lib/knowledge/application/search' +import { KnowledgeDocumentUnsupportedMediaTypeError } from '@/lib/knowledge/application/upload-sessions' +import { v2Error } from '@/app/api/v2/lib/response' + +function internalKnowledgeErrorPolicy(unhandledMessage: string): InternalErrorPolicy { + return { + project: internalPlainOrchestrationErrorPolicy.project, + unhandled: () => internalErrorResponse(500, { error: unhandledMessage }), + } +} + +const internalKnowledgeUploadErrorPolicy: InternalErrorPolicy = { + project(error) { + if (error instanceof KnowledgeDocumentUnsupportedMediaTypeError) { + return internalErrorResponse(415, { error: error.message }) + } + if (error instanceof KnowledgeUsageLimitExceededError) { + return internalErrorResponse(402, { error: error.message }) + } + return internalPlainOrchestrationErrorPolicy.project(error) + }, + unhandled: () => + internalErrorResponse(500, { error: 'Failed to process knowledge upload request' }), +} + +const internalKnowledgeSearchErrorPolicy: InternalErrorPolicy = { + project(error) { + if (error instanceof KnowledgeUsageLimitExceededError) { + return internalErrorResponse(402, { error: error.message }) + } + if (error instanceof KnowledgeSearchProvenanceUnavailableError) { + return internalErrorResponse(422, { error: error.message }) + } + return internalPlainOrchestrationErrorPolicy.project(error) + }, + unhandled: () => internalErrorResponse(500, { error: 'Failed to perform vector search' }), +} + +export const internalKnowledgeSessionOrExecutorAuth = createInternalSessionOrExecutorAuth({ + audience: KNOWLEDGE_DELEGATION_AUDIENCE, +}) + +export const internalKnowledgeErrorPolicies = { + list: internalKnowledgeErrorPolicy('Failed to fetch knowledge bases'), + read: internalKnowledgeErrorPolicy('Failed to fetch knowledge base'), + create: internalKnowledgeErrorPolicy('Failed to create knowledge base'), + update: internalKnowledgeErrorPolicy('Failed to update knowledge base'), + delete: internalKnowledgeErrorPolicy('Failed to delete knowledge base'), + restore: internalKnowledgeErrorPolicy('Internal server error'), + default: internalKnowledgeErrorPolicy('Internal server error'), + documents: internalKnowledgeErrorPolicy('Failed to process knowledge document request'), + chunks: internalKnowledgeErrorPolicy('Failed to process knowledge chunk request'), + upsert: internalKnowledgeUploadErrorPolicy, + search: internalKnowledgeSearchErrorPolicy, + tags: internalKnowledgeErrorPolicy('Failed to process knowledge tag request'), + connectors: internalKnowledgeErrorPolicy('Internal server error'), + uploads: internalKnowledgeUploadErrorPolicy, +} as const + +const v2KnowledgeUsageErrorPolicy = { + render(error) { + if (error instanceof KnowledgeUsageLimitExceededError) { + return v2Error('USAGE_LIMIT_EXCEEDED', error.message) + } + return v2OrchestrationErrorPolicy.render(error) + }, +} satisfies V2ErrorPolicy export const v2KnowledgeErrorPolicies = { default: v2OrchestrationErrorPolicy, + usage: v2KnowledgeUsageErrorPolicy, + documentUpload: { + render(error) { + if (error instanceof KnowledgeDocumentUnsupportedMediaTypeError) { + return v2Error('UNSUPPORTED_MEDIA_TYPE', error.message) + } + if (isPayloadSizeLimitError(error)) { + return v2Error('PAYLOAD_TOO_LARGE', error.message) + } + return v2KnowledgeUsageErrorPolicy.render(error) + }, + } satisfies V2ErrorPolicy, concealKnowledgeBaseAuthorization: createV2ResourceConcealmentPolicy({ notFoundMessage: 'Knowledge base not found', }), diff --git a/apps/sim/lib/knowledge/application/add-workspace-files.test.ts b/apps/sim/lib/knowledge/application/add-workspace-files.test.ts new file mode 100644 index 00000000000..c245ef702ac --- /dev/null +++ b/apps/sim/lib/knowledge/application/add-workspace-files.test.ts @@ -0,0 +1,342 @@ +/** + * @vitest-environment node + */ + +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + resolveKnowledgeBase: vi.fn(), + resolvePermission: vi.fn(), + resolveFile: vi.fn(), + loadFileContext: vi.fn(), + getProvenance: vi.fn(), + presign: vi.fn(), + resolveBilling: vi.fn(), + checkUsage: vi.fn(), + createDocument: vi.fn(), + processQueue: vi.fn(), + recordAudit: vi.fn(), + platformUploaded: vi.fn(), + captureServerEvent: vi.fn(), +})) + +vi.mock('@sim/audit', () => ({ + AuditAction: { DOCUMENT_UPLOADED: 'document.uploaded' }, + AuditResourceType: { DOCUMENT: 'document' }, + recordAudit: mocks.recordAudit, +})) + +vi.mock('@sim/platform-authz/workspace', () => ({ + permissionSatisfies: (actual: string | null, required: string) => { + const rank = { read: 1, write: 2, admin: 3 } as const + return ( + actual !== null && rank[actual as keyof typeof rank] >= rank[required as keyof typeof rank] + ) + }, + resolveEffectiveWorkspacePermission: mocks.resolvePermission, +})) + +vi.mock('@/lib/billing/core/billing-attribution', () => ({ + resolveBillingAttribution: mocks.resolveBilling, + resolveSystemBillingAttribution: mocks.resolveBilling, + checkAttributedUsageLimits: mocks.checkUsage, +})) + +vi.mock('@/lib/core/telemetry', () => ({ + PlatformEvents: { knowledgeBaseDocumentsUploaded: mocks.platformUploaded }, +})) + +vi.mock('@/lib/knowledge/application/contexts', () => ({ + resolveActiveKnowledgeBaseContext: mocks.resolveKnowledgeBase, +})) + +vi.mock('@/lib/knowledge/documents/service', () => ({ + createSingleDocument: mocks.createDocument, + processDocumentsWithQueue: mocks.processQueue, +})) + +vi.mock('@/lib/posthog/server', () => ({ captureServerEvent: mocks.captureServerEvent })) + +vi.mock('@/lib/uploads', () => ({ + StorageService: { generatePresignedDownloadUrl: mocks.presign }, +})) + +vi.mock('@/lib/uploads/contexts/workspace/workspace-file-secret-provenance', () => ({ + getBoundWorkspaceFileSecretProvenance: mocks.getProvenance, +})) + +vi.mock('@/lib/uploads/contexts/workspace/workspace-file-manager', () => ({ + loadActiveWorkspaceFileContext: mocks.loadFileContext, + resolveWorkspaceFileReference: mocks.resolveFile, +})) + +vi.mock('@/lib/uploads/utils/validation', () => ({ validateFileType: () => null })) + +import { OrchestrationError } from '@/lib/core/orchestration/types' +import { addWorkspaceFilesToKnowledgeBase } from '@/lib/knowledge/application/add-workspace-files' + +const knowledgeContext = { + workspaceId: 'workspace-1', + workspaceOrganizationId: null, + allowPersonalApiKeys: true, + billedAccountUserId: 'billing-owner-1', + knowledgeBaseId: 'knowledge-1', + knowledgeBase: { id: 'knowledge-1', name: 'Docs' }, +} + +const workspaceFile = { + id: 'file-1', + workspaceId: 'workspace-1', + key: 'workspace/workspace-1/file-1-report.pdf', + name: 'report.pdf', + path: '/api/files/serve/file-1', + size: 100, + type: 'application/pdf', + uploadedBy: 'user-1', + uploadedAt: new Date('2026-01-01T00:00:00Z'), + updatedAt: new Date('2026-01-01T00:00:00Z'), +} + +const delegatedPrincipal = { + kind: 'delegated', + serviceId: 'copilot', + subjectUserId: 'dual-workspace-user', + workspaceId: 'workspace-1', + delegationId: 'tool-call-1', + audience: 'sim:knowledge', + issuedAt: new Date(), + expiresAt: new Date(Date.now() + 60_000), +} as const + +describe('add workspace files to knowledge base application command', () => { + beforeEach(() => { + vi.clearAllMocks() + mocks.resolveKnowledgeBase.mockResolvedValue(knowledgeContext) + mocks.resolvePermission.mockResolvedValue('write') + mocks.resolveFile.mockResolvedValue(workspaceFile) + mocks.loadFileContext.mockResolvedValue({ + fileId: workspaceFile.id, + workspaceId: 'workspace-1', + workspaceOrganizationId: null, + allowPersonalApiKeys: true, + billedAccountUserId: 'billing-owner-1', + }) + mocks.getProvenance.mockResolvedValue({ status: 'exact', entries: [] }) + mocks.presign.mockResolvedValue('https://storage.test/report.pdf') + mocks.resolveBilling.mockResolvedValue({ + actorUserId: 'dual-workspace-user', + workspaceId: 'workspace-1', + }) + mocks.checkUsage.mockResolvedValue({ isExceeded: false }) + mocks.createDocument.mockResolvedValue({ + id: 'document-1', + filename: workspaceFile.name, + fileUrl: 'https://storage.test/report.pdf', + fileSize: workspaceFile.size, + mimeType: workspaceFile.type, + }) + mocks.processQueue.mockResolvedValue(undefined) + }) + + it('bounds file references before canonical knowledge loading', async () => { + await expect( + addWorkspaceFilesToKnowledgeBase.execute({ + principal: delegatedPrincipal, + input: { + knowledgeBaseId: 'knowledge-1', + assertedWorkspaceId: 'workspace-1', + fileReferences: Array.from({ length: 101 }, (_, index) => `files/file-${index}.pdf`), + }, + }) + ).rejects.toMatchObject({ code: 'validation' }) + + expect(mocks.resolveKnowledgeBase).not.toHaveBeenCalled() + expect(mocks.createDocument).not.toHaveBeenCalled() + }) + + it('authorizes canonical scope and admits usage before document creation', async () => { + await addWorkspaceFilesToKnowledgeBase.execute({ + principal: delegatedPrincipal, + input: { + knowledgeBaseId: 'knowledge-1', + assertedWorkspaceId: 'workspace-1', + fileReferences: ['files/report.pdf'], + source: 'agent', + }, + }) + + expect(mocks.resolvePermission.mock.invocationCallOrder[0]).toBeLessThan( + mocks.resolveFile.mock.invocationCallOrder[0] + ) + expect(mocks.getProvenance.mock.invocationCallOrder[0]).toBeLessThan( + mocks.resolveBilling.mock.invocationCallOrder[0] + ) + expect(mocks.checkUsage.mock.invocationCallOrder[0]).toBeLessThan( + mocks.createDocument.mock.invocationCallOrder[0] + ) + expect(mocks.resolvePermission).toHaveBeenCalledTimes(2) + expect(mocks.createDocument).toHaveBeenCalledWith( + expect.objectContaining({ filename: 'report.pdf' }), + 'knowledge-1', + expect.any(String), + 'dual-workspace-user', + undefined, + undefined, + { expectedWorkspaceId: 'workspace-1' } + ) + }) + + it('conceals a cross-workspace file before provenance, storage, or mutation', async () => { + mocks.loadFileContext.mockResolvedValueOnce({ + fileId: 'workspace-2-file', + workspaceId: 'workspace-2', + workspaceOrganizationId: null, + allowPersonalApiKeys: true, + billedAccountUserId: 'billing-owner-2', + }) + + const result = await addWorkspaceFilesToKnowledgeBase.execute({ + principal: delegatedPrincipal, + input: { + knowledgeBaseId: 'knowledge-1', + assertedWorkspaceId: 'workspace-1', + fileReferences: ['workspace-2-file'], + }, + }) + + expect(result).toMatchObject({ added: [], failed: ['workspace-2-file'] }) + expect(mocks.getProvenance).not.toHaveBeenCalled() + expect(mocks.presign).not.toHaveBeenCalled() + expect(mocks.checkUsage).not.toHaveBeenCalled() + expect(mocks.createDocument).not.toHaveBeenCalled() + expect(mocks.recordAudit).not.toHaveBeenCalled() + }) + + it('returns partial outcomes and keeps product analytics out of the application', async () => { + mocks.resolveFile + .mockResolvedValueOnce(workspaceFile) + .mockRejectedValueOnce(new OrchestrationError('not_found', 'File not found')) + + const result = await addWorkspaceFilesToKnowledgeBase.execute({ + principal: delegatedPrincipal, + input: { + knowledgeBaseId: 'knowledge-1', + assertedWorkspaceId: 'workspace-1', + fileReferences: ['files/report.pdf', 'files/missing.pdf'], + source: 'agent', + }, + }) + + expect(result).toMatchObject({ + added: [{ documentId: 'document-1', filename: 'report.pdf' }], + failed: ['files/missing.pdf'], + cancelled: false, + }) + expect(mocks.recordAudit).toHaveBeenCalledOnce() + expect(mocks.recordAudit).toHaveBeenCalledWith( + expect.objectContaining({ + resourceId: 'document-1', + metadata: expect.objectContaining({ + operation: 'knowledge.documents.add_workspace_files', + }), + }) + ) + expect(mocks.platformUploaded).not.toHaveBeenCalled() + expect(mocks.captureServerEvent).not.toHaveBeenCalled() + }) + + it('stops between document creations while auditing completed items', async () => { + const controller = new AbortController() + mocks.resolveFile + .mockResolvedValueOnce(workspaceFile) + .mockResolvedValueOnce({ ...workspaceFile, id: 'file-2', name: 'second.pdf' }) + mocks.loadFileContext + .mockResolvedValueOnce({ + fileId: 'file-1', + workspaceId: 'workspace-1', + workspaceOrganizationId: null, + allowPersonalApiKeys: true, + billedAccountUserId: 'billing-owner-1', + }) + .mockResolvedValueOnce({ + fileId: 'file-2', + workspaceId: 'workspace-1', + workspaceOrganizationId: null, + allowPersonalApiKeys: true, + billedAccountUserId: 'billing-owner-1', + }) + mocks.createDocument.mockImplementationOnce(async () => { + controller.abort('user stopped') + return { + id: 'document-1', + filename: 'report.pdf', + fileUrl: 'https://storage.test/report.pdf', + fileSize: 100, + mimeType: 'application/pdf', + } + }) + + const result = await addWorkspaceFilesToKnowledgeBase.execute({ + principal: delegatedPrincipal, + input: { + knowledgeBaseId: 'knowledge-1', + assertedWorkspaceId: 'workspace-1', + fileReferences: ['files/report.pdf', 'files/second.pdf'], + cancellationSignal: controller.signal, + }, + }) + + expect(result).toMatchObject({ added: [{ documentId: 'document-1' }], cancelled: true }) + expect(mocks.createDocument).toHaveBeenCalledOnce() + expect(mocks.recordAudit).toHaveBeenCalledOnce() + }) + + it('audits completed documents before propagating a later infrastructure failure', async () => { + const failure = new Error('document store unavailable') + mocks.resolveFile + .mockResolvedValueOnce(workspaceFile) + .mockResolvedValueOnce({ ...workspaceFile, id: 'file-2', name: 'second.pdf' }) + mocks.loadFileContext + .mockResolvedValueOnce({ + fileId: 'file-1', + workspaceId: 'workspace-1', + workspaceOrganizationId: null, + allowPersonalApiKeys: true, + billedAccountUserId: 'billing-owner-1', + }) + .mockResolvedValueOnce({ + fileId: 'file-2', + workspaceId: 'workspace-1', + workspaceOrganizationId: null, + allowPersonalApiKeys: true, + billedAccountUserId: 'billing-owner-1', + }) + mocks.createDocument + .mockResolvedValueOnce({ + id: 'document-1', + filename: 'report.pdf', + fileUrl: 'https://storage.test/report.pdf', + fileSize: 100, + mimeType: 'application/pdf', + }) + .mockRejectedValueOnce(failure) + + await expect( + addWorkspaceFilesToKnowledgeBase.execute({ + principal: delegatedPrincipal, + input: { + knowledgeBaseId: 'knowledge-1', + assertedWorkspaceId: 'workspace-1', + fileReferences: ['files/report.pdf', 'files/second.pdf'], + }, + }) + ).rejects.toBe(failure) + + expect(mocks.recordAudit).toHaveBeenCalledOnce() + expect(mocks.recordAudit).toHaveBeenCalledWith( + expect.objectContaining({ resourceId: 'document-1' }) + ) + expect(mocks.platformUploaded).not.toHaveBeenCalled() + expect(mocks.captureServerEvent).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/knowledge/application/add-workspace-files.ts b/apps/sim/lib/knowledge/application/add-workspace-files.ts new file mode 100644 index 00000000000..c0400c14647 --- /dev/null +++ b/apps/sim/lib/knowledge/application/add-workspace-files.ts @@ -0,0 +1,270 @@ +import { AuditAction, AuditResourceType } from '@sim/audit' +import type { Principal } from '@sim/auth/principal' +import { createLogger } from '@sim/logger' +import { checkAttributedUsageLimits } from '@/lib/billing/core/billing-attribution' +import { authorizeWorkspaceOperation } from '@/lib/core/application' +import { asOrchestrationError, OrchestrationError } from '@/lib/core/orchestration/types' +import { generateRequestId } from '@/lib/core/utils/request' +import { knowledgeDelegationPolicy } from '@/lib/knowledge/application/authorization' +import { defineAuthorizedKnowledgeUseCase } from '@/lib/knowledge/application/authorized-knowledge-use-case' +import { + ADD_WORKSPACE_FILES_COST_POLICY, + type KnowledgeBatchExecutionResult, + requireBoundedKnowledgeBatch, + rethrowKnowledgeBatchTerminalFailure, +} from '@/lib/knowledge/application/batch-policy' +import { + KnowledgeUsageLimitExceededError, + resolveKnowledgeAttributedUserId, + resolveKnowledgeBillingAttribution, +} from '@/lib/knowledge/application/billing' +import { + type ActiveKnowledgeBaseContext, + resolveActiveKnowledgeBaseContext, +} from '@/lib/knowledge/application/contexts' +import { knowledgeOperations } from '@/lib/knowledge/application/operations' +import { + createSingleDocument, + type DocumentData, + processDocumentsWithQueue, +} from '@/lib/knowledge/documents/service' +import { StorageService } from '@/lib/uploads' +import { + loadActiveWorkspaceFileContext, + resolveWorkspaceFileReference, + type WorkspaceFileRecord, +} from '@/lib/uploads/contexts/workspace/workspace-file-manager' +import { getBoundWorkspaceFileSecretProvenance } from '@/lib/uploads/contexts/workspace/workspace-file-secret-provenance' +import { MAX_KNOWLEDGE_DOCUMENT_FILE_SIZE } from '@/lib/uploads/shared/types' +import { validateFileType } from '@/lib/uploads/utils/validation' + +const logger = createLogger('AddWorkspaceFilesToKnowledgeBase') + +export interface AddWorkspaceFilesToKnowledgeBaseInput { + knowledgeBaseId: string + assertedWorkspaceId?: string + fileReferences: string[] + cancellationSignal?: AbortSignal + source?: string +} + +interface AddedWorkspaceFileDocument { + documentId: string + filename: string + mimeType: string + fileSize: number +} + +export interface AddWorkspaceFilesToKnowledgeBaseResult { + knowledgeBaseId: string + knowledgeBaseName: string + added: AddedWorkspaceFileDocument[] + failed: string[] + cancelled: boolean +} + +interface AddWorkspaceFilesExecutionResult + extends AddWorkspaceFilesToKnowledgeBaseResult, + KnowledgeBatchExecutionResult {} + +interface AddWorkspaceFilesContext extends ActiveKnowledgeBaseContext { + fileReferences: string[] +} + +interface PreparedWorkspaceFile { + reference: string + file: WorkspaceFileRecord + fileUrl: string +} + +async function prepareWorkspaceFile( + principal: Principal, + context: AddWorkspaceFilesContext, + reference: string +): Promise { + const file = await resolveWorkspaceFileReference(context.workspaceId, reference) + if (!file) throw new OrchestrationError('not_found', 'File not found') + const canonical = await loadActiveWorkspaceFileContext(file.id) + if (!canonical || canonical.workspaceId !== context.workspaceId) { + throw new OrchestrationError('not_found', 'File not found') + } + await authorizeWorkspaceOperation(principal, knowledgeOperations.addWorkspaceFiles, canonical, { + delegation: knowledgeDelegationPolicy, + }) + if (file.size < 0 || file.size > MAX_KNOWLEDGE_DOCUMENT_FILE_SIZE) { + throw new OrchestrationError('payload_too_large', 'Knowledge document exceeds the 100MB limit') + } + const fileTypeError = validateFileType(file.name, file.type) + if (fileTypeError) throw new OrchestrationError('validation', fileTypeError.message) + + const provenance = await getBoundWorkspaceFileSecretProvenance(context.workspaceId, { + fileId: file.id, + key: file.key, + context: 'workspace', + }) + if (provenance.status !== 'exact' || provenance.entries.length > 0) { + throw new OrchestrationError( + 'validation', + 'Workspace file secret provenance prevents knowledge ingestion' + ) + } + + return { + reference, + file, + fileUrl: await StorageService.generatePresignedDownloadUrl(file.key, 'workspace', 5 * 60), + } +} + +export const addWorkspaceFilesToKnowledgeBase = defineAuthorizedKnowledgeUseCase({ + operation: knowledgeOperations.addWorkspaceFiles, + async resolveContext({ + input, + }: { + input: AddWorkspaceFilesToKnowledgeBaseInput + }): Promise { + const fileReferences = requireBoundedKnowledgeBatch( + input.fileReferences, + 'files', + ADD_WORKSPACE_FILES_COST_POLICY.maxItems + ) + return { + ...(await resolveActiveKnowledgeBaseContext(input)), + fileReferences, + } + }, + async execute({ principal, input, context }): Promise { + const prepared: PreparedWorkspaceFile[] = [] + const failed: string[] = [] + const canonicalFileIds = new Set() + + for (const reference of context.fileReferences) { + if (input.cancellationSignal?.aborted) break + try { + const candidate = await prepareWorkspaceFile(principal, context, reference) + if (canonicalFileIds.has(candidate.file.id)) continue + canonicalFileIds.add(candidate.file.id) + prepared.push(candidate) + } catch (error) { + const classified = asOrchestrationError(error) + if (classified && classified.code !== 'internal') { + failed.push(reference) + continue + } + throw error + } + } + + if (prepared.length === 0) { + return { + knowledgeBaseId: context.knowledgeBaseId, + knowledgeBaseName: context.knowledgeBase.name, + added: [], + failed, + cancelled: input.cancellationSignal?.aborted ?? false, + } + } + + if (input.cancellationSignal?.aborted) { + return { + knowledgeBaseId: context.knowledgeBaseId, + knowledgeBaseName: context.knowledgeBase.name, + added: [], + failed, + cancelled: true, + } + } + + const billingAttribution = await resolveKnowledgeBillingAttribution(principal, context) + const usage = await checkAttributedUsageLimits(billingAttribution) + if (usage.isExceeded) { + throw new KnowledgeUsageLimitExceededError( + usage.message || 'Usage limit exceeded. Please upgrade your plan to continue.' + ) + } + const uploadedBy = resolveKnowledgeAttributedUserId(principal, context) + const added: AddedWorkspaceFileDocument[] = [] + let terminalFailure: KnowledgeBatchExecutionResult['terminalFailure'] + + for (const candidate of prepared) { + if (input.cancellationSignal?.aborted) break + try { + const requestId = generateRequestId() + const document = await createSingleDocument( + { + filename: candidate.file.name, + fileUrl: candidate.fileUrl, + fileSize: candidate.file.size, + mimeType: candidate.file.type, + }, + context.knowledgeBaseId, + requestId, + uploadedBy, + undefined, + undefined, + { expectedWorkspaceId: context.workspaceId } + ) + const processingDocument: DocumentData = { + documentId: document.id, + filename: document.filename, + fileUrl: document.fileUrl, + fileSize: document.fileSize, + mimeType: document.mimeType, + } + processDocumentsWithQueue( + [processingDocument], + context.knowledgeBaseId, + {}, + requestId, + billingAttribution + ).catch((error: unknown) => { + logger.error('Knowledge document processing pipeline failed', { + knowledgeBaseId: context.knowledgeBaseId, + documentId: document.id, + error, + }) + }) + added.push({ + documentId: document.id, + filename: document.filename, + mimeType: document.mimeType, + fileSize: document.fileSize, + }) + } catch (error) { + const classified = asOrchestrationError(error) + if (classified && classified.code !== 'internal') { + failed.push(candidate.reference) + continue + } + terminalFailure = { error } + break + } + } + + return { + knowledgeBaseId: context.knowledgeBaseId, + knowledgeBaseName: context.knowledgeBase.name, + added, + failed, + cancelled: input.cancellationSignal?.aborted ?? false, + ...(terminalFailure && { terminalFailure }), + } + }, + projectAudit: ({ input, context, result }) => + result.added.map((document) => ({ + action: AuditAction.DOCUMENT_UPLOADED, + resourceType: AuditResourceType.DOCUMENT, + resourceId: document.documentId, + resourceName: document.filename, + description: `Uploaded document "${document.filename}" to knowledge base "${context.knowledgeBase.name}"`, + metadata: { + source: input.source, + knowledgeBaseId: context.knowledgeBaseId, + knowledgeBaseName: context.knowledgeBase.name, + fileName: document.filename, + fileType: document.mimeType, + fileSize: document.fileSize, + }, + })), + afterSuccess: ({ result }) => rethrowKnowledgeBatchTerminalFailure(result), +}) diff --git a/apps/sim/lib/knowledge/application/authorization.test.ts b/apps/sim/lib/knowledge/application/authorization.test.ts index bb5ea1030d9..50744f68ff2 100644 --- a/apps/sim/lib/knowledge/application/authorization.test.ts +++ b/apps/sim/lib/knowledge/application/authorization.test.ts @@ -51,4 +51,25 @@ describe('knowledge delegation policy', () => { expect(principal.audience).not.toBe(knowledgeDelegationPolicy.audience) }) + + it('accepts a correctly scoped executor delegation for executor-enabled operations', () => { + const principal: DelegatedPrincipal = { + kind: 'delegated', + serviceId: 'executor', + subjectUserId: 'user-1', + workspaceId: 'workspace-1', + delegationId: 'execution-1', + audience: KNOWLEDGE_DELEGATION_AUDIENCE, + issuedAt: new Date(), + expiresAt: new Date(Date.now() + 60_000), + } + + expect( + knowledgeDelegationPolicy.isWithinScope(principal, { + workspaceId: 'workspace-1', + workspaceOrganizationId: null, + allowPersonalApiKeys: true, + }) + ).toBe(true) + }) }) diff --git a/apps/sim/lib/knowledge/application/authorization.ts b/apps/sim/lib/knowledge/application/authorization.ts index e75d1c496d4..60320ea04d8 100644 --- a/apps/sim/lib/knowledge/application/authorization.ts +++ b/apps/sim/lib/knowledge/application/authorization.ts @@ -9,6 +9,9 @@ export const KNOWLEDGE_DELEGATION_AUDIENCE = 'sim:knowledge' export interface KnowledgeAuthorizationContext extends WorkspaceAuthorizationContext { knowledgeBaseId?: string documentId?: string + chunkId?: string + tagDefinitionId?: string + connectorId?: string } export type KnowledgeAuthorizationOptions = Omit< diff --git a/apps/sim/lib/knowledge/application/batch-policy.ts b/apps/sim/lib/knowledge/application/batch-policy.ts new file mode 100644 index 00000000000..3e995996c01 --- /dev/null +++ b/apps/sim/lib/knowledge/application/batch-policy.ts @@ -0,0 +1,47 @@ +import { OrchestrationError } from '@/lib/core/orchestration/types' + +export const MAX_KNOWLEDGE_BATCH_ITEMS = 100 + +export const ADD_WORKSPACE_FILES_COST_POLICY = { + maxItems: MAX_KNOWLEDGE_BATCH_ITEMS, + usageAdmission: 'once_before_processing', +} as const + +export const BULK_DELETE_KNOWLEDGE_BASES_COST_POLICY = { + maxItems: MAX_KNOWLEDGE_BATCH_ITEMS, + execution: 'sequential_best_effort', +} as const + +export const BULK_DELETE_KNOWLEDGE_DOCUMENTS_COST_POLICY = { + maxItems: MAX_KNOWLEDGE_BATCH_ITEMS, + execution: 'sequential_best_effort', +} as const + +export interface KnowledgeBatchTerminalFailure { + error: unknown +} + +export interface KnowledgeBatchExecutionResult { + terminalFailure?: KnowledgeBatchTerminalFailure +} + +export function rethrowKnowledgeBatchTerminalFailure(result: KnowledgeBatchExecutionResult): void { + if (result.terminalFailure) throw result.terminalFailure.error +} + +export function requireBoundedKnowledgeBatch( + items: readonly string[], + resource: string, + maxItems: number +): string[] { + if (items.length === 0) { + throw new OrchestrationError('validation', `At least one ${resource} is required`) + } + if (items.length > maxItems) { + throw new OrchestrationError( + 'validation', + `Too many ${resource} (${items.length}). Maximum is ${maxItems}.` + ) + } + return [...new Set(items)] +} diff --git a/apps/sim/lib/knowledge/application/chunks.ts b/apps/sim/lib/knowledge/application/chunks.ts new file mode 100644 index 00000000000..38d4d56dcca --- /dev/null +++ b/apps/sim/lib/knowledge/application/chunks.ts @@ -0,0 +1,260 @@ +import { createLogger } from '@sim/logger' +import { OrchestrationError } from '@/lib/core/orchestration/types' +import { generateRequestId } from '@/lib/core/utils/request' +import { + createDurableSecretProvenanceRegistry, + type DurableSecretProvenance, +} from '@/lib/execution/durable-secret-provenance' +import { defineAuthorizedKnowledgeUseCase } from '@/lib/knowledge/application/authorized-knowledge-use-case' +import { resolveKnowledgeAttributedUserId } from '@/lib/knowledge/application/billing' +import { + type ActiveKnowledgeDocumentContext, + resolveActiveKnowledgeChunkContext, + resolveCanonicalActiveKnowledgeDocumentContext, +} from '@/lib/knowledge/application/contexts' +import { knowledgeOperations } from '@/lib/knowledge/application/operations' +import { + batchChunkOperation, + createChunk, + deleteChunk, + queryChunks, + updateChunk, +} from '@/lib/knowledge/chunks/service' +import type { ChunkFilters } from '@/lib/knowledge/chunks/types' +import { runWithKnowledgeModelInputProvenance } from '@/lib/knowledge/model-input-provenance' +import { calculateCost } from '@/providers/utils' + +const logger = createLogger('KnowledgeChunkApplication') + +interface KnowledgeDocumentChunkInput { + knowledgeBaseId: string + documentId: string + assertedWorkspaceId?: string +} + +interface KnowledgeChunkInput extends KnowledgeDocumentChunkInput { + chunkId: string +} + +interface ResolveChunkProvenanceInput { + userId: string + workspaceId: string +} + +export interface ListKnowledgeChunksInput extends KnowledgeDocumentChunkInput, ChunkFilters {} + +export interface CreateKnowledgeChunkInput extends KnowledgeDocumentChunkInput { + content: string + enabled?: boolean + resolveContentProvenance(input: ResolveChunkProvenanceInput): DurableSecretProvenance | undefined +} + +export interface UpdateKnowledgeChunkInput extends KnowledgeChunkInput { + content?: string + enabled?: boolean + resolveContentProvenance(input: ResolveChunkProvenanceInput): DurableSecretProvenance | undefined +} + +export interface BulkKnowledgeChunksInput extends KnowledgeDocumentChunkInput { + operation: 'enable' | 'disable' | 'delete' + chunkIds: string[] +} + +function requireChunkReadable(context: ActiveKnowledgeDocumentContext): void { + if (context.document.processingStatus !== 'completed') { + throw new OrchestrationError( + 'validation', + `Document is not ready for access (status: ${context.document.processingStatus})` + ) + } +} + +function requireChunkWritable(context: ActiveKnowledgeDocumentContext): void { + if (context.document.connectorId) { + throw new OrchestrationError( + 'forbidden', + 'Chunks from connector-synced documents are read-only' + ) + } +} + +function documentTags(context: ActiveKnowledgeDocumentContext) { + const document = context.document + return { + tag1: document.tag1 ?? null, + tag2: document.tag2 ?? null, + tag3: document.tag3 ?? null, + tag4: document.tag4 ?? null, + tag5: document.tag5 ?? null, + tag6: document.tag6 ?? null, + tag7: document.tag7 ?? null, + number1: document.number1 ?? null, + number2: document.number2 ?? null, + number3: document.number3 ?? null, + number4: document.number4 ?? null, + number5: document.number5 ?? null, + date1: document.date1 ?? null, + date2: document.date2 ?? null, + boolean1: document.boolean1 ?? null, + boolean2: document.boolean2 ?? null, + boolean3: document.boolean3 ?? null, + } +} + +export const listKnowledgeChunks = defineAuthorizedKnowledgeUseCase({ + operation: knowledgeOperations.listChunks, + resolveContext: ({ input }: { input: ListKnowledgeChunksInput }) => + resolveCanonicalActiveKnowledgeDocumentContext(input), + async execute({ input, context }) { + requireChunkReadable(context) + const { + knowledgeBaseId: _knowledgeBaseId, + documentId, + assertedWorkspaceId: _scope, + ...filters + } = input + const result = await queryChunks(documentId, filters, generateRequestId()) + return { ...result, workspaceId: context.workspaceId, documentId } + }, +}) + +export const readKnowledgeChunk = defineAuthorizedKnowledgeUseCase({ + operation: knowledgeOperations.readChunk, + resolveContext: ({ input }: { input: KnowledgeChunkInput }) => + resolveActiveKnowledgeChunkContext(input), + async execute({ context }) { + requireChunkReadable(context) + return { + chunk: context.chunk, + workspaceId: context.workspaceId, + documentId: context.documentId, + } + }, +}) + +export const createKnowledgeChunk = defineAuthorizedKnowledgeUseCase({ + operation: knowledgeOperations.createChunk, + resolveContext: ({ input }: { input: CreateKnowledgeChunkInput }) => + resolveCanonicalActiveKnowledgeDocumentContext(input), + async execute({ principal, input, context }) { + requireChunkWritable(context) + if (context.document.processingStatus === 'failed') { + throw new OrchestrationError('validation', 'Cannot add chunks to failed document') + } + const userId = resolveKnowledgeAttributedUserId(principal, context) + const provenance = input.resolveContentProvenance({ userId, workspaceId: context.workspaceId }) + if (provenance?.status === 'unknown') { + throw new OrchestrationError('validation', 'Knowledge chunk secret provenance is unavailable') + } + const registry = provenance + ? await createDurableSecretProvenanceRegistry(provenance, { + userId, + workspaceId: context.workspaceId, + }) + : undefined + const chunk = await runWithKnowledgeModelInputProvenance(registry, () => + createChunk( + context.knowledgeBaseId, + context.documentId, + documentTags(context), + { content: input.content, enabled: input.enabled }, + generateRequestId(), + context.workspaceId, + provenance + ) + ) + let cost: ReturnType | null = null + try { + cost = calculateCost(context.knowledgeBase.embeddingModel, chunk.tokenCount, 0, false) + } catch (error) { + logger.warn('Failed to calculate cost for chunk upload', { error }) + } + return { + chunk: { + ...chunk, + documentId: context.documentId, + documentName: context.document.filename, + ...(cost + ? { + cost: { + input: cost.input, + output: cost.output, + total: cost.total, + tokens: { prompt: chunk.tokenCount, completion: 0, total: chunk.tokenCount }, + model: context.knowledgeBase.embeddingModel, + pricing: cost.pricing, + }, + } + : {}), + }, + provenance, + workspaceId: context.workspaceId, + userId, + } + }, +}) + +export const updateKnowledgeChunk = defineAuthorizedKnowledgeUseCase({ + operation: knowledgeOperations.updateChunk, + resolveContext: ({ input }: { input: UpdateKnowledgeChunkInput }) => + resolveActiveKnowledgeChunkContext(input), + async execute({ principal, input, context }) { + requireChunkReadable(context) + requireChunkWritable(context) + const userId = resolveKnowledgeAttributedUserId(principal, context) + const provenance = input.resolveContentProvenance({ userId, workspaceId: context.workspaceId }) + if (provenance?.status === 'unknown') { + throw new OrchestrationError('validation', 'Knowledge chunk secret provenance is unavailable') + } + const registry = provenance + ? await createDurableSecretProvenanceRegistry(provenance, { + userId, + workspaceId: context.workspaceId, + }) + : undefined + const chunk = await runWithKnowledgeModelInputProvenance(registry, () => + updateChunk( + context.chunkId, + { content: input.content, enabled: input.enabled }, + generateRequestId(), + context.workspaceId, + provenance + ) + ) + return { chunk, workspaceId: context.workspaceId, documentId: context.documentId } + }, +}) + +export const deleteKnowledgeChunk = defineAuthorizedKnowledgeUseCase({ + operation: knowledgeOperations.deleteChunk, + resolveContext: ({ input }: { input: KnowledgeChunkInput }) => + resolveActiveKnowledgeChunkContext(input), + async execute({ context }) { + requireChunkReadable(context) + requireChunkWritable(context) + await deleteChunk(context.chunkId, context.documentId, generateRequestId()) + return { deleted: true as const } + }, +}) + +export const bulkUpdateKnowledgeChunks = defineAuthorizedKnowledgeUseCase({ + operation: knowledgeOperations.bulkChunks, + resolveContext: ({ input }: { input: BulkKnowledgeChunksInput }) => + resolveCanonicalActiveKnowledgeDocumentContext(input), + async execute({ input, context }) { + requireChunkWritable(context) + const result = await batchChunkOperation( + context.documentId, + input.operation, + input.chunkIds, + generateRequestId() + ) + return { + operation: input.operation, + successCount: result.processed, + errorCount: result.errors.length, + processed: result.processed, + errors: result.errors, + } + }, +}) diff --git a/apps/sim/lib/knowledge/application/connectors.test.ts b/apps/sim/lib/knowledge/application/connectors.test.ts new file mode 100644 index 00000000000..816565d737e --- /dev/null +++ b/apps/sim/lib/knowledge/application/connectors.test.ts @@ -0,0 +1,476 @@ +/** + * @vitest-environment node + */ + +import { document } from '@sim/db/schema' +import { dbChainMockFns, queueTableRows, resetDbChainMock } from '@sim/testing' +import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + resolveKnowledgeBase: vi.fn(), + resolveConnector: vi.fn(), + resolvePermission: vi.fn(), + createConnector: vi.fn(), + updateConnector: vi.fn(), + deleteConnector: vi.fn(), + syncConnector: vi.fn(), + resolveBilling: vi.fn(), + resolveTokenIdentity: vi.fn(), + refreshToken: vi.fn(), + validateConnectorConfig: vi.fn(), + recordAudit: vi.fn(), +})) + +vi.mock('@sim/audit', () => ({ + AuditAction: { + CONNECTOR_CREATED: 'connector.created', + CONNECTOR_UPDATED: 'connector.updated', + CONNECTOR_DELETED: 'connector.deleted', + CONNECTOR_SYNCED: 'connector.synced', + }, + AuditResourceType: { CONNECTOR: 'connector' }, + recordAudit: mocks.recordAudit, +})) + +vi.mock('@sim/platform-authz/workspace', () => ({ + permissionSatisfies: (actual: string | null, required: string) => { + const rank = { read: 1, write: 2, admin: 3 } as const + return ( + actual !== null && rank[actual as keyof typeof rank] >= rank[required as keyof typeof rank] + ) + }, + resolveEffectiveWorkspacePermission: mocks.resolvePermission, +})) + +vi.mock('@/lib/knowledge/application/contexts', () => ({ + resolveActiveKnowledgeBaseContext: mocks.resolveKnowledgeBase, + resolveActiveKnowledgeConnectorContext: mocks.resolveConnector, +})) + +vi.mock('@/lib/knowledge/orchestration/connectors', () => ({ + performCreateKnowledgeConnector: mocks.createConnector, + performUpdateKnowledgeConnector: mocks.updateConnector, + performDeleteKnowledgeConnector: mocks.deleteConnector, + performSyncKnowledgeConnector: mocks.syncConnector, +})) + +vi.mock('@/lib/credentials/access', () => ({ + resolveCredentialTokenIdentity: mocks.resolveTokenIdentity, +})) + +vi.mock('@/lib/oauth/credential-service', () => ({ + refreshAccessTokenIfNeeded: mocks.refreshToken, +})) + +vi.mock('@/connectors/registry.server', () => ({ + CONNECTOR_REGISTRY: { + confluence: { + auth: { mode: 'oauth' }, + validateConfig: mocks.validateConnectorConfig, + }, + }, +})) + +import { + createKnowledgeConnector, + deleteKnowledgeConnector, + listKnowledgeConnectorDocuments, + syncKnowledgeConnector, + updateKnowledgeConnector, + updateKnowledgeConnectorDocuments, +} from '@/lib/knowledge/application/connectors' + +const crossWorkspaceContext = { + workspaceId: 'workspace-b', + workspaceOrganizationId: null, + allowPersonalApiKeys: true, + billedAccountUserId: 'billing-owner-b', + knowledgeBaseId: 'knowledge-b', + knowledgeBase: { id: 'knowledge-b', name: 'Workspace B docs' }, +} + +const connectorContext = { + ...crossWorkspaceContext, + connectorId: 'connector-b', + connector: { + id: 'connector-b', + knowledgeBaseId: 'knowledge-b', + connectorType: 'confluence', + status: 'active', + }, +} + +const delegatedPrincipal = { + kind: 'delegated' as const, + serviceId: 'copilot', + subjectUserId: 'shared-user', + workspaceId: 'workspace-a', + delegationId: 'tool-call-1', + audience: 'sim:knowledge', + issuedAt: new Date(), + expiresAt: new Date(Date.now() + 60_000), + resourceScope: {}, +} + +describe('knowledge connector application use cases', () => { + beforeEach(() => { + vi.clearAllMocks() + resetDbChainMock() + mocks.resolvePermission.mockResolvedValue('write') + mocks.resolveKnowledgeBase.mockResolvedValue(crossWorkspaceContext) + mocks.resolveConnector.mockResolvedValue(connectorContext) + mocks.resolveTokenIdentity.mockResolvedValue({ kind: 'oauth', userId: 'credential-owner' }) + mocks.refreshToken.mockResolvedValue('access-token') + mocks.validateConnectorConfig.mockResolvedValue({ valid: true }) + }) + + afterAll(resetDbChainMock) + + it.each([ + [ + 'create', + createKnowledgeConnector, + { + knowledgeBaseId: 'knowledge-b', + assertedWorkspaceId: 'workspace-a', + connectorType: 'confluence', + credentialId: 'credential-1', + sourceConfig: {}, + syncIntervalMinutes: 1440, + resolveBillingAttribution: mocks.resolveBilling, + }, + ], + [ + 'update', + updateKnowledgeConnector, + { + connectorId: 'connector-b', + assertedWorkspaceId: 'workspace-a', + updates: { status: 'paused' as const }, + }, + ], + [ + 'delete', + deleteKnowledgeConnector, + { connectorId: 'connector-b', assertedWorkspaceId: 'workspace-a' }, + ], + [ + 'sync', + syncKnowledgeConnector, + { + connectorId: 'connector-b', + assertedWorkspaceId: 'workspace-a', + resolveBillingAttribution: mocks.resolveBilling, + }, + ], + ])( + 'rejects cross-workspace %s before membership, billing, or orchestration', + async (_name, useCase, input) => { + await expect(useCase.execute({ principal: delegatedPrincipal, input })).rejects.toMatchObject( + { + name: 'DelegatedWorkspaceAuthorizationError', + code: 'forbidden', + } + ) + + expect(mocks.resolvePermission).not.toHaveBeenCalled() + expect(mocks.resolveBilling).not.toHaveBeenCalled() + expect(mocks.resolveTokenIdentity).not.toHaveBeenCalled() + expect(mocks.createConnector).not.toHaveBeenCalled() + expect(mocks.updateConnector).not.toHaveBeenCalled() + expect(mocks.deleteConnector).not.toHaveBeenCalled() + expect(mocks.syncConnector).not.toHaveBeenCalled() + expect(mocks.recordAudit).not.toHaveBeenCalled() + } + ) + + it('authorizes current delegated membership before orchestration and owns semantic audit', async () => { + const sameWorkspaceContext = { + ...connectorContext, + workspaceId: 'workspace-a', + knowledgeBaseId: 'knowledge-a', + knowledgeBase: { id: 'knowledge-a', name: 'Workspace A docs' }, + connector: { ...connectorContext.connector, knowledgeBaseId: 'knowledge-a' }, + } + const updatedConnector = { + ...sameWorkspaceContext.connector, + credentialId: 'credential-1', + sourceConfig: {}, + syncIntervalMinutes: 1440, + } + mocks.resolveConnector.mockResolvedValueOnce(sameWorkspaceContext) + mocks.updateConnector.mockResolvedValueOnce({ + success: true, + connector: updatedConnector, + }) + + const result = await updateKnowledgeConnector.execute({ + principal: delegatedPrincipal, + input: { + connectorId: 'connector-b', + assertedWorkspaceId: 'workspace-a', + updates: { status: 'paused' }, + source: 'agent', + }, + }) + + expect(result.connector).toEqual(updatedConnector) + expect(mocks.resolvePermission).toHaveBeenCalledWith( + 'shared-user', + 'workspace-a', + null, + undefined, + { forUpdate: undefined } + ) + expect(mocks.resolvePermission.mock.invocationCallOrder[0]).toBeLessThan( + mocks.updateConnector.mock.invocationCallOrder[0] + ) + expect(mocks.updateConnector).toHaveBeenCalledWith( + expect.objectContaining({ + connectorId: 'connector-b', + userId: 'shared-user', + source: 'agent', + recordSemanticAudit: false, + }) + ) + expect(mocks.recordAudit).toHaveBeenCalledWith( + expect.objectContaining({ + workspaceId: 'workspace-a', + action: 'connector.updated', + metadata: expect.objectContaining({ + operation: 'knowledge.connectors.update', + actor: expect.objectContaining({ kind: 'delegated', serviceId: 'copilot' }), + }), + }) + ) + }) + + it('owns source-config credential resolution and validation after authorization', async () => { + const sameWorkspaceContext = { + ...connectorContext, + workspaceId: 'workspace-a', + knowledgeBaseId: 'knowledge-a', + knowledgeBase: { id: 'knowledge-a', name: 'Workspace A docs' }, + connector: { ...connectorContext.connector, knowledgeBaseId: 'knowledge-a' }, + } + mocks.resolveConnector.mockResolvedValueOnce(sameWorkspaceContext) + mocks.updateConnector.mockResolvedValueOnce({ + success: true, + connector: { ...sameWorkspaceContext.connector, sourceConfig: { space: 'ENG' } }, + }) + + await updateKnowledgeConnector.execute({ + principal: delegatedPrincipal, + input: { + connectorId: 'connector-b', + assertedWorkspaceId: 'workspace-a', + updates: { sourceConfig: { space: 'ENG' } }, + source: 'agent', + }, + }) + + const orchestrationInput = mocks.updateConnector.mock.calls[0]?.[0] as { + validateSourceConfig?: ( + connector: { + connectorType: string + credentialId: string + encryptedApiKey: null + }, + sourceConfig: Record + ) => Promise + } + if (!orchestrationInput.validateSourceConfig) { + throw new Error('Application command did not provide source-config validation') + } + await expect( + orchestrationInput.validateSourceConfig( + { + connectorType: 'confluence', + credentialId: 'credential-1', + encryptedApiKey: null, + }, + { space: 'ENG' } + ) + ).resolves.toBeNull() + expect(mocks.resolvePermission.mock.invocationCallOrder[0]).toBeLessThan( + mocks.updateConnector.mock.invocationCallOrder[0] + ) + expect(mocks.resolveTokenIdentity).toHaveBeenCalledWith('credential-1', 'workspace-a') + expect(mocks.refreshToken).toHaveBeenCalledWith( + 'credential-1', + 'credential-owner', + expect.any(String) + ) + expect(mocks.validateConnectorConfig).toHaveBeenCalledWith('access-token', { space: 'ENG' }) + }) + + it.each([ + [ + 'create', + createKnowledgeConnector, + mocks.createConnector, + { + knowledgeBaseId: 'knowledge-a', + assertedWorkspaceId: 'workspace-a', + connectorType: 'confluence', + credentialId: 'credential-1', + sourceConfig: {}, + syncIntervalMinutes: 1440, + resolveBillingAttribution: mocks.resolveBilling, + }, + { + success: true, + connector: { ...connectorContext.connector, knowledgeBaseId: 'knowledge-a' }, + }, + ], + [ + 'delete', + deleteKnowledgeConnector, + mocks.deleteConnector, + { connectorId: 'connector-b', assertedWorkspaceId: 'workspace-a' }, + { success: true, documentsDeleted: 0, documentsKept: 1 }, + ], + [ + 'sync', + syncKnowledgeConnector, + mocks.syncConnector, + { + connectorId: 'connector-b', + assertedWorkspaceId: 'workspace-a', + resolveBillingAttribution: mocks.resolveBilling, + }, + { success: true }, + ], + ])( + 'disables legacy semantic audit and product analytics for %s', + async (_name, useCase, orchestration, input, outcome) => { + const sameWorkspaceContext = { + ...connectorContext, + workspaceId: 'workspace-a', + knowledgeBaseId: 'knowledge-a', + knowledgeBase: { id: 'knowledge-a', name: 'Workspace A docs' }, + connector: { ...connectorContext.connector, knowledgeBaseId: 'knowledge-a' }, + } + mocks.resolveKnowledgeBase.mockResolvedValueOnce(sameWorkspaceContext) + mocks.resolveConnector.mockResolvedValueOnce(sameWorkspaceContext) + orchestration.mockResolvedValueOnce(outcome) + + await useCase.execute({ principal: delegatedPrincipal, input }) + + expect(orchestration).toHaveBeenCalledWith( + expect.objectContaining({ + recordSemanticAudit: false, + recordProductAnalytics: false, + }) + ) + expect(mocks.recordAudit).toHaveBeenCalledOnce() + } + ) + + it('paginates connector documents while returning authoritative total counts', async () => { + const sameWorkspaceContext = { + ...connectorContext, + workspaceId: 'workspace-a', + knowledgeBaseId: 'knowledge-a', + knowledgeBase: { id: 'knowledge-a', name: 'Workspace A docs' }, + connector: { ...connectorContext.connector, knowledgeBaseId: 'knowledge-a' }, + } + mocks.resolveConnector.mockResolvedValueOnce(sameWorkspaceContext) + queueTableRows(document, [{ value: 5 }]) + queueTableRows(document, [{ value: 2 }]) + queueTableRows(document, [ + { id: 'document-3', filename: 'c.txt', userExcluded: false }, + { id: 'document-4', filename: 'd.txt', userExcluded: true }, + ]) + + const result = await listKnowledgeConnectorDocuments.execute({ + principal: delegatedPrincipal, + input: { + knowledgeBaseId: 'knowledge-a', + connectorId: 'connector-b', + assertedWorkspaceId: 'workspace-a', + includeExcluded: true, + limit: 2, + offset: 2, + }, + }) + + expect(result).toEqual({ + documents: [ + { id: 'document-3', filename: 'c.txt', userExcluded: false }, + { id: 'document-4', filename: 'd.txt', userExcluded: true }, + ], + counts: { active: 5, excluded: 2 }, + }) + expect(dbChainMockFns.limit).toHaveBeenCalledWith(2) + expect(dbChainMockFns.offset).toHaveBeenCalledWith(2) + }) + + it('caps connector document mutations before persistence', async () => { + const sameWorkspaceContext = { + ...connectorContext, + workspaceId: 'workspace-a', + knowledgeBaseId: 'knowledge-a', + knowledgeBase: { id: 'knowledge-a', name: 'Workspace A docs' }, + connector: { ...connectorContext.connector, knowledgeBaseId: 'knowledge-a' }, + } + mocks.resolveConnector.mockResolvedValueOnce(sameWorkspaceContext) + + await expect( + updateKnowledgeConnectorDocuments.execute({ + principal: delegatedPrincipal, + input: { + knowledgeBaseId: 'knowledge-a', + connectorId: 'connector-b', + assertedWorkspaceId: 'workspace-a', + operation: 'exclude', + documentIds: Array.from({ length: 101 }, (_, index) => `document-${index}`), + }, + }) + ).rejects.toMatchObject({ code: 'validation' }) + + expect(dbChainMockFns.update).not.toHaveBeenCalled() + expect(mocks.recordAudit).not.toHaveBeenCalled() + }) + + it('deduplicates connector document IDs before mutation and audit', async () => { + const sameWorkspaceContext = { + ...connectorContext, + workspaceId: 'workspace-a', + knowledgeBaseId: 'knowledge-a', + knowledgeBase: { id: 'knowledge-a', name: 'Workspace A docs' }, + connector: { ...connectorContext.connector, knowledgeBaseId: 'knowledge-a' }, + } + mocks.resolveConnector.mockResolvedValueOnce(sameWorkspaceContext) + dbChainMockFns.returning.mockResolvedValueOnce([{ id: 'document-1' }, { id: 'document-2' }]) + + const result = await updateKnowledgeConnectorDocuments.execute({ + principal: delegatedPrincipal, + input: { + knowledgeBaseId: 'knowledge-a', + connectorId: 'connector-b', + assertedWorkspaceId: 'workspace-a', + operation: 'exclude', + documentIds: ['document-1', 'document-1', 'document-2'], + }, + }) + + expect(result.documentIds).toEqual(['document-1', 'document-2']) + const where = dbChainMockFns.where.mock.calls.at(-1)?.[0] + expect(where).toEqual( + expect.objectContaining({ + conditions: expect.arrayContaining([ + expect.objectContaining({ + type: 'inArray', + values: ['document-1', 'document-2'], + }), + ]), + }) + ) + expect(mocks.recordAudit).toHaveBeenCalledWith( + expect.objectContaining({ + metadata: expect.objectContaining({ documentIds: ['document-1', 'document-2'] }), + }) + ) + }) +}) diff --git a/apps/sim/lib/knowledge/application/connectors.ts b/apps/sim/lib/knowledge/application/connectors.ts new file mode 100644 index 00000000000..15e9e24b4c7 --- /dev/null +++ b/apps/sim/lib/knowledge/application/connectors.ts @@ -0,0 +1,533 @@ +import { AuditAction, AuditResourceType } from '@sim/audit' +import { db } from '@sim/db' +import { document, knowledgeConnector, knowledgeConnectorSyncLog } from '@sim/db/schema' +import { and, asc, count, desc, eq, inArray, isNull } from 'drizzle-orm' +import { decryptApiKey } from '@/lib/api-key/crypto' +import type { BillingAttributionSnapshot } from '@/lib/billing/core/billing-attribution' +import { OrchestrationError } from '@/lib/core/orchestration/types' +import { generateRequestId } from '@/lib/core/utils/request' +import { resolveCredentialTokenIdentity } from '@/lib/credentials/access' +import { defineAuthorizedKnowledgeUseCase } from '@/lib/knowledge/application/authorized-knowledge-use-case' +import { resolveKnowledgeAttributedUserId } from '@/lib/knowledge/application/billing' +import { + type ActiveKnowledgeBaseContext, + resolveActiveKnowledgeBaseContext, + resolveActiveKnowledgeConnectorContext, +} from '@/lib/knowledge/application/contexts' +import { knowledgeOperations } from '@/lib/knowledge/application/operations' +import { + DEFAULT_KNOWLEDGE_CONNECTOR_DOCUMENT_PAGE_SIZE, + MAX_KNOWLEDGE_CONNECTOR_DOCUMENT_MUTATION_ITEMS, + MAX_KNOWLEDGE_CONNECTOR_DOCUMENT_PAGE_SIZE, +} from '@/lib/knowledge/constants' +import { + getKnowledgeConnector, + type KnowledgeConnectorRow, + performCreateKnowledgeConnector, + performDeleteKnowledgeConnector, + performSyncKnowledgeConnector, + performUpdateKnowledgeConnector, + type SourceConfigRejection, +} from '@/lib/knowledge/orchestration/connectors' +import type { + KnowledgeOperationSource, + KnowledgeOrchestrationResult, +} from '@/lib/knowledge/orchestration/shared' +import { refreshAccessTokenIfNeeded } from '@/lib/oauth/credential-service' + +interface KnowledgeConnectorApplicationInput { + assertedWorkspaceId?: string + source?: KnowledgeOperationSource +} + +export interface ListKnowledgeConnectorsInput extends KnowledgeConnectorApplicationInput { + knowledgeBaseId: string +} + +export interface ReadKnowledgeConnectorInput extends KnowledgeConnectorApplicationInput { + knowledgeBaseId: string + connectorId: string +} + +export interface CreateKnowledgeConnectorInput extends KnowledgeConnectorApplicationInput { + knowledgeBaseId: string + connectorType: string + credentialId?: string + apiKey?: string + sourceConfig: Record + syncIntervalMinutes: number + resolveBillingAttribution(workspaceId: string): Promise +} + +export interface UpdateKnowledgeConnectorInput extends KnowledgeConnectorApplicationInput { + connectorId: string + updates: { + sourceConfig?: Record + syncIntervalMinutes?: number + status?: 'active' | 'paused' + } +} + +export interface DeleteKnowledgeConnectorInput extends KnowledgeConnectorApplicationInput { + connectorId: string + deleteDocuments?: boolean +} + +export interface SyncKnowledgeConnectorInput extends KnowledgeConnectorApplicationInput { + connectorId: string + rehydrate?: boolean + resolveBillingAttribution(workspaceId: string): Promise +} + +export interface ListKnowledgeConnectorDocumentsInput extends ReadKnowledgeConnectorInput { + includeExcluded?: boolean + limit?: number + offset?: number +} + +export interface UpdateKnowledgeConnectorDocumentsInput extends ReadKnowledgeConnectorInput { + operation: 'restore' | 'exclude' + documentIds: string[] +} + +function requireSuccessfulOutcome( + outcome: KnowledgeOrchestrationResult, + fallback: string +): asserts outcome is { success: true } & T { + if (outcome.success) return + if (outcome.errorCode === 'internal') { + throw new Error(fallback, { cause: new Error(outcome.error) }) + } + throw new OrchestrationError(outcome.errorCode, outcome.error) +} + +function connectorTarget(context: ActiveKnowledgeBaseContext) { + return { + id: context.knowledgeBaseId, + name: context.knowledgeBase.name, + workspaceId: context.workspaceId, + } +} + +async function resolveConnectorCredentialAccessToken(input: { + credentialId: string + workspaceId: string + actingUserId: string + requestId: string +}): Promise { + const identity = await resolveCredentialTokenIdentity(input.credentialId, input.workspaceId) + if (!identity) return null + return refreshAccessTokenIfNeeded( + input.credentialId, + identity.kind === 'oauth' ? identity.userId : input.actingUserId, + input.requestId + ) +} + +async function validateConnectorSourceConfig(input: { + connector: KnowledgeConnectorRow + sourceConfig: Record + workspaceId: string + actingUserId: string + requestId: string +}): Promise { + const { CONNECTOR_REGISTRY } = await import('@/connectors/registry.server') + const connectorConfig = CONNECTOR_REGISTRY[input.connector.connectorType] + if (!connectorConfig) { + return { + message: `Unknown connector type: ${input.connector.connectorType}`, + errorCode: 'validation', + } + } + + let accessToken: string | null = null + if (connectorConfig.auth.mode === 'apiKey') { + if (!input.connector.encryptedApiKey) { + return { + message: 'API key not found. Please reconfigure the connector.', + errorCode: 'validation', + } + } + accessToken = (await decryptApiKey(input.connector.encryptedApiKey)).decrypted + } else { + if (!input.connector.credentialId) { + return { + message: 'OAuth credential not found. Please reconfigure the connector.', + errorCode: 'validation', + } + } + const identity = await resolveCredentialTokenIdentity( + input.connector.credentialId, + input.workspaceId + ) + if (!identity) { + return { + message: 'Credential is no longer usable in this workspace. Please reconnect it.', + errorCode: 'validation', + } + } + accessToken = await refreshAccessTokenIfNeeded( + input.connector.credentialId, + identity.kind === 'oauth' ? identity.userId : input.actingUserId, + input.requestId + ) + if (!accessToken) { + return { + message: 'Failed to refresh access token. Please reconnect your account.', + errorCode: 'unauthorized', + } + } + } + + const validation = await connectorConfig.validateConfig(accessToken, input.sourceConfig) + return validation.valid + ? null + : { message: validation.error || 'Invalid source configuration', errorCode: 'validation' } +} + +export const listKnowledgeConnectors = defineAuthorizedKnowledgeUseCase({ + operation: knowledgeOperations.listConnectors, + resolveContext: ({ input }: { input: ListKnowledgeConnectorsInput }) => + resolveActiveKnowledgeBaseContext(input), + async execute({ context }) { + const connectors = await db + .select() + .from(knowledgeConnector) + .where( + and( + eq(knowledgeConnector.knowledgeBaseId, context.knowledgeBaseId), + isNull(knowledgeConnector.archivedAt), + isNull(knowledgeConnector.deletedAt) + ) + ) + .orderBy(desc(knowledgeConnector.createdAt)) + return { connectors: connectors.map(({ encryptedApiKey: _encryptedApiKey, ...rest }) => rest) } + }, +}) + +export const readKnowledgeConnector = defineAuthorizedKnowledgeUseCase({ + operation: knowledgeOperations.readConnector, + resolveContext: ({ input }: { input: ReadKnowledgeConnectorInput }) => + resolveActiveKnowledgeConnectorContext(input), + async execute({ context }) { + const connector = await getKnowledgeConnector(context.knowledgeBaseId, context.connectorId) + if (!connector) throw new OrchestrationError('not_found', 'Connector not found') + const syncLogs = await db + .select() + .from(knowledgeConnectorSyncLog) + .where(eq(knowledgeConnectorSyncLog.connectorId, context.connectorId)) + .orderBy(desc(knowledgeConnectorSyncLog.startedAt)) + .limit(10) + const { encryptedApiKey: _encryptedApiKey, ...connectorData } = connector + return { connector: { ...connectorData, syncLogs } } + }, +}) + +export const createKnowledgeConnector = defineAuthorizedKnowledgeUseCase({ + operation: knowledgeOperations.createConnector, + resolveContext: ({ input }: { input: CreateKnowledgeConnectorInput }) => + resolveActiveKnowledgeBaseContext(input), + async execute({ principal, input, context, request }) { + const requestId = generateRequestId() + const actingUserId = resolveKnowledgeAttributedUserId(principal, context) + const outcome = await performCreateKnowledgeConnector({ + knowledgeBase: connectorTarget(context), + connectorType: input.connectorType, + credentialId: input.credentialId, + apiKey: input.apiKey, + sourceConfig: input.sourceConfig, + syncIntervalMinutes: input.syncIntervalMinutes, + resolveBillingAttribution: () => input.resolveBillingAttribution(context.workspaceId), + resolveAccessToken: (credentialId) => + resolveConnectorCredentialAccessToken({ + credentialId, + workspaceId: context.workspaceId, + actingUserId, + requestId, + }), + userId: actingUserId, + source: input.source ?? 'agent', + requestId, + request, + recordSemanticAudit: false, + recordProductAnalytics: false, + }) + requireSuccessfulOutcome(outcome, 'Knowledge connector creation failed') + return { connector: outcome.connector, workspaceId: context.workspaceId } + }, + projectAudit: ({ input, context, result }) => ({ + action: AuditAction.CONNECTOR_CREATED, + resourceType: AuditResourceType.CONNECTOR, + resourceId: result.connector.id, + resourceName: result.connector.connectorType, + description: `Created ${result.connector.connectorType} connector for knowledge base "${context.knowledgeBase.name}"`, + metadata: { + source: input.source, + knowledgeBaseId: context.knowledgeBaseId, + knowledgeBaseName: context.knowledgeBase.name, + connectorType: result.connector.connectorType, + syncIntervalMinutes: result.connector.syncIntervalMinutes, + authMode: result.connector.credentialId ? 'oauth' : 'apiKey', + }, + }), +}) + +export const updateKnowledgeConnector = defineAuthorizedKnowledgeUseCase({ + operation: knowledgeOperations.updateConnector, + resolveContext: ({ input }: { input: UpdateKnowledgeConnectorInput }) => + resolveActiveKnowledgeConnectorContext(input), + async execute({ principal, input, context, request }) { + const requestId = generateRequestId() + const actingUserId = resolveKnowledgeAttributedUserId(principal, context) + const outcome = await performUpdateKnowledgeConnector({ + knowledgeBase: connectorTarget(context), + connectorId: context.connectorId, + updates: input.updates, + validateSourceConfig: (connector, sourceConfig) => + validateConnectorSourceConfig({ + connector, + sourceConfig, + workspaceId: context.workspaceId, + actingUserId, + requestId, + }), + userId: actingUserId, + source: input.source ?? 'agent', + requestId, + request, + recordSemanticAudit: false, + }) + requireSuccessfulOutcome(outcome, 'Knowledge connector update failed') + return { connector: outcome.connector } + }, + projectAudit: ({ input, context, result }) => ({ + action: AuditAction.CONNECTOR_UPDATED, + resourceType: AuditResourceType.CONNECTOR, + resourceId: result.connector.id, + resourceName: result.connector.connectorType, + description: `Updated connector for knowledge base "${context.knowledgeBase.name}"`, + metadata: { + source: input.source, + knowledgeBaseId: context.knowledgeBaseId, + knowledgeBaseName: context.knowledgeBase.name, + connectorType: result.connector.connectorType, + updatedFields: Object.keys(input.updates).filter( + (key) => input.updates[key as keyof UpdateKnowledgeConnectorInput['updates']] !== undefined + ), + ...(input.updates.syncIntervalMinutes !== undefined && { + syncIntervalMinutes: input.updates.syncIntervalMinutes, + }), + ...(input.updates.status !== undefined && { newStatus: input.updates.status }), + }, + }), +}) + +export const deleteKnowledgeConnector = defineAuthorizedKnowledgeUseCase({ + operation: knowledgeOperations.deleteConnector, + resolveContext: ({ input }: { input: DeleteKnowledgeConnectorInput }) => + resolveActiveKnowledgeConnectorContext(input), + async execute({ principal, input, context, request }) { + const outcome = await performDeleteKnowledgeConnector({ + knowledgeBase: connectorTarget(context), + connectorId: context.connectorId, + deleteDocuments: input.deleteDocuments, + userId: resolveKnowledgeAttributedUserId(principal, context), + source: input.source ?? 'agent', + requestId: generateRequestId(), + request, + recordSemanticAudit: false, + recordProductAnalytics: false, + }) + requireSuccessfulOutcome(outcome, 'Knowledge connector deletion failed') + return { + knowledgeBaseId: context.knowledgeBaseId, + workspaceId: context.workspaceId, + connectorId: context.connectorId, + connectorType: context.connector.connectorType, + documentsDeleted: outcome.documentsDeleted, + documentsKept: outcome.documentsKept, + } + }, + projectAudit: ({ input, context, result }) => ({ + action: AuditAction.CONNECTOR_DELETED, + resourceType: AuditResourceType.CONNECTOR, + resourceId: result.connectorId, + resourceName: context.connector.connectorType, + description: `Deleted connector from knowledge base "${context.knowledgeBase.name}"`, + metadata: { + source: input.source, + knowledgeBaseId: context.knowledgeBaseId, + knowledgeBaseName: context.knowledgeBase.name, + connectorType: context.connector.connectorType, + deleteDocuments: input.deleteDocuments ?? false, + documentsDeleted: result.documentsDeleted, + documentsKept: result.documentsKept, + }, + }), +}) + +export const syncKnowledgeConnector = defineAuthorizedKnowledgeUseCase({ + operation: knowledgeOperations.syncConnector, + resolveContext: ({ input }: { input: SyncKnowledgeConnectorInput }) => + resolveActiveKnowledgeConnectorContext(input), + async execute({ principal, input, context, request }) { + const outcome = await performSyncKnowledgeConnector({ + knowledgeBase: connectorTarget(context), + connectorId: context.connectorId, + resolveBillingAttribution: () => input.resolveBillingAttribution(context.workspaceId), + rehydrate: input.rehydrate, + userId: resolveKnowledgeAttributedUserId(principal, context), + source: input.source ?? 'agent', + requestId: generateRequestId(), + request, + recordSemanticAudit: false, + recordProductAnalytics: false, + }) + requireSuccessfulOutcome(outcome, 'Knowledge connector sync failed') + return { + knowledgeBaseId: context.knowledgeBaseId, + workspaceId: context.workspaceId, + connectorId: context.connectorId, + connectorType: context.connector.connectorType, + } + }, + projectAudit: ({ input, context, result }) => ({ + action: AuditAction.CONNECTOR_SYNCED, + resourceType: AuditResourceType.CONNECTOR, + resourceId: result.connectorId, + resourceName: context.connector.connectorType, + description: `Triggered manual sync for connector on knowledge base "${context.knowledgeBase.name}"`, + metadata: { + source: input.source, + knowledgeBaseId: context.knowledgeBaseId, + knowledgeBaseName: context.knowledgeBase.name, + connectorType: context.connector.connectorType, + connectorStatus: context.connector.status, + syncType: input.rehydrate ? 'manual-rehydrate' : 'manual', + }, + }), +}) + +const connectorDocumentSelection = { + id: document.id, + filename: document.filename, + externalId: document.externalId, + sourceUrl: document.sourceUrl, + enabled: document.enabled, + userExcluded: document.userExcluded, + uploadedAt: document.uploadedAt, + processingStatus: document.processingStatus, +} + +export const listKnowledgeConnectorDocuments = defineAuthorizedKnowledgeUseCase({ + operation: knowledgeOperations.listConnectorDocuments, + resolveContext: ({ input }: { input: ListKnowledgeConnectorDocumentsInput }) => + resolveActiveKnowledgeConnectorContext(input), + async execute({ input, context }) { + const limit = input.limit ?? DEFAULT_KNOWLEDGE_CONNECTOR_DOCUMENT_PAGE_SIZE + const offset = input.offset ?? 0 + if ( + !Number.isInteger(limit) || + limit < 1 || + limit > MAX_KNOWLEDGE_CONNECTOR_DOCUMENT_PAGE_SIZE + ) { + throw new OrchestrationError( + 'validation', + `Connector document limit must be between 1 and ${MAX_KNOWLEDGE_CONNECTOR_DOCUMENT_PAGE_SIZE}` + ) + } + if (!Number.isInteger(offset) || offset < 0) { + throw new OrchestrationError( + 'validation', + 'Connector document offset must be a non-negative integer' + ) + } + const baseConditions = [ + eq(document.connectorId, context.connectorId), + isNull(document.archivedAt), + isNull(document.deletedAt), + ] as const + const [[activeCount], excludedCountRows] = await Promise.all([ + db + .select({ value: count() }) + .from(document) + .where(and(...baseConditions, eq(document.userExcluded, false))), + input.includeExcluded + ? db + .select({ value: count() }) + .from(document) + .where(and(...baseConditions, eq(document.userExcluded, true))) + : Promise.resolve([{ value: 0 }]), + ]) + const excludedCount = excludedCountRows[0] + const documents = await db + .select(connectorDocumentSelection) + .from(document) + .where( + and(...baseConditions, input.includeExcluded ? undefined : eq(document.userExcluded, false)) + ) + .orderBy(asc(document.userExcluded), asc(document.filename)) + .limit(limit) + .offset(offset) + return { + documents, + counts: { active: activeCount?.value ?? 0, excluded: excludedCount?.value ?? 0 }, + } + }, +}) + +export const updateKnowledgeConnectorDocuments = defineAuthorizedKnowledgeUseCase({ + operation: knowledgeOperations.updateConnectorDocuments, + resolveContext: ({ input }: { input: UpdateKnowledgeConnectorDocumentsInput }) => + resolveActiveKnowledgeConnectorContext(input), + async execute({ input, context }) { + if (input.documentIds.length === 0) { + throw new OrchestrationError('validation', 'At least one connector document is required') + } + if (input.documentIds.length > MAX_KNOWLEDGE_CONNECTOR_DOCUMENT_MUTATION_ITEMS) { + throw new OrchestrationError( + 'validation', + `At most ${MAX_KNOWLEDGE_CONNECTOR_DOCUMENT_MUTATION_ITEMS} connector documents may be updated at once` + ) + } + const documentIds = [...new Set(input.documentIds)] + const restoring = input.operation === 'restore' + const updated = await db + .update(document) + .set({ userExcluded: !restoring, enabled: restoring }) + .where( + and( + eq(document.connectorId, context.connectorId), + inArray(document.id, documentIds), + eq(document.userExcluded, !restoring), + isNull(document.archivedAt), + isNull(document.deletedAt) + ) + ) + .returning({ id: document.id }) + return { + operation: input.operation, + count: updated.length, + documentIds: updated.map(({ id }) => id), + } + }, + projectAudit: ({ input, context, result }) => ({ + action: + input.operation === 'restore' + ? AuditAction.CONNECTOR_DOCUMENT_RESTORED + : AuditAction.CONNECTOR_DOCUMENT_EXCLUDED, + resourceType: AuditResourceType.CONNECTOR, + resourceId: context.connectorId, + description: + input.operation === 'restore' + ? `Restored ${result.count} excluded document(s) for knowledge base "${context.knowledgeBase.name}"` + : `Excluded ${result.count} document(s) from knowledge base "${context.knowledgeBase.name}"`, + metadata: { + knowledgeBaseId: context.knowledgeBaseId, + knowledgeBaseName: context.knowledgeBase.name, + operation: input.operation, + documentCount: result.count, + documentIds: result.documentIds, + }, + }), +}) diff --git a/apps/sim/lib/knowledge/application/contexts.test.ts b/apps/sim/lib/knowledge/application/contexts.test.ts index 950ae781ecd..0e04796c5b7 100644 --- a/apps/sim/lib/knowledge/application/contexts.test.ts +++ b/apps/sim/lib/knowledge/application/contexts.test.ts @@ -1,21 +1,47 @@ /** * @vitest-environment node */ + import { beforeEach, describe, expect, it, vi } from 'vitest' const mocks = vi.hoisted(() => ({ getKnowledgeBase: vi.fn(), + getDocument: vi.fn(), + getDocumentById: vi.fn(), + getTag: vi.fn(), + getConnector: vi.fn(), loadWorkspace: vi.fn(), + loadWorkspaceIncludingArchived: vi.fn(), +})) + +vi.mock('@/lib/knowledge/service', () => ({ + getKnowledgeBaseById: mocks.getKnowledgeBase, +})) + +vi.mock('@/lib/knowledge/documents/service', () => ({ + getKnowledgeDocument: mocks.getDocument, + getKnowledgeDocumentById: mocks.getDocumentById, +})) + +vi.mock('@/lib/knowledge/tags/service', () => ({ + getTagDefinitionById: mocks.getTag, +})) + +vi.mock('@/lib/knowledge/connectors/service', () => ({ + getActiveKnowledgeConnectorReference: mocks.getConnector, })) -vi.mock('@/lib/knowledge/service', () => ({ getKnowledgeBaseById: mocks.getKnowledgeBase })) -vi.mock('@/lib/knowledge/documents/service', () => ({ getKnowledgeDocument: vi.fn() })) vi.mock('@/lib/workspaces/application/workspace-context', () => ({ loadActiveWorkspaceApplicationContext: mocks.loadWorkspace, + loadWorkspaceApplicationContext: mocks.loadWorkspaceIncludingArchived, })) import { + loadKnowledgeWorkspaceAuthorizationContext, resolveActiveKnowledgeBaseContext, + resolveActiveKnowledgeConnectorContext, + resolveActiveKnowledgeTagContext, + resolveCanonicalActiveKnowledgeDocumentContext, resolveKnowledgeWorkspaceContext, } from '@/lib/knowledge/application/contexts' @@ -32,6 +58,7 @@ describe('knowledge application contexts', () => { vi.clearAllMocks() mocks.getKnowledgeBase.mockResolvedValue(knowledgeBase) mocks.loadWorkspace.mockResolvedValue(workspace) + mocks.loadWorkspaceIncludingArchived.mockResolvedValue(workspace) }) it('uses the canonical active-workspace loader', async () => { @@ -41,6 +68,15 @@ describe('knowledge application contexts', () => { expect(mocks.loadWorkspace).toHaveBeenCalledWith('workspace-1') }) + it('uses the neutral canonical loader when archived workspace authorization is explicit', async () => { + await expect( + loadKnowledgeWorkspaceAuthorizationContext('workspace-1', { includeArchived: true }) + ).resolves.toBe(workspace) + expect(mocks.loadWorkspaceIncludingArchived).toHaveBeenCalledWith('workspace-1', { + includeArchived: true, + }) + }) + it('conceals an inactive canonical workspace as knowledge-base absence', async () => { mocks.loadWorkspace.mockResolvedValueOnce(null) @@ -57,4 +93,65 @@ describe('knowledge application contexts', () => { resolveActiveKnowledgeBaseContext({ knowledgeBaseId: 'knowledge-1' }) ).rejects.toBe(failure) }) + + describe('canonical child resources', () => { + beforeEach(() => { + mocks.getKnowledgeBase.mockResolvedValue({ + id: 'knowledge-b', + name: 'Workspace B docs', + workspaceId: 'workspace-b', + }) + mocks.getDocumentById.mockResolvedValue({ + id: 'document-b', + knowledgeBaseId: 'knowledge-b', + }) + mocks.getTag.mockResolvedValue({ + id: 'tag-b', + knowledgeBaseId: 'knowledge-b', + }) + mocks.getConnector.mockResolvedValue({ + id: 'connector-b', + knowledgeBaseId: 'knowledge-b', + connectorType: 'confluence', + status: 'active', + }) + }) + + it('resolves a document parent canonically before comparing the trusted workspace', async () => { + await expect( + resolveCanonicalActiveKnowledgeDocumentContext({ + knowledgeBaseId: 'knowledge-b', + documentId: 'document-b', + assertedWorkspaceId: 'workspace-a', + }) + ).rejects.toMatchObject({ code: 'not_found' }) + + expect(mocks.getDocumentById).toHaveBeenCalledWith('document-b') + expect(mocks.getKnowledgeBase).toHaveBeenCalledWith('knowledge-b') + }) + + it('resolves a tag parent canonically before comparing the trusted workspace', async () => { + await expect( + resolveActiveKnowledgeTagContext({ + tagDefinitionId: 'tag-b', + assertedWorkspaceId: 'workspace-a', + }) + ).rejects.toMatchObject({ code: 'not_found' }) + + expect(mocks.getTag).toHaveBeenCalledWith('tag-b') + expect(mocks.getKnowledgeBase).toHaveBeenCalledWith('knowledge-b') + }) + + it('resolves a connector parent canonically before comparing the trusted workspace', async () => { + await expect( + resolveActiveKnowledgeConnectorContext({ + connectorId: 'connector-b', + assertedWorkspaceId: 'workspace-a', + }) + ).rejects.toMatchObject({ code: 'not_found' }) + + expect(mocks.getConnector).toHaveBeenCalledWith('connector-b') + expect(mocks.getKnowledgeBase).toHaveBeenCalledWith('knowledge-b') + }) + }) }) diff --git a/apps/sim/lib/knowledge/application/contexts.ts b/apps/sim/lib/knowledge/application/contexts.ts index 57a6cd81c64..b81227640a2 100644 --- a/apps/sim/lib/knowledge/application/contexts.ts +++ b/apps/sim/lib/knowledge/application/contexts.ts @@ -1,10 +1,23 @@ +import { db } from '@sim/db' +import { embedding } from '@sim/db/schema' +import { and, eq } from 'drizzle-orm' import { OrchestrationError } from '@/lib/core/orchestration/types' import type { KnowledgeAuthorizationContext } from '@/lib/knowledge/application/authorization' +import type { ChunkData } from '@/lib/knowledge/chunks/types' +import { + type ActiveKnowledgeConnectorReference, + getActiveKnowledgeConnectorReference, +} from '@/lib/knowledge/connectors/service' import type { ActiveKnowledgeDocument } from '@/lib/knowledge/documents/service' -import { getKnowledgeDocument } from '@/lib/knowledge/documents/service' +import { getKnowledgeDocument, getKnowledgeDocumentById } from '@/lib/knowledge/documents/service' import { getKnowledgeBaseById } from '@/lib/knowledge/service' +import { getTagDefinitionById } from '@/lib/knowledge/tags/service' +import type { DocumentTagDefinition } from '@/lib/knowledge/tags/types' import type { KnowledgeBaseWithCounts } from '@/lib/knowledge/types' -import { loadActiveWorkspaceApplicationContext } from '@/lib/workspaces/application/workspace-context' +import { + loadActiveWorkspaceApplicationContext, + loadWorkspaceApplicationContext, +} from '@/lib/workspaces/application/workspace-context' export interface KnowledgeWorkspaceContext extends KnowledgeAuthorizationContext { billedAccountUserId: string @@ -20,12 +33,34 @@ export interface ActiveKnowledgeDocumentContext extends ActiveKnowledgeBaseConte document: ActiveKnowledgeDocument } +export interface ActiveKnowledgeTagContext extends ActiveKnowledgeBaseContext { + tagDefinitionId: string + tagDefinition: DocumentTagDefinition +} + +export interface ActiveKnowledgeConnectorContext extends ActiveKnowledgeBaseContext { + connectorId: string + connector: ActiveKnowledgeConnectorReference +} + +export interface ActiveKnowledgeChunkContext extends ActiveKnowledgeDocumentContext { + chunkId: string + chunk: ChunkData +} + export async function loadKnowledgeWorkspaceContext( workspaceId: string ): Promise { return loadActiveWorkspaceApplicationContext(workspaceId) } +export async function loadKnowledgeWorkspaceAuthorizationContext( + workspaceId: string, + options: { includeArchived?: boolean } = {} +): Promise { + return loadWorkspaceApplicationContext(workspaceId, options) +} + export async function resolveKnowledgeWorkspaceContext(input: { workspaceId: string }): Promise { @@ -69,3 +104,91 @@ export async function resolveActiveKnowledgeDocumentContext(input: { document, } } + +export async function resolveCanonicalActiveKnowledgeDocumentContext(input: { + knowledgeBaseId: string + documentId: string + assertedWorkspaceId?: string +}): Promise { + const document = await getKnowledgeDocumentById(input.documentId) + if (!document || document.knowledgeBaseId !== input.knowledgeBaseId) { + throw new OrchestrationError('not_found', 'Document not found') + } + const context = await resolveActiveKnowledgeBaseContext({ + knowledgeBaseId: document.knowledgeBaseId, + assertedWorkspaceId: input.assertedWorkspaceId, + }) + return { + ...context, + documentId: document.id, + document, + } +} + +export async function resolveActiveKnowledgeChunkContext(input: { + knowledgeBaseId: string + documentId: string + chunkId: string + assertedWorkspaceId?: string +}): Promise { + const [chunk] = await db + .select() + .from(embedding) + .where(and(eq(embedding.id, input.chunkId), eq(embedding.documentId, input.documentId))) + .limit(1) + if (!chunk || chunk.knowledgeBaseId !== input.knowledgeBaseId) { + throw new OrchestrationError('not_found', 'Chunk not found') + } + const context = await resolveCanonicalActiveKnowledgeDocumentContext(input) + return { + ...context, + chunkId: chunk.id, + chunk: chunk as ChunkData, + } +} + +export async function resolveActiveKnowledgeTagContext(input: { + tagDefinitionId: string + knowledgeBaseId?: string + assertedWorkspaceId?: string +}): Promise { + const tagDefinition = await getTagDefinitionById(input.tagDefinitionId) + if ( + !tagDefinition || + (input.knowledgeBaseId && tagDefinition.knowledgeBaseId !== input.knowledgeBaseId) + ) { + throw new OrchestrationError('not_found', 'Tag definition not found') + } + const context = await resolveActiveKnowledgeBaseContext({ + knowledgeBaseId: tagDefinition.knowledgeBaseId, + assertedWorkspaceId: input.assertedWorkspaceId, + }) + return { + ...context, + tagDefinitionId: tagDefinition.id, + tagDefinition, + } +} + +export async function resolveActiveKnowledgeConnectorContext(input: { + connectorId: string + knowledgeBaseId?: string + assertedWorkspaceId?: string +}): Promise { + const connector = await getActiveKnowledgeConnectorReference(input.connectorId) + if ( + !connector || + (input.knowledgeBaseId && connector.knowledgeBaseId !== input.knowledgeBaseId) + ) { + throw new OrchestrationError('not_found', 'Connector not found') + } + const context = await resolveActiveKnowledgeBaseContext({ + knowledgeBaseId: connector.knowledgeBaseId, + assertedWorkspaceId: input.assertedWorkspaceId, + }) + return { + ...context, + connectorId: connector.id, + connector, + } +} diff --git a/apps/sim/lib/knowledge/application/documents.test.ts b/apps/sim/lib/knowledge/application/documents.test.ts index acfeea66d47..20806987c63 100644 --- a/apps/sim/lib/knowledge/application/documents.test.ts +++ b/apps/sim/lib/knowledge/application/documents.test.ts @@ -7,6 +7,7 @@ import { beforeEach, describe, expect, it, vi } from 'vitest' const mocks = vi.hoisted(() => ({ resolveKnowledgeBase: vi.fn(), resolveDocument: vi.fn(), + resolveCanonicalDocument: vi.fn(), resolvePermission: vi.fn(), resolveHumanBilling: vi.fn(), resolveSystemBilling: vi.fn(), @@ -14,14 +15,22 @@ const mocks = vi.hoisted(() => ({ getDocuments: vi.fn(), createDocument: vi.fn(), deleteDocument: vi.fn(), + updateDocument: vi.fn(), processQueue: vi.fn(), + getProcessingConfig: vi.fn(), + performSingleUpload: vi.fn(), + performBulkUpload: vi.fn(), + markTimedOut: vi.fn(), + retryProcessing: vi.fn(), recordAudit: vi.fn(), + captureServerEvent: vi.fn(), })) vi.mock('@sim/audit', () => ({ AuditAction: { DOCUMENT_UPLOADED: 'document.uploaded', DOCUMENT_DELETED: 'document.deleted', + DOCUMENT_UPDATED: 'document.updated', }, AuditResourceType: { DOCUMENT: 'document' }, recordAudit: mocks.recordAudit, @@ -46,19 +55,34 @@ vi.mock('@/lib/billing/core/billing-attribution', () => ({ vi.mock('@/lib/knowledge/application/contexts', () => ({ resolveActiveKnowledgeBaseContext: mocks.resolveKnowledgeBase, resolveActiveKnowledgeDocumentContext: mocks.resolveDocument, + resolveCanonicalActiveKnowledgeDocumentContext: mocks.resolveCanonicalDocument, })) vi.mock('@/lib/knowledge/documents/service', () => ({ getDocuments: mocks.getDocuments, createSingleDocument: mocks.createDocument, deleteKnowledgeDocumentInKnowledgeBase: mocks.deleteDocument, + updateDocument: mocks.updateDocument, processDocumentsWithQueue: mocks.processQueue, + getProcessingConfig: mocks.getProcessingConfig, })) +vi.mock('@/lib/knowledge/orchestration/documents', () => ({ + performUploadKnowledgeDocument: mocks.performSingleUpload, + performUploadKnowledgeDocuments: mocks.performBulkUpload, + performMarkKnowledgeDocumentTimedOut: mocks.markTimedOut, + performRetryKnowledgeDocumentProcessing: mocks.retryProcessing, +})) + +vi.mock('@/lib/posthog/server', () => ({ captureServerEvent: mocks.captureServerEvent })) + import { OrchestrationError } from '@/lib/core/orchestration/types' import { + bulkDeleteKnowledgeDocuments, + createKnowledgeDocuments, deleteKnowledgeDocument, listKnowledgeDocuments, + updateKnowledgeDocument, uploadKnowledgeDocument, } from '@/lib/knowledge/application/documents' @@ -92,6 +116,11 @@ describe('knowledge document application use cases', () => { documentId: document.id, document, }) + mocks.resolveCanonicalDocument.mockResolvedValue({ + ...context, + documentId: document.id, + document, + }) mocks.resolveSystemBilling.mockResolvedValue({ actorUserId: 'billing-owner-1', workspaceId: 'workspace-1', @@ -102,7 +131,21 @@ describe('knowledge document application use cases', () => { }) mocks.checkUsage.mockResolvedValue({ isExceeded: false }) mocks.createDocument.mockResolvedValue(document) + mocks.updateDocument.mockResolvedValue(document) mocks.processQueue.mockResolvedValue(undefined) + mocks.getProcessingConfig.mockReturnValue({ batchSize: 10, maxConcurrentDocuments: 2 }) + mocks.performBulkUpload.mockResolvedValue({ + success: true, + documents: [ + { + documentId: document.id, + filename: document.filename, + fileUrl: document.fileUrl, + fileSize: document.fileSize, + mimeType: document.mimeType, + }, + ], + }) mocks.getDocuments.mockResolvedValue({ documents: [], pagination: { total: 0, limit: 50, offset: 0, hasMore: false }, @@ -231,6 +274,89 @@ describe('knowledge document application use cases', () => { ) }) + it('rejects a cross-workspace document update before current membership or mutation', async () => { + mocks.resolveCanonicalDocument.mockResolvedValueOnce({ + ...context, + workspaceId: 'workspace-b', + billedAccountUserId: 'billing-owner-b', + knowledgeBaseId: 'knowledge-b', + knowledgeBase: { id: 'knowledge-b', name: 'Workspace B docs' }, + documentId: 'document-b', + document: { ...document, id: 'document-b', knowledgeBaseId: 'knowledge-b' }, + }) + + await expect( + updateKnowledgeDocument.execute({ + principal: { + kind: 'delegated', + serviceId: 'copilot', + subjectUserId: 'shared-user', + workspaceId: 'workspace-a', + delegationId: 'tool-call-1', + audience: 'sim:knowledge', + issuedAt: new Date(), + expiresAt: new Date(Date.now() + 60_000), + resourceScope: {}, + }, + input: { + knowledgeBaseId: 'knowledge-b', + documentId: 'document-b', + assertedWorkspaceId: 'workspace-a', + filename: 'renamed.pdf', + }, + }) + ).rejects.toMatchObject({ + name: 'DelegatedWorkspaceAuthorizationError', + code: 'forbidden', + }) + + expect(mocks.resolvePermission).not.toHaveBeenCalled() + expect(mocks.updateDocument).not.toHaveBeenCalled() + expect(mocks.recordAudit).not.toHaveBeenCalled() + }) + + it('authorizes and audits a same-workspace delegated document update', async () => { + await updateKnowledgeDocument.execute({ + principal: { + kind: 'delegated', + serviceId: 'copilot', + subjectUserId: 'shared-user', + workspaceId: 'workspace-1', + delegationId: 'tool-call-1', + audience: 'sim:knowledge', + issuedAt: new Date(), + expiresAt: new Date(Date.now() + 60_000), + resourceScope: {}, + }, + input: { + knowledgeBaseId: 'knowledge-1', + documentId: 'document-1', + assertedWorkspaceId: 'workspace-1', + enabled: false, + source: 'agent', + }, + }) + + expect(mocks.resolvePermission.mock.invocationCallOrder[0]).toBeLessThan( + mocks.updateDocument.mock.invocationCallOrder[0] + ) + expect(mocks.updateDocument).toHaveBeenCalledWith( + 'document-1', + { filename: undefined, enabled: false }, + expect.any(String) + ) + expect(mocks.recordAudit).toHaveBeenCalledWith( + expect.objectContaining({ + action: 'document.updated', + metadata: expect.objectContaining({ + operation: 'knowledge.documents.update', + enabled: false, + actor: expect.objectContaining({ kind: 'delegated', serviceId: 'copilot' }), + }), + }) + ) + }) + it('propagates document infrastructure failures without audit', async () => { const failure = new Error('storage ledger unavailable') mocks.createDocument.mockRejectedValueOnce(failure) @@ -248,4 +374,217 @@ describe('knowledge document application use cases', () => { expect(mocks.recordAudit).not.toHaveBeenCalled() }) + + it('bounds bulk document creation before billing or orchestration', async () => { + await expect( + createKnowledgeDocuments.execute({ + principal: { kind: 'session', userId: 'user-1', sessionId: 'session-1' }, + input: { + knowledgeBaseId: 'knowledge-1', + assertedWorkspaceId: 'workspace-1', + documents: Array.from({ length: 101 }, (_, index) => ({ + filename: `document-${index}.txt`, + fileUrl: `/document-${index}.txt`, + fileSize: 1, + mimeType: 'text/plain', + })), + bulk: true, + resolveSecretProvenances: () => undefined, + }, + }) + ).rejects.toMatchObject({ code: 'validation' }) + + expect(mocks.checkUsage).not.toHaveBeenCalled() + expect(mocks.performBulkUpload).not.toHaveBeenCalled() + expect(mocks.recordAudit).not.toHaveBeenCalled() + }) + + it('disables legacy analytics and projects delegated audit for bulk creation', async () => { + await createKnowledgeDocuments.execute({ + principal: { + kind: 'delegated', + serviceId: 'copilot', + subjectUserId: 'shared-user', + workspaceId: 'workspace-1', + delegationId: 'tool-call-1', + audience: 'sim:knowledge', + issuedAt: new Date(), + expiresAt: new Date(Date.now() + 60_000), + }, + input: { + knowledgeBaseId: 'knowledge-1', + assertedWorkspaceId: 'workspace-1', + documents: [ + { + filename: document.filename, + fileUrl: document.fileUrl, + fileSize: document.fileSize, + mimeType: document.mimeType, + }, + ], + bulk: true, + source: 'agent', + resolveSecretProvenances: () => undefined, + }, + }) + + expect(mocks.performBulkUpload).toHaveBeenCalledWith( + expect.objectContaining({ + userId: 'shared-user', + recordSemanticAudit: false, + recordProductAnalytics: false, + }) + ) + expect(mocks.recordAudit).toHaveBeenCalledWith( + expect.objectContaining({ + workspaceId: 'workspace-1', + action: 'document.uploaded', + metadata: expect.objectContaining({ + operation: 'knowledge.documents.upload', + actor: expect.objectContaining({ kind: 'delegated', serviceId: 'copilot' }), + }), + }) + ) + expect(mocks.captureServerEvent).not.toHaveBeenCalled() + }) + + it('bounds best-effort document deletion before canonical knowledge loading', async () => { + await expect( + bulkDeleteKnowledgeDocuments.execute({ + principal: { kind: 'session', userId: 'user-1', sessionId: 'session-1' }, + input: { + knowledgeBaseId: 'knowledge-1', + assertedWorkspaceId: 'workspace-1', + documentIds: Array.from({ length: 101 }, (_, index) => `document-${index}`), + }, + }) + ).rejects.toMatchObject({ code: 'validation' }) + + expect(mocks.resolveKnowledgeBase).not.toHaveBeenCalled() + expect(mocks.deleteDocument).not.toHaveBeenCalled() + }) + + it('conceals a cross-knowledge-base bulk document before mutation for a dual-workspace subject', async () => { + mocks.resolveCanonicalDocument.mockRejectedValueOnce( + new OrchestrationError('not_found', 'Document not found') + ) + + const result = await bulkDeleteKnowledgeDocuments.execute({ + principal: { + kind: 'delegated', + serviceId: 'copilot', + subjectUserId: 'dual-workspace-user', + workspaceId: 'workspace-1', + delegationId: 'tool-call-1', + audience: 'sim:knowledge', + issuedAt: new Date(), + expiresAt: new Date(Date.now() + 60_000), + }, + input: { + knowledgeBaseId: 'knowledge-1', + assertedWorkspaceId: 'workspace-1', + documentIds: ['workspace-2-document'], + }, + }) + + expect(result).toMatchObject({ deleted: [], failed: ['workspace-2-document'] }) + expect(mocks.resolvePermission).toHaveBeenCalledWith( + 'dual-workspace-user', + 'workspace-1', + null, + undefined, + { forUpdate: undefined } + ) + expect(mocks.deleteDocument).not.toHaveBeenCalled() + expect(mocks.recordAudit).not.toHaveBeenCalled() + }) + + it('returns partial document outcomes and keeps product analytics out of the application', async () => { + mocks.resolveCanonicalDocument.mockImplementation(async ({ documentId }) => ({ + ...context, + documentId, + document: { ...document, id: documentId, filename: `${documentId}.pdf` }, + })) + mocks.deleteDocument + .mockResolvedValueOnce(undefined) + .mockRejectedValueOnce(new OrchestrationError('conflict', 'Document is locked')) + + const result = await bulkDeleteKnowledgeDocuments.execute({ + principal: { kind: 'session', userId: 'user-1', sessionId: 'session-1' }, + input: { + knowledgeBaseId: 'knowledge-1', + assertedWorkspaceId: 'workspace-1', + documentIds: ['document-1', 'document-2'], + source: 'agent', + }, + }) + + expect(result).toMatchObject({ + deleted: ['document-1'], + failed: ['document-2'], + cancelled: false, + }) + expect(mocks.resolvePermission).toHaveBeenCalledTimes(3) + expect(mocks.recordAudit).toHaveBeenCalledOnce() + expect(mocks.recordAudit).toHaveBeenCalledWith( + expect.objectContaining({ + resourceId: 'document-1', + metadata: expect.objectContaining({ operation: 'knowledge.documents.bulk_delete' }), + }) + ) + expect(mocks.captureServerEvent).not.toHaveBeenCalled() + }) + + it('stops between document deletions while auditing completed items', async () => { + const controller = new AbortController() + mocks.resolveCanonicalDocument.mockImplementation(async ({ documentId }) => ({ + ...context, + documentId, + document: { ...document, id: documentId }, + })) + mocks.deleteDocument.mockImplementationOnce(async () => { + controller.abort('user stopped') + }) + + const result = await bulkDeleteKnowledgeDocuments.execute({ + principal: { kind: 'session', userId: 'user-1', sessionId: 'session-1' }, + input: { + knowledgeBaseId: 'knowledge-1', + assertedWorkspaceId: 'workspace-1', + documentIds: ['document-1', 'document-2'], + cancellationSignal: controller.signal, + }, + }) + + expect(result).toMatchObject({ deleted: ['document-1'], cancelled: true }) + expect(mocks.deleteDocument).toHaveBeenCalledOnce() + expect(mocks.recordAudit).toHaveBeenCalledOnce() + }) + + it('audits completed document deletions before propagating infrastructure failure', async () => { + const failure = new Error('document store unavailable') + mocks.resolveCanonicalDocument.mockImplementation(async ({ documentId }) => ({ + ...context, + documentId, + document: { ...document, id: documentId }, + })) + mocks.deleteDocument.mockResolvedValueOnce(undefined).mockRejectedValueOnce(failure) + + await expect( + bulkDeleteKnowledgeDocuments.execute({ + principal: { kind: 'session', userId: 'user-1', sessionId: 'session-1' }, + input: { + knowledgeBaseId: 'knowledge-1', + assertedWorkspaceId: 'workspace-1', + documentIds: ['document-1', 'document-2'], + }, + }) + ).rejects.toBe(failure) + + expect(mocks.recordAudit).toHaveBeenCalledOnce() + expect(mocks.recordAudit).toHaveBeenCalledWith( + expect.objectContaining({ resourceId: 'document-1' }) + ) + expect(mocks.captureServerEvent).not.toHaveBeenCalled() + }) }) diff --git a/apps/sim/lib/knowledge/application/documents.ts b/apps/sim/lib/knowledge/application/documents.ts index 76f81c22e7d..70a9f8af830 100644 --- a/apps/sim/lib/knowledge/application/documents.ts +++ b/apps/sim/lib/knowledge/application/documents.ts @@ -1,29 +1,57 @@ import { AuditAction, AuditResourceType } from '@sim/audit' +import { db } from '@sim/db' +import { document as documentTable } from '@sim/db/schema' import { createLogger } from '@sim/logger' +import { and, eq, isNull } from 'drizzle-orm' import { checkAttributedUsageLimits } from '@/lib/billing/core/billing-attribution' -import { OrchestrationError } from '@/lib/core/orchestration/types' +import { authorizeWorkspaceOperation } from '@/lib/core/application' +import { asOrchestrationError, OrchestrationError } from '@/lib/core/orchestration/types' import { generateRequestId } from '@/lib/core/utils/request' +import { knowledgeDelegationPolicy } from '@/lib/knowledge/application/authorization' import { defineAuthorizedKnowledgeUseCase } from '@/lib/knowledge/application/authorized-knowledge-use-case' +import { + BULK_DELETE_KNOWLEDGE_DOCUMENTS_COST_POLICY, + type KnowledgeBatchExecutionResult, + requireBoundedKnowledgeBatch, + rethrowKnowledgeBatchTerminalFailure, +} from '@/lib/knowledge/application/batch-policy' import { KnowledgeUsageLimitExceededError, resolveKnowledgeAttributedUserId, resolveKnowledgeBillingAttribution, } from '@/lib/knowledge/application/billing' import { + type ActiveKnowledgeBaseContext, type ActiveKnowledgeDocumentContext, resolveActiveKnowledgeBaseContext, resolveActiveKnowledgeDocumentContext, + resolveCanonicalActiveKnowledgeDocumentContext, } from '@/lib/knowledge/application/contexts' import { knowledgeOperations } from '@/lib/knowledge/application/operations' +import { MAX_KNOWLEDGE_DOCUMENTS_PER_CREATE } from '@/lib/knowledge/constants' import { + bulkDocumentOperation, + bulkDocumentOperationByFilter, + createDocumentRecords, createSingleDocument, type DocumentData, + deleteDocument, deleteKnowledgeDocumentInKnowledgeBase, getDocuments, + getProcessingConfig, type ProcessingOptions, processDocumentsWithQueue, + updateDocument, } from '@/lib/knowledge/documents/service' +import type { TagFilterCondition } from '@/lib/knowledge/documents/tag-filter' import type { DocumentSortField, SortOrder } from '@/lib/knowledge/documents/types' +import { + performMarkKnowledgeDocumentTimedOut, + performRetryKnowledgeDocumentProcessing, + performUploadKnowledgeDocument, + performUploadKnowledgeDocuments, +} from '@/lib/knowledge/orchestration/documents' +import type { KnowledgeDocumentWriteSecretProvenance } from '@/lib/knowledge/secret-provenance' import { MAX_KNOWLEDGE_DOCUMENT_FILE_SIZE } from '@/lib/uploads/shared/types' import { validateFileType } from '@/lib/uploads/utils/validation' @@ -38,6 +66,7 @@ export interface ListKnowledgeDocumentsInput { offset?: number sortBy?: DocumentSortField sortOrder?: SortOrder + tagFilters?: TagFilterCondition[] } export interface ReadKnowledgeDocumentInput { @@ -75,10 +104,89 @@ export interface UploadKnowledgeDocumentInput extends UploadKnowledgeDocumentAdm source?: string } +export interface CreateKnowledgeDocumentsInput extends UploadKnowledgeDocumentAdmissionInput { + documents: KnowledgeDocumentInput[] + bulk: boolean + processingOptions?: ProcessingOptions + source?: 'ui' | 'api' | 'agent' + resolveBillingAttribution?( + workspaceId: string + ): Promise>> + resolveSecretProvenances(input: { + userId: string + workspaceId: string + }): KnowledgeDocumentWriteSecretProvenance[] | undefined +} + export interface DeleteKnowledgeDocumentInput extends ReadKnowledgeDocumentInput { source?: string } +export interface BulkDeleteKnowledgeDocumentsInput extends UploadKnowledgeDocumentAdmissionInput { + documentIds: string[] + cancellationSignal?: AbortSignal + source?: string +} + +interface DeletedKnowledgeDocument { + id: string + filename: string + fileSize: number + mimeType: string +} + +export interface BulkDeleteKnowledgeDocumentsResult { + knowledgeBaseId: string + deleted: string[] + failed: string[] + deletedDocuments: DeletedKnowledgeDocument[] + cancelled: boolean +} + +interface BulkDeleteKnowledgeDocumentsExecutionResult + extends BulkDeleteKnowledgeDocumentsResult, + KnowledgeBatchExecutionResult {} + +interface BulkDeleteKnowledgeDocumentsContext extends ActiveKnowledgeBaseContext { + documentIds: string[] +} + +export interface UpdateKnowledgeDocumentInput extends ReadKnowledgeDocumentInput { + filename?: string + enabled?: boolean + updates?: Parameters[1] + markFailedDueToTimeout?: boolean + retryProcessing?: boolean + resolveBillingAttribution?( + workspaceId: string + ): Promise>> + source?: string +} + +export interface BulkKnowledgeDocumentsInput extends UploadKnowledgeDocumentAdmissionInput { + operation: 'enable' | 'disable' | 'delete' + documentIds?: string[] + selectAll?: boolean + enabledFilter?: 'all' | 'enabled' | 'disabled' +} + +export interface UpsertKnowledgeDocumentInput extends UploadKnowledgeDocumentAdmissionInput { + documentId?: string + filename: string + fileUrl: string + fileSize: number + mimeType: string + documentTagsData?: string + processingOptions?: ProcessingOptions + resolveBillingAttribution( + workspaceId: string + ): Promise>> + resolveSecretProvenances(input: { + userId: string + workspaceId: string + }): KnowledgeDocumentWriteSecretProvenance[] | undefined +} + export const listKnowledgeDocuments = defineAuthorizedKnowledgeUseCase({ operation: knowledgeOperations.listDocuments, resolveContext: ({ input }: { input: ListKnowledgeDocumentsInput }) => @@ -92,7 +200,7 @@ export const listKnowledgeDocuments = defineAuthorizedKnowledgeUseCase({ if (!Number.isInteger(offset) || offset < 0) { throw new OrchestrationError('validation', 'Document offset must be a non-negative integer') } - return getDocuments( + const result = await getDocuments( context.knowledgeBaseId, { enabledFilter: input.enabledFilter === 'all' ? undefined : input.enabledFilter, @@ -101,9 +209,11 @@ export const listKnowledgeDocuments = defineAuthorizedKnowledgeUseCase({ offset, sortBy: input.sortBy, sortOrder: input.sortOrder, + tagFilters: input.tagFilters, }, generateRequestId() ) + return { ...result, workspaceId: context.workspaceId } }, }) @@ -112,7 +222,7 @@ export const readKnowledgeDocument = defineAuthorizedKnowledgeUseCase({ resolveContext: ({ input }: { input: ReadKnowledgeDocumentInput }) => resolveActiveKnowledgeDocumentContext(input), async execute({ context }: { context: ActiveKnowledgeDocumentContext }) { - return { document: context.document } + return { document: context.document, workspaceId: context.workspaceId } }, }) @@ -211,6 +321,265 @@ export const uploadKnowledgeDocument = defineAuthorizedKnowledgeUseCase({ }), }) +export const createKnowledgeDocuments = defineAuthorizedKnowledgeUseCase({ + operation: knowledgeOperations.uploadDocument, + resolveContext: ({ input }: { input: CreateKnowledgeDocumentsInput }) => + resolveActiveKnowledgeBaseContext(input), + async execute({ principal, input, context, request }) { + if (input.documents.length === 0) { + throw new OrchestrationError('validation', 'No documents specified') + } + if (input.documents.length > MAX_KNOWLEDGE_DOCUMENTS_PER_CREATE) { + throw new OrchestrationError( + 'validation', + `At most ${MAX_KNOWLEDGE_DOCUMENTS_PER_CREATE} documents may be created at once` + ) + } + const billingAttribution = input.resolveBillingAttribution + ? await input.resolveBillingAttribution(context.workspaceId) + : await resolveKnowledgeBillingAttribution(principal, context) + const usage = await checkAttributedUsageLimits(billingAttribution) + if (usage.isExceeded) { + throw new KnowledgeUsageLimitExceededError( + usage.message || 'Usage limit exceeded. Please upgrade your plan to continue.' + ) + } + const userId = resolveKnowledgeAttributedUserId(principal, context) + const secretProvenances = input.resolveSecretProvenances({ + userId, + workspaceId: context.workspaceId, + }) + const knowledgeBase = { + id: context.knowledgeBaseId, + name: context.knowledgeBase.name, + workspaceId: context.workspaceId, + } + if (input.bulk) { + const outcome = await performUploadKnowledgeDocuments({ + knowledgeBase, + documents: input.documents, + processingOptions: input.processingOptions, + billingAttribution, + uploadedBy: userId, + secretProvenances, + userId, + source: input.source ?? 'ui', + request, + recordSemanticAudit: false, + recordProductAnalytics: false, + }) + if (!outcome.success) { + if (outcome.errorCode === 'internal') throw new Error('Knowledge document creation failed') + throw new OrchestrationError(outcome.errorCode, outcome.error) + } + const { batchSize, maxConcurrentDocuments } = getProcessingConfig() + return { + kind: 'bulk' as const, + data: { + total: outcome.documents.length, + documentsCreated: outcome.documents.map((document) => ({ + documentId: document.documentId, + filename: document.filename, + status: 'pending' as const, + })), + processingMethod: 'background', + processingConfig: { + maxConcurrentDocuments, + batchSize, + totalBatches: Math.ceil(outcome.documents.length / batchSize), + }, + }, + workspaceId: context.workspaceId, + knowledgeBaseId: context.knowledgeBaseId, + userId, + secretProvenances, + } + } + + const document = input.documents[0] + if (!document) throw new OrchestrationError('validation', 'No documents specified') + const outcome = await performUploadKnowledgeDocument({ + knowledgeBase, + document, + billingAttribution, + uploadedBy: userId, + secretProvenance: secretProvenances?.[0], + userId, + source: input.source ?? 'ui', + request, + recordSemanticAudit: false, + recordProductAnalytics: false, + }) + if (!outcome.success) { + if (outcome.errorCode === 'internal') throw new Error('Knowledge document creation failed') + throw new OrchestrationError(outcome.errorCode, outcome.error) + } + return { + kind: 'single' as const, + data: outcome.document, + workspaceId: context.workspaceId, + userId, + secretProvenances, + } + }, + projectAudit: ({ input, context, result }) => { + if (result.kind === 'bulk') { + return { + action: AuditAction.DOCUMENT_UPLOADED, + resourceType: AuditResourceType.DOCUMENT, + resourceId: context.knowledgeBaseId, + resourceName: `${result.data.total} document(s)`, + description: `Uploaded ${result.data.total} document(s) to knowledge base "${context.knowledgeBase.name}"`, + metadata: { + source: input.source, + knowledgeBaseId: context.knowledgeBaseId, + knowledgeBaseName: context.knowledgeBase.name, + fileCount: result.data.total, + }, + } + } + return { + action: AuditAction.DOCUMENT_UPLOADED, + resourceType: AuditResourceType.DOCUMENT, + resourceId: result.data.id, + resourceName: result.data.filename, + description: `Uploaded document "${result.data.filename}" to knowledge base "${context.knowledgeBase.name}"`, + metadata: { + source: input.source, + knowledgeBaseId: context.knowledgeBaseId, + knowledgeBaseName: context.knowledgeBase.name, + fileName: result.data.filename, + fileType: result.data.mimeType, + fileSize: result.data.fileSize, + }, + } + }, +}) + +export const upsertKnowledgeDocument = defineAuthorizedKnowledgeUseCase({ + operation: knowledgeOperations.uploadDocument, + resolveContext: ({ input }: { input: UpsertKnowledgeDocumentInput }) => + resolveActiveKnowledgeBaseContext(input), + async execute({ principal, input, context }) { + const billingAttribution = await input.resolveBillingAttribution(context.workspaceId) + const usage = await checkAttributedUsageLimits(billingAttribution) + if (usage.isExceeded) { + throw new KnowledgeUsageLimitExceededError( + usage.message || 'Usage limit exceeded. Please upgrade your plan to continue.' + ) + } + const userId = resolveKnowledgeAttributedUserId(principal, context) + const secretProvenances = input.resolveSecretProvenances({ + userId, + workspaceId: context.workspaceId, + }) + let existingDocumentId: string | null = null + if (input.documentId) { + const [existing] = await db + .select({ id: documentTable.id }) + .from(documentTable) + .where( + and( + eq(documentTable.id, input.documentId), + eq(documentTable.knowledgeBaseId, context.knowledgeBaseId), + isNull(documentTable.deletedAt) + ) + ) + .limit(1) + existingDocumentId = existing?.id ?? null + } else { + const [existing] = await db + .select({ id: documentTable.id }) + .from(documentTable) + .where( + and( + eq(documentTable.filename, input.filename), + eq(documentTable.knowledgeBaseId, context.knowledgeBaseId), + isNull(documentTable.deletedAt) + ) + ) + .limit(1) + existingDocumentId = existing?.id ?? null + } + const requestId = generateRequestId() + const createdDocuments = await createDocumentRecords( + [ + { + filename: input.filename, + fileUrl: input.fileUrl, + fileSize: input.fileSize, + mimeType: input.mimeType, + ...(input.documentTagsData ? { documentTagsData: input.documentTagsData } : {}), + }, + ], + context.knowledgeBaseId, + requestId, + userId, + secretProvenances + ) + const createdDocument = createdDocuments[0] + if (!createdDocument) throw new Error('Knowledge document upsert created no document record') + if (existingDocumentId) { + try { + await deleteDocument(existingDocumentId, requestId) + } catch (error) { + try { + await deleteDocument(createdDocument.documentId, requestId) + } catch (rollbackError) { + logger.error('Failed to remove replacement after document upsert failure', { + knowledgeBaseId: context.knowledgeBaseId, + documentId: createdDocument.documentId, + rollbackError, + }) + } + throw new Error('Failed to replace existing document', { cause: error }) + } + } + processDocumentsWithQueue( + createdDocuments, + context.knowledgeBaseId, + input.processingOptions ?? {}, + requestId, + billingAttribution + ).catch((error: unknown) => { + logger.error('Knowledge document upsert processing pipeline failed', { + knowledgeBaseId: context.knowledgeBaseId, + documentId: createdDocument.documentId, + error, + }) + }) + const isUpdate = existingDocumentId !== null + const { maxConcurrentDocuments, batchSize } = getProcessingConfig() + return { + document: createdDocument, + knowledgeBaseId: context.knowledgeBaseId, + isUpdate, + previousDocumentId: existingDocumentId, + processingConfig: { maxConcurrentDocuments, batchSize }, + workspaceId: context.workspaceId, + userId, + secretProvenances, + } + }, + projectAudit: ({ input, context, result }) => ({ + action: result.isUpdate ? AuditAction.DOCUMENT_UPDATED : AuditAction.DOCUMENT_UPLOADED, + resourceType: AuditResourceType.DOCUMENT, + resourceId: context.knowledgeBaseId, + resourceName: input.filename, + description: result.isUpdate + ? `Upserted (replaced) document "${input.filename}" in knowledge base "${context.knowledgeBaseId}"` + : `Upserted (created) document "${input.filename}" in knowledge base "${context.knowledgeBaseId}"`, + metadata: { + knowledgeBaseName: context.knowledgeBase.name, + fileName: input.filename, + fileType: input.mimeType, + fileSize: input.fileSize, + previousDocumentId: result.previousDocumentId, + isUpdate: result.isUpdate, + }, + }), +}) + export const deleteKnowledgeDocument = defineAuthorizedKnowledgeUseCase({ operation: knowledgeOperations.deleteDocument, resolveContext: ({ input }: { input: DeleteKnowledgeDocumentInput }) => @@ -223,6 +592,8 @@ export const deleteKnowledgeDocument = defineAuthorizedKnowledgeUseCase({ ) return { id: context.documentId, + knowledgeBaseId: context.knowledgeBaseId, + workspaceId: context.workspaceId, filename: context.document.filename, fileSize: context.document.fileSize, mimeType: context.document.mimeType, @@ -244,3 +615,184 @@ export const deleteKnowledgeDocument = defineAuthorizedKnowledgeUseCase({ }, }), }) + +export const bulkDeleteKnowledgeDocuments = defineAuthorizedKnowledgeUseCase({ + operation: knowledgeOperations.bulkDeleteDocuments, + async resolveContext({ + input, + }: { + input: BulkDeleteKnowledgeDocumentsInput + }): Promise { + const documentIds = requireBoundedKnowledgeBatch( + input.documentIds, + 'document IDs', + BULK_DELETE_KNOWLEDGE_DOCUMENTS_COST_POLICY.maxItems + ) + return { + ...(await resolveActiveKnowledgeBaseContext(input)), + documentIds, + } + }, + async execute({ + principal, + input, + context, + }): Promise { + const deletedDocuments: DeletedKnowledgeDocument[] = [] + const failed: string[] = [] + let terminalFailure: KnowledgeBatchExecutionResult['terminalFailure'] + + for (const documentId of context.documentIds) { + if (input.cancellationSignal?.aborted) break + try { + const canonical = await resolveCanonicalActiveKnowledgeDocumentContext({ + knowledgeBaseId: context.knowledgeBaseId, + documentId, + assertedWorkspaceId: context.workspaceId, + }) + await authorizeWorkspaceOperation( + principal, + knowledgeOperations.bulkDeleteDocuments, + canonical, + { delegation: knowledgeDelegationPolicy } + ) + if (input.cancellationSignal?.aborted) break + await deleteKnowledgeDocumentInKnowledgeBase( + canonical.knowledgeBaseId, + canonical.documentId, + generateRequestId() + ) + deletedDocuments.push({ + id: canonical.documentId, + filename: canonical.document.filename, + fileSize: canonical.document.fileSize, + mimeType: canonical.document.mimeType, + }) + } catch (error) { + const classified = asOrchestrationError(error) + if (classified && classified.code !== 'internal') { + failed.push(documentId) + continue + } + terminalFailure = { error } + break + } + } + + return { + knowledgeBaseId: context.knowledgeBaseId, + deleted: deletedDocuments.map((document) => document.id), + failed, + deletedDocuments, + cancelled: input.cancellationSignal?.aborted ?? false, + ...(terminalFailure && { terminalFailure }), + } + }, + projectAudit: ({ input, context, result }) => + result.deletedDocuments.map((document) => ({ + action: AuditAction.DOCUMENT_DELETED, + resourceType: AuditResourceType.DOCUMENT, + resourceId: document.id, + resourceName: document.filename, + description: `Deleted document "${document.filename}" from knowledge base "${context.knowledgeBase.name}"`, + metadata: { + source: input.source, + knowledgeBaseId: context.knowledgeBaseId, + knowledgeBaseName: context.knowledgeBase.name, + fileName: document.filename, + fileSize: document.fileSize, + mimeType: document.mimeType, + }, + })), + afterSuccess: ({ result }) => rethrowKnowledgeBatchTerminalFailure(result), +}) + +export const updateKnowledgeDocument = defineAuthorizedKnowledgeUseCase({ + operation: knowledgeOperations.updateDocument, + resolveContext: ({ input }: { input: UpdateKnowledgeDocumentInput }) => + resolveCanonicalActiveKnowledgeDocumentContext(input), + async execute({ principal, input, context }) { + if (input.markFailedDueToTimeout || input.retryProcessing) { + const outcome = input.markFailedDueToTimeout + ? await performMarkKnowledgeDocumentTimedOut({ document: context.document }) + : await performRetryKnowledgeDocumentProcessing({ + knowledgeBaseId: context.knowledgeBaseId, + document: context.document, + billingAttribution: input.resolveBillingAttribution + ? await input.resolveBillingAttribution(context.workspaceId) + : await resolveKnowledgeBillingAttribution(principal, context), + }) + if (!outcome.success) { + if (outcome.errorCode === 'internal') { + throw new Error('Knowledge document processing operation failed') + } + throw new OrchestrationError(outcome.errorCode, outcome.error) + } + return { + kind: 'processing' as const, + documentId: context.documentId, + status: outcome.status, + message: outcome.message, + } + } + const updates = input.updates ?? { filename: input.filename, enabled: input.enabled } + const updatedFields = Object.keys(updates).filter( + (key) => updates[key as keyof typeof updates] !== undefined + ) + if (updatedFields.length === 0) { + throw new OrchestrationError('validation', 'No updates specified') + } + return { + kind: 'updated' as const, + document: await updateDocument(context.documentId, updates, generateRequestId()), + updatedFields, + } + }, + projectAudit: ({ input, context, result }) => { + if (result.kind === 'processing') return [] + return { + action: AuditAction.DOCUMENT_UPDATED, + resourceType: AuditResourceType.DOCUMENT, + resourceId: result.document.id, + resourceName: result.document.filename, + description: `Updated document "${result.document.filename}" in knowledge base "${context.knowledgeBase.name}"`, + metadata: { + source: input.source, + knowledgeBaseId: context.knowledgeBaseId, + knowledgeBaseName: context.knowledgeBase.name, + fileName: result.document.filename, + updatedFields: result.updatedFields, + ...(input.enabled !== undefined && { enabled: input.enabled }), + }, + } + }, +}) + +export const bulkUpdateKnowledgeDocuments = defineAuthorizedKnowledgeUseCase({ + operation: knowledgeOperations.bulkDocuments, + resolveContext: ({ input }: { input: BulkKnowledgeDocumentsInput }) => + resolveActiveKnowledgeBaseContext(input), + async execute({ input, context }) { + const result = input.selectAll + ? await bulkDocumentOperationByFilter( + context.knowledgeBaseId, + input.operation, + input.enabledFilter, + generateRequestId() + ) + : input.documentIds?.length + ? await bulkDocumentOperation( + context.knowledgeBaseId, + input.operation, + input.documentIds, + generateRequestId() + ) + : null + if (!result) throw new OrchestrationError('validation', 'No documents specified') + return { + operation: input.operation, + successCount: result.successCount, + updatedDocuments: result.updatedDocuments, + } + }, +}) diff --git a/apps/sim/lib/knowledge/application/folders.test.ts b/apps/sim/lib/knowledge/application/folders.test.ts index 53e873224f7..33442e38f6d 100644 --- a/apps/sim/lib/knowledge/application/folders.test.ts +++ b/apps/sim/lib/knowledge/application/folders.test.ts @@ -13,7 +13,6 @@ const mocks = vi.hoisted(() => ({ relocateByPath: vi.fn(), deleteByPath: vi.fn(), recordAudit: vi.fn(), - notify: vi.fn(), })) vi.mock('@sim/audit', () => ({ @@ -53,10 +52,6 @@ vi.mock('@/lib/folders/orchestration', () => ({ deleteFolderByPath: mocks.deleteByPath, })) -vi.mock('@/lib/realtime/notify', () => ({ - notifyFolderResourceChanged: mocks.notify, -})) - import { createKnowledgeFolder, deleteKnowledgeFolder, @@ -101,7 +96,6 @@ describe('knowledge folder application use cases', () => { path: '/Docs', deletedItems: { folders: 2, knowledgeBases: 3 }, }) - mocks.notify.mockResolvedValue(undefined) }) it('resolves a canonical parent path before listing', async () => { @@ -161,9 +155,7 @@ describe('knowledge folder application use cases', () => { }), }) ) - expect(mocks.recordAudit.mock.invocationCallOrder[0]).toBeLessThan( - mocks.notify.mock.invocationCallOrder[0] - ) + expect(mocks.recordAudit).toHaveBeenCalledOnce() }) it('preserves recursive cascade counts', async () => { @@ -178,7 +170,7 @@ describe('knowledge folder application use cases', () => { expect(result.deletedItems).toEqual({ folders: 2, knowledgeBases: 3 }) }) - it('propagates infrastructure failures without audit or notification', async () => { + it('propagates infrastructure failures without audit', async () => { const failure = new Error('folder database unavailable') mocks.createAtPath.mockRejectedValueOnce(failure) @@ -190,6 +182,5 @@ describe('knowledge folder application use cases', () => { ).rejects.toBe(failure) expect(mocks.recordAudit).not.toHaveBeenCalled() - expect(mocks.notify).not.toHaveBeenCalled() }) }) diff --git a/apps/sim/lib/knowledge/application/folders.ts b/apps/sim/lib/knowledge/application/folders.ts index 5485322c039..a3f530c25bb 100644 --- a/apps/sim/lib/knowledge/application/folders.ts +++ b/apps/sim/lib/knowledge/application/folders.ts @@ -17,7 +17,6 @@ import { resolveKnowledgeAttributedUserId } from '@/lib/knowledge/application/bi import { resolveKnowledgeWorkspaceContext } from '@/lib/knowledge/application/contexts' import { knowledgeOperations } from '@/lib/knowledge/application/operations' import { MAX_KNOWLEDGE_FOLDERS_PER_WORKSPACE } from '@/lib/knowledge/constants' -import { notifyFolderResourceChanged } from '@/lib/realtime/notify' type KnowledgeFolder = typeof folder.$inferSelect & { path: string } @@ -120,7 +119,6 @@ export const createKnowledgeFolder = defineAuthorizedKnowledgeUseCase({ folderResourceType: 'knowledge_base', }, }), - afterSuccess: ({ context }) => notifyFolderResourceChanged('knowledge_base', context.workspaceId), }) export const relocateKnowledgeFolder = defineAuthorizedKnowledgeUseCase({ @@ -154,7 +152,6 @@ export const relocateKnowledgeFolder = defineAuthorizedKnowledgeUseCase({ folderResourceType: 'knowledge_base', }, }), - afterSuccess: ({ context }) => notifyFolderResourceChanged('knowledge_base', context.workspaceId), }) export const deleteKnowledgeFolder = defineAuthorizedKnowledgeUseCase({ @@ -205,5 +202,4 @@ export const deleteKnowledgeFolder = defineAuthorizedKnowledgeUseCase({ deletedItems: result.deletedItems, }, }), - afterSuccess: ({ context }) => notifyFolderResourceChanged('knowledge_base', context.workspaceId), }) diff --git a/apps/sim/lib/knowledge/application/knowledge-bases.test.ts b/apps/sim/lib/knowledge/application/knowledge-bases.test.ts index 9ae6b3f7fc9..c3318e7daa1 100644 --- a/apps/sim/lib/knowledge/application/knowledge-bases.test.ts +++ b/apps/sim/lib/knowledge/application/knowledge-bases.test.ts @@ -2,10 +2,12 @@ * @vitest-environment node */ +import { dbChainMockFns } from '@sim/testing' import { beforeEach, describe, expect, it, vi } from 'vitest' const mocks = vi.hoisted(() => ({ resolveWorkspace: vi.fn(), + loadAuthorizationWorkspace: vi.fn(), resolveKnowledgeBase: vi.fn(), resolvePermission: vi.fn(), resolveFolderPath: vi.fn(), @@ -13,8 +15,15 @@ const mocks = vi.hoisted(() => ({ updateRecord: vi.fn(), deleteRecord: vi.fn(), listRecords: vi.fn(), + listInternalRecords: vi.fn(), + getRecord: vi.fn(), + getRestorableRecord: vi.fn(), + performUpdate: vi.fn(), + performDelete: vi.fn(), + performRestore: vi.fn(), loadFolderIndex: vi.fn(), recordAudit: vi.fn(), + knowledgeBaseDeleted: vi.fn(), })) vi.mock('@sim/audit', () => ({ @@ -37,11 +46,16 @@ vi.mock('@sim/platform-authz/workspace', () => ({ resolveEffectiveWorkspacePermission: mocks.resolvePermission, })) +vi.mock('@/lib/core/telemetry', () => ({ + PlatformEvents: { knowledgeBaseDeleted: mocks.knowledgeBaseDeleted }, +})) + vi.mock('@/lib/folders/queries', () => ({ loadActiveFolderPathIndex: mocks.loadFolderIndex, })) vi.mock('@/lib/knowledge/application/contexts', () => ({ + loadKnowledgeWorkspaceAuthorizationContext: mocks.loadAuthorizationWorkspace, resolveKnowledgeWorkspaceContext: mocks.resolveWorkspace, resolveActiveKnowledgeBaseContext: mocks.resolveKnowledgeBase, })) @@ -60,13 +74,30 @@ vi.mock('@/lib/knowledge/service', () => ({ createAuthorizedKnowledgeBase: mocks.createRecord, updateKnowledgeBase: mocks.updateRecord, deleteKnowledgeBase: mocks.deleteRecord, + getKnowledgeBaseById: mocks.getRecord, + getKnowledgeBases: mocks.listInternalRecords, getWorkspaceKnowledgeBases: mocks.listRecords, })) +vi.mock('@/lib/knowledge/orchestration', () => ({ + getRestorableKnowledgeBase: mocks.getRestorableRecord, + performUpdateKnowledgeBase: mocks.performUpdate, + performDeleteKnowledgeBase: mocks.performDelete, + performRestoreKnowledgeBase: mocks.performRestore, +})) + import { OrchestrationError } from '@/lib/core/orchestration/types' import { + bulkDeleteKnowledgeBases, createKnowledgeBase, + deleteInternalKnowledgeBase, + listArchivedKnowledgeBases, + listInternalKnowledgeBases, + listKnowledgeBaseCatalog, + readInternalKnowledgeBase, readKnowledgeBase, + restoreInternalKnowledgeBase, + updateInternalKnowledgeBase, updateKnowledgeBaseOperation, } from '@/lib/knowledge/application/knowledge-bases' @@ -99,6 +130,7 @@ describe('knowledge base application use cases', () => { beforeEach(() => { vi.clearAllMocks() mocks.resolveWorkspace.mockResolvedValue(context) + mocks.loadAuthorizationWorkspace.mockResolvedValue(context) mocks.resolveKnowledgeBase.mockResolvedValue({ ...context, knowledgeBaseId: knowledgeBase.id, @@ -111,7 +143,137 @@ describe('knowledge base application use cases', () => { }) mocks.loadFolderIndex.mockResolvedValue({ pathById: new Map() }) mocks.createRecord.mockResolvedValue(knowledgeBase) + mocks.listRecords.mockResolvedValue([]) + mocks.listInternalRecords.mockResolvedValue([knowledgeBase]) + mocks.getRecord.mockResolvedValue(knowledgeBase) + mocks.getRestorableRecord.mockResolvedValue(knowledgeBase) + mocks.performUpdate.mockResolvedValue({ + success: true, + knowledgeBase: { ...knowledgeBase, name: 'Renamed' }, + }) + mocks.performDelete.mockResolvedValue({ success: true }) + mocks.performRestore.mockResolvedValue({ success: true, knowledgeBase }) mocks.updateRecord.mockResolvedValue({ ...knowledgeBase, name: 'Renamed' }) + mocks.deleteRecord.mockResolvedValue(undefined) + }) + + it('lists legacy personal knowledge bases through the explicit session-only operation', async () => { + await expect( + listInternalKnowledgeBases.execute({ + principal: { kind: 'session', userId: 'user-1', sessionId: 'session-1' }, + input: { scope: 'all' }, + }) + ).resolves.toEqual({ knowledgeBases: [knowledgeBase] }) + + expect(mocks.resolveWorkspace).not.toHaveBeenCalled() + expect(mocks.resolvePermission).not.toHaveBeenCalled() + expect(mocks.listInternalRecords).toHaveBeenCalledWith('user-1', undefined, 'all') + }) + + it('authorizes a canonical workspace before listing its internal knowledge bases', async () => { + await listInternalKnowledgeBases.execute({ + principal: { kind: 'session', userId: 'user-1', sessionId: 'session-1' }, + input: { workspaceId: 'workspace-1', scope: 'archived' }, + }) + + expect(mocks.resolveWorkspace).toHaveBeenCalledWith({ workspaceId: 'workspace-1' }) + expect(mocks.resolvePermission).toHaveBeenCalledWith( + 'user-1', + 'workspace-1', + 'organization-1', + undefined, + { forUpdate: undefined } + ) + expect(mocks.listInternalRecords).toHaveBeenCalledWith('user-1', 'workspace-1', 'archived') + }) + + it('loads the active knowledge catalog and tag metadata only after workspace authorization', async () => { + mocks.listRecords.mockResolvedValueOnce([knowledgeBase]) + dbChainMockFns.orderBy.mockResolvedValueOnce([ + { + knowledgeBaseId: 'knowledge-1', + tagSlot: 'tag1', + displayName: 'Department', + fieldType: 'text', + }, + ]) + + const result = await listKnowledgeBaseCatalog.execute({ + principal: { + kind: 'delegated', + serviceId: 'copilot', + subjectUserId: 'user-1', + workspaceId: 'workspace-1', + delegationId: 'vfs-1', + audience: 'sim:knowledge', + issuedAt: new Date(), + expiresAt: new Date(Date.now() + 60_000), + }, + input: { workspaceId: 'workspace-1' }, + }) + + expect(mocks.resolvePermission.mock.invocationCallOrder[0]).toBeLessThan( + mocks.listRecords.mock.invocationCallOrder[0] + ) + expect(result.knowledgeBases).toEqual([ + expect.objectContaining({ + knowledgeBase, + tagDefinitions: [ + { + knowledgeBaseId: 'knowledge-1', + tagSlot: 'tag1', + displayName: 'Department', + fieldType: 'text', + }, + ], + }), + ]) + }) + + it('rejects an archived Knowledge list bound to another trusted workspace before reading', async () => { + await expect( + listArchivedKnowledgeBases.execute({ + principal: { + kind: 'delegated', + serviceId: 'copilot', + subjectUserId: 'dual-workspace-user', + workspaceId: 'workspace-b', + delegationId: 'vfs-1', + audience: 'sim:knowledge', + issuedAt: new Date(), + expiresAt: new Date(Date.now() + 60_000), + }, + input: { workspaceId: 'workspace-1' }, + }) + ).rejects.toMatchObject({ code: 'forbidden' }) + + expect(mocks.resolvePermission).not.toHaveBeenCalled() + expect(mocks.listRecords).not.toHaveBeenCalled() + }) + + it('rejects a workspace listing before reading when current access is insufficient', async () => { + mocks.resolvePermission.mockResolvedValueOnce(null) + + await expect( + listInternalKnowledgeBases.execute({ + principal: { kind: 'session', userId: 'user-1', sessionId: 'session-1' }, + input: { workspaceId: 'workspace-1', scope: 'active' }, + }) + ).rejects.toMatchObject({ code: 'forbidden' }) + + expect(mocks.listInternalRecords).not.toHaveBeenCalled() + }) + + it('rejects non-session principals before resolving internal list input', async () => { + await expect( + listInternalKnowledgeBases.execute({ + principal: { kind: 'personal_api_key', userId: 'user-1', keyId: 'key-1' }, + input: { scope: 'active' }, + }) + ).rejects.toMatchObject({ code: 'forbidden' }) + + expect(mocks.resolveWorkspace).not.toHaveBeenCalled() + expect(mocks.listInternalRecords).not.toHaveBeenCalled() }) it('rejects an insufficient role before the protected mutation', async () => { @@ -159,6 +321,29 @@ describe('knowledge base application use cases', () => { ) }) + it('passes the internal folder ID through only after workspace authorization', async () => { + await createKnowledgeBase.execute({ + principal: { kind: 'session', userId: 'user-1', sessionId: 'session-1' }, + input: { + workspaceId: 'workspace-1', + name: 'Docs', + folderId: 'folder-1', + source: 'ui', + }, + }) + + expect(mocks.loadFolderIndex).toHaveBeenCalledWith( + 'workspace-1', + 'knowledge_base', + undefined, + expect.any(Object) + ) + expect(mocks.createRecord).toHaveBeenCalledWith( + expect.objectContaining({ folderId: 'folder-1' }), + expect.any(String) + ) + }) + it('conceals a canonical scope mismatch and never audits it', async () => { mocks.resolveKnowledgeBase.mockRejectedValueOnce( new OrchestrationError('not_found', 'Knowledge base not found') @@ -206,4 +391,217 @@ describe('knowledge base application use cases', () => { { assertedWorkspaceId: 'workspace-1' } ) }) + + it('reads a legacy personal knowledge base only for its owning session', async () => { + const personalKnowledgeBase = { + ...knowledgeBase, + userId: 'user-1', + workspaceId: null, + } + mocks.getRecord.mockResolvedValueOnce(personalKnowledgeBase) + + await expect( + readInternalKnowledgeBase.execute({ + principal: { kind: 'session', userId: 'user-1', sessionId: 'session-1' }, + input: { knowledgeBaseId: 'knowledge-1' }, + }) + ).resolves.toEqual({ knowledgeBase: personalKnowledgeBase }) + expect(mocks.loadAuthorizationWorkspace).not.toHaveBeenCalled() + }) + + it('authorizes the canonical workspace before an internal detail read', async () => { + await readInternalKnowledgeBase.execute({ + principal: { kind: 'session', userId: 'user-1', sessionId: 'session-1' }, + input: { knowledgeBaseId: 'knowledge-1' }, + }) + + expect(mocks.loadAuthorizationWorkspace).toHaveBeenCalledWith('workspace-1') + expect(mocks.resolvePermission).toHaveBeenCalled() + }) + + it('authorizes both canonical workspaces before moving a knowledge base', async () => { + mocks.resolveWorkspace.mockResolvedValueOnce({ ...context, workspaceId: 'workspace-2' }) + + await updateInternalKnowledgeBase.execute({ + principal: { kind: 'session', userId: 'user-1', sessionId: 'session-1' }, + input: { knowledgeBaseId: 'knowledge-1', workspaceId: 'workspace-2' }, + }) + + expect(mocks.resolvePermission).toHaveBeenCalledTimes(2) + expect(mocks.performUpdate).toHaveBeenCalledWith( + expect.objectContaining({ + knowledgeBaseId: 'knowledge-1', + assertedWorkspaceId: 'workspace-1', + updates: expect.objectContaining({ workspaceId: 'workspace-2' }), + }) + ) + }) + + it('rejects a destination workspace before an internal move mutation', async () => { + mocks.resolveWorkspace.mockResolvedValueOnce({ ...context, workspaceId: 'workspace-2' }) + mocks.resolvePermission.mockResolvedValueOnce('write').mockResolvedValueOnce(null) + + await expect( + updateInternalKnowledgeBase.execute({ + principal: { kind: 'session', userId: 'user-1', sessionId: 'session-1' }, + input: { knowledgeBaseId: 'knowledge-1', workspaceId: 'workspace-2' }, + }) + ).rejects.toMatchObject({ code: 'forbidden' }) + expect(mocks.performUpdate).not.toHaveBeenCalled() + }) + + it('carries canonical scope into internal delete and restores only after authorization', async () => { + const principal = { kind: 'session', userId: 'user-1', sessionId: 'session-1' } as const + await deleteInternalKnowledgeBase.execute({ + principal, + input: { knowledgeBaseId: 'knowledge-1' }, + }) + await restoreInternalKnowledgeBase.execute({ + principal, + input: { knowledgeBaseId: 'knowledge-1' }, + }) + + expect(mocks.performDelete).toHaveBeenCalledWith( + expect.objectContaining({ assertedWorkspaceId: 'workspace-1' }) + ) + expect(mocks.loadAuthorizationWorkspace).toHaveBeenLastCalledWith('workspace-1', { + includeArchived: true, + }) + expect(mocks.performRestore).toHaveBeenCalledOnce() + }) + + it('bounds bulk deletion before canonical workspace loading', async () => { + await expect( + bulkDeleteKnowledgeBases.execute({ + principal: { kind: 'session', userId: 'user-1', sessionId: 'session-1' }, + input: { + assertedWorkspaceId: 'workspace-1', + knowledgeBaseIds: Array.from({ length: 101 }, (_, index) => `knowledge-${index}`), + }, + }) + ).rejects.toMatchObject({ code: 'validation' }) + + expect(mocks.resolveWorkspace).not.toHaveBeenCalled() + expect(mocks.deleteRecord).not.toHaveBeenCalled() + }) + + it('conceals a cross-workspace bulk target before mutation for a dual-workspace subject', async () => { + mocks.resolveKnowledgeBase.mockRejectedValueOnce( + new OrchestrationError('not_found', 'Knowledge base not found') + ) + + const result = await bulkDeleteKnowledgeBases.execute({ + principal: { + kind: 'delegated', + serviceId: 'copilot', + subjectUserId: 'dual-workspace-user', + workspaceId: 'workspace-1', + delegationId: 'tool-call-1', + audience: 'sim:knowledge', + issuedAt: new Date(), + expiresAt: new Date(Date.now() + 60_000), + }, + input: { + assertedWorkspaceId: 'workspace-1', + knowledgeBaseIds: ['workspace-2-knowledge'], + }, + }) + + expect(result).toMatchObject({ deleted: [], notFound: ['workspace-2-knowledge'] }) + expect(mocks.resolvePermission).toHaveBeenCalledWith( + 'dual-workspace-user', + 'workspace-1', + 'organization-1', + undefined, + { forUpdate: undefined } + ) + expect(mocks.deleteRecord).not.toHaveBeenCalled() + expect(mocks.recordAudit).not.toHaveBeenCalled() + }) + + it('returns explicit best-effort outcomes and audits only authoritative deletions', async () => { + mocks.resolveKnowledgeBase.mockImplementation(async ({ knowledgeBaseId }) => ({ + ...context, + knowledgeBaseId, + knowledgeBase: { ...knowledgeBase, id: knowledgeBaseId, name: `Name ${knowledgeBaseId}` }, + })) + mocks.deleteRecord + .mockResolvedValueOnce(undefined) + .mockRejectedValueOnce(new OrchestrationError('conflict', 'Knowledge base is locked')) + + const result = await bulkDeleteKnowledgeBases.execute({ + principal: { kind: 'session', userId: 'user-1', sessionId: 'session-1' }, + input: { + assertedWorkspaceId: 'workspace-1', + knowledgeBaseIds: ['knowledge-1', 'knowledge-2'], + source: 'agent', + }, + }) + + expect(result).toMatchObject({ + deleted: [{ id: 'knowledge-1', name: 'Name knowledge-1' }], + failed: [{ id: 'knowledge-2', name: 'Name knowledge-2', reason: 'Knowledge base is locked' }], + cancelled: false, + }) + expect(mocks.resolvePermission).toHaveBeenCalledTimes(3) + expect(mocks.recordAudit).toHaveBeenCalledOnce() + expect(mocks.recordAudit).toHaveBeenCalledWith( + expect.objectContaining({ + resourceId: 'knowledge-1', + metadata: expect.objectContaining({ operation: 'knowledge.bulk_delete' }), + }) + ) + expect(mocks.knowledgeBaseDeleted).toHaveBeenCalledWith({ knowledgeBaseId: 'knowledge-1' }) + }) + + it('stops between bulk mutations while auditing completed items', async () => { + const controller = new AbortController() + mocks.resolveKnowledgeBase.mockImplementation(async ({ knowledgeBaseId }) => ({ + ...context, + knowledgeBaseId, + knowledgeBase: { ...knowledgeBase, id: knowledgeBaseId, name: knowledgeBaseId }, + })) + mocks.deleteRecord.mockImplementationOnce(async () => { + controller.abort('user stopped') + }) + + const result = await bulkDeleteKnowledgeBases.execute({ + principal: { kind: 'session', userId: 'user-1', sessionId: 'session-1' }, + input: { + assertedWorkspaceId: 'workspace-1', + knowledgeBaseIds: ['knowledge-1', 'knowledge-2'], + cancellationSignal: controller.signal, + }, + }) + + expect(result).toMatchObject({ deleted: [{ id: 'knowledge-1' }], cancelled: true }) + expect(mocks.deleteRecord).toHaveBeenCalledOnce() + expect(mocks.recordAudit).toHaveBeenCalledOnce() + }) + + it('audits completed knowledge base deletions before propagating infrastructure failure', async () => { + const failure = new Error('knowledge store unavailable') + mocks.resolveKnowledgeBase.mockImplementation(async ({ knowledgeBaseId }) => ({ + ...context, + knowledgeBaseId, + knowledgeBase: { ...knowledgeBase, id: knowledgeBaseId, name: knowledgeBaseId }, + })) + mocks.deleteRecord.mockResolvedValueOnce(undefined).mockRejectedValueOnce(failure) + + await expect( + bulkDeleteKnowledgeBases.execute({ + principal: { kind: 'session', userId: 'user-1', sessionId: 'session-1' }, + input: { + assertedWorkspaceId: 'workspace-1', + knowledgeBaseIds: ['knowledge-1', 'knowledge-2'], + }, + }) + ).rejects.toBe(failure) + + expect(mocks.recordAudit).toHaveBeenCalledOnce() + expect(mocks.recordAudit).toHaveBeenCalledWith( + expect.objectContaining({ resourceId: 'knowledge-1' }) + ) + expect(mocks.knowledgeBaseDeleted).toHaveBeenCalledOnce() + }) }) diff --git a/apps/sim/lib/knowledge/application/knowledge-bases.ts b/apps/sim/lib/knowledge/application/knowledge-bases.ts index 56d4ab1c2db..44180f1a1b7 100644 --- a/apps/sim/lib/knowledge/application/knowledge-bases.ts +++ b/apps/sim/lib/knowledge/application/knowledge-bases.ts @@ -1,13 +1,32 @@ import { AuditAction, AuditResourceType } from '@sim/audit' +import type { Principal, SessionPrincipal } from '@sim/auth/principal' +import { db } from '@sim/db' +import { knowledgeBaseTagDefinitions } from '@sim/db/schema' import { createLogger } from '@sim/logger' -import { OrchestrationError } from '@/lib/core/orchestration/types' +import { inArray } from 'drizzle-orm' +import { + authorizeWorkspaceOperation, + type OperationUseCase, + PrincipalKindAuthorizationError, + type WorkspaceOperation, +} from '@/lib/core/application' +import { asOrchestrationError, OrchestrationError } from '@/lib/core/orchestration/types' +import { PlatformEvents } from '@/lib/core/telemetry' import { generateRequestId } from '@/lib/core/utils/request' import { loadActiveFolderPathIndex } from '@/lib/folders/queries' +import { knowledgeDelegationPolicy } from '@/lib/knowledge/application/authorization' import { defineAuthorizedKnowledgeUseCase } from '@/lib/knowledge/application/authorized-knowledge-use-case' +import { + BULK_DELETE_KNOWLEDGE_BASES_COST_POLICY, + type KnowledgeBatchExecutionResult, + requireBoundedKnowledgeBatch, + rethrowKnowledgeBatchTerminalFailure, +} from '@/lib/knowledge/application/batch-policy' import { resolveKnowledgeAttributedUserId } from '@/lib/knowledge/application/billing' import { type ActiveKnowledgeBaseContext, type KnowledgeWorkspaceContext, + loadKnowledgeWorkspaceAuthorizationContext, resolveActiveKnowledgeBaseContext, resolveKnowledgeWorkspaceContext, } from '@/lib/knowledge/application/contexts' @@ -15,16 +34,29 @@ import { knowledgeFolderPathForId, resolveKnowledgeFolderPath, } from '@/lib/knowledge/application/folder-paths' -import { knowledgeOperations } from '@/lib/knowledge/application/operations' +import { + knowledgeOperations, + knowledgeSessionOperations, +} from '@/lib/knowledge/application/operations' import { DEFAULT_CHUNKING_CONFIG, MAX_KNOWLEDGE_FOLDERS_PER_WORKSPACE, } from '@/lib/knowledge/constants' import { EMBEDDING_DIMENSIONS, getConfiguredEmbeddingModel } from '@/lib/knowledge/embeddings' +import { + getRestorableKnowledgeBase, + performDeleteKnowledgeBase, + performRestoreKnowledgeBase, + performUpdateKnowledgeBase, +} from '@/lib/knowledge/orchestration' +import type { KnowledgeOrchestrationResult } from '@/lib/knowledge/orchestration/shared' import { createAuthorizedKnowledgeBase, deleteKnowledgeBase, + getKnowledgeBaseById, + getKnowledgeBases, getWorkspaceKnowledgeBases, + type KnowledgeBaseScope, updateKnowledgeBase, } from '@/lib/knowledge/service' import type { ChunkingConfig, KnowledgeBaseWithCounts } from '@/lib/knowledge/types' @@ -48,15 +80,42 @@ export interface ListKnowledgeBasesResult { knowledgeBases: KnowledgeBaseResult[] } +export interface ListArchivedKnowledgeBasesResult { + knowledgeBases: KnowledgeBaseWithCounts[] +} + +export interface KnowledgeBaseCatalogTagDefinition { + knowledgeBaseId: string + tagSlot: string + displayName: string + fieldType: string +} + +export interface ListKnowledgeBaseCatalogResult { + knowledgeBases: Array< + KnowledgeBaseResult & { tagDefinitions: KnowledgeBaseCatalogTagDefinition[] } + > +} + export interface CreateKnowledgeBaseInput { workspaceId: string name: string description?: string chunkingConfig?: Partial folderPath?: string + folderId?: string | null source?: string } +export interface ListInternalKnowledgeBasesInput { + workspaceId?: string + scope: KnowledgeBaseScope +} + +export interface ListInternalKnowledgeBasesResult { + knowledgeBases: KnowledgeBaseWithCounts[] +} + export interface ReadKnowledgeBaseInput { knowledgeBaseId: string assertedWorkspaceId?: string @@ -74,6 +133,87 @@ export interface DeleteKnowledgeBaseInput extends ReadKnowledgeBaseInput { source?: string } +export interface BulkDeleteKnowledgeBasesInput { + assertedWorkspaceId: string + knowledgeBaseIds: string[] + cancellationSignal?: AbortSignal + source?: string +} + +export interface BulkDeleteKnowledgeBasesResult { + deleted: Array<{ id: string; name: string }> + notFound: string[] + failed: Array<{ id: string; name: string; reason: string }> + cancelled: boolean +} + +interface BulkDeleteKnowledgeBasesExecutionResult + extends BulkDeleteKnowledgeBasesResult, + KnowledgeBatchExecutionResult {} + +interface BulkDeleteKnowledgeBasesContext extends KnowledgeWorkspaceContext { + knowledgeBaseIds: string[] +} + +export interface ReadInternalKnowledgeBaseInput { + knowledgeBaseId: string +} + +export interface UpdateInternalKnowledgeBaseInput extends ReadInternalKnowledgeBaseInput { + name?: string + description?: string + workspaceId?: string | null + folderId?: string | null + chunkingConfig?: ChunkingConfig +} + +export interface RestoreInternalKnowledgeBaseInput extends ReadInternalKnowledgeBaseInput {} + +export interface InternalKnowledgeBaseResult { + knowledgeBase: KnowledgeBaseWithCounts +} + +function requireSessionPrincipal( + principal: Principal, + operationId: string +): asserts principal is SessionPrincipal { + if (principal.kind !== 'session') { + throw new PrincipalKindAuthorizationError(principal.kind, operationId) + } +} + +function throwKnowledgeOrchestrationFailure( + outcome: Extract, + fallback: string +): never { + if (outcome.errorCode === 'internal') throw new Error(fallback) + throw new OrchestrationError(outcome.errorCode, outcome.error) +} + +async function loadInternalActiveKnowledgeBase( + knowledgeBaseId: string +): Promise { + const knowledgeBase = await getKnowledgeBaseById(knowledgeBaseId) + if (!knowledgeBase) throw new OrchestrationError('not_found', 'Knowledge base not found') + return knowledgeBase +} + +async function authorizeInternalKnowledgeBase( + principal: SessionPrincipal, + knowledgeBase: Pick, + operation: WorkspaceOperation +): Promise { + if (!knowledgeBase.workspaceId) { + if (knowledgeBase.userId !== principal.userId) { + throw new OrchestrationError('unauthorized', 'Unauthorized') + } + return + } + const context = await loadKnowledgeWorkspaceAuthorizationContext(knowledgeBase.workspaceId) + if (!context) throw new OrchestrationError('not_found', 'Knowledge base not found') + await authorizeWorkspaceOperation(principal, operation, context) +} + async function executeListKnowledgeBases(args: { input: ListKnowledgeBasesInput context: KnowledgeWorkspaceContext @@ -109,8 +249,21 @@ async function executeCreateKnowledgeBase(args: { input: CreateKnowledgeBaseInput context: KnowledgeWorkspaceContext }): Promise { - const path = args.input.folderPath ?? '/' - const { folderId, index } = await resolveKnowledgeFolderPath(args.context.workspaceId, path) + if (args.input.folderId !== undefined && args.input.folderPath !== undefined) { + throw new OrchestrationError('validation', 'Specify either folderId or folderPath, not both') + } + const { folderId, index } = + args.input.folderId !== undefined + ? { + folderId: args.input.folderId, + index: await loadActiveFolderPathIndex( + args.context.workspaceId, + 'knowledge_base', + undefined, + { maxRows: MAX_KNOWLEDGE_FOLDERS_PER_WORKSPACE } + ), + } + : await resolveKnowledgeFolderPath(args.context.workspaceId, args.input.folderPath ?? '/') const chunkingConfig: ChunkingConfig = { ...DEFAULT_CHUNKING_CONFIG, ...args.input.chunkingConfig, @@ -203,6 +356,78 @@ export const listKnowledgeBases = defineAuthorizedKnowledgeUseCase({ execute: executeListKnowledgeBases, }) +export const listKnowledgeBaseCatalog = defineAuthorizedKnowledgeUseCase({ + operation: knowledgeOperations.list, + resolveContext: ({ input }: { input: ListKnowledgeBasesInput }) => + resolveKnowledgeWorkspaceContext(input), + async execute({ input, context }): Promise { + const result = await executeListKnowledgeBases({ input, context }) + const knowledgeBaseIds = result.knowledgeBases.map(({ knowledgeBase }) => knowledgeBase.id) + const tagDefinitions = + knowledgeBaseIds.length === 0 + ? [] + : await db + .select({ + knowledgeBaseId: knowledgeBaseTagDefinitions.knowledgeBaseId, + tagSlot: knowledgeBaseTagDefinitions.tagSlot, + displayName: knowledgeBaseTagDefinitions.displayName, + fieldType: knowledgeBaseTagDefinitions.fieldType, + }) + .from(knowledgeBaseTagDefinitions) + .where(inArray(knowledgeBaseTagDefinitions.knowledgeBaseId, knowledgeBaseIds)) + .orderBy(knowledgeBaseTagDefinitions.tagSlot) + const tagsByKnowledgeBase = new Map() + for (const definition of tagDefinitions) { + const existing = tagsByKnowledgeBase.get(definition.knowledgeBaseId) + if (existing) existing.push(definition) + else tagsByKnowledgeBase.set(definition.knowledgeBaseId, [definition]) + } + return { + knowledgeBases: result.knowledgeBases.map((entry) => ({ + ...entry, + tagDefinitions: tagsByKnowledgeBase.get(entry.knowledgeBase.id) ?? [], + })), + } + }, +}) + +export const listArchivedKnowledgeBases = defineAuthorizedKnowledgeUseCase({ + operation: knowledgeOperations.listArchived, + resolveContext: ({ input }: { input: { workspaceId: string } }) => + resolveKnowledgeWorkspaceContext(input), + async execute({ context }): Promise { + return { + knowledgeBases: await getWorkspaceKnowledgeBases(context.workspaceId, 'archived'), + } + }, +}) + +export const listInternalKnowledgeBases = { + operation: knowledgeSessionOperations.list, + async execute({ + principal, + input, + }: { + principal: Principal + input: ListInternalKnowledgeBasesInput + }): Promise { + if (principal.kind !== 'session') { + throw new PrincipalKindAuthorizationError(principal.kind, knowledgeSessionOperations.list.id) + } + if (input.workspaceId !== undefined) { + const context = await resolveKnowledgeWorkspaceContext({ workspaceId: input.workspaceId }) + await authorizeWorkspaceOperation(principal, knowledgeOperations.list, context) + } + return { + knowledgeBases: await getKnowledgeBases(principal.userId, input.workspaceId, input.scope), + } + }, +} satisfies OperationUseCase< + (typeof knowledgeSessionOperations)['list'], + ListInternalKnowledgeBasesInput, + ListInternalKnowledgeBasesResult +> + export const createKnowledgeBase = defineAuthorizedKnowledgeUseCase({ operation: knowledgeOperations.create, resolveContext: ({ input }: { input: CreateKnowledgeBaseInput }) => @@ -266,3 +491,245 @@ export const deleteKnowledgeBaseOperation = defineAuthorizedKnowledgeUseCase({ metadata: { source: input.source, knowledgeBaseName: result.name }, }), }) + +export const bulkDeleteKnowledgeBases = defineAuthorizedKnowledgeUseCase({ + operation: knowledgeOperations.bulkDelete, + async resolveContext({ + input, + }: { + input: BulkDeleteKnowledgeBasesInput + }): Promise { + const knowledgeBaseIds = requireBoundedKnowledgeBatch( + input.knowledgeBaseIds, + 'knowledge base IDs', + BULK_DELETE_KNOWLEDGE_BASES_COST_POLICY.maxItems + ) + return { + ...(await resolveKnowledgeWorkspaceContext({ workspaceId: input.assertedWorkspaceId })), + knowledgeBaseIds, + } + }, + async execute({ principal, input, context }): Promise { + const deleted: BulkDeleteKnowledgeBasesResult['deleted'] = [] + const notFound: string[] = [] + const failed: BulkDeleteKnowledgeBasesResult['failed'] = [] + let terminalFailure: KnowledgeBatchExecutionResult['terminalFailure'] + + for (const knowledgeBaseId of context.knowledgeBaseIds) { + if (input.cancellationSignal?.aborted) break + let knowledgeBaseName = knowledgeBaseId + try { + const canonical = await resolveActiveKnowledgeBaseContext({ + knowledgeBaseId, + assertedWorkspaceId: context.workspaceId, + }) + knowledgeBaseName = canonical.knowledgeBase.name + await authorizeWorkspaceOperation(principal, knowledgeOperations.bulkDelete, canonical, { + delegation: knowledgeDelegationPolicy, + }) + if (input.cancellationSignal?.aborted) break + deleted.push(await executeDeleteKnowledgeBase({ context: canonical })) + } catch (error) { + const classified = asOrchestrationError(error) + if ( + classified?.code === 'not_found' || + classified?.code === 'forbidden' || + classified?.code === 'unauthorized' + ) { + notFound.push(knowledgeBaseId) + continue + } + if (classified && classified.code !== 'internal') { + failed.push({ + id: knowledgeBaseId, + name: knowledgeBaseName, + reason: classified.message, + }) + continue + } + terminalFailure = { error } + break + } + } + + return { + deleted, + notFound, + failed, + cancelled: input.cancellationSignal?.aborted ?? false, + ...(terminalFailure && { terminalFailure }), + } + }, + projectAudit: ({ input, result }) => + result.deleted.map((knowledgeBase) => ({ + action: AuditAction.KNOWLEDGE_BASE_DELETED, + resourceType: AuditResourceType.KNOWLEDGE_BASE, + resourceId: knowledgeBase.id, + resourceName: knowledgeBase.name, + description: `Deleted knowledge base "${knowledgeBase.name}"`, + metadata: { source: input.source, knowledgeBaseName: knowledgeBase.name }, + })), + afterSuccess: ({ result }) => { + try { + for (const knowledgeBase of result.deleted) { + PlatformEvents.knowledgeBaseDeleted({ knowledgeBaseId: knowledgeBase.id }) + } + } finally { + rethrowKnowledgeBatchTerminalFailure(result) + } + }, +}) + +export const readInternalKnowledgeBase = { + operation: knowledgeSessionOperations.read, + async execute({ + principal, + input, + }: { + principal: Principal + input: ReadInternalKnowledgeBaseInput + }): Promise { + requireSessionPrincipal(principal, knowledgeSessionOperations.read.id) + const knowledgeBase = await loadInternalActiveKnowledgeBase(input.knowledgeBaseId) + await authorizeInternalKnowledgeBase(principal, knowledgeBase, knowledgeOperations.read) + return { knowledgeBase } + }, +} satisfies OperationUseCase< + (typeof knowledgeSessionOperations)['read'], + ReadInternalKnowledgeBaseInput, + InternalKnowledgeBaseResult +> + +export const updateInternalKnowledgeBase = { + operation: knowledgeSessionOperations.update, + async execute({ + principal, + input, + request, + }: { + principal: Principal + input: UpdateInternalKnowledgeBaseInput + request?: { headers: { get(name: string): string | null } } + }): Promise { + requireSessionPrincipal(principal, knowledgeSessionOperations.update.id) + const knowledgeBase = await loadInternalActiveKnowledgeBase(input.knowledgeBaseId) + await authorizeInternalKnowledgeBase(principal, knowledgeBase, knowledgeOperations.update) + + if (input.workspaceId !== undefined && input.workspaceId !== knowledgeBase.workspaceId) { + if (input.workspaceId === null) { + if (knowledgeBase.userId !== principal.userId) { + throw new OrchestrationError( + 'forbidden', + 'Only the knowledge base owner can remove it from a workspace' + ) + } + } else { + const destination = await resolveKnowledgeWorkspaceContext({ + workspaceId: input.workspaceId, + }) + await authorizeWorkspaceOperation(principal, knowledgeOperations.update, destination) + } + } + + const outcome = await performUpdateKnowledgeBase({ + knowledgeBaseId: knowledgeBase.id, + workspaceId: knowledgeBase.workspaceId, + assertedWorkspaceId: knowledgeBase.workspaceId ?? undefined, + userId: principal.userId, + source: 'ui', + updates: { + name: input.name, + description: input.description, + workspaceId: input.workspaceId, + folderId: input.folderId, + chunkingConfig: input.chunkingConfig, + }, + request, + }) + if (!outcome.success) { + throwKnowledgeOrchestrationFailure(outcome, 'Failed to update knowledge base') + } + return { knowledgeBase: outcome.knowledgeBase } + }, +} satisfies OperationUseCase< + (typeof knowledgeSessionOperations)['update'], + UpdateInternalKnowledgeBaseInput, + InternalKnowledgeBaseResult +> + +export const deleteInternalKnowledgeBase = { + operation: knowledgeSessionOperations.delete, + async execute({ + principal, + input, + request, + }: { + principal: Principal + input: ReadInternalKnowledgeBaseInput + request?: { headers: { get(name: string): string | null } } + }): Promise<{ success: true }> { + requireSessionPrincipal(principal, knowledgeSessionOperations.delete.id) + const knowledgeBase = await loadInternalActiveKnowledgeBase(input.knowledgeBaseId) + await authorizeInternalKnowledgeBase(principal, knowledgeBase, knowledgeOperations.delete) + const outcome = await performDeleteKnowledgeBase({ + knowledgeBase: { + id: knowledgeBase.id, + name: knowledgeBase.name, + workspaceId: knowledgeBase.workspaceId, + }, + assertedWorkspaceId: knowledgeBase.workspaceId ?? undefined, + userId: principal.userId, + source: 'ui', + request, + }) + if (!outcome.success) { + throwKnowledgeOrchestrationFailure(outcome, 'Failed to delete knowledge base') + } + return { success: true } + }, +} satisfies OperationUseCase< + (typeof knowledgeSessionOperations)['delete'], + ReadInternalKnowledgeBaseInput, + { success: true } +> + +export const restoreInternalKnowledgeBase = { + operation: knowledgeSessionOperations.restore, + async execute({ + principal, + input, + request, + }: { + principal: Principal + input: RestoreInternalKnowledgeBaseInput + request?: { headers: { get(name: string): string | null } } + }): Promise<{ success: true }> { + requireSessionPrincipal(principal, knowledgeSessionOperations.restore.id) + const knowledgeBase = await getRestorableKnowledgeBase(input.knowledgeBaseId) + if (!knowledgeBase) throw new OrchestrationError('not_found', 'Knowledge base not found') + if (knowledgeBase.workspaceId) { + const context = await loadKnowledgeWorkspaceAuthorizationContext(knowledgeBase.workspaceId, { + includeArchived: true, + }) + if (!context) throw new OrchestrationError('not_found', 'Knowledge base not found') + await authorizeWorkspaceOperation(principal, knowledgeOperations.update, context) + } else if (knowledgeBase.userId !== principal.userId) { + throw new OrchestrationError('unauthorized', 'Unauthorized') + } + + const outcome = await performRestoreKnowledgeBase({ + knowledgeBaseId: knowledgeBase.id, + userId: principal.userId, + source: 'ui', + request, + }) + if (!outcome.success) { + throwKnowledgeOrchestrationFailure(outcome, 'Failed to restore knowledge base') + } + return { success: true } + }, +} satisfies OperationUseCase< + (typeof knowledgeSessionOperations)['restore'], + RestoreInternalKnowledgeBaseInput, + { success: true } +> diff --git a/apps/sim/lib/knowledge/application/operations.test.ts b/apps/sim/lib/knowledge/application/operations.test.ts index c8d39ec283b..b7b48188f8f 100644 --- a/apps/sim/lib/knowledge/application/operations.test.ts +++ b/apps/sim/lib/knowledge/application/operations.test.ts @@ -11,10 +11,12 @@ describe('knowledge operation registry', () => { const ids = Object.values(knowledgeOperations).map((operation) => operation.id) expect(ids).toEqual([ 'knowledge.list', + 'knowledge.list_archived', 'knowledge.read', 'knowledge.create', 'knowledge.update', 'knowledge.delete', + 'knowledge.bulk_delete', 'knowledge.search', 'knowledge.folders.list', 'knowledge.folders.create', @@ -23,7 +25,34 @@ describe('knowledge operation registry', () => { 'knowledge.documents.list', 'knowledge.documents.read', 'knowledge.documents.upload', + 'knowledge.documents.add_workspace_files', 'knowledge.documents.delete', + 'knowledge.documents.bulk_delete', + 'knowledge.documents.update', + 'knowledge.documents.bulk', + 'knowledge.chunks.list', + 'knowledge.chunks.read', + 'knowledge.chunks.create', + 'knowledge.chunks.update', + 'knowledge.chunks.delete', + 'knowledge.chunks.bulk', + 'knowledge.tags.list', + 'knowledge.tags.create', + 'knowledge.tags.update', + 'knowledge.tags.delete', + 'knowledge.tags.read_usage', + 'knowledge.tags.read_detailed_usage', + 'knowledge.tags.read_next_slot', + 'knowledge.tags.save_document_definitions', + 'knowledge.tags.delete_document_definitions', + 'knowledge.connectors.list', + 'knowledge.connectors.read', + 'knowledge.connectors.create', + 'knowledge.connectors.update', + 'knowledge.connectors.delete', + 'knowledge.connectors.sync', + 'knowledge.connectors.documents.list', + 'knowledge.connectors.documents.update', 'knowledge.documents.upload.create', 'knowledge.documents.upload.parts', 'knowledge.documents.upload.complete', @@ -33,20 +62,49 @@ describe('knowledge operation registry', () => { }) it('keeps workspace keys within their fixed write ceiling', () => { - for (const operation of Object.values(knowledgeOperations)) { + const workspaceKeyOperations = Object.values(knowledgeOperations).filter( + (operation) => operation.workspaceApiKey === 'allow' + ) + for (const operation of workspaceKeyOperations) { expect(operation.workspaceApiKey).toBe('allow') expect(operation.principalKinds).toContain('workspace_api_key') expect(permissionSatisfies('write', operation.minimumRole)).toBe(true) } }) + it('keeps human-delegated tag, connector, and composed document operations off workspace keys', () => { + const operations = [ + knowledgeOperations.updateDocument, + knowledgeOperations.addWorkspaceFiles, + knowledgeOperations.bulkDeleteDocuments, + knowledgeOperations.listTags, + knowledgeOperations.createTag, + knowledgeOperations.updateTag, + knowledgeOperations.deleteTag, + knowledgeOperations.readTagUsage, + knowledgeOperations.createConnector, + knowledgeOperations.updateConnector, + knowledgeOperations.deleteConnector, + knowledgeOperations.syncConnector, + ] + for (const operation of operations) { + expect(operation.workspaceApiKey).toBe('deny') + expect(operation.principalKinds).not.toContain('workspace_api_key') + expect(operation.principalKinds).toContain('delegated') + } + }) + it('allows delegated callers only on semantic knowledge and document operations', () => { expect(knowledgeOperations.list.principalKinds).toContain('delegated') expect(knowledgeOperations.search.principalKinds).toContain('delegated') expect(knowledgeOperations.uploadDocument.principalKinds).toContain('delegated') + expect(knowledgeOperations.updateDocument.principalKinds).toContain('delegated') + expect(knowledgeOperations.updateTag.principalKinds).toContain('delegated') + expect(knowledgeOperations.syncConnector.principalKinds).toContain('delegated') expect(knowledgeOperations.listFolders.principalKinds).not.toContain('delegated') expect(knowledgeOperations.uploadComplete.principalKinds).not.toContain('delegated') expect(knowledgeOperations.list.delegatedServices).toEqual(['copilot']) + expect(knowledgeOperations.search.delegatedServices).toEqual(['copilot', 'executor']) expect(knowledgeOperations.uploadComplete.delegatedServices).toBeUndefined() }) }) diff --git a/apps/sim/lib/knowledge/application/operations.ts b/apps/sim/lib/knowledge/application/operations.ts index dda120bb467..4bacaa7a01b 100644 --- a/apps/sim/lib/knowledge/application/operations.ts +++ b/apps/sim/lib/knowledge/application/operations.ts @@ -5,8 +5,25 @@ const ALL_PRINCIPAL_POLICY = { delegatedServices: ['copilot'], } as const +const ALL_PRINCIPAL_WITH_EXECUTOR_POLICY = { + principalKinds: ['session', 'personal_api_key', 'workspace_api_key', 'delegated'], + delegatedServices: ['copilot', 'executor'], +} as const + const HTTP_PRINCIPAL_KINDS = ['session', 'personal_api_key', 'workspace_api_key'] as const +const HUMAN_AND_DELEGATED_PRINCIPAL_KINDS = ['session', 'personal_api_key', 'delegated'] as const + +const HUMAN_AND_COPILOT_PRINCIPAL_POLICY = { + principalKinds: HUMAN_AND_DELEGATED_PRINCIPAL_KINDS, + delegatedServices: ['copilot'], +} as const + +const HUMAN_COPILOT_AND_EXECUTOR_PRINCIPAL_POLICY = { + principalKinds: HUMAN_AND_DELEGATED_PRINCIPAL_KINDS, + delegatedServices: ['copilot', 'executor'], +} as const + export const knowledgeOperations = { list: defineWorkspaceOperation({ id: 'knowledge.list', @@ -14,6 +31,12 @@ export const knowledgeOperations = { workspaceApiKey: 'allow', ...ALL_PRINCIPAL_POLICY, }), + listArchived: defineWorkspaceOperation({ + id: 'knowledge.list_archived', + minimumRole: 'read', + workspaceApiKey: 'deny', + ...HUMAN_AND_COPILOT_PRINCIPAL_POLICY, + }), read: defineWorkspaceOperation({ id: 'knowledge.read', minimumRole: 'read', @@ -38,11 +61,17 @@ export const knowledgeOperations = { workspaceApiKey: 'allow', ...ALL_PRINCIPAL_POLICY, }), + bulkDelete: defineWorkspaceOperation({ + id: 'knowledge.bulk_delete', + minimumRole: 'write', + workspaceApiKey: 'allow', + ...ALL_PRINCIPAL_POLICY, + }), search: defineWorkspaceOperation({ id: 'knowledge.search', minimumRole: 'read', workspaceApiKey: 'allow', - ...ALL_PRINCIPAL_POLICY, + ...ALL_PRINCIPAL_WITH_EXECUTOR_POLICY, }), listFolders: defineWorkspaceOperation({ id: 'knowledge.folders.list', @@ -72,25 +101,187 @@ export const knowledgeOperations = { id: 'knowledge.documents.list', minimumRole: 'read', workspaceApiKey: 'allow', - ...ALL_PRINCIPAL_POLICY, + ...ALL_PRINCIPAL_WITH_EXECUTOR_POLICY, }), readDocument: defineWorkspaceOperation({ id: 'knowledge.documents.read', minimumRole: 'read', workspaceApiKey: 'allow', - ...ALL_PRINCIPAL_POLICY, + ...ALL_PRINCIPAL_WITH_EXECUTOR_POLICY, }), uploadDocument: defineWorkspaceOperation({ id: 'knowledge.documents.upload', minimumRole: 'write', workspaceApiKey: 'allow', - ...ALL_PRINCIPAL_POLICY, + ...ALL_PRINCIPAL_WITH_EXECUTOR_POLICY, + }), + addWorkspaceFiles: defineWorkspaceOperation({ + id: 'knowledge.documents.add_workspace_files', + minimumRole: 'write', + workspaceApiKey: 'deny', + ...HUMAN_AND_COPILOT_PRINCIPAL_POLICY, }), deleteDocument: defineWorkspaceOperation({ id: 'knowledge.documents.delete', minimumRole: 'write', workspaceApiKey: 'allow', - ...ALL_PRINCIPAL_POLICY, + ...ALL_PRINCIPAL_WITH_EXECUTOR_POLICY, + }), + bulkDeleteDocuments: defineWorkspaceOperation({ + id: 'knowledge.documents.bulk_delete', + minimumRole: 'write', + workspaceApiKey: 'deny', + ...HUMAN_AND_COPILOT_PRINCIPAL_POLICY, + }), + updateDocument: defineWorkspaceOperation({ + id: 'knowledge.documents.update', + minimumRole: 'write', + workspaceApiKey: 'deny', + ...HUMAN_COPILOT_AND_EXECUTOR_PRINCIPAL_POLICY, + }), + bulkDocuments: defineWorkspaceOperation({ + id: 'knowledge.documents.bulk', + minimumRole: 'write', + workspaceApiKey: 'deny', + ...HUMAN_AND_COPILOT_PRINCIPAL_POLICY, + }), + listChunks: defineWorkspaceOperation({ + id: 'knowledge.chunks.list', + minimumRole: 'read', + workspaceApiKey: 'deny', + ...HUMAN_COPILOT_AND_EXECUTOR_PRINCIPAL_POLICY, + }), + readChunk: defineWorkspaceOperation({ + id: 'knowledge.chunks.read', + minimumRole: 'read', + workspaceApiKey: 'deny', + ...HUMAN_AND_COPILOT_PRINCIPAL_POLICY, + }), + createChunk: defineWorkspaceOperation({ + id: 'knowledge.chunks.create', + minimumRole: 'write', + workspaceApiKey: 'deny', + ...HUMAN_COPILOT_AND_EXECUTOR_PRINCIPAL_POLICY, + }), + updateChunk: defineWorkspaceOperation({ + id: 'knowledge.chunks.update', + minimumRole: 'write', + workspaceApiKey: 'deny', + ...HUMAN_COPILOT_AND_EXECUTOR_PRINCIPAL_POLICY, + }), + deleteChunk: defineWorkspaceOperation({ + id: 'knowledge.chunks.delete', + minimumRole: 'write', + workspaceApiKey: 'deny', + ...HUMAN_COPILOT_AND_EXECUTOR_PRINCIPAL_POLICY, + }), + bulkChunks: defineWorkspaceOperation({ + id: 'knowledge.chunks.bulk', + minimumRole: 'write', + workspaceApiKey: 'deny', + ...HUMAN_AND_COPILOT_PRINCIPAL_POLICY, + }), + listTags: defineWorkspaceOperation({ + id: 'knowledge.tags.list', + minimumRole: 'read', + workspaceApiKey: 'deny', + ...HUMAN_COPILOT_AND_EXECUTOR_PRINCIPAL_POLICY, + }), + createTag: defineWorkspaceOperation({ + id: 'knowledge.tags.create', + minimumRole: 'write', + workspaceApiKey: 'deny', + ...HUMAN_AND_COPILOT_PRINCIPAL_POLICY, + }), + updateTag: defineWorkspaceOperation({ + id: 'knowledge.tags.update', + minimumRole: 'write', + workspaceApiKey: 'deny', + ...HUMAN_AND_COPILOT_PRINCIPAL_POLICY, + }), + deleteTag: defineWorkspaceOperation({ + id: 'knowledge.tags.delete', + minimumRole: 'write', + workspaceApiKey: 'deny', + ...HUMAN_AND_COPILOT_PRINCIPAL_POLICY, + }), + readTagUsage: defineWorkspaceOperation({ + id: 'knowledge.tags.read_usage', + minimumRole: 'read', + workspaceApiKey: 'deny', + ...HUMAN_AND_COPILOT_PRINCIPAL_POLICY, + }), + readDetailedTagUsage: defineWorkspaceOperation({ + id: 'knowledge.tags.read_detailed_usage', + minimumRole: 'read', + workspaceApiKey: 'deny', + ...HUMAN_AND_COPILOT_PRINCIPAL_POLICY, + }), + readNextTagSlot: defineWorkspaceOperation({ + id: 'knowledge.tags.read_next_slot', + minimumRole: 'read', + workspaceApiKey: 'deny', + ...HUMAN_AND_COPILOT_PRINCIPAL_POLICY, + }), + saveDocumentTagDefinitions: defineWorkspaceOperation({ + id: 'knowledge.tags.save_document_definitions', + minimumRole: 'write', + workspaceApiKey: 'deny', + ...HUMAN_AND_COPILOT_PRINCIPAL_POLICY, + }), + deleteDocumentTagDefinitions: defineWorkspaceOperation({ + id: 'knowledge.tags.delete_document_definitions', + minimumRole: 'write', + workspaceApiKey: 'deny', + ...HUMAN_AND_COPILOT_PRINCIPAL_POLICY, + }), + listConnectors: defineWorkspaceOperation({ + id: 'knowledge.connectors.list', + minimumRole: 'read', + workspaceApiKey: 'deny', + ...HUMAN_COPILOT_AND_EXECUTOR_PRINCIPAL_POLICY, + }), + readConnector: defineWorkspaceOperation({ + id: 'knowledge.connectors.read', + minimumRole: 'read', + workspaceApiKey: 'deny', + ...HUMAN_COPILOT_AND_EXECUTOR_PRINCIPAL_POLICY, + }), + createConnector: defineWorkspaceOperation({ + id: 'knowledge.connectors.create', + minimumRole: 'write', + workspaceApiKey: 'deny', + ...HUMAN_AND_COPILOT_PRINCIPAL_POLICY, + }), + updateConnector: defineWorkspaceOperation({ + id: 'knowledge.connectors.update', + minimumRole: 'write', + workspaceApiKey: 'deny', + ...HUMAN_AND_COPILOT_PRINCIPAL_POLICY, + }), + deleteConnector: defineWorkspaceOperation({ + id: 'knowledge.connectors.delete', + minimumRole: 'write', + workspaceApiKey: 'deny', + ...HUMAN_AND_COPILOT_PRINCIPAL_POLICY, + }), + syncConnector: defineWorkspaceOperation({ + id: 'knowledge.connectors.sync', + minimumRole: 'write', + workspaceApiKey: 'deny', + ...HUMAN_COPILOT_AND_EXECUTOR_PRINCIPAL_POLICY, + }), + listConnectorDocuments: defineWorkspaceOperation({ + id: 'knowledge.connectors.documents.list', + minimumRole: 'read', + workspaceApiKey: 'deny', + ...HUMAN_AND_COPILOT_PRINCIPAL_POLICY, + }), + updateConnectorDocuments: defineWorkspaceOperation({ + id: 'knowledge.connectors.documents.update', + minimumRole: 'write', + workspaceApiKey: 'deny', + ...HUMAN_AND_COPILOT_PRINCIPAL_POLICY, }), uploadCreate: defineWorkspaceOperation({ id: 'knowledge.documents.upload.create', @@ -118,4 +309,12 @@ export const knowledgeOperations = { }), } as const +export const knowledgeSessionOperations = { + list: Object.freeze({ id: 'knowledge.session.list' as const }), + read: Object.freeze({ id: 'knowledge.session.read' as const }), + update: Object.freeze({ id: 'knowledge.session.update' as const }), + delete: Object.freeze({ id: 'knowledge.session.delete' as const }), + restore: Object.freeze({ id: 'knowledge.session.restore' as const }), +} as const + export type KnowledgeOperation = (typeof knowledgeOperations)[keyof typeof knowledgeOperations] diff --git a/apps/sim/lib/knowledge/application/search.test.ts b/apps/sim/lib/knowledge/application/search.test.ts index a56e6350ac4..90b6e7fb32a 100644 --- a/apps/sim/lib/knowledge/application/search.test.ts +++ b/apps/sim/lib/knowledge/application/search.test.ts @@ -77,6 +77,7 @@ const workspace = { const knowledgeBase = { id: 'knowledge-1', + name: 'Docs', workspaceId: 'workspace-1', embeddingModel: 'text-embedding-3-small', } @@ -157,6 +158,7 @@ describe('knowledge search application use case', () => { documentId: 'document-1', similarity: 0.8, }) + expect(result.knowledgeBases).toEqual([{ id: 'knowledge-1', name: 'Docs' }]) }) it('rejects a cross-workspace knowledge base before authorization or spend', async () => { diff --git a/apps/sim/lib/knowledge/application/search.ts b/apps/sim/lib/knowledge/application/search.ts index 778806335df..e3aecf60b6f 100644 --- a/apps/sim/lib/knowledge/application/search.ts +++ b/apps/sim/lib/knowledge/application/search.ts @@ -1,6 +1,16 @@ -import { checkAttributedUsageLimits } from '@/lib/billing/core/billing-attribution' +import { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' +import { + type BillingAttributionSnapshot, + checkAttributedUsageLimits, + toBillingContext, +} from '@/lib/billing/core/billing-attribution' +import { recordUsage } from '@/lib/billing/core/usage-log' +import { checkAndBillPayerOverageThreshold } from '@/lib/billing/threshold-billing' import { OrchestrationError } from '@/lib/core/orchestration/types' +import { PlatformEvents } from '@/lib/core/telemetry' import { generateRequestId } from '@/lib/core/utils/request' +import { importDurableSecretProvenance } from '@/lib/execution/durable-secret-provenance' import { defineAuthorizedKnowledgeUseCase } from '@/lib/knowledge/application/authorized-knowledge-use-case' import { KnowledgeUsageLimitExceededError, @@ -13,7 +23,9 @@ import { } from '@/lib/knowledge/application/contexts' import { knowledgeOperations } from '@/lib/knowledge/application/operations' import { ALL_TAG_SLOTS } from '@/lib/knowledge/constants' -import { recordSearchEmbeddingUsage } from '@/lib/knowledge/embeddings' +import { getEmbeddingModelInfo } from '@/lib/knowledge/embedding-models' +import { runWithKnowledgeModelInputProvenance } from '@/lib/knowledge/model-input-provenance' +import { rerank } from '@/lib/knowledge/reranker' import { executeKnowledgeSearch, generateSearchEmbedding, @@ -25,7 +37,25 @@ import { getKnowledgeBaseById } from '@/lib/knowledge/service' import { getDocumentTagDefinitions } from '@/lib/knowledge/tags/service' import { buildUndefinedTagsError, validateTagValue } from '@/lib/knowledge/tags/utils' import type { KnowledgeBaseWithCounts, StructuredFilter } from '@/lib/knowledge/types' -import type { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry' +import { estimateTokenCount } from '@/lib/tokenization/estimators' +import { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry' +import { getRerankModelPricing } from '@/providers/models' +import { calculateCost } from '@/providers/utils' + +const logger = createLogger('KnowledgeSearchApplication') + +export const KNOWLEDGE_SEARCH_COST_POLICY = { + maxKnowledgeBases: 20, + maxTopK: 100, + usageAdmission: 'before_model_execution', +} as const + +export class KnowledgeSearchProvenanceUnavailableError extends Error { + constructor() { + super('Knowledge result secret provenance is unavailable') + this.name = 'KnowledgeSearchProvenanceUnavailableError' + } +} export interface KnowledgeSearchTagFilter { tagName: string @@ -36,11 +66,24 @@ export interface KnowledgeSearchTagFilter { } export interface SearchKnowledgeInput { - workspaceId: string + /** Optional assertion from a trusted adapter or public contract. */ + workspaceId?: string knowledgeBaseIds: string[] query?: string topK: number tagFilters?: KnowledgeSearchTagFilter[] + searchMode?: 'vector' | 'hybrid' + rerankerEnabled?: boolean + rerankerModel?: string + rerankerInputCount?: number + rerankerApiKey?: string + /** Honored only for an authenticated executor delegation. */ + skipUsageBilling?: boolean + resolveBillingAttribution?(workspaceId: string): Promise + prepareModelInputProvenance?(input: { + userId: string + workspaceId: string + }): Promise /** Trusted execution provenance sink; never sourced from an HTTP or model payload. */ resultSecretRegistry?: ResolvedSecretTraceRegistry } @@ -59,56 +102,140 @@ export interface KnowledgeSearchItem { chunkIndex: number metadata: Record similarity: number + rerankerScore?: number +} + +interface KnowledgeSearchCost { + input: number + output: number + total: number + tokens: { prompt: number; completion: number; total: number } + model: string + pricing: { input: number; output: number; updatedAt?: string } + rerankerCost?: number + rerankerModel?: string + rerankerSearchUnits?: number } export interface SearchKnowledgeResult { results: KnowledgeSearchItem[] query: string knowledgeBaseIds: string[] + knowledgeBases: Array<{ id: string; name: string }> + knowledgeBaseId: string topK: number totalResults: number + cost?: KnowledgeSearchCost + workspaceId: string + userId: string + resultSecretRegistry?: ResolvedSecretTraceRegistry } async function resolveKnowledgeSearchContext( input: SearchKnowledgeInput ): Promise { - if (input.knowledgeBaseIds.length < 1 || input.knowledgeBaseIds.length > 20) { + if ( + input.knowledgeBaseIds.length < 1 || + input.knowledgeBaseIds.length > KNOWLEDGE_SEARCH_COST_POLICY.maxKnowledgeBases + ) { throw new OrchestrationError( 'validation', - 'Knowledge search requires between 1 and 20 knowledge bases' + `Knowledge search requires between 1 and ${KNOWLEDGE_SEARCH_COST_POLICY.maxKnowledgeBases} knowledge bases` ) } - if (!Number.isInteger(input.topK) || input.topK < 1 || input.topK > 100) { - throw new OrchestrationError('validation', 'topK must be an integer between 1 and 100') + if ( + !Number.isInteger(input.topK) || + input.topK < 1 || + input.topK > KNOWLEDGE_SEARCH_COST_POLICY.maxTopK + ) { + throw new OrchestrationError( + 'validation', + `topK must be an integer between 1 and ${KNOWLEDGE_SEARCH_COST_POLICY.maxTopK}` + ) } - const workspaceContext = await resolveKnowledgeWorkspaceContext(input) const knowledgeBases = await Promise.all(input.knowledgeBaseIds.map(getKnowledgeBaseById)) - const inaccessibleIds = input.knowledgeBaseIds.filter( - (_id, index) => knowledgeBases[index]?.workspaceId !== workspaceContext.workspaceId + const missingIds = input.knowledgeBaseIds.filter( + (_, index) => !knowledgeBases[index]?.workspaceId ) - if (inaccessibleIds.length > 0) { + if (missingIds.length > 0) { + throw new OrchestrationError( + 'not_found', + `Knowledge bases not found or access denied: ${missingIds.join(', ')}` + ) + } + const canonicalWorkspaceIds = new Set(knowledgeBases.map((kb) => kb?.workspaceId)) + if (canonicalWorkspaceIds.size !== 1) { + throw new OrchestrationError( + 'validation', + 'Selected knowledge bases must belong to the same workspace' + ) + } + const canonicalWorkspaceId = knowledgeBases[0]?.workspaceId + if (!canonicalWorkspaceId || (input.workspaceId && input.workspaceId !== canonicalWorkspaceId)) { throw new OrchestrationError( 'not_found', - `Knowledge bases not found or access denied: ${inaccessibleIds.join(', ')}` + `Knowledge bases not found or access denied: ${input.knowledgeBaseIds.join(', ')}` ) } + const workspaceContext = await resolveKnowledgeWorkspaceContext({ + workspaceId: canonicalWorkspaceId, + }) return { ...workspaceContext, knowledgeBases: knowledgeBases as KnowledgeBaseWithCounts[], } } -function buildStructuredFilters( +async function buildStructuredFilters( filters: KnowledgeSearchTagFilter[], - tagDefinitions: Awaited> -): StructuredFilter[] { - const definitionsByName = new Map( - tagDefinitions.map((definition) => [definition.displayName, definition]) + knowledgeBaseIds: string[] +): Promise<{ + structuredFilters: StructuredFilter[] + definitionsByKnowledgeBase: Map>> +}> { + const definitionEntries = await Promise.all( + knowledgeBaseIds.map( + async (knowledgeBaseId) => + [knowledgeBaseId, await getDocumentTagDefinitions(knowledgeBaseId)] as const + ) ) + const definitionsByKnowledgeBase = new Map(definitionEntries) + const sharedDefinitions = new Map() + for (const [, definitions] of definitionEntries) { + const currentByName = new Map( + definitions.map((definition) => [ + definition.displayName, + { tagSlot: definition.tagSlot, fieldType: definition.fieldType }, + ]) + ) + for (const filter of filters) { + const current = currentByName.get(filter.tagName) + if (!current) { + if (knowledgeBaseIds.length > 1) { + throw new OrchestrationError( + 'validation', + `Tag "${filter.tagName}" does not exist in all selected knowledge bases. Search those knowledge bases separately.` + ) + } + continue + } + const existing = sharedDefinitions.get(filter.tagName) + if ( + existing && + (existing.tagSlot !== current.tagSlot || existing.fieldType !== current.fieldType) + ) { + throw new OrchestrationError( + 'validation', + `Tag "${filter.tagName}" is not mapped consistently across the selected knowledge bases. Search those knowledge bases separately.` + ) + } + sharedDefinitions.set(filter.tagName, current) + } + } const undefinedTags: string[] = [] const typeErrors: string[] = [] for (const filter of filters) { - const definition = definitionsByName.get(filter.tagName) + const definition = sharedDefinitions.get(filter.tagName) if (!definition) { undefinedTags.push(filter.tagName) continue @@ -121,23 +248,28 @@ function buildStructuredFilters( if (validationError) typeErrors.push(validationError) } if (undefinedTags.length > 0 || typeErrors.length > 0) { - const messages = [ - ...(undefinedTags.length > 0 ? [buildUndefinedTagsError(undefinedTags)] : []), - ...typeErrors, - ] - throw new OrchestrationError('validation', messages.join('\n')) + throw new OrchestrationError( + 'validation', + [ + ...(undefinedTags.length > 0 ? [buildUndefinedTagsError(undefinedTags)] : []), + ...typeErrors, + ].join('\n') + ) + } + return { + structuredFilters: filters.map((filter) => { + const definition = sharedDefinitions.get(filter.tagName) + if (!definition) throw new Error('Validated knowledge tag definition disappeared') + return { + tagSlot: definition.tagSlot, + fieldType: definition.fieldType, + operator: filter.operator, + value: filter.value, + valueTo: filter.valueTo, + } + }), + definitionsByKnowledgeBase, } - return filters.map((filter) => { - const definition = definitionsByName.get(filter.tagName) - if (!definition) throw new Error('Validated knowledge tag definition disappeared') - return { - tagSlot: definition.tagSlot, - fieldType: definition.fieldType, - operator: filter.operator, - value: filter.value, - valueTo: filter.valueTo, - } - }) } export const searchKnowledge = defineAuthorizedKnowledgeUseCase({ @@ -145,22 +277,27 @@ export const searchKnowledge = defineAuthorizedKnowledgeUseCase({ resolveContext: ({ input }: { input: SearchKnowledgeInput }) => resolveKnowledgeSearchContext(input), async execute({ principal, input, context }) { + const requestId = generateRequestId() const hasQuery = Boolean(input.query?.trim()) const filters = input.tagFilters ?? [] if (!hasQuery && filters.length === 0) { - throw new OrchestrationError('validation', 'Either query or tagFilters must be provided') - } - if (filters.length > 0 && context.knowledgeBases.length > 1) { throw new OrchestrationError( 'validation', - 'Tag filters are only supported when searching a single knowledge base' + 'Please provide either a search query or tag filters to search your knowledge base' ) } - + const userId = resolveKnowledgeAttributedUserId(principal, context) + const shouldMeter = !( + input.skipUsageBilling && + principal.kind === 'delegated' && + principal.serviceId === 'executor' + ) const billingAttribution = hasQuery - ? await resolveKnowledgeBillingAttribution(principal, context) + ? input.resolveBillingAttribution + ? await input.resolveBillingAttribution(context.workspaceId) + : await resolveKnowledgeBillingAttribution(principal, context) : undefined - if (billingAttribution) { + if (shouldMeter && billingAttribution) { const usage = await checkAttributedUsageLimits(billingAttribution) if (usage.isExceeded) { throw new KnowledgeUsageLimitExceededError( @@ -169,16 +306,16 @@ export const searchKnowledge = defineAuthorizedKnowledgeUseCase({ } } - const tagDefinitionsByKnowledgeBase = new Map< + const knowledgeBaseIds = context.knowledgeBases.map((knowledgeBase) => knowledgeBase.id) + let structuredFilters: StructuredFilter[] = [] + let definitionsByKnowledgeBase = new Map< string, Awaited> >() - let structuredFilters: StructuredFilter[] = [] if (filters.length > 0) { - const knowledgeBaseId = context.knowledgeBases[0].id - const definitions = await getDocumentTagDefinitions(knowledgeBaseId) - tagDefinitionsByKnowledgeBase.set(knowledgeBaseId, definitions) - structuredFilters = buildStructuredFilters(filters, definitions) + const built = await buildStructuredFilters(filters, knowledgeBaseIds) + structuredFilters = built.structuredFilters + definitionsByKnowledgeBase = built.definitionsByKnowledgeBase } const embeddingModels = [...new Set(context.knowledgeBases.map((kb) => kb.embeddingModel))] @@ -189,55 +326,156 @@ export const searchKnowledge = defineAuthorizedKnowledgeUseCase({ ) } const embeddingModel = embeddingModels[0] - let queryEmbeddingIsBYOK: boolean | null = null - let queryVector: string | undefined - if (hasQuery) { - const generated = await generateSearchEmbedding( - input.query!, - embeddingModel, - context.workspaceId - ) - queryEmbeddingIsBYOK = generated.isBYOK - queryVector = JSON.stringify(generated.embedding) - } - - const knowledgeBaseIds = context.knowledgeBases.map((kb) => kb.id) - const rows = await executeKnowledgeSearch({ + const preparedRegistry = input.prepareModelInputProvenance + ? await input.prepareModelInputProvenance({ userId, workspaceId: context.workspaceId }) + : undefined + const resultSecretRegistry = preparedRegistry ?? input.resultSecretRegistry + const queryEmbeddingPromise = hasQuery + ? runWithKnowledgeModelInputProvenance(resultSecretRegistry, () => + generateSearchEmbedding(input.query!, embeddingModel, context.workspaceId) + ) + : Promise.resolve(null) + const useReranker = Boolean(input.rerankerEnabled && hasQuery) + const candidateTopK = useReranker + ? input.rerankerInputCount !== undefined + ? Math.min( + KNOWLEDGE_SEARCH_COST_POLICY.maxTopK, + Math.max(input.topK, input.rerankerInputCount) + ) + : Math.min(KNOWLEDGE_SEARCH_COST_POLICY.maxTopK, input.topK * 4) + : input.topK + let rows = await executeKnowledgeSearch({ knowledgeBaseIds, - topK: input.topK, - searchMode: 'vector', + topK: candidateTopK, + searchMode: input.searchMode ?? 'vector', query: input.query, - queryVector, - structuredFilters, + queryVector: hasQuery + ? JSON.stringify((await queryEmbeddingPromise)?.embedding ?? null) + : undefined, + structuredFilters: structuredFilters.length > 0 ? structuredFilters : undefined, }) - if (input.resultSecretRegistry) { - const provenance = await importKnowledgeSearchResultSecretProvenance({ - registry: input.resultSecretRegistry, + const registry = + resultSecretRegistry ?? + (input.prepareModelInputProvenance + ? new ResolvedSecretTraceRegistry([], { + userId, + workspaceId: context.workspaceId, + }) + : undefined) + let provenanceSnapshot: Awaited< + ReturnType + > | null = null + if (registry) { + provenanceSnapshot = await importKnowledgeSearchResultSecretProvenance({ + registry, results: rows, }) - if (!provenance.imported) { - input.resultSecretRegistry.markIncomplete() - throw new Error('Knowledge result secret provenance is unavailable') + if (!provenanceSnapshot.imported) { + registry.markIncomplete() + if (useReranker) throw new KnowledgeSearchProvenanceUnavailableError() } } - if (queryEmbeddingIsBYOK !== null && billingAttribution) { - await recordSearchEmbeddingUsage({ - userId: resolveKnowledgeAttributedUserId(principal, context), - workspaceId: context.workspaceId, - embeddingModel, - query: input.query!, - isBYOK: queryEmbeddingIsBYOK, - sourceReference: `v2-kb-search:${generateRequestId()}`, - billingAttribution, - }) + const rerankerScores = new Map() + let rerankerBilled = false + let rerankerIsBYOK = false + if (useReranker && input.rerankerModel && rows.length > 0) { + const candidateCount = rows.length + try { + const reranked = await runWithKnowledgeModelInputProvenance(registry, () => + rerank( + input.query!, + rows.map((row) => ({ id: row.id, text: row.content })), + { + model: input.rerankerModel!, + topN: input.topK, + workspaceId: context.workspaceId, + apiKey: input.rerankerApiKey, + } + ) + ) + rerankerBilled = true + rerankerIsBYOK = reranked.isBYOK + if (reranked.results.length === 0) { + rows = rows.slice(0, input.topK) + } else { + const byId = new Map(rows.map((row) => [row.id, row])) + rows = reranked.results + .map((ranked) => byId.get(ranked.item.id)) + .filter((row): row is SearchResult => Boolean(row)) + for (const ranked of reranked.results) { + rerankerScores.set(ranked.item.id, ranked.relevanceScore) + } + } + } catch (error) { + if (registry?.isPermanentlyIncomplete()) throw error + logger.warn('Knowledge reranker failed; using vector ordering', { + error: getErrorMessage(error), + model: input.rerankerModel, + candidateCount, + }) + rows = rows.slice(0, input.topK) + } + } else if (useReranker) { + rows = rows.slice(0, input.topK) + } + + const queryEmbedding = await queryEmbeddingPromise + let tokenCount = 0 + let baseCost: ReturnType | null = null + if (hasQuery) { + tokenCount = estimateTokenCount( + input.query!, + getEmbeddingModelInfo(embeddingModel).tokenizerProvider + ).count + if (!queryEmbedding?.isBYOK) baseCost = calculateCost(embeddingModel, tokenCount, 0, false) + } + let rerankerCost = 0 + if (rerankerBilled && input.rerankerModel && !rerankerIsBYOK) { + const pricing = getRerankModelPricing(input.rerankerModel) + if (pricing) { + rerankerCost = pricing.perSearchUnit + baseCost = baseCost + ? { + ...baseCost, + input: baseCost.input + rerankerCost, + total: baseCost.total + rerankerCost, + } + : { + input: rerankerCost, + output: 0, + total: rerankerCost, + pricing: { input: 0, output: 0, updatedAt: pricing.updatedAt }, + } + } + } + if (shouldMeter && billingAttribution && baseCost && baseCost.total > 0) { + try { + await recordUsage({ + userId, + workspaceId: context.workspaceId, + ...toBillingContext(billingAttribution), + entries: [ + { + category: 'model', + source: 'knowledge-base', + description: embeddingModel, + cost: baseCost.total, + sourceReference: `kb-search:${requestId}`, + }, + ], + }) + await checkAndBillPayerOverageThreshold(billingAttribution.billingEntity) + } catch (error) { + logger.error('Failed to record Knowledge search usage', { error }) + } } const tagDefinitionEntries = await Promise.all( knowledgeBaseIds.map(async (knowledgeBaseId) => { const definitions = - tagDefinitionsByKnowledgeBase.get(knowledgeBaseId) ?? + definitionsByKnowledgeBase.get(knowledgeBaseId) ?? (await getDocumentTagDefinitions(knowledgeBaseId)) return [ knowledgeBaseId, @@ -246,16 +484,24 @@ export const searchKnowledge = defineAuthorizedKnowledgeUseCase({ }) ) const tagMaps = new Map(tagDefinitionEntries) - const documentMetadata = await getDocumentMetadataByIds(rows.map((row) => row.documentId)) - - const results = rows.map((row: SearchResult): KnowledgeSearchItem => { + const basicDocumentMetadata = provenanceSnapshot + ? {} + : await getDocumentMetadataByIds(rows.map((row) => row.documentId)) + const results = rows.map((row): KnowledgeSearchItem => { const metadata: Record = {} const tagMap = tagMaps.get(row.knowledgeBaseId) + const provenanceDocument = provenanceSnapshot?.documentMetadata[row.documentId] + const document = provenanceDocument ?? basicDocumentMetadata[row.documentId] for (const slot of ALL_TAG_SLOTS) { - const value = row[slot] + const value = + provenanceDocument && slot.startsWith('tag') + ? provenanceDocument[ + slot as 'tag1' | 'tag2' | 'tag3' | 'tag4' | 'tag5' | 'tag6' | 'tag7' + ] + : row[slot] if (value !== null && value !== undefined) metadata[tagMap?.get(slot) ?? slot] = value } - const document = documentMetadata[row.documentId] + const rerankerScore = rerankerScores.get(row.id) return { embeddingId: row.id, documentId: row.documentId, @@ -265,14 +511,65 @@ export const searchKnowledge = defineAuthorizedKnowledgeUseCase({ chunkIndex: row.chunkIndex, metadata, similarity: hasQuery ? 1 - row.distance : 1, + ...(rerankerScore !== undefined ? { rerankerScore } : {}), } }) + if (registry && provenanceSnapshot) { + for (const [documentId, document] of Object.entries(provenanceSnapshot.documentMetadata)) { + const renderedMetadata = results + .filter((result) => result.documentId === documentId) + .map((result) => ({ + documentName: result.documentName, + sourceUrl: result.sourceUrl, + metadata: result.metadata, + })) + if ( + renderedMetadata.length > 0 && + !(await importDurableSecretProvenance(registry, document.provenance, renderedMetadata)) + ) { + registry.markIncomplete() + } + } + } + const cost = baseCost + ? { + input: baseCost.input, + output: baseCost.output, + total: baseCost.total, + tokens: { prompt: tokenCount, completion: 0, total: tokenCount }, + model: embeddingModel, + pricing: baseCost.pricing, + ...(rerankerBilled && !rerankerIsBYOK + ? { + rerankerCost, + rerankerModel: input.rerankerModel, + rerankerSearchUnits: 1, + } + : {}), + } + : undefined return { results, query: input.query ?? '', knowledgeBaseIds, + knowledgeBases: context.knowledgeBases.map((knowledgeBase) => ({ + id: knowledgeBase.id, + name: knowledgeBase.name, + })), + knowledgeBaseId: knowledgeBaseIds[0], topK: input.topK, totalResults: results.length, + cost, + workspaceId: context.workspaceId, + userId, + resultSecretRegistry: registry, } }, + afterSuccess: ({ context, result }) => { + PlatformEvents.knowledgeBaseSearched({ + knowledgeBaseId: result.knowledgeBaseId, + resultsCount: result.totalResults, + workspaceId: context.workspaceId, + }) + }, }) diff --git a/apps/sim/lib/knowledge/application/tags.test.ts b/apps/sim/lib/knowledge/application/tags.test.ts new file mode 100644 index 00000000000..836e3991057 --- /dev/null +++ b/apps/sim/lib/knowledge/application/tags.test.ts @@ -0,0 +1,203 @@ +/** + * @vitest-environment node + */ + +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + resolveKnowledgeBase: vi.fn(), + resolveTag: vi.fn(), + resolvePermission: vi.fn(), + listTags: vi.fn(), + nextSlot: vi.fn(), + createTag: vi.fn(), + updateTag: vi.fn(), + deleteTag: vi.fn(), + readUsage: vi.fn(), + recordAudit: vi.fn(), +})) + +vi.mock('@sim/audit', () => ({ + AuditAction: { KNOWLEDGE_BASE_UPDATED: 'knowledge_base.updated' }, + AuditResourceType: { KNOWLEDGE_BASE: 'knowledge_base' }, + recordAudit: mocks.recordAudit, +})) + +vi.mock('@sim/platform-authz/workspace', () => ({ + permissionSatisfies: (actual: string | null, required: string) => { + const rank = { read: 1, write: 2, admin: 3 } as const + return ( + actual !== null && rank[actual as keyof typeof rank] >= rank[required as keyof typeof rank] + ) + }, + resolveEffectiveWorkspacePermission: mocks.resolvePermission, +})) + +vi.mock('@/lib/knowledge/application/contexts', () => ({ + resolveActiveKnowledgeBaseContext: mocks.resolveKnowledgeBase, + resolveActiveKnowledgeTagContext: mocks.resolveTag, +})) + +vi.mock('@/lib/knowledge/tags/service', () => ({ + getDocumentTagDefinitions: mocks.listTags, + getNextAvailableSlot: mocks.nextSlot, + createTagDefinition: mocks.createTag, + updateTagDefinition: mocks.updateTag, + deleteTagDefinition: mocks.deleteTag, + getTagUsageStats: mocks.readUsage, +})) + +import { + createKnowledgeTag, + deleteKnowledgeTag, + listKnowledgeTags, + readKnowledgeTagUsage, + updateKnowledgeTag, +} from '@/lib/knowledge/application/tags' + +const crossWorkspaceContext = { + workspaceId: 'workspace-b', + workspaceOrganizationId: null, + allowPersonalApiKeys: true, + billedAccountUserId: 'billing-owner-b', + knowledgeBaseId: 'knowledge-b', + knowledgeBase: { id: 'knowledge-b', name: 'Workspace B docs' }, +} + +const tagContext = { + ...crossWorkspaceContext, + tagDefinitionId: 'tag-b', + tagDefinition: { + id: 'tag-b', + knowledgeBaseId: 'knowledge-b', + tagSlot: 'tag1', + displayName: 'Region', + fieldType: 'text', + }, +} + +const delegatedPrincipal = { + kind: 'delegated' as const, + serviceId: 'copilot', + subjectUserId: 'shared-user', + workspaceId: 'workspace-a', + delegationId: 'tool-call-1', + audience: 'sim:knowledge', + issuedAt: new Date(), + expiresAt: new Date(Date.now() + 60_000), + resourceScope: {}, +} + +describe('knowledge tag application use cases', () => { + beforeEach(() => { + vi.clearAllMocks() + mocks.resolvePermission.mockResolvedValue('write') + mocks.resolveKnowledgeBase.mockResolvedValue(crossWorkspaceContext) + mocks.resolveTag.mockResolvedValue(tagContext) + }) + + it.each([ + [ + 'list', + listKnowledgeTags, + { knowledgeBaseId: 'knowledge-b', assertedWorkspaceId: 'workspace-a' }, + ], + [ + 'create', + createKnowledgeTag, + { + knowledgeBaseId: 'knowledge-b', + assertedWorkspaceId: 'workspace-a', + displayName: 'Region', + }, + ], + [ + 'update', + updateKnowledgeTag, + { + tagDefinitionId: 'tag-b', + assertedWorkspaceId: 'workspace-a', + updates: { displayName: 'Market' }, + }, + ], + [ + 'delete', + deleteKnowledgeTag, + { + knowledgeBaseId: 'knowledge-b', + tagDefinitionId: 'tag-b', + assertedWorkspaceId: 'workspace-a', + }, + ], + [ + 'read usage', + readKnowledgeTagUsage, + { knowledgeBaseId: 'knowledge-b', assertedWorkspaceId: 'workspace-a' }, + ], + ])( + 'rejects cross-workspace %s before current membership or tag work', + async (_name, useCase, input) => { + await expect(useCase.execute({ principal: delegatedPrincipal, input })).rejects.toMatchObject( + { + name: 'DelegatedWorkspaceAuthorizationError', + code: 'forbidden', + } + ) + + expect(mocks.resolvePermission).not.toHaveBeenCalled() + expect(mocks.listTags).not.toHaveBeenCalled() + expect(mocks.nextSlot).not.toHaveBeenCalled() + expect(mocks.createTag).not.toHaveBeenCalled() + expect(mocks.updateTag).not.toHaveBeenCalled() + expect(mocks.deleteTag).not.toHaveBeenCalled() + expect(mocks.readUsage).not.toHaveBeenCalled() + expect(mocks.recordAudit).not.toHaveBeenCalled() + } + ) + + it('authorizes current delegated membership before mutation and records semantic audit', async () => { + const sameWorkspaceContext = { + ...tagContext, + workspaceId: 'workspace-a', + knowledgeBaseId: 'knowledge-a', + knowledgeBase: { id: 'knowledge-a', name: 'Workspace A docs' }, + tagDefinition: { ...tagContext.tagDefinition, knowledgeBaseId: 'knowledge-a' }, + } + const updatedTag = { ...sameWorkspaceContext.tagDefinition, displayName: 'Market' } + mocks.resolveTag.mockResolvedValueOnce(sameWorkspaceContext) + mocks.updateTag.mockResolvedValueOnce(updatedTag) + + const result = await updateKnowledgeTag.execute({ + principal: delegatedPrincipal, + input: { + tagDefinitionId: 'tag-b', + assertedWorkspaceId: 'workspace-a', + updates: { displayName: 'Market' }, + source: 'agent', + }, + }) + + expect(result.tagDefinition).toEqual(updatedTag) + expect(mocks.resolvePermission).toHaveBeenCalledWith( + 'shared-user', + 'workspace-a', + null, + undefined, + { forUpdate: undefined } + ) + expect(mocks.resolvePermission.mock.invocationCallOrder[0]).toBeLessThan( + mocks.updateTag.mock.invocationCallOrder[0] + ) + expect(mocks.recordAudit).toHaveBeenCalledWith( + expect.objectContaining({ + workspaceId: 'workspace-a', + action: 'knowledge_base.updated', + metadata: expect.objectContaining({ + operation: 'knowledge.tags.update', + change: 'tag_updated', + actor: expect.objectContaining({ kind: 'delegated', serviceId: 'copilot' }), + }), + }) + ) + }) +}) diff --git a/apps/sim/lib/knowledge/application/tags.ts b/apps/sim/lib/knowledge/application/tags.ts new file mode 100644 index 00000000000..7aec057f414 --- /dev/null +++ b/apps/sim/lib/knowledge/application/tags.ts @@ -0,0 +1,311 @@ +import { AuditAction, AuditResourceType } from '@sim/audit' +import { OrchestrationError } from '@/lib/core/orchestration/types' +import { generateRequestId } from '@/lib/core/utils/request' +import { defineAuthorizedKnowledgeUseCase } from '@/lib/knowledge/application/authorized-knowledge-use-case' +import { + resolveActiveKnowledgeBaseContext, + resolveActiveKnowledgeTagContext, + resolveCanonicalActiveKnowledgeDocumentContext, +} from '@/lib/knowledge/application/contexts' +import { knowledgeOperations } from '@/lib/knowledge/application/operations' +import { SUPPORTED_FIELD_TYPES } from '@/lib/knowledge/constants' +import { + cleanupUnusedTagDefinitions, + createOrUpdateTagDefinitionsBulk, + createTagDefinition, + deleteAllTagDefinitions, + deleteTagDefinition, + getDocumentTagDefinitions, + getNextAvailableSlot, + getTagDefinitions, + getTagUsage, + getTagUsageStats, + updateTagDefinition, +} from '@/lib/knowledge/tags/service' +import type { BulkTagDefinitionsData } from '@/lib/knowledge/tags/types' +import type { TagDefinition, UpdateTagDefinitionData } from '@/lib/knowledge/types' + +export interface ListKnowledgeTagsInput { + knowledgeBaseId: string + assertedWorkspaceId?: string +} + +export interface CreateKnowledgeTagInput extends ListKnowledgeTagsInput { + tagSlot?: string + displayName: string + fieldType?: string + source?: string +} + +export interface UpdateKnowledgeTagInput { + tagDefinitionId: string + assertedWorkspaceId?: string + updates: UpdateTagDefinitionData + source?: string +} + +export interface DeleteKnowledgeTagInput extends ListKnowledgeTagsInput { + tagDefinitionId: string + source?: string +} + +export interface ReadNextKnowledgeTagSlotInput extends ListKnowledgeTagsInput { + fieldType: string +} + +export interface KnowledgeDocumentTagDefinitionsInput extends ListKnowledgeTagsInput { + documentId: string +} + +export interface SaveKnowledgeDocumentTagDefinitionsInput + extends KnowledgeDocumentTagDefinitionsInput { + definitions: BulkTagDefinitionsData['definitions'] +} + +export interface DeleteKnowledgeDocumentTagDefinitionsInput + extends KnowledgeDocumentTagDefinitionsInput { + action?: 'cleanup' | 'all' +} + +export const listKnowledgeTags = defineAuthorizedKnowledgeUseCase({ + operation: knowledgeOperations.listTags, + resolveContext: ({ input }: { input: ListKnowledgeTagsInput }) => + resolveActiveKnowledgeBaseContext(input), + async execute({ context }) { + return { tagDefinitions: await getDocumentTagDefinitions(context.knowledgeBaseId) } + }, +}) + +export const createKnowledgeTag = defineAuthorizedKnowledgeUseCase({ + operation: knowledgeOperations.createTag, + resolveContext: ({ input }: { input: CreateKnowledgeTagInput }) => + resolveActiveKnowledgeBaseContext(input), + async execute({ input, context }): Promise<{ + tagDefinition: TagDefinition + knowledgeBaseId: string + }> { + const fieldType = input.fieldType ?? 'text' + if (!(SUPPORTED_FIELD_TYPES as readonly string[]).includes(fieldType)) { + throw new OrchestrationError('validation', 'Invalid field type') + } + const tagSlot = + input.tagSlot ?? (await getNextAvailableSlot(context.knowledgeBaseId, fieldType)) + if (!tagSlot) { + throw new OrchestrationError( + 'validation', + `No available slots for field type "${fieldType}". Maximum tags of this type reached.` + ) + } + const tagDefinition = await createTagDefinition( + { + knowledgeBaseId: context.knowledgeBaseId, + tagSlot, + displayName: input.displayName, + fieldType, + }, + generateRequestId() + ) + return { tagDefinition, knowledgeBaseId: context.knowledgeBaseId } + }, + projectAudit: ({ input, context, result }) => ({ + action: AuditAction.KNOWLEDGE_BASE_UPDATED, + resourceType: AuditResourceType.KNOWLEDGE_BASE, + resourceId: context.knowledgeBaseId, + resourceName: context.knowledgeBase.name, + description: `Created tag "${result.tagDefinition.displayName}" in knowledge base "${context.knowledgeBase.name}"`, + metadata: { + source: input.source, + change: 'tag_created', + tagDefinitionId: result.tagDefinition.id, + tagSlot: result.tagDefinition.tagSlot, + fieldType: result.tagDefinition.fieldType, + }, + }), +}) + +export const updateKnowledgeTag = defineAuthorizedKnowledgeUseCase({ + operation: knowledgeOperations.updateTag, + resolveContext: ({ input }: { input: UpdateKnowledgeTagInput }) => + resolveActiveKnowledgeTagContext(input), + async execute({ input, context }): Promise<{ + tagDefinition: TagDefinition + knowledgeBaseId: string + }> { + if (input.updates.displayName === undefined && input.updates.fieldType === undefined) { + throw new OrchestrationError('validation', 'No tag updates specified') + } + return { + tagDefinition: await updateTagDefinition( + context.tagDefinitionId, + input.updates, + generateRequestId() + ), + knowledgeBaseId: context.knowledgeBaseId, + } + }, + projectAudit: ({ input, context, result }) => ({ + action: AuditAction.KNOWLEDGE_BASE_UPDATED, + resourceType: AuditResourceType.KNOWLEDGE_BASE, + resourceId: context.knowledgeBaseId, + resourceName: context.knowledgeBase.name, + description: `Updated tag "${result.tagDefinition.displayName}" in knowledge base "${context.knowledgeBase.name}"`, + metadata: { + source: input.source, + change: 'tag_updated', + tagDefinitionId: result.tagDefinition.id, + updatedFields: Object.keys(input.updates).filter( + (key) => input.updates[key as keyof UpdateTagDefinitionData] !== undefined + ), + }, + }), +}) + +export const deleteKnowledgeTag = defineAuthorizedKnowledgeUseCase({ + operation: knowledgeOperations.deleteTag, + resolveContext: ({ input }: { input: DeleteKnowledgeTagInput }) => + resolveActiveKnowledgeTagContext(input), + async execute({ context }) { + const deleted = await deleteTagDefinition( + context.knowledgeBaseId, + context.tagDefinitionId, + generateRequestId() + ) + return { ...deleted, tagDefinitionId: context.tagDefinitionId } + }, + projectAudit: ({ input, context, result }) => ({ + action: AuditAction.KNOWLEDGE_BASE_UPDATED, + resourceType: AuditResourceType.KNOWLEDGE_BASE, + resourceId: context.knowledgeBaseId, + resourceName: context.knowledgeBase.name, + description: `Deleted tag "${result.displayName}" from knowledge base "${context.knowledgeBase.name}"`, + metadata: { + source: input.source, + change: 'tag_deleted', + tagDefinitionId: result.tagDefinitionId, + tagSlot: result.tagSlot, + }, + }), +}) + +export const readKnowledgeTagUsage = defineAuthorizedKnowledgeUseCase({ + operation: knowledgeOperations.readTagUsage, + resolveContext: ({ input }: { input: ListKnowledgeTagsInput }) => + resolveActiveKnowledgeBaseContext(input), + async execute({ context }) { + return { usage: await getTagUsageStats(context.knowledgeBaseId, generateRequestId()) } + }, +}) + +export const readDetailedKnowledgeTagUsage = defineAuthorizedKnowledgeUseCase({ + operation: knowledgeOperations.readDetailedTagUsage, + resolveContext: ({ input }: { input: ListKnowledgeTagsInput }) => + resolveActiveKnowledgeBaseContext(input), + async execute({ context }) { + return { usage: await getTagUsage(context.knowledgeBaseId, generateRequestId()) } + }, +}) + +export const readNextKnowledgeTagSlot = defineAuthorizedKnowledgeUseCase({ + operation: knowledgeOperations.readNextTagSlot, + resolveContext: ({ input }: { input: ReadNextKnowledgeTagSlotInput }) => + resolveActiveKnowledgeBaseContext(input), + async execute({ input, context }) { + if (!(SUPPORTED_FIELD_TYPES as readonly string[]).includes(input.fieldType)) { + throw new OrchestrationError('validation', 'Invalid field type') + } + const existingDefinitions = await getTagDefinitions(context.knowledgeBaseId) + const usedSlots = existingDefinitions + .filter((definition) => definition.fieldType === input.fieldType) + .map((definition) => definition.tagSlot) + const existingBySlot = new Map( + existingDefinitions.map((definition) => [definition.tagSlot, definition]) + ) + const nextAvailableSlot = await getNextAvailableSlot( + context.knowledgeBaseId, + input.fieldType, + existingBySlot + ) + return { + nextAvailableSlot, + fieldType: input.fieldType, + usedSlots, + totalSlots: 7, + availableSlots: nextAvailableSlot ? 7 - usedSlots.length : 0, + } + }, +}) + +export const listKnowledgeDocumentTagDefinitions = defineAuthorizedKnowledgeUseCase({ + operation: knowledgeOperations.listTags, + resolveContext: ({ input }: { input: KnowledgeDocumentTagDefinitionsInput }) => + resolveCanonicalActiveKnowledgeDocumentContext(input), + async execute({ context }) { + return { tagDefinitions: await getDocumentTagDefinitions(context.knowledgeBaseId) } + }, +}) + +export const saveKnowledgeDocumentTagDefinitions = defineAuthorizedKnowledgeUseCase({ + operation: knowledgeOperations.saveDocumentTagDefinitions, + resolveContext: ({ input }: { input: SaveKnowledgeDocumentTagDefinitionsInput }) => + resolveCanonicalActiveKnowledgeDocumentContext(input), + async execute({ input, context }) { + for (const definition of input.definitions) { + if (!(SUPPORTED_FIELD_TYPES as readonly string[]).includes(definition.fieldType)) { + throw new OrchestrationError( + 'validation', + `Unsupported field type: ${definition.fieldType}` + ) + } + } + return createOrUpdateTagDefinitionsBulk( + context.knowledgeBaseId, + { definitions: input.definitions }, + generateRequestId() + ) + }, + projectAudit: ({ context, result }) => ({ + action: AuditAction.KNOWLEDGE_BASE_UPDATED, + resourceType: AuditResourceType.KNOWLEDGE_BASE, + resourceId: context.knowledgeBaseId, + resourceName: context.knowledgeBase.name, + description: `Updated tag definitions in knowledge base "${context.knowledgeBase.name}"`, + metadata: { + change: 'document_tag_definitions_saved', + createdCount: result.created.length, + updatedCount: result.updated.length, + errorCount: result.errors.length, + }, + }), +}) + +export const deleteKnowledgeDocumentTagDefinitions = defineAuthorizedKnowledgeUseCase({ + operation: knowledgeOperations.deleteDocumentTagDefinitions, + resolveContext: ({ input }: { input: DeleteKnowledgeDocumentTagDefinitionsInput }) => + resolveCanonicalActiveKnowledgeDocumentContext(input), + async execute({ input, context }) { + if (input.action === 'cleanup') { + return { + action: 'cleanup' as const, + count: await cleanupUnusedTagDefinitions(context.knowledgeBaseId, generateRequestId()), + } + } + return { + action: 'all' as const, + count: await deleteAllTagDefinitions(context.knowledgeBaseId, generateRequestId()), + } + }, + projectAudit: ({ input, context, result }) => ({ + action: AuditAction.KNOWLEDGE_BASE_UPDATED, + resourceType: AuditResourceType.KNOWLEDGE_BASE, + resourceId: context.knowledgeBaseId, + resourceName: context.knowledgeBase.name, + description: + input.action === 'cleanup' + ? `Cleaned unused tag definitions in knowledge base "${context.knowledgeBase.name}"` + : `Deleted tag definitions in knowledge base "${context.knowledgeBase.name}"`, + metadata: { + change: result.action === 'cleanup' ? 'tag_definitions_cleaned' : 'tag_definitions_deleted', + count: result.count, + }, + }), +}) diff --git a/apps/sim/lib/knowledge/application/upload-sessions.ts b/apps/sim/lib/knowledge/application/upload-sessions.ts index 3d8586ad9da..73526ddf1f5 100644 --- a/apps/sim/lib/knowledge/application/upload-sessions.ts +++ b/apps/sim/lib/knowledge/application/upload-sessions.ts @@ -1,7 +1,5 @@ import { AuditAction, AuditResourceType } from '@sim/audit' import type { Principal } from '@sim/auth/principal' -import type { V2KnowledgeDocumentUploadMetadata } from '@/lib/api/contracts/v2/knowledge' -import { v2KnowledgeDocumentUploadMetadataSchema } from '@/lib/api/contracts/v2/knowledge' import { checkAttributedUsageLimits } from '@/lib/billing/core/billing-attribution' import { authorizeWorkspaceOperation, type WorkspaceOperation } from '@/lib/core/application' import { OrchestrationError } from '@/lib/core/orchestration/types' @@ -25,6 +23,10 @@ import { } from '@/lib/knowledge/documents/service' import type { CreatedKnowledgeDocument } from '@/lib/knowledge/orchestration/documents' import { findBoundKnowledgeDocument } from '@/lib/knowledge/orchestration/documents' +import { + type KnowledgeDocumentUploadMetadata, + knowledgeDocumentUploadMetadataSchema, +} from '@/lib/knowledge/upload-metadata' import { recordKnowledgeBaseFileOwnership } from '@/lib/uploads/server/metadata' import { requestOrigin } from '@/lib/uploads/upload-session/application' import { @@ -60,7 +62,7 @@ export interface CreateKnowledgeDocumentUploadInput { name: string contentType: string size: number - metadata: V2KnowledgeDocumentUploadMetadata + metadata: KnowledgeDocumentUploadMetadata } export interface KnowledgeDocumentUploadControlInput { @@ -384,7 +386,7 @@ export const completeKnowledgeDocumentUpload = defineAuthorizedKnowledgeUseCase( async function dispatchKnowledgeDocumentProcessing( document: CreatedKnowledgeDocument, knowledgeBaseId: string, - processingOptions: V2KnowledgeDocumentUploadMetadata['processingOptions'], + processingOptions: KnowledgeDocumentUploadMetadata['processingOptions'], requestId: string, billingAttribution: Awaited> ): Promise { @@ -447,7 +449,7 @@ async function reauthorizeKnowledgeDocumentUpload( function knowledgeDocumentMetadataFor(session: UploadSessionRecord) { const { authBinding: _authBinding, ...metadata } = session.metadata - return v2KnowledgeDocumentUploadMetadataSchema.parse(metadata) + return knowledgeDocumentUploadMetadataSchema.parse(metadata) } function knowledgeDocumentInputFor(session: UploadSessionRecord) { diff --git a/apps/sim/lib/knowledge/connectors/service.ts b/apps/sim/lib/knowledge/connectors/service.ts new file mode 100644 index 00000000000..9d454e62109 --- /dev/null +++ b/apps/sim/lib/knowledge/connectors/service.ts @@ -0,0 +1,34 @@ +import { db } from '@sim/db' +import { knowledgeConnector } from '@sim/db/schema' +import { and, eq, isNull } from 'drizzle-orm' + +export interface ActiveKnowledgeConnectorReference { + id: string + knowledgeBaseId: string + connectorType: string + status: string +} + +/** Resolves a connector's canonical active parent without trusting a caller-supplied KB ID. */ +export async function getActiveKnowledgeConnectorReference( + connectorId: string +): Promise { + const [connector] = await db + .select({ + id: knowledgeConnector.id, + knowledgeBaseId: knowledgeConnector.knowledgeBaseId, + connectorType: knowledgeConnector.connectorType, + status: knowledgeConnector.status, + }) + .from(knowledgeConnector) + .where( + and( + eq(knowledgeConnector.id, connectorId), + isNull(knowledgeConnector.archivedAt), + isNull(knowledgeConnector.deletedAt) + ) + ) + .limit(1) + + return connector ?? null +} diff --git a/apps/sim/lib/knowledge/connectors/sync-engine.test.ts b/apps/sim/lib/knowledge/connectors/sync-engine.test.ts index 540ae694595..b4d3a162480 100644 --- a/apps/sim/lib/knowledge/connectors/sync-engine.test.ts +++ b/apps/sim/lib/knowledge/connectors/sync-engine.test.ts @@ -18,7 +18,7 @@ vi.mock('@/lib/knowledge/documents/service', () => ({ processDocumentAsync: vi.fn(), })) vi.mock('@/lib/uploads', () => ({ StorageService: {} })) -vi.mock('@/app/api/auth/oauth/utils', () => authOAuthUtilsMock) +vi.mock('@/lib/oauth/credential-service', () => authOAuthUtilsMock) vi.mock('@/background/knowledge-connector-sync', () => ({ knowledgeConnectorSync: { trigger: vi.fn() }, })) diff --git a/apps/sim/lib/knowledge/connectors/sync-engine.ts b/apps/sim/lib/knowledge/connectors/sync-engine.ts index 8ca978ec829..109167e2edc 100644 --- a/apps/sim/lib/knowledge/connectors/sync-engine.ts +++ b/apps/sim/lib/knowledge/connectors/sync-engine.ts @@ -20,11 +20,11 @@ import { getInternalApiBaseUrl } from '@/lib/core/utils/urls' import { resolveCredentialTokenIdentity } from '@/lib/credentials/access' import type { DocumentData } from '@/lib/knowledge/documents/service' import { hardDeleteDocuments, processDocumentsWithQueue } from '@/lib/knowledge/documents/service' +import { refreshAccessTokenIfNeeded } from '@/lib/oauth/credential-service' import { StorageService } from '@/lib/uploads' import { deleteFile } from '@/lib/uploads/core/storage-service' import { deleteFileMetadata } from '@/lib/uploads/server/metadata' import { extractStorageKey } from '@/lib/uploads/utils/file-utils' -import { refreshAccessTokenIfNeeded } from '@/app/api/auth/oauth/utils' import { CONNECTOR_REGISTRY } from '@/connectors/registry.server' import type { ConnectorAuthConfig, diff --git a/apps/sim/lib/knowledge/constants.ts b/apps/sim/lib/knowledge/constants.ts index 54cdf56ec97..71a03f5d0ed 100644 --- a/apps/sim/lib/knowledge/constants.ts +++ b/apps/sim/lib/knowledge/constants.ts @@ -8,6 +8,13 @@ export const MAX_KNOWLEDGE_BASES_PER_WORKSPACE = 10_000 export const MAX_KNOWLEDGE_FOLDERS_PER_WORKSPACE = MAX_FOLDERS_PER_WORKSPACE /** Hard bound for connector-type rows projected onto one knowledge-base list. */ export const MAX_KNOWLEDGE_CONNECTOR_TYPE_ROWS_PER_LIST = 100_000 +/** Maximum documents accepted by one internal bulk-create command. */ +export const MAX_KNOWLEDGE_DOCUMENTS_PER_CREATE = 100 +/** Maximum connector documents mutated atomically by one command. */ +export const MAX_KNOWLEDGE_CONNECTOR_DOCUMENT_MUTATION_ITEMS = 100 +/** Default and maximum bounded connector-document list page sizes. */ +export const DEFAULT_KNOWLEDGE_CONNECTOR_DOCUMENT_PAGE_SIZE = 100 +export const MAX_KNOWLEDGE_CONNECTOR_DOCUMENT_PAGE_SIZE = 200 /** * Chunking a knowledge base gets when its creator names no configuration. diff --git a/apps/sim/lib/knowledge/documents/service.ts b/apps/sim/lib/knowledge/documents/service.ts index 7425819f571..8f9219df7ab 100644 --- a/apps/sim/lib/knowledge/documents/service.ts +++ b/apps/sim/lib/knowledge/documents/service.ts @@ -1819,6 +1819,30 @@ export async function getKnowledgeDocument( return row ? { ...row, connectorType: row.connectorType ?? null } : null } +/** Loads one visible document by its canonical ID before any asserted parent is trusted. */ +export async function getKnowledgeDocumentById( + documentId: string +): Promise { + const [row] = await db + .select({ + ...getTableColumns(document), + connectorType: knowledgeConnector.connectorType, + }) + .from(document) + .leftJoin(knowledgeConnector, eq(document.connectorId, knowledgeConnector.id)) + .where( + and( + eq(document.id, documentId), + eq(document.userExcluded, false), + isNull(document.archivedAt), + isNull(document.deletedAt) + ) + ) + .limit(1) + + return row ? { ...row, connectorType: row.connectorType ?? null } : null +} + export async function createSingleDocument( documentData: { filename: string diff --git a/apps/sim/lib/knowledge/orchestration/connectors.test.ts b/apps/sim/lib/knowledge/orchestration/connectors.test.ts index 68f7a10f242..6131c462c93 100644 --- a/apps/sim/lib/knowledge/orchestration/connectors.test.ts +++ b/apps/sim/lib/knowledge/orchestration/connectors.test.ts @@ -112,6 +112,24 @@ describe('performDeleteKnowledgeConnector', () => { expect(outcome).toMatchObject({ success: false, errorCode: 'not_found' }) expect(mockRecordAudit).not.toHaveBeenCalled() }) + + it('returns authoritative delete counts without legacy audit or analytics when disabled', async () => { + dbChainMockFns.limit.mockResolvedValueOnce([{ id: 'conn-1', connectorType: 'notion' }]) + queueTableRows(document, [{ id: 'doc-1', fileUrl: '/a.txt' }]) + dbChainMockFns.returning.mockResolvedValueOnce([{ id: 'conn-1' }]) + + const outcome = await performDeleteKnowledgeConnector({ + ...ACTOR, + knowledgeBase: KB, + connectorId: 'conn-1', + recordSemanticAudit: false, + recordProductAnalytics: false, + }) + + expect(outcome).toMatchObject({ success: true, documentsKept: 1 }) + expect(mockRecordAudit).not.toHaveBeenCalled() + expect(mockCaptureServerEvent).not.toHaveBeenCalled() + }) }) describe('performUpdateKnowledgeConnector', () => { @@ -207,6 +225,24 @@ describe('performUpdateKnowledgeConnector', () => { expect.objectContaining({ consecutiveFailures: 0, lastSyncError: null }) ) }) + + it('leaves semantic audit to an authorized application caller when requested', async () => { + dbChainMockFns.limit.mockResolvedValueOnce([{ id: 'conn-1', connectorType: 'notion' }]) + dbChainMockFns.returning.mockResolvedValueOnce([ + { id: 'conn-1', connectorType: 'notion', status: 'paused' }, + ]) + + const outcome = await performUpdateKnowledgeConnector({ + ...ACTOR, + knowledgeBase: KB, + connectorId: 'conn-1', + updates: { status: 'paused' }, + recordSemanticAudit: false, + }) + + expect(outcome).toMatchObject({ success: true }) + expect(mockRecordAudit).not.toHaveBeenCalled() + }) }) describe('performSyncKnowledgeConnector', () => { @@ -295,4 +331,24 @@ describe('performSyncKnowledgeConnector', () => { expect(outcome).toMatchObject({ success: false, errorCode: 'conflict' }) }) + + it('dispatches while leaving semantic audit and product analytics to the application surface', async () => { + dbChainMockFns.limit.mockResolvedValueOnce([ + { id: 'conn-1', connectorType: 'notion', status: 'active' }, + ]) + + const outcome = await performSyncKnowledgeConnector({ + ...ACTOR, + knowledgeBase: KB, + connectorId: 'conn-1', + resolveBillingAttribution, + recordSemanticAudit: false, + recordProductAnalytics: false, + }) + + expect(outcome).toMatchObject({ success: true }) + expect(mockDispatchSync).toHaveBeenCalledOnce() + expect(mockRecordAudit).not.toHaveBeenCalled() + expect(mockCaptureServerEvent).not.toHaveBeenCalled() + }) }) diff --git a/apps/sim/lib/knowledge/orchestration/connectors.ts b/apps/sim/lib/knowledge/orchestration/connectors.ts index 01d18f5ccea..16ca5d23d8c 100644 --- a/apps/sim/lib/knowledge/orchestration/connectors.ts +++ b/apps/sim/lib/knowledge/orchestration/connectors.ts @@ -98,6 +98,10 @@ export interface PerformCreateKnowledgeConnectorParams extends KnowledgeOperatio * because credential lookup is scoped to the requesting identity. */ resolveAccessToken: (credentialId: string) => Promise + /** False only when an authorized application use case projects the semantic audit. */ + recordSemanticAudit?: boolean + /** False when the calling HTTP/tool adapter owns product analytics. */ + recordProductAnalytics?: boolean } export type PerformConnectorResult = KnowledgeOrchestrationResult<{ @@ -302,39 +306,43 @@ export async function performCreateKnowledgeConnector( logger.info(`[${requestId}] Created connector ${connectorId} for KB ${kb.id}`) - captureServerEvent( - params.userId, - 'knowledge_base_connector_added', - { - knowledge_base_id: kb.id, - workspace_id: workspaceId, - connector_type: connectorType, - sync_interval_minutes: syncIntervalMinutes, - }, - { - groups: { workspace: workspaceId }, - setOnce: { first_connector_added_at: new Date().toISOString() }, - } - ) + if (params.recordProductAnalytics !== false) { + captureServerEvent( + params.userId, + 'knowledge_base_connector_added', + { + knowledge_base_id: kb.id, + workspace_id: workspaceId, + connector_type: connectorType, + sync_interval_minutes: syncIntervalMinutes, + }, + { + groups: { workspace: workspaceId }, + setOnce: { first_connector_added_at: new Date().toISOString() }, + } + ) + } - recordAudit({ - workspaceId, - ...auditActorFields(params), - action: AuditAction.CONNECTOR_CREATED, - resourceType: AuditResourceType.CONNECTOR, - resourceId: connectorId, - resourceName: connectorType, - description: `Created ${connectorType} connector for knowledge base "${kb.name}"`, - metadata: { - source, - knowledgeBaseId: kb.id, - knowledgeBaseName: kb.name, - connectorType, - syncIntervalMinutes, - authMode: connectorConfig.auth.mode, - }, - ...(request ? { request } : {}), - }) + if (params.recordSemanticAudit !== false) { + recordAudit({ + workspaceId, + ...auditActorFields(params), + action: AuditAction.CONNECTOR_CREATED, + resourceType: AuditResourceType.CONNECTOR, + resourceId: connectorId, + resourceName: connectorType, + description: `Created ${connectorType} connector for knowledge base "${kb.name}"`, + metadata: { + source, + knowledgeBaseId: kb.id, + knowledgeBaseName: kb.name, + connectorType, + syncIntervalMinutes, + authMode: connectorConfig.auth.mode, + }, + ...(request ? { request } : {}), + }) + } const dispatchSync = await loadDispatchSync() dispatchSync(connectorId, { billingAttribution, requestId }).catch((error) => { @@ -368,6 +376,8 @@ export interface PerformUpdateKnowledgeConnectorParams extends KnowledgeOperatio connector: KnowledgeConnectorRow, sourceConfig: Record ) => Promise + /** False only when an authorized application use case projects the semantic audit. */ + recordSemanticAudit?: boolean } /** Loads an active connector scoped to its knowledge base. */ @@ -480,27 +490,29 @@ export async function performUpdateKnowledgeConnector( return classifyKnowledgeFailure(error, requestId, `Update connector ${connectorId}`) } - recordAudit({ - workspaceId: kb.workspaceId, - ...auditActorFields(params), - action: AuditAction.CONNECTOR_UPDATED, - resourceType: AuditResourceType.CONNECTOR, - resourceId: connectorId, - resourceName: updated.connectorType, - description: `Updated connector for knowledge base "${kb.name}"`, - metadata: { - source, - knowledgeBaseId: kb.id, - knowledgeBaseName: kb.name, - connectorType: updated.connectorType, - updatedFields, - ...(updates.syncIntervalMinutes !== undefined && { - syncIntervalMinutes: updates.syncIntervalMinutes, - }), - ...(updates.status !== undefined && { newStatus: updates.status }), - }, - ...(request ? { request } : {}), - }) + if (params.recordSemanticAudit !== false) { + recordAudit({ + workspaceId: kb.workspaceId, + ...auditActorFields(params), + action: AuditAction.CONNECTOR_UPDATED, + resourceType: AuditResourceType.CONNECTOR, + resourceId: connectorId, + resourceName: updated.connectorType, + description: `Updated connector for knowledge base "${kb.name}"`, + metadata: { + source, + knowledgeBaseId: kb.id, + knowledgeBaseName: kb.name, + connectorType: updated.connectorType, + updatedFields, + ...(updates.syncIntervalMinutes !== undefined && { + syncIntervalMinutes: updates.syncIntervalMinutes, + }), + ...(updates.status !== undefined && { newStatus: updates.status }), + }, + ...(request ? { request } : {}), + }) + } return { success: true, connector: withoutSecret(updated) } } @@ -513,6 +525,10 @@ export interface PerformDeleteKnowledgeConnectorParams extends KnowledgeOperatio * them, which turns them into ordinary standalone knowledge base entries. */ deleteDocuments?: boolean + /** False only when an authorized application use case projects the semantic audit. */ + recordSemanticAudit?: boolean + /** False when the calling HTTP/tool adapter owns product analytics. */ + recordProductAnalytics?: boolean } /** What actually happened to the connector's documents, for the caller to report. */ @@ -609,37 +625,41 @@ export async function performDeleteKnowledgeConnector( `[${requestId}] Deleted connector ${connectorId}${deleteDocuments ? ` and ${docCount} documents` : `, kept ${docCount} documents`}` ) - captureServerEvent( - params.userId, - 'knowledge_base_connector_removed', - { - knowledge_base_id: kb.id, - workspace_id: kb.workspaceId ?? '', - connector_type: existing.connectorType, - documents_deleted: deleteDocuments ? docCount : 0, - }, - kb.workspaceId ? { groups: { workspace: kb.workspaceId } } : undefined - ) + if (params.recordProductAnalytics !== false) { + captureServerEvent( + params.userId, + 'knowledge_base_connector_removed', + { + knowledge_base_id: kb.id, + workspace_id: kb.workspaceId ?? '', + connector_type: existing.connectorType, + documents_deleted: deleteDocuments ? docCount : 0, + }, + kb.workspaceId ? { groups: { workspace: kb.workspaceId } } : undefined + ) + } - recordAudit({ - workspaceId: kb.workspaceId, - ...auditActorFields(params), - action: AuditAction.CONNECTOR_DELETED, - resourceType: AuditResourceType.CONNECTOR, - resourceId: connectorId, - resourceName: existing.connectorType, - description: `Deleted connector from knowledge base "${kb.name}"`, - metadata: { - source, - knowledgeBaseId: kb.id, - knowledgeBaseName: kb.name, - connectorType: existing.connectorType, - deleteDocuments, - documentsDeleted: deleteDocuments ? docCount : 0, - documentsKept: deleteDocuments ? 0 : docCount, - }, - ...(request ? { request } : {}), - }) + if (params.recordSemanticAudit !== false) { + recordAudit({ + workspaceId: kb.workspaceId, + ...auditActorFields(params), + action: AuditAction.CONNECTOR_DELETED, + resourceType: AuditResourceType.CONNECTOR, + resourceId: connectorId, + resourceName: existing.connectorType, + description: `Deleted connector from knowledge base "${kb.name}"`, + metadata: { + source, + knowledgeBaseId: kb.id, + knowledgeBaseName: kb.name, + connectorType: existing.connectorType, + deleteDocuments, + documentsDeleted: deleteDocuments ? docCount : 0, + documentsKept: deleteDocuments ? 0 : docCount, + }, + ...(request ? { request } : {}), + }) + } return { success: true, @@ -658,6 +678,10 @@ export interface PerformSyncKnowledgeConnectorParams extends KnowledgeOperationC resolveBillingAttribution: () => Promise /** Re-fetch and re-index every already-synced document, not only changed ones. */ rehydrate?: boolean + /** False only when an authorized application use case projects the semantic audit. */ + recordSemanticAudit?: boolean + /** False when the calling HTTP/tool adapter owns product analytics. */ + recordProductAnalytics?: boolean } export type PerformSyncKnowledgeConnectorResult = KnowledgeOrchestrationResult @@ -693,35 +717,39 @@ export async function performSyncKnowledgeConnector( `[${requestId}] Manual sync${rehydrate ? ' (full rehydrate)' : ''} triggered for connector ${connectorId}` ) - captureServerEvent( - params.userId, - 'knowledge_base_connector_synced', - { - knowledge_base_id: kb.id, - workspace_id: kb.workspaceId ?? '', - connector_type: connector.connectorType, - }, - kb.workspaceId ? { groups: { workspace: kb.workspaceId } } : undefined - ) + if (params.recordProductAnalytics !== false) { + captureServerEvent( + params.userId, + 'knowledge_base_connector_synced', + { + knowledge_base_id: kb.id, + workspace_id: kb.workspaceId ?? '', + connector_type: connector.connectorType, + }, + kb.workspaceId ? { groups: { workspace: kb.workspaceId } } : undefined + ) + } - recordAudit({ - workspaceId: kb.workspaceId, - ...auditActorFields(params), - action: AuditAction.CONNECTOR_SYNCED, - resourceType: AuditResourceType.CONNECTOR, - resourceId: connectorId, - resourceName: connector.connectorType, - description: `Triggered manual sync for connector on knowledge base "${kb.name}"`, - metadata: { - source, - knowledgeBaseId: kb.id, - knowledgeBaseName: kb.name, - connectorType: connector.connectorType, - connectorStatus: connector.status, - syncType: rehydrate ? 'manual-rehydrate' : 'manual', - }, - ...(request ? { request } : {}), - }) + if (params.recordSemanticAudit !== false) { + recordAudit({ + workspaceId: kb.workspaceId, + ...auditActorFields(params), + action: AuditAction.CONNECTOR_SYNCED, + resourceType: AuditResourceType.CONNECTOR, + resourceId: connectorId, + resourceName: connector.connectorType, + description: `Triggered manual sync for connector on knowledge base "${kb.name}"`, + metadata: { + source, + knowledgeBaseId: kb.id, + knowledgeBaseName: kb.name, + connectorType: connector.connectorType, + connectorStatus: connector.status, + syncType: rehydrate ? 'manual-rehydrate' : 'manual', + }, + ...(request ? { request } : {}), + }) + } const dispatchSync = await loadDispatchSync() dispatchSync(connectorId, { billingAttribution, requestId, rehydrate }).catch((error) => { diff --git a/apps/sim/lib/knowledge/orchestration/documents.test.ts b/apps/sim/lib/knowledge/orchestration/documents.test.ts index 42a8b250e05..e4a66fec7f7 100644 --- a/apps/sim/lib/knowledge/orchestration/documents.test.ts +++ b/apps/sim/lib/knowledge/orchestration/documents.test.ts @@ -12,6 +12,7 @@ const { mockMarkDocumentAsFailedTimeout, mockProcessDocumentAsync, mockProcessDocumentsWithQueue, + mockPlatformUpload, mockRecordAudit, mockRetryDocumentProcessing, mockUpdateDocument, @@ -24,6 +25,7 @@ const { mockMarkDocumentAsFailedTimeout: vi.fn(), mockProcessDocumentAsync: vi.fn(), mockProcessDocumentsWithQueue: vi.fn(), + mockPlatformUpload: vi.fn(), mockRecordAudit: vi.fn(), mockRetryDocumentProcessing: vi.fn(), mockUpdateDocument: vi.fn(), @@ -39,7 +41,7 @@ vi.mock('@sim/audit', () => ({ recordAudit: mockRecordAudit, })) vi.mock('@/lib/core/telemetry', () => ({ - PlatformEvents: { knowledgeBaseDocumentsUploaded: vi.fn() }, + PlatformEvents: { knowledgeBaseDocumentsUploaded: mockPlatformUpload }, })) vi.mock('@/lib/knowledge/documents/service', () => ({ createDocumentRecords: mockCreateDocumentRecords, @@ -110,7 +112,14 @@ describe('performUploadKnowledgeDocument', () => { uploadedBy: 'workspace-owner', }) - expect(mockCreateSingleDocument).toHaveBeenCalledWith(FILE, 'kb-1', 'req-1', 'workspace-owner') + expect(mockCreateSingleDocument).toHaveBeenCalledWith( + FILE, + 'kb-1', + 'req-1', + 'workspace-owner', + undefined, + undefined + ) }) it('starts no indexing unless the caller asks for it', async () => { @@ -160,6 +169,21 @@ describe('performUploadKnowledgeDocument', () => { ).toBe('forbidden') }) + it('returns the authoritative upload without legacy audit or product analytics when disabled', async () => { + const outcome = await performUploadKnowledgeDocument({ + ...ACTOR, + knowledgeBase: KB, + document: FILE, + recordSemanticAudit: false, + recordProductAnalytics: false, + }) + + expect(outcome).toMatchObject({ success: true, document: { id: 'doc-1' } }) + expect(mockRecordAudit).not.toHaveBeenCalled() + expect(mockPlatformUpload).not.toHaveBeenCalled() + expect(mockCaptureServerEvent).not.toHaveBeenCalled() + }) + it('returns the document already bound to a stateless upload id without duplicating work', async () => { const existing = { id: 'upload-1', @@ -219,7 +243,8 @@ describe('performUploadKnowledgeDocument', () => { 'kb-1', 'req-1', 'user-1', - 'upload-1' + 'upload-1', + undefined ) expect(mockProcessDocumentsWithQueue).not.toHaveBeenCalled() expect(mockRecordAudit).not.toHaveBeenCalled() @@ -261,6 +286,22 @@ describe('performUploadKnowledgeDocuments', () => { expect(outcome).toMatchObject({ success: false, errorCode: 'validation' }) expect(mockCreateDocumentRecords).not.toHaveBeenCalled() }) + + it('returns the authoritative batch without legacy audit or product analytics when disabled', async () => { + const outcome = await performUploadKnowledgeDocuments({ + ...ACTOR, + knowledgeBase: KB, + documents: [FILE], + recordSemanticAudit: false, + recordProductAnalytics: false, + }) + + expect(outcome).toMatchObject({ success: true }) + expect(outcome.success && outcome.documents[0]).toMatchObject({ documentId: 'doc-1' }) + expect(mockRecordAudit).not.toHaveBeenCalled() + expect(mockPlatformUpload).not.toHaveBeenCalled() + expect(mockCaptureServerEvent).not.toHaveBeenCalled() + }) }) describe('performUpdateKnowledgeDocument', () => { diff --git a/apps/sim/lib/knowledge/orchestration/documents.ts b/apps/sim/lib/knowledge/orchestration/documents.ts index d4b0d00556c..72c7eb51cf7 100644 --- a/apps/sim/lib/knowledge/orchestration/documents.ts +++ b/apps/sim/lib/knowledge/orchestration/documents.ts @@ -77,6 +77,10 @@ export interface PerformUploadKnowledgeDocumentParams extends KnowledgeOperation /** Deterministic id carried by a stateless upload token for completion retries. */ documentId?: string secretProvenance?: KnowledgeDocumentWriteSecretProvenance + /** False when an authorized application use case projects the semantic audit. */ + recordSemanticAudit?: boolean + /** False when the calling HTTP/tool adapter owns product analytics. */ + recordProductAnalytics?: boolean } export type PerformUploadKnowledgeDocumentResult = KnowledgeOrchestrationResult<{ @@ -266,25 +270,29 @@ export async function performUploadKnowledgeDocument( }) } - PlatformEvents.knowledgeBaseDocumentsUploaded({ - knowledgeBaseId: knowledgeBase.id, - documentsCount: 1, - uploadType: 'single', - mimeType: document.mimeType, - fileSize: document.fileSize, - }) - captureUpload(params, 1, 'single') - - auditUpload(params, { - resourceId: created.id, - resourceName: document.filename, - description: `Uploaded document "${document.filename}" to knowledge base "${knowledgeBase.name ?? knowledgeBase.id}"`, - metadata: { - fileName: document.filename, - fileType: document.mimeType, + if (params.recordProductAnalytics !== false) { + PlatformEvents.knowledgeBaseDocumentsUploaded({ + knowledgeBaseId: knowledgeBase.id, + documentsCount: 1, + uploadType: 'single', + mimeType: document.mimeType, fileSize: document.fileSize, - }, - }) + }) + captureUpload(params, 1, 'single') + } + + if (params.recordSemanticAudit !== false) { + auditUpload(params, { + resourceId: created.id, + resourceName: document.filename, + description: `Uploaded document "${document.filename}" to knowledge base "${knowledgeBase.name ?? knowledgeBase.id}"`, + metadata: { + fileName: document.filename, + fileType: document.mimeType, + fileSize: document.fileSize, + }, + }) + } return { success: true, document: created, created: true } } @@ -296,6 +304,10 @@ export interface PerformUploadKnowledgeDocumentsParams extends KnowledgeOperatio billingAttribution?: BillingAttributionSnapshot uploadedBy?: string | null secretProvenances?: readonly KnowledgeDocumentWriteSecretProvenance[] + /** False when an authorized application use case projects the semantic audit. */ + recordSemanticAudit?: boolean + /** False when the calling HTTP/tool adapter owns product analytics. */ + recordProductAnalytics?: boolean } export type PerformUploadKnowledgeDocumentsResult = KnowledgeOrchestrationResult<{ @@ -349,20 +361,24 @@ export async function performUploadKnowledgeDocuments( logger.error(`[${requestId}] Critical error in document processing pipeline`, { error }) }) - PlatformEvents.knowledgeBaseDocumentsUploaded({ - knowledgeBaseId: knowledgeBase.id, - documentsCount: created.length, - uploadType: 'bulk', - recipe: processingOptions?.recipe, - }) - captureUpload(params, created.length, 'bulk') + if (params.recordProductAnalytics !== false) { + PlatformEvents.knowledgeBaseDocumentsUploaded({ + knowledgeBaseId: knowledgeBase.id, + documentsCount: created.length, + uploadType: 'bulk', + recipe: processingOptions?.recipe, + }) + captureUpload(params, created.length, 'bulk') + } - auditUpload(params, { - resourceId: knowledgeBase.id, - resourceName: `${created.length} document(s)`, - description: `Uploaded ${created.length} document(s) to knowledge base "${knowledgeBase.name ?? knowledgeBase.id}"`, - metadata: { fileCount: created.length }, - }) + if (params.recordSemanticAudit !== false) { + auditUpload(params, { + resourceId: knowledgeBase.id, + resourceName: `${created.length} document(s)`, + description: `Uploaded ${created.length} document(s) to knowledge base "${knowledgeBase.name ?? knowledgeBase.id}"`, + metadata: { fileCount: created.length }, + }) + } return { success: true, documents: created } } diff --git a/apps/sim/lib/knowledge/orchestration/knowledge-bases.test.ts b/apps/sim/lib/knowledge/orchestration/knowledge-bases.test.ts index 6aed845a9be..74ed2b10951 100644 --- a/apps/sim/lib/knowledge/orchestration/knowledge-bases.test.ts +++ b/apps/sim/lib/knowledge/orchestration/knowledge-bases.test.ts @@ -235,7 +235,9 @@ describe('performDeleteKnowledgeBase', () => { }) expect(outcome.success).toBe(true) - expect(mockDeleteKnowledgeBase).toHaveBeenCalledWith('kb-1', 'req-1') + expect(mockDeleteKnowledgeBase).toHaveBeenCalledWith('kb-1', 'req-1', { + assertedWorkspaceId: undefined, + }) expect(mockRecordAudit).toHaveBeenCalledWith( expect.objectContaining({ actorId: 'user-1', resourceId: 'kb-1' }) ) diff --git a/apps/sim/lib/knowledge/orchestration/knowledge-bases.ts b/apps/sim/lib/knowledge/orchestration/knowledge-bases.ts index b808f820e97..b2f6d873764 100644 --- a/apps/sim/lib/knowledge/orchestration/knowledge-bases.ts +++ b/apps/sim/lib/knowledge/orchestration/knowledge-bases.ts @@ -121,6 +121,7 @@ export async function performCreateKnowledgeBase( export interface PerformUpdateKnowledgeBaseParams extends KnowledgeOperationContext { knowledgeBaseId: string + assertedWorkspaceId?: string /** Workspace the knowledge base currently belongs to, for the audit row. */ workspaceId: string | null updates: { @@ -157,6 +158,7 @@ export async function performUpdateKnowledgeBase( try { updated = await updateKnowledgeBase(knowledgeBaseId, updates, requestId, { actorUserId: params.userId, + assertedWorkspaceId: params.assertedWorkspaceId, }) } catch (error) { return classifyKnowledgeFailure(error, requestId, `Update knowledge base ${knowledgeBaseId}`) @@ -193,6 +195,7 @@ export async function performUpdateKnowledgeBase( export interface PerformDeleteKnowledgeBaseParams extends KnowledgeOperationContext { knowledgeBase: { id: string; name: string; workspaceId: string | null } + assertedWorkspaceId?: string } export type PerformDeleteKnowledgeBaseResult = KnowledgeOrchestrationResult @@ -211,7 +214,9 @@ export async function performDeleteKnowledgeBase( const requestId = params.requestId ?? generateRequestId() try { - await deleteKnowledgeBase(knowledgeBase.id, requestId) + await deleteKnowledgeBase(knowledgeBase.id, requestId, { + assertedWorkspaceId: params.assertedWorkspaceId, + }) } catch (error) { return classifyKnowledgeFailure(error, requestId, `Delete knowledge base ${knowledgeBase.id}`) } diff --git a/apps/sim/lib/knowledge/tags/secret-provenance-delete.test.ts b/apps/sim/lib/knowledge/tags/secret-provenance-delete.test.ts index 11a8551abb0..73439d09062 100644 --- a/apps/sim/lib/knowledge/tags/secret-provenance-delete.test.ts +++ b/apps/sim/lib/knowledge/tags/secret-provenance-delete.test.ts @@ -4,6 +4,7 @@ import { document, embedding, knowledgeBase, knowledgeBaseTagDefinitions } from '@sim/db/schema' import { dbChainMockFns, queueTableRows, resetDbChainMock } from '@sim/testing' import { beforeEach, describe, expect, it, vi } from 'vitest' +import { asOrchestrationError } from '@/lib/core/orchestration/types' import { deleteAllTagDefinitions, deleteTagDefinition, @@ -60,6 +61,14 @@ describe('knowledge tag deletion provenance', () => { expect(dbChainMockFns.delete).not.toHaveBeenCalled() }) + it('classifies provenance rejection as a caller-actionable conflict', () => { + expect(asOrchestrationError(new KnowledgeTagProvenanceConflictError())).toMatchObject({ + code: 'conflict', + message: + 'Tag definitions cannot be deleted while resolved-secret document provenance is present', + }) + }) + it('clears every bounded tag slot through the same guarded mutation path', async () => { queueTableRows(knowledgeBase, [{ id: KNOWLEDGE_BASE_ID }]) queueTableRows(knowledgeBaseTagDefinitions, [ diff --git a/apps/sim/lib/knowledge/tags/service.ts b/apps/sim/lib/knowledge/tags/service.ts index 7fe061670ea..b7f559da7e9 100644 --- a/apps/sim/lib/knowledge/tags/service.ts +++ b/apps/sim/lib/knowledge/tags/service.ts @@ -9,6 +9,7 @@ import { import { createLogger } from '@sim/logger' import { generateId } from '@sim/utils/id' import { and, eq, isNotNull, isNull, or, sql } from 'drizzle-orm' +import { OrchestrationError } from '@/lib/core/orchestration/types' import type { DbOrTx, DbTransaction } from '@/lib/db/types' import { getSlotsForFieldType, SUPPORTED_FIELD_TYPES } from '@/lib/knowledge/constants' import type { BulkTagDefinitionsData, DocumentTagDefinition } from '@/lib/knowledge/tags/types' @@ -43,9 +44,12 @@ const TAG_MUTATION_STATEMENT_TIMEOUT_MS = 120_000 const TAG_MUTATION_LOCK_TIMEOUT_MS = 5_000 const TAG_MUTATION_IDLE_TIMEOUT_MS = 30_000 -export class KnowledgeTagProvenanceConflictError extends Error { +export class KnowledgeTagProvenanceConflictError extends OrchestrationError { constructor() { - super('Tag definitions cannot be deleted while resolved-secret document provenance is present') + super( + 'conflict', + 'Tag definitions cannot be deleted while resolved-secret document provenance is present' + ) this.name = 'KnowledgeTagProvenanceConflictError' } } diff --git a/apps/sim/lib/knowledge/upload-metadata.ts b/apps/sim/lib/knowledge/upload-metadata.ts new file mode 100644 index 00000000000..ec5761b925d --- /dev/null +++ b/apps/sim/lib/knowledge/upload-metadata.ts @@ -0,0 +1,28 @@ +import { z } from 'zod' + +const knowledgeDocumentUploadTagSchema = z + .string() + .max(1000, 'Knowledge document tag values cannot exceed 1000 characters') + .optional() + +/** Persisted metadata stored with a resumable Knowledge document upload session. */ +export const knowledgeDocumentUploadMetadataSchema = z + .object({ + tag1: knowledgeDocumentUploadTagSchema, + tag2: knowledgeDocumentUploadTagSchema, + tag3: knowledgeDocumentUploadTagSchema, + tag4: knowledgeDocumentUploadTagSchema, + tag5: knowledgeDocumentUploadTagSchema, + tag6: knowledgeDocumentUploadTagSchema, + tag7: knowledgeDocumentUploadTagSchema, + processingOptions: z + .object({ + recipe: z.string().max(255, 'recipe cannot exceed 255 characters').optional(), + lang: z.string().max(35, 'lang cannot exceed 35 characters').optional(), + }) + .strict() + .optional(), + }) + .strict() + +export type KnowledgeDocumentUploadMetadata = z.output diff --git a/apps/sim/app/api/auth/oauth/utils.ts b/apps/sim/lib/oauth/credential-service.ts similarity index 99% rename from apps/sim/app/api/auth/oauth/utils.ts rename to apps/sim/lib/oauth/credential-service.ts index b004e1d3148..e9b8bafcf85 100644 --- a/apps/sim/app/api/auth/oauth/utils.ts +++ b/apps/sim/lib/oauth/credential-service.ts @@ -20,13 +20,13 @@ import { parseTokenServiceAccountSecretBlob, type TokenServiceAccountSecretBlob, } from '@/lib/credentials/token-service-accounts/server' -import { refreshOAuthToken } from '@/lib/oauth' import { isInstagramProvider, shouldProactivelyRefreshInstagramToken } from '@/lib/oauth/instagram' import { getMicrosoftRefreshTokenExpiry, isMicrosoftProvider, PROACTIVE_REFRESH_THRESHOLD_DAYS, } from '@/lib/oauth/microsoft' +import { refreshOAuthToken } from '@/lib/oauth/oauth' import { extractSlackTeamId, fanOutSlackTokenChain, @@ -46,7 +46,7 @@ import { SLACK_CUSTOM_BOT_PROVIDER_ID, } from '@/lib/oauth/types' -const logger = createLogger('OAuthUtilsAPI') +const logger = createLogger('OAuthCredentialService') export class ServiceAccountTokenError extends Error { constructor( diff --git a/apps/sim/lib/uploads/client/session-upload.ts b/apps/sim/lib/uploads/client/session-upload.ts index 244703a8dbe..55d1a7466f6 100644 --- a/apps/sim/lib/uploads/client/session-upload.ts +++ b/apps/sim/lib/uploads/client/session-upload.ts @@ -4,6 +4,10 @@ import { completeKnowledgeDocumentUploadContract, createKnowledgeDocumentUploadContract, createKnowledgeDocumentUploadPartUrlsContract, + type KnowledgeDocumentUploadMetadataBody, + type KnowledgeDocumentUploadPartUrl, + type KnowledgeDocumentUploadSummary, + type KnowledgeDocumentUploadTransfer, } from '@/lib/api/contracts/knowledge/upload-sessions' import { abortInternalFileUploadContract, @@ -13,11 +17,6 @@ import { createInternalFileUploadPartUrlsContract, type InternalFileUploadSession, } from '@/lib/api/contracts/upload-sessions' -import type { - V2KnowledgeDocumentSummary, - V2KnowledgeDocumentUploadMetadata, -} from '@/lib/api/contracts/v2/knowledge' -import type { V2UploadPartUrl, V2UploadTransfer } from '@/lib/api/contracts/v2/uploads' import type { UploadProgressEvent } from '@/lib/uploads/client/types' import { uploadFileSession } from '@/lib/uploads/client/upload-session' import { getFileContentType } from '@/lib/uploads/utils/file-utils' @@ -55,7 +54,7 @@ type InternalUploadResult = NonNullable< export type UploadInternalFileSessionParams = InternalUploadCommonParams & InternalUploadContext -interface UploadKnowledgeDocumentSessionParams extends V2KnowledgeDocumentUploadMetadata { +interface UploadKnowledgeDocumentSessionParams extends KnowledgeDocumentUploadMetadataBody { workspaceId: string knowledgeBaseId: string file: File @@ -65,10 +64,10 @@ interface UploadKnowledgeDocumentSessionParams extends V2KnowledgeDocumentUpload interface RunCreatedUploadParams { file: File - transfer: V2UploadTransfer + transfer: KnowledgeDocumentUploadTransfer signal?: AbortSignal onProgress?: (event: UploadProgressEvent) => void - getPartUrls: (partNumbers: number[]) => Promise + getPartUrls: (partNumbers: number[]) => Promise complete: () => Promise abort: () => Promise } @@ -182,7 +181,7 @@ function internalUploadBody(params: UploadInternalFileSessionParams): CreateInte export async function uploadKnowledgeDocumentSession( params: UploadKnowledgeDocumentSessionParams -): Promise { +): Promise { const { workspaceId, knowledgeBaseId, file, signal, onProgress, ...metadata } = params const created = await requestJson(createKnowledgeDocumentUploadContract, { params: { id: knowledgeBaseId }, diff --git a/apps/sim/lib/webhooks/deploy.test.ts b/apps/sim/lib/webhooks/deploy.test.ts index 4aa780ceeac..eb81920d0c6 100644 --- a/apps/sim/lib/webhooks/deploy.test.ts +++ b/apps/sim/lib/webhooks/deploy.test.ts @@ -41,7 +41,7 @@ const { mockRefreshAccessTokenIfNeeded: vi.fn(), mockFetchSlackTeamId: vi.fn(), })) -vi.mock('@/app/api/auth/oauth/utils', () => ({ +vi.mock('@/lib/oauth/credential-service', () => ({ getSlackBotCredential: mockGetSlackBotCredential, resolveOAuthAccountId: mockResolveOAuthAccountId, refreshAccessTokenIfNeeded: mockRefreshAccessTokenIfNeeded, diff --git a/apps/sim/lib/webhooks/deploy.ts b/apps/sim/lib/webhooks/deploy.ts index 52e3e5244d2..7827315ec9e 100644 --- a/apps/sim/lib/webhooks/deploy.ts +++ b/apps/sim/lib/webhooks/deploy.ts @@ -6,6 +6,11 @@ import { generateShortId } from '@sim/utils/id' import { and, eq, inArray, isNull, or } from 'drizzle-orm' import type { NextRequest } from 'next/server' import { getProviderIdFromServiceId } from '@/lib/oauth' +import { + getSlackBotCredential, + refreshAccessTokenIfNeeded, + resolveOAuthAccountId, +} from '@/lib/oauth/credential-service' import { WebhookPathClaimConflictError } from '@/lib/webhooks/path-claims' import { PendingWebhookVerificationTracker } from '@/lib/webhooks/pending-verification' import { @@ -27,11 +32,6 @@ import { isCanonicalPair, resolveActiveCanonicalValue, } from '@/lib/workflows/subblocks/visibility' -import { - getSlackBotCredential, - refreshAccessTokenIfNeeded, - resolveOAuthAccountId, -} from '@/app/api/auth/oauth/utils' import type { SubBlockConfig } from '@/blocks/types' import type { BlockState } from '@/stores/workflows/workflow/types' import { getTrigger, isTriggerValid } from '@/triggers' diff --git a/apps/sim/lib/webhooks/polling/utils.test.ts b/apps/sim/lib/webhooks/polling/utils.test.ts index ace42a66e9d..04dd995442e 100644 --- a/apps/sim/lib/webhooks/polling/utils.test.ts +++ b/apps/sim/lib/webhooks/polling/utils.test.ts @@ -27,20 +27,20 @@ vi.mock('drizzle-orm', () => { or: vi.fn(), } }) -vi.mock('@/app/api/auth/oauth/utils', () => ({ +vi.mock('@/lib/oauth/credential-service', () => ({ getOAuthToken: vi.fn(), refreshAccessTokenIfNeeded: vi.fn(), resolveOAuthAccountId: vi.fn(), })) vi.mock('@/triggers/constants', () => ({ MAX_CONSECUTIVE_FAILURES: 5 })) -import type { WebhookRecord } from '@/lib/webhooks/polling/types' -import { resolveOAuthCredential, updateWebhookProviderConfig } from '@/lib/webhooks/polling/utils' import { getOAuthToken, refreshAccessTokenIfNeeded, resolveOAuthAccountId, -} from '@/app/api/auth/oauth/utils' +} from '@/lib/oauth/credential-service' +import type { WebhookRecord } from '@/lib/webhooks/polling/types' +import { resolveOAuthCredential, updateWebhookProviderConfig } from '@/lib/webhooks/polling/utils' afterAll(resetDbChainMock) diff --git a/apps/sim/lib/webhooks/polling/utils.ts b/apps/sim/lib/webhooks/polling/utils.ts index 8064d8ab928..df082862cf8 100644 --- a/apps/sim/lib/webhooks/polling/utils.ts +++ b/apps/sim/lib/webhooks/polling/utils.ts @@ -2,14 +2,14 @@ import { db } from '@sim/db' import { account, webhook, workflow, workflowDeploymentVersion } from '@sim/db/schema' import type { Logger } from '@sim/logger' import { and, eq, isNull, ne, or, sql } from 'drizzle-orm' -import { deliverableWebhookPredicate } from '@/lib/webhooks/delivery-predicate' -import type { WebhookRecord, WorkflowRecord } from '@/lib/webhooks/polling/types' import { getOAuthToken, refreshAccessTokenIfNeeded, resolveOAuthAccountId, resolveServiceAccountToken, -} from '@/app/api/auth/oauth/utils' +} from '@/lib/oauth/credential-service' +import { deliverableWebhookPredicate } from '@/lib/webhooks/delivery-predicate' +import type { WebhookRecord, WorkflowRecord } from '@/lib/webhooks/polling/types' import { MAX_CONSECUTIVE_FAILURES } from '@/triggers/constants' /** Concurrency limit for parallel webhook processing. Standardized across all providers. */ diff --git a/apps/sim/lib/webhooks/provider-subscription-utils.ts b/apps/sim/lib/webhooks/provider-subscription-utils.ts index e52e1eeefa1..b6f02f880ff 100644 --- a/apps/sim/lib/webhooks/provider-subscription-utils.ts +++ b/apps/sim/lib/webhooks/provider-subscription-utils.ts @@ -3,7 +3,7 @@ import { account } from '@sim/db/schema' import { createLogger } from '@sim/logger' import { eq } from 'drizzle-orm' import { getBaseUrl } from '@/lib/core/utils/urls' -import { resolveOAuthAccountId } from '@/app/api/auth/oauth/utils' +import { resolveOAuthAccountId } from '@/lib/oauth/credential-service' const logger = createLogger('WebhookProviderSubscriptions') diff --git a/apps/sim/lib/webhooks/providers/airtable.ts b/apps/sim/lib/webhooks/providers/airtable.ts index fd0463b4b95..99b77074cf6 100644 --- a/apps/sim/lib/webhooks/providers/airtable.ts +++ b/apps/sim/lib/webhooks/providers/airtable.ts @@ -4,6 +4,11 @@ import { createLogger } from '@sim/logger' import { eq } from 'drizzle-orm' import { validateAirtableId } from '@/lib/core/security/input-validation' import { getBaseUrl } from '@/lib/core/utils/urls' +import { + getOAuthToken, + refreshAccessTokenIfNeeded, + resolveOAuthAccountId, +} from '@/lib/oauth/credential-service' import { getCredentialOwner, getNotificationUrl, @@ -16,11 +21,6 @@ import type { SubscriptionResult, WebhookProviderHandler, } from '@/lib/webhooks/providers/types' -import { - getOAuthToken, - refreshAccessTokenIfNeeded, - resolveOAuthAccountId, -} from '@/app/api/auth/oauth/utils' const logger = createLogger('WebhookProvider:Airtable') diff --git a/apps/sim/lib/webhooks/providers/attio.ts b/apps/sim/lib/webhooks/providers/attio.ts index 1e607f786f0..693e7e1f5e6 100644 --- a/apps/sim/lib/webhooks/providers/attio.ts +++ b/apps/sim/lib/webhooks/providers/attio.ts @@ -4,6 +4,7 @@ import { hmacSha256Hex } from '@sim/security/hmac' import { toError } from '@sim/utils/errors' import { NextResponse } from 'next/server' import { getBaseUrl } from '@/lib/core/utils/urls' +import { refreshAccessTokenIfNeeded } from '@/lib/oauth/credential-service' import { getCredentialOwner, getProviderConfig } from '@/lib/webhooks/provider-subscription-utils' import type { AuthContext, @@ -15,7 +16,6 @@ import type { SubscriptionResult, WebhookProviderHandler, } from '@/lib/webhooks/providers/types' -import { refreshAccessTokenIfNeeded } from '@/app/api/auth/oauth/utils' const logger = createLogger('WebhookProvider:Attio') diff --git a/apps/sim/lib/webhooks/providers/clickup.test.ts b/apps/sim/lib/webhooks/providers/clickup.test.ts index 00247cd12e4..af6ab66ac90 100644 --- a/apps/sim/lib/webhooks/providers/clickup.test.ts +++ b/apps/sim/lib/webhooks/providers/clickup.test.ts @@ -17,7 +17,7 @@ vi.mock('@/lib/webhooks/provider-subscription-utils', () => ({ getCredentialOwner: mockGetCredentialOwner, })) -vi.mock('@/app/api/auth/oauth/utils', () => ({ +vi.mock('@/lib/oauth/credential-service', () => ({ refreshAccessTokenIfNeeded: mockRefreshAccessTokenIfNeeded, })) diff --git a/apps/sim/lib/webhooks/providers/clickup.ts b/apps/sim/lib/webhooks/providers/clickup.ts index 63fa5c420ac..2ca160112a6 100644 --- a/apps/sim/lib/webhooks/providers/clickup.ts +++ b/apps/sim/lib/webhooks/providers/clickup.ts @@ -3,6 +3,7 @@ import { safeCompare } from '@sim/security/compare' import { hmacSha256Hex } from '@sim/security/hmac' import { toError } from '@sim/utils/errors' import { NextResponse } from 'next/server' +import { refreshAccessTokenIfNeeded } from '@/lib/oauth/credential-service' import { getCredentialOwner, getNotificationUrl, @@ -18,7 +19,6 @@ import type { WebhookProviderHandler, } from '@/lib/webhooks/providers/types' import { createHmacVerifier } from '@/lib/webhooks/providers/utils' -import { refreshAccessTokenIfNeeded } from '@/app/api/auth/oauth/utils' import { CLICKUP_API_BASE_URL, clickupAuthorizationHeader } from '@/tools/clickup/shared' const logger = createLogger('WebhookProvider:ClickUp') diff --git a/apps/sim/lib/webhooks/providers/gmail.ts b/apps/sim/lib/webhooks/providers/gmail.ts index ca5e1094bb4..4abc2caa6fb 100644 --- a/apps/sim/lib/webhooks/providers/gmail.ts +++ b/apps/sim/lib/webhooks/providers/gmail.ts @@ -2,13 +2,13 @@ import { db } from '@sim/db' import { account, webhook } from '@sim/db/schema' import { createLogger } from '@sim/logger' import { eq } from 'drizzle-orm' +import { refreshAccessTokenIfNeeded, resolveOAuthAccountId } from '@/lib/oauth/credential-service' import type { FormatInputContext, FormatInputResult, PollingConfigContext, WebhookProviderHandler, } from '@/lib/webhooks/providers/types' -import { refreshAccessTokenIfNeeded, resolveOAuthAccountId } from '@/app/api/auth/oauth/utils' const logger = createLogger('WebhookProvider:Gmail') diff --git a/apps/sim/lib/webhooks/providers/microsoft-teams.test.ts b/apps/sim/lib/webhooks/providers/microsoft-teams.test.ts index d209d33695a..93bf5642662 100644 --- a/apps/sim/lib/webhooks/providers/microsoft-teams.test.ts +++ b/apps/sim/lib/webhooks/providers/microsoft-teams.test.ts @@ -7,7 +7,7 @@ import { NextRequest } from 'next/server' import { beforeEach, describe, expect, it, vi } from 'vitest' vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock) -vi.mock('@/app/api/auth/oauth/utils', () => authOAuthUtilsMock) +vi.mock('@/lib/oauth/credential-service', () => authOAuthUtilsMock) import { microsoftTeamsHandler } from '@/lib/webhooks/providers/microsoft-teams' diff --git a/apps/sim/lib/webhooks/providers/microsoft-teams.ts b/apps/sim/lib/webhooks/providers/microsoft-teams.ts index eabb0cb656c..ccaa56eb12c 100644 --- a/apps/sim/lib/webhooks/providers/microsoft-teams.ts +++ b/apps/sim/lib/webhooks/providers/microsoft-teams.ts @@ -14,6 +14,7 @@ import { validateUrlWithDNS, } from '@/lib/core/security/input-validation.server' import { sanitizeUrlForLog } from '@/lib/core/utils/logging' +import { refreshAccessTokenIfNeeded, resolveOAuthAccountId } from '@/lib/oauth/credential-service' import { getCredentialOwner, getNotificationUrl, @@ -29,7 +30,6 @@ import type { SubscriptionResult, WebhookProviderHandler, } from '@/lib/webhooks/providers/types' -import { refreshAccessTokenIfNeeded, resolveOAuthAccountId } from '@/app/api/auth/oauth/utils' const logger = createLogger('WebhookProvider:MicrosoftTeams') diff --git a/apps/sim/lib/webhooks/providers/monday.ts b/apps/sim/lib/webhooks/providers/monday.ts index 87c1f49994a..b0ad31267d2 100644 --- a/apps/sim/lib/webhooks/providers/monday.ts +++ b/apps/sim/lib/webhooks/providers/monday.ts @@ -2,6 +2,7 @@ import { createLogger } from '@sim/logger' import { toError } from '@sim/utils/errors' import { NextResponse } from 'next/server' import { validateMondayNumericId } from '@/lib/core/security/input-validation' +import { getOAuthToken, refreshAccessTokenIfNeeded } from '@/lib/oauth/credential-service' import { getCredentialOwner, getNotificationUrl, @@ -15,7 +16,6 @@ import type { SubscriptionResult, WebhookProviderHandler, } from '@/lib/webhooks/providers/types' -import { getOAuthToken, refreshAccessTokenIfNeeded } from '@/app/api/auth/oauth/utils' const logger = createLogger('WebhookProvider:Monday') diff --git a/apps/sim/lib/webhooks/providers/outlook.ts b/apps/sim/lib/webhooks/providers/outlook.ts index f9d6727fb98..a6f2fa3d80e 100644 --- a/apps/sim/lib/webhooks/providers/outlook.ts +++ b/apps/sim/lib/webhooks/providers/outlook.ts @@ -2,13 +2,13 @@ import { db } from '@sim/db' import { account, webhook } from '@sim/db/schema' import { createLogger } from '@sim/logger' import { eq } from 'drizzle-orm' +import { refreshAccessTokenIfNeeded, resolveOAuthAccountId } from '@/lib/oauth/credential-service' import type { FormatInputContext, FormatInputResult, PollingConfigContext, WebhookProviderHandler, } from '@/lib/webhooks/providers/types' -import { refreshAccessTokenIfNeeded, resolveOAuthAccountId } from '@/app/api/auth/oauth/utils' const logger = createLogger('WebhookProvider:Outlook') diff --git a/apps/sim/lib/webhooks/providers/slack.ts b/apps/sim/lib/webhooks/providers/slack.ts index fcd88767072..9830f7a5a9a 100644 --- a/apps/sim/lib/webhooks/providers/slack.ts +++ b/apps/sim/lib/webhooks/providers/slack.ts @@ -11,6 +11,11 @@ import { secureFetchWithPinnedIP, validateUrlWithDNS, } from '@/lib/core/security/input-validation.server' +import { + getSlackBotCredential, + refreshAccessTokenIfNeeded, + resolveOAuthAccountId, +} from '@/lib/oauth/credential-service' import type { AuthContext, EventFilterContext, @@ -18,11 +23,6 @@ import type { FormatInputResult, WebhookProviderHandler, } from '@/lib/webhooks/providers/types' -import { - getSlackBotCredential, - refreshAccessTokenIfNeeded, - resolveOAuthAccountId, -} from '@/app/api/auth/oauth/utils' import { type SlackEventFilter, slackEventSupportsFilter } from '@/triggers/slack/shared' const logger = createLogger('WebhookProvider:Slack') diff --git a/apps/sim/lib/webhooks/providers/webflow.ts b/apps/sim/lib/webhooks/providers/webflow.ts index 7494ae39568..25fabfc36d4 100644 --- a/apps/sim/lib/webhooks/providers/webflow.ts +++ b/apps/sim/lib/webhooks/providers/webflow.ts @@ -1,6 +1,7 @@ import { createLogger } from '@sim/logger' import { validateAlphanumericId } from '@/lib/core/security/input-validation' import { getBaseUrl } from '@/lib/core/utils/urls' +import { getOAuthToken, refreshAccessTokenIfNeeded } from '@/lib/oauth/credential-service' import { getCredentialOwner, getProviderConfig } from '@/lib/webhooks/provider-subscription-utils' import type { DeleteSubscriptionContext, @@ -11,7 +12,6 @@ import type { SubscriptionResult, WebhookProviderHandler, } from '@/lib/webhooks/providers/types' -import { getOAuthToken, refreshAccessTokenIfNeeded } from '@/app/api/auth/oauth/utils' const logger = createLogger('WebhookProvider:Webflow') diff --git a/apps/sim/lib/webhooks/providers/zoho-desk.test.ts b/apps/sim/lib/webhooks/providers/zoho-desk.test.ts index 3c516cfdbab..fee16ac6612 100644 --- a/apps/sim/lib/webhooks/providers/zoho-desk.test.ts +++ b/apps/sim/lib/webhooks/providers/zoho-desk.test.ts @@ -3,7 +3,7 @@ */ import { afterEach, describe, expect, it, vi } from 'vitest' -vi.mock('@/app/api/auth/oauth/utils', () => ({ +vi.mock('@/lib/oauth/credential-service', () => ({ refreshAccessTokenIfNeeded: vi.fn(), })) @@ -12,13 +12,13 @@ vi.mock('@/lib/webhooks/provider-subscription-utils', () => ({ getNotificationUrl: vi.fn(() => 'https://example.com/api/webhooks/trigger/path'), })) +import { refreshAccessTokenIfNeeded } from '@/lib/oauth/credential-service' import { matchesPendingWebhookVerificationProbe, requiresPendingWebhookVerification, } from '@/lib/webhooks/pending-verification' import { getCredentialOwner } from '@/lib/webhooks/provider-subscription-utils' import { mapZohoWebhookError, zohoDeskHandler } from '@/lib/webhooks/providers/zoho-desk' -import { refreshAccessTokenIfNeeded } from '@/app/api/auth/oauth/utils' function errorStatus(err: unknown): number | undefined { return (err as { status?: number })?.status diff --git a/apps/sim/lib/webhooks/providers/zoho-desk.ts b/apps/sim/lib/webhooks/providers/zoho-desk.ts index a70935d4385..719038fe31b 100644 --- a/apps/sim/lib/webhooks/providers/zoho-desk.ts +++ b/apps/sim/lib/webhooks/providers/zoho-desk.ts @@ -6,6 +6,7 @@ import { truncate } from '@sim/utils/string' import { eq } from 'drizzle-orm' import * as jose from 'jose' import { NextResponse } from 'next/server' +import { refreshAccessTokenIfNeeded } from '@/lib/oauth/credential-service' import { getCredentialOwner, getNotificationUrl } from '@/lib/webhooks/provider-subscription-utils' import type { AuthContext, @@ -16,7 +17,6 @@ import type { SubscriptionResult, WebhookProviderHandler, } from '@/lib/webhooks/providers/types' -import { refreshAccessTokenIfNeeded } from '@/app/api/auth/oauth/utils' import { isZohoHost } from '@/tools/zoho_desk/host-allowlist' import { withDerivedContentText } from '@/tools/zoho_desk/utils' diff --git a/apps/sim/lib/workspaces/application/workspace-context.test.ts b/apps/sim/lib/workspaces/application/workspace-context.test.ts index c448ef5f91b..db644e1fb84 100644 --- a/apps/sim/lib/workspaces/application/workspace-context.test.ts +++ b/apps/sim/lib/workspaces/application/workspace-context.test.ts @@ -3,7 +3,10 @@ */ import { dbChainMockFns, resetDbChainMock, schemaMock } from '@sim/testing' import { beforeEach, describe, expect, it } from 'vitest' -import { loadActiveWorkspaceApplicationContext } from '@/lib/workspaces/application/workspace-context' +import { + loadActiveWorkspaceApplicationContext, + loadWorkspaceApplicationContext, +} from '@/lib/workspaces/application/workspace-context' describe('loadActiveWorkspaceApplicationContext', () => { beforeEach(() => { @@ -35,6 +38,26 @@ describe('loadActiveWorkspaceApplicationContext', () => { await expect(loadActiveWorkspaceApplicationContext('workspace-1')).resolves.toBeNull() }) + it('can explicitly include archived workspaces', async () => { + dbChainMockFns.limit.mockResolvedValueOnce([ + { + id: 'workspace-1', + organizationId: null, + allowPersonalApiKeys: false, + billedAccountUserId: 'billing-owner-1', + }, + ]) + + await expect( + loadWorkspaceApplicationContext('workspace-1', { includeArchived: true }) + ).resolves.toEqual({ + workspaceId: 'workspace-1', + workspaceOrganizationId: null, + allowPersonalApiKeys: false, + billedAccountUserId: 'billing-owner-1', + }) + }) + it('propagates database failures', async () => { const failure = new Error('database unavailable') dbChainMockFns.limit.mockRejectedValueOnce(failure) diff --git a/apps/sim/lib/workspaces/application/workspace-context.ts b/apps/sim/lib/workspaces/application/workspace-context.ts index fb01e591ac0..af4194a992b 100644 --- a/apps/sim/lib/workspaces/application/workspace-context.ts +++ b/apps/sim/lib/workspaces/application/workspace-context.ts @@ -7,9 +7,10 @@ export interface ActiveWorkspaceApplicationContext extends WorkspaceAuthorizatio billedAccountUserId: string } -/** Loads the active canonical workspace state required by application authorization. */ -export async function loadActiveWorkspaceApplicationContext( - workspaceId: string +/** Loads canonical workspace state required by application authorization. */ +export async function loadWorkspaceApplicationContext( + workspaceId: string, + options: { includeArchived?: boolean } = {} ): Promise { const [row] = await db .select({ @@ -19,7 +20,12 @@ export async function loadActiveWorkspaceApplicationContext( billedAccountUserId: workspace.billedAccountUserId, }) .from(workspace) - .where(and(eq(workspace.id, workspaceId), isNull(workspace.archivedAt))) + .where( + and( + eq(workspace.id, workspaceId), + options.includeArchived ? undefined : isNull(workspace.archivedAt) + ) + ) .limit(1) if (!row) return null @@ -30,3 +36,10 @@ export async function loadActiveWorkspaceApplicationContext( billedAccountUserId: row.billedAccountUserId, } } + +/** Loads active canonical workspace state required by application authorization. */ +export async function loadActiveWorkspaceApplicationContext( + workspaceId: string +): Promise { + return loadWorkspaceApplicationContext(workspaceId) +} diff --git a/apps/sim/tools/index.test.ts b/apps/sim/tools/index.test.ts index a0e4f2d90cc..6972002f77d 100644 --- a/apps/sim/tools/index.test.ts +++ b/apps/sim/tools/index.test.ts @@ -48,6 +48,7 @@ const { mockListCustomTools, mockMarkWorkspaceFileSecretProvenanceUnknown, mockGetCustomToolByIdOrTitle, + mockGenerateInternalDelegationToken, mockGenerateInternalToken, mockResolveWorkspaceFileReference, } = vi.hoisted(() => ({ @@ -62,6 +63,7 @@ const { mockListCustomTools: vi.fn(), mockMarkWorkspaceFileSecretProvenanceUnknown: vi.fn(), mockGetCustomToolByIdOrTitle: vi.fn(), + mockGenerateInternalDelegationToken: vi.fn(), mockGenerateInternalToken: vi.fn(), mockResolveWorkspaceFileReference: vi.fn(), })) @@ -76,6 +78,8 @@ vi.mock('@/lib/api-key/byok', () => ({ })) vi.mock('@/lib/auth/internal', () => ({ + generateInternalDelegationToken: (...args: unknown[]) => + mockGenerateInternalDelegationToken(...args), generateInternalToken: (...args: unknown[]) => mockGenerateInternalToken(...args), })) @@ -209,6 +213,25 @@ const mockRegistryTools: Record = { result: { type: 'json', description: 'Execution result' }, }, }, + test_executor_delegation: { + id: 'test_executor_delegation', + name: 'Executor Delegation Test', + description: 'Exercises scoped internal executor authentication', + version: '1.0.0', + params: {}, + request: { + url: '/api/knowledge/test', + method: 'POST', + internalAuth: 'executor_delegation', + headers: () => ({ 'Content-Type': 'application/json' }), + body: () => ({}), + }, + transformResponse: async (response: Response) => ({ + success: response.ok, + output: await response.json(), + }), + outputs: {}, + }, gmail_read: { id: 'gmail_read', name: 'Gmail Read', @@ -423,6 +446,7 @@ vi.spyOn(getQueryClientModule, 'getQueryClient').mockImplementation(createMockQu beforeEach(() => { vi.spyOn(getQueryClientModule, 'getQueryClient').mockImplementation(createMockQueryClient) + mockGenerateInternalDelegationToken.mockResolvedValue('executor-token') // Suites below call vi.resetAllMocks(), which wipes the shared env/urls mock // implementations — restore their defaults and re-pin the base URL each test. resetEnvMock() @@ -707,6 +731,58 @@ describe('executeTool Function', () => { expect(new Headers(request?.headers).get('authorization')).toBe('Bearer mothership-token') }) + it('uses server-authored executor identity for protected internal tools', async () => { + global.fetch = Object.assign( + vi.fn().mockResolvedValue( + new Response(JSON.stringify({ success: true }), { + status: 200, + headers: { 'content-type': 'application/json' }, + }) + ), + { preconnect: vi.fn() } + ) as typeof fetch + + const executionContext = createToolExecutionContext({ + userId: 'trusted-user', + workflowId: 'trusted-workflow', + executionId: 'trusted-execution', + }) + await executeTool( + 'test_executor_delegation', + { + _context: { + userId: 'model-user', + workflowId: 'model-workflow', + }, + }, + { executionContext } + ) + + expect(mockGenerateInternalDelegationToken).toHaveBeenCalledWith({ + subjectUserId: 'trusted-user', + workflowId: 'trusted-workflow', + executionId: 'trusted-execution', + }) + const request = vi.mocked(global.fetch).mock.calls[0]?.[1] + expect(new Headers(request?.headers).get('authorization')).toBe('Bearer executor-token') + }) + + it('rejects protected internal tools without trusted executor scope before transport', async () => { + const result = await executeTool('test_executor_delegation', { + _context: { + userId: 'model-user', + workflowId: 'model-workflow', + }, + }) + + expect(result).toMatchObject({ + success: false, + error: 'Executor delegation requires a trusted workflow execution context', + }) + expect(mockGenerateInternalDelegationToken).not.toHaveBeenCalled() + expect(global.fetch).not.toHaveBeenCalled() + }) + it('imports File Get Content provenance without exposing private transport metadata', async () => { const registry = new ResolvedSecretTraceRegistry([], { userId: 'user-1', diff --git a/apps/sim/tools/index.ts b/apps/sim/tools/index.ts index 3575ccad894..dbdf118f084 100644 --- a/apps/sim/tools/index.ts +++ b/apps/sim/tools/index.ts @@ -5,6 +5,7 @@ import { isPlainRecord } from '@sim/utils/object' import { backoffWithJitter, parseRetryAfter } from '@sim/utils/retry' import { getBYOKKey } from '@/lib/api-key/byok' import { + type GenerateInternalDelegationTokenInput, generateInternalToken, type InternalSandboxProfile, type InternalTokenClaims, @@ -62,6 +63,7 @@ import { assertPermissionsAllowed } from '@/ee/access-control/utils/permission-c import { isCustomTool, isMcpTool } from '@/executor/constants' import { resolveSkillContent } from '@/executor/handlers/agent/skills-resolver' import type { ExecutionContext, UserFile } from '@/executor/types' +import { buildExecutorDelegationHeaders } from '@/executor/utils/http' import { resolveEnvVarReferences } from '@/executor/utils/reference-validation' import { projectResolvedSecretDiagnosticContent, @@ -179,6 +181,28 @@ function resolveToolScope( } } +function resolveInternalExecutorDelegation( + tool: ToolConfig, + executionContext: ExecutionContext | undefined, + supplied: GenerateInternalDelegationTokenInput | undefined +): GenerateInternalDelegationTokenInput | undefined { + if (tool.request.internalAuth !== 'executor_delegation') return undefined + if (supplied) { + if (!supplied.subjectUserId || !supplied.workflowId) { + throw new Error('Executor delegation requires an authenticated user and workflow') + } + return supplied + } + if (!executionContext?.userId || !executionContext.workflowId) { + throw new Error('Executor delegation requires a trusted workflow execution context') + } + return { + subjectUserId: executionContext.userId, + workflowId: executionContext.workflowId, + ...(executionContext.executionId ? { executionId: executionContext.executionId } : {}), + } +} + function toUserFileFromWorkspaceRecord(record: { id: string name: string @@ -1147,6 +1171,8 @@ export interface ExecuteToolOptions { resolvedSecretTraceRegistry?: ResolvedSecretTraceRegistry /** Trusted base image claim for an internal Function execution. */ internalSandboxProfile?: InternalSandboxProfile + /** Trusted executor identity supplied by a server adapter without entering model parameters. */ + internalExecutorDelegation?: GenerateInternalDelegationTokenInput } interface PrivateToolResponseMetadataResult { @@ -1518,6 +1544,7 @@ async function executeToolImplementation( signal, resolvedSecretTraceRegistry: explicitResolvedSecretTraceRegistry, internalSandboxProfile, + internalExecutorDelegation: suppliedInternalExecutorDelegation, } = options const resolvedSecretTraceRegistry = explicitResolvedSecretTraceRegistry ?? executionContext?.resolvedSecretTraceRegistry @@ -1649,6 +1676,12 @@ async function executeToolImplementation( throw new Error(`Tool not found: ${toolId}`) } + const internalExecutorDelegation = resolveInternalExecutorDelegation( + tool, + executionContext, + suppliedInternalExecutorDelegation + ) + await normalizeCopilotFileParams(tool, contextParams, scope) normalizeCopilotCredentialParams(contextParams) enforceCopilotCredentialSelection(toolId, tool, contextParams, scope) @@ -1938,7 +1971,8 @@ async function executeToolImplementation( privateToolMetadataType, privateToolMetadataPolicy?.incomplete, resolvedSecretTraceRegistry, - internalSandboxProfile + internalSandboxProfile, + internalExecutorDelegation ), { requestId, @@ -1966,7 +2000,8 @@ async function executeToolImplementation( privateToolMetadataType, privateToolMetadataPolicy?.incomplete, resolvedSecretTraceRegistry, - internalSandboxProfile + internalSandboxProfile, + internalExecutorDelegation ) }, } @@ -1979,7 +2014,8 @@ async function executeToolImplementation( privateToolMetadataType, privateToolMetadataPolicy?.incomplete, resolvedSecretTraceRegistry, - internalSandboxProfile + internalSandboxProfile, + internalExecutorDelegation ) // Apply post-processing if available and not skipped @@ -2213,14 +2249,19 @@ async function addInternalAuthIfNeeded( requestId: string, context: string, userId?: string, - claims?: InternalTokenClaims + claims?: InternalTokenClaims, + executorDelegation?: GenerateInternalDelegationTokenInput ): Promise { if (typeof window === 'undefined') { if (isInternalRoute) { try { - const internalToken = claims - ? await generateInternalToken(userId, claims) - : await generateInternalToken(userId) + const internalToken = executorDelegation + ? (await buildExecutorDelegationHeaders(executorDelegation)).Authorization.slice( + 'Bearer '.length + ) + : claims + ? await generateInternalToken(userId, claims) + : await generateInternalToken(userId) if (headers instanceof Headers) { headers.set('Authorization', `Bearer ${internalToken}`) } else { @@ -2229,6 +2270,7 @@ async function addInternalAuthIfNeeded( logger.info(`[${requestId}] Added internal auth token for ${context}`) } catch (error) { logger.error(`[${requestId}] Failed to generate internal token for ${context}:`, error) + if (executorDelegation) throw error } } else { logger.info(`[${requestId}] Skipping internal auth token for external URL: ${context}`) @@ -2306,7 +2348,8 @@ async function executeToolRequest( privateToolMetadataType?: PrivateToolMetadataType, privateToolMetadataIncomplete: 'reject' | 'propagate' = 'reject', resolvedSecretTraceRegistry?: ResolvedSecretTraceRegistry, - internalSandboxProfile?: InternalSandboxProfile + internalSandboxProfile?: InternalSandboxProfile, + internalExecutorDelegation?: GenerateInternalDelegationTokenInput ): Promise { const requestId = generateRequestId() const structuralOnlyToolLogs = @@ -2368,7 +2411,8 @@ async function executeToolRequest( requestId, toolId, params._context?.userId, - internalSandboxProfile ? { sandboxProfile: internalSandboxProfile } : undefined + internalSandboxProfile ? { sandboxProfile: internalSandboxProfile } : undefined, + internalExecutorDelegation ) if (isInternalRoute && params._context?.billingAttribution) { headers.set( diff --git a/apps/sim/tools/knowledge/create_document.ts b/apps/sim/tools/knowledge/create_document.ts index 002cfe3e95a..2046f738f43 100644 --- a/apps/sim/tools/knowledge/create_document.ts +++ b/apps/sim/tools/knowledge/create_document.ts @@ -48,6 +48,7 @@ export const knowledgeCreateDocumentTool: ToolConfig `/api/knowledge/${params.knowledgeBaseId}/documents`, method: 'POST', secretProvenance: { diff --git a/apps/sim/tools/knowledge/delete_chunk.ts b/apps/sim/tools/knowledge/delete_chunk.ts index 3bc759af63f..1ddcef291e1 100644 --- a/apps/sim/tools/knowledge/delete_chunk.ts +++ b/apps/sim/tools/knowledge/delete_chunk.ts @@ -29,6 +29,7 @@ export const knowledgeDeleteChunkTool: ToolConfig `/api/knowledge/${params.knowledgeBaseId}/documents/${params.documentId}/chunks/${params.chunkId}`, method: 'DELETE', diff --git a/apps/sim/tools/knowledge/delete_document.ts b/apps/sim/tools/knowledge/delete_document.ts index 39493e38283..e48891f1892 100644 --- a/apps/sim/tools/knowledge/delete_document.ts +++ b/apps/sim/tools/knowledge/delete_document.ts @@ -23,6 +23,7 @@ export const knowledgeDeleteDocumentTool: ToolConfig `/api/knowledge/${params.knowledgeBaseId}/documents/${params.documentId}`, method: 'DELETE', headers: () => ({ diff --git a/apps/sim/tools/knowledge/get_connector.ts b/apps/sim/tools/knowledge/get_connector.ts index 9ae1e03e548..bfe919128cb 100644 --- a/apps/sim/tools/knowledge/get_connector.ts +++ b/apps/sim/tools/knowledge/get_connector.ts @@ -24,6 +24,7 @@ export const knowledgeGetConnectorTool: ToolConfig `/api/knowledge/${params.knowledgeBaseId}/connectors/${params.connectorId}`, method: 'GET', headers: () => ({ diff --git a/apps/sim/tools/knowledge/get_document.ts b/apps/sim/tools/knowledge/get_document.ts index 2ac840ca032..2ddc3bf3efe 100644 --- a/apps/sim/tools/knowledge/get_document.ts +++ b/apps/sim/tools/knowledge/get_document.ts @@ -24,6 +24,7 @@ export const knowledgeGetDocumentTool: ToolConfig `/api/knowledge/${params.knowledgeBaseId}/documents/${params.documentId}`, method: 'GET', secretProvenance: { response: { incomplete: 'reject' } }, diff --git a/apps/sim/tools/knowledge/list_chunks.ts b/apps/sim/tools/knowledge/list_chunks.ts index 7198b63fa6b..2b0b019a75d 100644 --- a/apps/sim/tools/knowledge/list_chunks.ts +++ b/apps/sim/tools/knowledge/list_chunks.ts @@ -48,6 +48,7 @@ export const knowledgeListChunksTool: ToolConfig { const queryParams = new URLSearchParams() if (params.search) queryParams.set('search', params.search) diff --git a/apps/sim/tools/knowledge/list_connectors.ts b/apps/sim/tools/knowledge/list_connectors.ts index 5acf7081e4a..155ce61f416 100644 --- a/apps/sim/tools/knowledge/list_connectors.ts +++ b/apps/sim/tools/knowledge/list_connectors.ts @@ -18,6 +18,7 @@ export const knowledgeListConnectorsTool: ToolConfig `/api/knowledge/${params.knowledgeBaseId}/connectors`, method: 'GET', headers: () => ({ diff --git a/apps/sim/tools/knowledge/list_documents.ts b/apps/sim/tools/knowledge/list_documents.ts index 6bf491a6521..cf300122b82 100644 --- a/apps/sim/tools/knowledge/list_documents.ts +++ b/apps/sim/tools/knowledge/list_documents.ts @@ -41,6 +41,7 @@ export const knowledgeListDocumentsTool: ToolConfig { const queryParams = new URLSearchParams() if (params.search) queryParams.set('search', params.search) diff --git a/apps/sim/tools/knowledge/list_tags.ts b/apps/sim/tools/knowledge/list_tags.ts index fbe95a6a2bc..7a5bb95ab21 100644 --- a/apps/sim/tools/knowledge/list_tags.ts +++ b/apps/sim/tools/knowledge/list_tags.ts @@ -17,6 +17,7 @@ export const knowledgeListTagsTool: ToolConfig = }, request: { + internalAuth: 'executor_delegation', url: (params) => `/api/knowledge/${params.knowledgeBaseId}/tag-definitions`, method: 'GET', headers: () => ({ diff --git a/apps/sim/tools/knowledge/search.ts b/apps/sim/tools/knowledge/search.ts index ebc8aed6944..f5ea1022622 100644 --- a/apps/sim/tools/knowledge/search.ts +++ b/apps/sim/tools/knowledge/search.ts @@ -85,6 +85,7 @@ export const knowledgeSearchTool: ToolConfig = { }, request: { + internalAuth: 'executor_delegation', url: () => '/api/knowledge/search', method: 'POST', modelInput: { diff --git a/apps/sim/tools/knowledge/trigger_sync.ts b/apps/sim/tools/knowledge/trigger_sync.ts index 127c37a0c1a..5d584e980ae 100644 --- a/apps/sim/tools/knowledge/trigger_sync.ts +++ b/apps/sim/tools/knowledge/trigger_sync.ts @@ -23,6 +23,7 @@ export const knowledgeTriggerSyncTool: ToolConfig `/api/knowledge/${params.knowledgeBaseId}/connectors/${params.connectorId}/sync`, method: 'POST', diff --git a/apps/sim/tools/knowledge/update_chunk.ts b/apps/sim/tools/knowledge/update_chunk.ts index e0de0164596..d4b5345464c 100644 --- a/apps/sim/tools/knowledge/update_chunk.ts +++ b/apps/sim/tools/knowledge/update_chunk.ts @@ -41,6 +41,7 @@ export const knowledgeUpdateChunkTool: ToolConfig `/api/knowledge/${params.knowledgeBaseId}/documents/${params.documentId}/chunks/${params.chunkId}`, method: 'PUT', diff --git a/apps/sim/tools/knowledge/upload_chunk.ts b/apps/sim/tools/knowledge/upload_chunk.ts index 85cf9ba79aa..9887778f78e 100644 --- a/apps/sim/tools/knowledge/upload_chunk.ts +++ b/apps/sim/tools/knowledge/upload_chunk.ts @@ -29,6 +29,7 @@ export const knowledgeUploadChunkTool: ToolConfig `/api/knowledge/${params.knowledgeBaseId}/documents/${params.documentId}/chunks`, method: 'POST', diff --git a/apps/sim/tools/knowledge/upsert_document.ts b/apps/sim/tools/knowledge/upsert_document.ts index 2cfec98070e..1c7f8edd8da 100644 --- a/apps/sim/tools/knowledge/upsert_document.ts +++ b/apps/sim/tools/knowledge/upsert_document.ts @@ -60,6 +60,7 @@ export const knowledgeUpsertDocumentTool: ToolConfig< }, request: { + internalAuth: 'executor_delegation', url: (params) => `/api/knowledge/${params.knowledgeBaseId}/documents/upsert`, method: 'POST', secretProvenance: { diff --git a/apps/sim/tools/types.ts b/apps/sim/tools/types.ts index fc2e25436c7..bffcb82c6f0 100644 --- a/apps/sim/tools/types.ts +++ b/apps/sim/tools/types.ts @@ -177,6 +177,8 @@ export interface ToolConfig

{ method: HttpMethod | ((params: P) => HttpMethod) headers: (params: P) => Record body?: (params: P) => Record | string | FormData | undefined + /** Selects the signed, workflow-scoped identity required by protected internal routes. */ + internalAuth?: 'executor_delegation' /** Defines the exact request fields that may become model-visible. */ modelInput?: | {