improvement(search): speed up organization integration and source queries (#7830)

Author: waleedlatif1

apps/sim/app/api/files/authorization.test.ts

@@ -436,21 +436,28 @@ describe('KB file live source authorization', () => {
     async (allowed) => {
       const scope = { kind: 'user' as const, userId: USER_ID, tokens: ['reader-token'] }
       const getForConnectors = vi.fn().mockResolvedValue(scope)
+      const liveSources = { type: 'live-sources' }
       const access: KnowledgeAccessProvider = {
         get: async () => scope,
         getForConnectors,
         getForDocuments: async () => scope,
+        liveSourceConnectorCondition: async () => liveSources as never,
       }
       queueTableRows(schemaMock.document, [])
-      queueTableRows(schemaMock.document, [{ connectorId: 'confluence-source' }])
+      queueTableRows(schemaMock.knowledgeConnector, [{ connectorId: 'confluence-source' }])
       queueTableRows(schemaMock.document, allowed ? [{ id: 'doc-1' }] : [])
       await expect(
         verifyFileAccess(CLOUD_KEY, USER_ID, undefined, 'knowledge-base', false, {
           knowledgeAccess: access,
         })
       ).resolves.toBe(allowed)
       expect(getForConnectors).toHaveBeenCalledExactlyOnceWith(['confluence-source'], undefined)
+      const discovery = dbChainMockFns.where.mock.calls.filter(([condition]) =>
+        hasMockCondition(condition, (node) => node === liveSources)
+      )
+      expect(discovery).toHaveLength(1)
       for (const [condition] of dbChainMockFns.where.mock.calls) {
+        if (discovery.some(([live]) => live === condition)) continue
         expect(
           hasMockCondition(
             condition,

apps/sim/app/api/knowledge/sim-search/sources/route.test.ts

@@ -59,7 +59,7 @@ const source = {
   isSyncing: false,
   lastSyncAt: null,
   hasSyncError: false,
-  viewerDocumentCount: 0,
+  hasViewerDocuments: false,
   viewerFailedDocumentCount: 0,
   viewerEmailVerified: true,
   viewerAccounts: [],

apps/sim/app/o/[organizationId]/integrations/integrations.test.tsx

@@ -149,7 +149,7 @@ const memberSource: SearchSourceSummary = {
   isSyncing: false,
   lastSyncAt: null,
   hasSyncError: false,
-  viewerDocumentCount: 0,
+  hasViewerDocuments: false,
   viewerFailedDocumentCount: 0,
   viewerEmailVerified: true,
   viewerAccounts: [],

apps/sim/app/o/[organizationId]/settings/integrations/providers/[connectorType]/provider-detail.test.tsx

@@ -136,7 +136,7 @@ const source = {
   lastSyncAt: '2026-09-08T12:00:00.000Z',
   connectionRequired: true,
   viewerMembership: 'needs_reauth',
-  viewerDocumentCount: 0,
+  hasViewerDocuments: false,
 }
 const credentialGroup = {
   id: 'accounts-one',
@@ -429,7 +429,6 @@ describe('organization provider management', () => {
     )
     expect(container.textContent).not.toContain('Connect account')
     expect(container.textContent).not.toContain('Reconnect')
-    expect(container.textContent).not.toContain('0 searchable documents')
   })
 
   it('loads sources for a nonpersonal provider even when an accounts view URL is supplied', async () => {

apps/sim/hooks/queries/kb/search-source-progress.test.tsx

@@ -23,7 +23,7 @@ function Probe() {
   const sync = useTriggerSync()
   return (
     <div>
-      <span>{result.data?.[0]?.viewerDocumentCount ?? 0}</span>
+      <span>{String(result.data?.[0]?.hasViewerDocuments ?? false)}</span>
       <button
         disabled={sync.isPending}
         onClick={() => sync.mutate({ knowledgeBaseId: 'kb', connectorId: 'source' })}
@@ -83,7 +83,7 @@ beforeEach(() => {
             isSyncing: syncing,
             hasSyncError: false,
             viewerFailedDocumentCount: 0,
-            viewerDocumentCount: syncing ? 0 : 1,
+            hasViewerDocuments: !syncing,
           },
         ],
         nextCursor: null,
@@ -127,7 +127,7 @@ describe('source progress polling', () => {
     await advance(1)
     expect(
       client.getQueryData(searchSourceKeys.pages('workspace', { search: '', mine: false }))
-    ).toMatchObject({ pages: [{ sources: [{ isSyncing: false, viewerDocumentCount: 1 }] }] })
+    ).toMatchObject({ pages: [{ sources: [{ isSyncing: false, hasViewerDocuments: true }] }] })
     const total = mocks.requestJson.mock.calls.length
     await advance(60_000)
     expect(mocks.requestJson).toHaveBeenCalledTimes(total)
@@ -157,7 +157,9 @@ describe('source progress polling', () => {
     expect(
       client.getQueryData(searchSourceKeys.pages('workspace', { search: '', mine: false }))
     ).toMatchObject({
-      pages: [{ sources: [expect.objectContaining({ isSyncing: false, viewerDocumentCount: 1 })] }],
+      pages: [
+        { sources: [expect.objectContaining({ isSyncing: false, hasViewerDocuments: true })] },
+      ],
     })
     const total = mocks.requestJson.mock.calls.length
     await advance(60_000)
@@ -173,7 +175,9 @@ describe('source progress polling', () => {
     expect(
       client.getQueryData(searchSourceKeys.pages('workspace', { search: '', mine: false }))
     ).toMatchObject({
-      pages: [{ sources: [expect.objectContaining({ isSyncing: false, viewerDocumentCount: 1 })] }],
+      pages: [
+        { sources: [expect.objectContaining({ isSyncing: false, hasViewerDocuments: true })] },
+      ],
     })
   })
   it('lets a slow summary refresh finish while progress keeps polling', async () => {
@@ -201,7 +205,9 @@ describe('source progress polling', () => {
     expect(
       client.getQueryData(searchSourceKeys.pages('workspace', { search: '', mine: false }))
     ).toMatchObject({
-      pages: [{ sources: [expect.objectContaining({ isSyncing: false, viewerDocumentCount: 1 })] }],
+      pages: [
+        { sources: [expect.objectContaining({ isSyncing: false, hasViewerDocuments: true })] },
+      ],
     })
     const total = mocks.requestJson.mock.calls.length
     await advance(60_000)

apps/sim/lib/api/contracts/knowledge/connectors.ts

@@ -359,7 +359,11 @@ const searchSourceSummaryFields = {
   isSyncing: z.boolean(),
   lastSyncAt: z.string().datetime().nullable(),
   hasSyncError: z.boolean(),
-  viewerDocumentCount: z.number().int().nonnegative(),
+  /**
+   * Whether the viewer can search at least one indexed document from this source. An
+   * existence flag rather than a count: counting means access-checking every visible document.
+   */
+  hasViewerDocuments: z.boolean(),
   viewerFailedDocumentCount: z.number().int().nonnegative().default(0),
   viewerEmailVerified: z.boolean(),
   viewerAccounts: z

apps/sim/lib/api/contracts/knowledge/personal-integrations.ts

@@ -23,7 +23,6 @@ export const personalSearchIntegrationSchema = z.object({
     .max(100),
   connectionStatus: z.enum(['connected', 'reconnect_needed', 'not_connected', 'unavailable']),
   indexingStatus: z.enum(['indexing', 'indexed', 'not_indexed', 'sync_failed', 'paused']),
-  searchableDocuments: z.number().int().nonnegative(),
   action: searchConnectionTargetSchema.nullable(),
 })
 

apps/sim/lib/copilot/application/load-search-integrations.test.ts

@@ -84,7 +84,6 @@ describe('loadCopilotSearchIntegrations', () => {
       ],
       connectionStatus: 'reconnect_needed',
       indexingStatus: 'indexed',
-      searchableDocuments: 7,
       action: null,
     }
     listIntegrations

apps/sim/lib/knowledge/__integration__/github-member.integration.ts

@@ -886,7 +886,7 @@ describe('fixture-backed GitHub member search in PostgreSQL', () => {
         {
           connectorId,
           viewerMembership: 'connected',
-          viewerDocumentCount: userId === ids.aliceId ? 1 : 0,
+          hasViewerDocuments: userId === ids.aliceId,
         },
       ])
     }
@@ -919,7 +919,7 @@ describe('fixture-backed GitHub member search in PostgreSQL', () => {
         })
       ).sources[0]
     expect(await summary(ids.aliceId)).toMatchObject({
-      viewerDocumentCount: 1,
+      hasViewerDocuments: true,
       viewerFailedDocumentCount: 0,
       hasSyncError: false,
     })
@@ -977,7 +977,7 @@ describe('fixture-backed GitHub member search in PostgreSQL', () => {
         )
       }
       expect(await summary(userId)).toMatchObject({
-        viewerDocumentCount: userId === ids.aliceId ? 1 : 0,
+        hasViewerDocuments: userId === ids.aliceId,
         viewerFailedDocumentCount: userId === ids.aliceId ? 1 : 0,
       })
     }
@@ -1327,13 +1327,13 @@ describe('fixture-backed GitHub member search in PostgreSQL', () => {
       expect(
         summaries.sources.map((source) => ({
           connectorId: source.connectorId,
-          viewerDocumentCount: source.viewerDocumentCount,
+          hasViewerDocuments: source.hasViewerDocuments,
         }))
       ).toEqual(
         expect.arrayContaining([
-          { connectorId: enrolled.connectorId, viewerDocumentCount: 1 },
-          { connectorId: privateId, viewerDocumentCount: userId === ids.aliceId ? 1 : 0 },
-          { connectorId: blockedId, viewerDocumentCount: 0 },
+          { connectorId: enrolled.connectorId, hasViewerDocuments: true },
+          { connectorId: privateId, hasViewerDocuments: userId === ids.aliceId },
+          { connectorId: blockedId, hasViewerDocuments: false },
         ])
       )
     }

apps/sim/lib/knowledge/__integration__/organization-search-overview.integration.ts

@@ -206,7 +206,7 @@ describe('organization operational overview with real SQL', () => {
       input: { ...input, connectorType: 'google_drive' },
     })
     expect(visible.sources).toHaveLength(2)
-    expect(visible.sources.every((source) => source.viewerDocumentCount === 0)).toBe(true)
+    expect(visible.sources.every((source) => !source.hasViewerDocuments)).toBe(true)
   })
   it('keeps explicit approvals and deactivations visible before source creation', async () => {
     await db.insert(organizationSearchIntegration).values([