Fix auto model

Author: Sg312

apps/sim/executor/handlers/evaluator/evaluator-handler.test.ts

@@ -3,6 +3,10 @@ import '@sim/testing/mocks/executor'
 import { authOAuthUtilsMock, authOAuthUtilsMockFns } from '@sim/testing'
 import { beforeEach, describe, expect, it, type Mock, vi } from 'vitest'
 
+const { mockResolveAutoModel } = vi.hoisted(() => ({
+  mockResolveAutoModel: vi.fn(),
+}))
+
 vi.mock('@/app/api/auth/oauth/utils', () => authOAuthUtilsMock)
 
 vi.mock('@/lib/credentials/access', () => ({
@@ -20,6 +24,13 @@ vi.mock('@/lib/credentials/access', () => ({
   }),
 }))
 
+vi.mock('@/lib/model-router/resolve', () => ({
+  addAutoRoutingCost: (cost: Record<string, number>, routingCost: number) =>
+    routingCost > 0 ? { ...cost, routing: routingCost, total: cost.total + routingCost } : cost,
+  resolveAutoModel: mockResolveAutoModel,
+  SIM_AUTO_SYSTEM_PREAMBLE: 'Sim auto system preamble',
+}))
+
 import { BlockType } from '@/executor/constants'
 import { EvaluatorBlockHandler } from '@/executor/handlers/evaluator/evaluator-handler'
 import type { ExecutionContext } from '@/executor/types'
@@ -82,6 +93,12 @@ describe('EvaluatorBlockHandler', () => {
       refreshed: false,
     })
     mockGetProviderFromModel.mockReturnValue('openai')
+    mockResolveAutoModel.mockResolvedValue({
+      model: 'fireworks/glm-5.2',
+      tier: '2',
+      decidedBy: 'llm',
+      billableRoutingCost: 0.002,
+    })
 
     // Set up fetch mock to return a successful response
     mockFetch.mockImplementation(() => {
@@ -101,16 +118,27 @@ describe('EvaluatorBlockHandler', () => {
 
   it('should handle evaluator blocks', () => {
     expect(handler.canHandle(mockBlock)).toBe(true)
-    const nonEvalBlock: SerializedBlock = { ...mockBlock, metadata: { id: 'other' } }
+    const nonEvalBlock: SerializedBlock = {
+      ...mockBlock,
+      metadata: { id: 'other' },
+    }
     expect(handler.canHandle(nonEvalBlock)).toBe(false)
   })
 
   it('should execute evaluator block correctly with basic inputs', async () => {
     const inputs = {
       content: 'This is the content to evaluate.',
       metrics: [
-        { name: 'score1', description: 'First score', range: { min: 0, max: 10 } },
-        { name: 'score2', description: 'Second score', range: { min: 0, max: 10 } },
+        {
+          name: 'score1',
+          description: 'First score',
+          range: { min: 0, max: 10 },
+        },
+        {
+          name: 'score2',
+          description: 'Second score',
+          range: { min: 0, max: 10 },
+        },
       ],
       model: 'gpt-4o',
       apiKey: 'test-api-key',
@@ -163,6 +191,64 @@ describe('EvaluatorBlockHandler', () => {
     })
   })
 
+  it('resolves sim-auto before executing evaluator and preserves its public identity', async () => {
+    const inputs = {
+      content: 'A clear and accurate answer.',
+      metrics: [
+        {
+          name: 'quality',
+          description: 'Overall answer quality',
+          range: { min: 1, max: 5 },
+        },
+      ],
+      model: 'sim-auto',
+    }
+
+    mockFetch.mockResolvedValueOnce({
+      ok: true,
+      json: () =>
+        Promise.resolve({
+          content: JSON.stringify({ quality: 5 }),
+          model: 'fireworks/glm-5.2',
+          tokens: { input: 80, output: 10, total: 90 },
+          cost: { input: 0.001, output: 0.0005, total: 0.0015 },
+        }),
+    })
+
+    const result = await handler.execute(mockContext, mockBlock, inputs)
+
+    expect(mockResolveAutoModel).toHaveBeenCalledWith({
+      ctx: mockContext,
+      blockId: mockBlock.id,
+      signals: expect.objectContaining({
+        lastMessage: inputs.content,
+        messageCount: 1,
+        toolNames: [],
+        mediaKind: 'none',
+        hasResponseFormat: true,
+      }),
+      fallbackModel: 'claude-sonnet-5',
+    })
+    expect(mockGetProviderFromModel).toHaveBeenCalledWith('fireworks/glm-5.2')
+
+    const requestBody = JSON.parse(mockFetch.mock.calls[0][1].body)
+    expect(requestBody).toMatchObject({
+      provider: 'openai',
+      model: 'fireworks/glm-5.2',
+      systemPrompt: expect.stringMatching(/^Sim auto system preamble\n\n/),
+    })
+    expect(result).toMatchObject({
+      model: 'sim-auto',
+      quality: 5,
+      cost: {
+        input: 0.001,
+        output: 0.0005,
+        routing: 0.002,
+        total: 0.0035,
+      },
+    })
+  })
+
   it('bills the cost the provider proxy decided rather than recomputing it', async () => {
     // The proxy already resolved key provenance and the margin; recomputing
     // here would re-charge a BYOK caller the proxy correctly zeroed.
@@ -195,7 +281,13 @@ describe('EvaluatorBlockHandler', () => {
     const contentObj = { text: 'Evaluate this JSON.', value: 42 }
     const inputs = {
       content: JSON.stringify(contentObj),
-      metrics: [{ name: 'clarity', description: 'Clarity score', range: { min: 1, max: 5 } }],
+      metrics: [
+        {
+          name: 'clarity',
+          description: 'Clarity score',
+          range: { min: 1, max: 5 },
+        },
+      ],
       apiKey: 'test-api-key',
     }
 
@@ -227,7 +319,11 @@ describe('EvaluatorBlockHandler', () => {
     const inputs = {
       content: contentObj,
       metrics: [
-        { name: 'completeness', description: 'Data completeness', range: { min: 0, max: 1 } },
+        {
+          name: 'completeness',
+          description: 'Data completeness',
+          range: { min: 0, max: 1 },
+        },
       ],
       apiKey: 'test-api-key',
     }
@@ -258,7 +354,13 @@ describe('EvaluatorBlockHandler', () => {
   it('should parse valid JSON response correctly', async () => {
     const inputs = {
       content: 'Test content',
-      metrics: [{ name: 'quality', description: 'Quality score', range: { min: 1, max: 10 } }],
+      metrics: [
+        {
+          name: 'quality',
+          description: 'Quality score',
+          range: { min: 1, max: 10 },
+        },
+      ],
       apiKey: 'test-api-key',
     }
 
@@ -339,7 +441,13 @@ describe('EvaluatorBlockHandler', () => {
   it('should extract metric scores ignoring case', async () => {
     const inputs = {
       content: 'Test',
-      metrics: [{ name: 'CamelCaseScore', description: 'Desc', range: { min: 0, max: 10 } }],
+      metrics: [
+        {
+          name: 'CamelCaseScore',
+          description: 'Desc',
+          range: { min: 0, max: 10 },
+        },
+      ],
       apiKey: 'test-api-key',
     }
 
@@ -366,8 +474,16 @@ describe('EvaluatorBlockHandler', () => {
     const inputs = {
       content: 'Test',
       metrics: [
-        { name: 'presentScore', description: 'Desc1', range: { min: 0, max: 5 } },
-        { name: 'missingScore', description: 'Desc2', range: { min: 0, max: 5 } },
+        {
+          name: 'presentScore',
+          description: 'Desc1',
+          range: { min: 0, max: 5 },
+        },
+        {
+          name: 'missingScore',
+          description: 'Desc2',
+          range: { min: 0, max: 5 },
+        },
       ],
       apiKey: 'test-api-key',
     }
@@ -410,7 +526,13 @@ describe('EvaluatorBlockHandler', () => {
   it('should handle Azure OpenAI models with endpoint and API version', async () => {
     const inputs = {
       content: 'Test content to evaluate',
-      metrics: [{ name: 'quality', description: 'Quality score', range: { min: 1, max: 10 } }],
+      metrics: [
+        {
+          name: 'quality',
+          description: 'Quality score',
+          range: { min: 1, max: 10 },
+        },
+      ],
       model: 'gpt-4o',
       apiKey: 'test-azure-key',
       azureEndpoint: 'https://test.openai.azure.com',
@@ -450,7 +572,13 @@ describe('EvaluatorBlockHandler', () => {
   it('should handle Vertex AI models with OAuth credential', async () => {
     const inputs = {
       content: 'Test content to evaluate',
-      metrics: [{ name: 'quality', description: 'Quality score', range: { min: 1, max: 10 } }],
+      metrics: [
+        {
+          name: 'quality',
+          description: 'Quality score',
+          range: { min: 1, max: 10 },
+        },
+      ],
       model: 'gemini-2.0-flash-exp',
       vertexCredential: 'test-vertex-credential-id',
       vertexProject: 'test-gcp-project',

apps/sim/executor/handlers/evaluator/evaluator-handler.ts

@@ -1,4 +1,10 @@
 import { createLogger } from '@sim/logger'
+import {
+  type AutoRoutingResult,
+  addAutoRoutingCost,
+  resolveAutoModel,
+  SIM_AUTO_SYSTEM_PREAMBLE,
+} from '@/lib/model-router/resolve'
 import type { BlockOutput } from '@/blocks/types'
 import { validateModelProvider } from '@/ee/access-control/utils/permission-check'
 import { BlockType, DEFAULTS, EVALUATOR } from '@/executor/constants'
@@ -7,6 +13,7 @@ import { buildAPIUrl, buildAuthHeaders, extractAPIErrorMessage } from '@/executo
 import { isJSONString, parseJSON, stringifyJSON } from '@/executor/utils/json'
 import { resolveVertexCredential } from '@/executor/utils/vertex-credential'
 import { resolveProxiedModelCost } from '@/providers/cost-policy'
+import { isAutoModel, SIM_AUTO_MODEL_ID } from '@/providers/models'
 import { getProviderFromModel } from '@/providers/utils'
 import type { SerializedBlock } from '@/serializer/types'
 
@@ -36,19 +43,6 @@ export class EvaluatorBlockHandler implements BlockHandler {
       bedrockRegion: inputs.bedrockRegion,
     }
 
-    await validateModelProvider(ctx.userId, ctx.workspaceId, evaluatorConfig.model, ctx)
-
-    const providerId = getProviderFromModel(evaluatorConfig.model)
-
-    let finalApiKey: string | undefined = evaluatorConfig.apiKey
-    if (providerId === 'vertex' && evaluatorConfig.vertexCredential) {
-      finalApiKey = await resolveVertexCredential(
-        evaluatorConfig.vertexCredential,
-        ctx.userId,
-        'vertex-evaluator'
-      )
-    }
-
     const processedContent = this.processContent(inputs.content)
 
     let systemPromptObj: { systemPrompt: string; responseFormat: any } = {
@@ -105,12 +99,55 @@ export class EvaluatorBlockHandler implements BlockHandler {
         'Evaluate the content and provide scores for each metric as JSON.'
     }
 
+    let model = evaluatorConfig.model
+    let autoRouting: AutoRoutingResult | null = null
+    if (isAutoModel(model)) {
+      autoRouting = await resolveAutoModel({
+        ctx,
+        blockId: block.id,
+        signals: {
+          systemPrompt: systemPromptObj.systemPrompt,
+          lastMessage: processedContent,
+          messageCount: 1,
+          toolNames: [],
+          mediaKind: 'none',
+          hasResponseFormat: true,
+          approxInputTokens: Math.ceil(
+            (systemPromptObj.systemPrompt.length + processedContent.length) / 4
+          ),
+        },
+        fallbackModel: EVALUATOR.DEFAULT_MODEL,
+      })
+      model = autoRouting.model
+      systemPromptObj.systemPrompt = [SIM_AUTO_SYSTEM_PREAMBLE, systemPromptObj.systemPrompt]
+        .filter(Boolean)
+        .join('\n\n')
+      logger.info('Resolved sim-auto model for evaluator', {
+        blockId: block.id,
+        model,
+        tier: autoRouting.tier,
+        decidedBy: autoRouting.decidedBy,
+      })
+    }
+
+    await validateModelProvider(ctx.userId, ctx.workspaceId, model, ctx)
+    const providerId = getProviderFromModel(model)
+
+    let finalApiKey: string | undefined = evaluatorConfig.apiKey
+    if (providerId === 'vertex' && evaluatorConfig.vertexCredential) {
+      finalApiKey = await resolveVertexCredential(
+        evaluatorConfig.vertexCredential,
+        ctx.userId,
+        'vertex-evaluator'
+      )
+    }
+
     try {
       const url = buildAPIUrl('/api/providers', ctx.userId ? { userId: ctx.userId } : {})
 
       const providerRequest: Record<string, any> = {
         provider: providerId,
-        model: evaluatorConfig.model,
+        model,
         systemPrompt: systemPromptObj.systemPrompt,
         responseFormat: systemPromptObj.responseFormat,
         context: stringifyJSON([
@@ -155,11 +192,14 @@ export class EvaluatorBlockHandler implements BlockHandler {
       const outputTokens =
         result.tokens?.output || result.tokens?.completion || DEFAULTS.TOKENS.COMPLETION
 
-      const cost = resolveProxiedModelCost(result.cost)
+      const cost = addAutoRoutingCost(
+        resolveProxiedModelCost(result.cost),
+        autoRouting?.billableRoutingCost ?? 0
+      )
 
       return {
         content: inputs.content,
-        model: result.model,
+        model: autoRouting ? SIM_AUTO_MODEL_ID : result.model,
         tokens: {
           input: inputTokens,
           output: outputTokens,
@@ -169,6 +209,7 @@ export class EvaluatorBlockHandler implements BlockHandler {
           input: cost.input,
           output: cost.output,
           total: cost.total,
+          ...(cost.routing === undefined ? {} : { routing: cost.routing }),
         },
         ...metricScores,
       }