Skip to content

Commit 9aa8fe0

Browse files
committed
refactor(providers): drop the sanitizer's level diagnostics
The two warnings logged server-side, where the workflow author who set the level never sees them, and the surprising case they described — a level discarded for a model newer than the catalogue — is now fixed at the source rather than narrated. They also carried the redaction that leaked resolved content before it was caught, so removing them removes that surface entirely. Levels still normalize, and still drop for a catalogued model that does not take the field. `describeModelLevel` stays for Anthropic's unsupported-thinking warning, which is a pre-existing log this feature newly exposes to resolved reference content.
1 parent a4c7abb commit 9aa8fe0

2 files changed

Lines changed: 11 additions & 211 deletions

File tree

apps/sim/providers/index.test.ts

Lines changed: 1 addition & 111 deletions
Original file line numberDiff line numberDiff line change
@@ -4,32 +4,11 @@
44
import { envFlagsMockFns, resetEnvFlagsMock } from '@sim/testing'
55
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
66

7-
const { mockGetApiKeyWithBYOK, mockExecuteRequest, mockLoggerWarn } = vi.hoisted(() => ({
7+
const { mockGetApiKeyWithBYOK, mockExecuteRequest } = vi.hoisted(() => ({
88
mockGetApiKeyWithBYOK: vi.fn(),
99
mockExecuteRequest: vi.fn(),
10-
mockLoggerWarn: vi.fn(),
1110
}))
1211

13-
/** Overrides the global logger mock so the sanitizer's warnings are assertable. */
14-
vi.mock('@sim/logger', () => {
15-
const createLogger = () => ({
16-
info: vi.fn(),
17-
warn: mockLoggerWarn,
18-
error: vi.fn(),
19-
debug: vi.fn(),
20-
trace: vi.fn(),
21-
fatal: vi.fn(),
22-
child: () => createLogger(),
23-
withMetadata: () => createLogger(),
24-
})
25-
return {
26-
createLogger,
27-
logger: createLogger(),
28-
runWithRequestContext: <T>(_ctx: unknown, fn: () => T): T => fn(),
29-
getRequestContext: () => undefined,
30-
}
31-
})
32-
3312
vi.mock('@/lib/api-key/byok', () => ({
3413
getApiKeyWithBYOK: (...args: unknown[]) => mockGetApiKeyWithBYOK(...args),
3514
}))
@@ -550,95 +529,6 @@ describe('executeProviderRequest — model level normalization', () => {
550529
expect(sentRequest().verbosity).toBeUndefined()
551530
})
552531

553-
/**
554-
* The model can itself be a reference, so it is only known at execution time. A run whose
555-
* reference resolved to a model outside Sim's catalogue must not fall back to that model's
556-
* default in silence.
557-
*/
558-
it('reports the level it drops when the resolved model does not support the field', async () => {
559-
await executeProviderRequest('anthropic', {
560-
model: 'claude-opus-4-6',
561-
workspaceId: 'ws-1',
562-
reasoningEffort: 'high',
563-
})
564-
565-
expect(mockLoggerWarn).toHaveBeenCalledWith(
566-
'Model does not support this level; dropping it from the request',
567-
expect.objectContaining({
568-
field: 'reasoningEffort',
569-
model: 'claude-opus-4-6',
570-
value: 'high',
571-
})
572-
)
573-
})
574-
575-
it('stays quiet when an unsupported model was never given a level', async () => {
576-
await executeProviderRequest('anthropic', {
577-
model: 'claude-opus-4-6',
578-
workspaceId: 'ws-1',
579-
})
580-
581-
expect(mockLoggerWarn).not.toHaveBeenCalled()
582-
})
583-
584-
it('reports a level the model accepts but does not declare', async () => {
585-
await executeProviderRequest('openai', {
586-
model: 'gpt-5',
587-
workspaceId: 'ws-1',
588-
reasoningEffort: 'xhigh',
589-
})
590-
591-
expect(mockLoggerWarn).toHaveBeenCalledWith(
592-
'Model level is not one this model declares; forwarding to the provider',
593-
expect.objectContaining({ field: 'reasoningEffort', model: 'gpt-5', value: 'xhigh' })
594-
)
595-
})
596-
597-
it('stays quiet for a declared level and for the auto and none sentinels', async () => {
598-
await executeProviderRequest('openai', {
599-
model: 'gpt-5',
600-
workspaceId: 'ws-1',
601-
reasoningEffort: 'auto',
602-
verbosity: 'high',
603-
})
604-
605-
expect(mockLoggerWarn).not.toHaveBeenCalled()
606-
})
607-
608-
/**
609-
* These fields take environment and block references, so a mistyped reference resolves the
610-
* secret into the level. The diagnostics must never echo it.
611-
*/
612-
it('redacts a level that is not a catalogue level before logging it', async () => {
613-
const secret = 'sk-proj-abcdef0123456789'
614-
615-
await executeProviderRequest('openai', {
616-
model: 'gpt-5',
617-
workspaceId: 'ws-1',
618-
reasoningEffort: secret,
619-
})
620-
621-
expect(mockLoggerWarn).toHaveBeenCalledWith(
622-
expect.any(String),
623-
expect.objectContaining({ value: `[redacted ${secret.length} chars]` })
624-
)
625-
const loggedText = JSON.stringify(mockLoggerWarn.mock.calls)
626-
expect(loggedText).not.toContain(secret)
627-
})
628-
629-
it('keeps the auto sentinel readable in a drop diagnostic', async () => {
630-
await executeProviderRequest('anthropic', {
631-
model: 'claude-opus-4-6',
632-
workspaceId: 'ws-1',
633-
reasoningEffort: 'auto',
634-
})
635-
636-
expect(mockLoggerWarn).toHaveBeenCalledWith(
637-
'Model does not support this level; dropping it from the request',
638-
expect.objectContaining({ value: 'auto' })
639-
)
640-
})
641-
642532
/**
643533
* A model the catalogue has never seen is unknown, not known-incapable — which is exactly
644534
* how a newly released model arrives through a reference before Sim catalogues it. The

apps/sim/providers/index.ts

Lines changed: 10 additions & 100 deletions
Original file line numberDiff line numberDiff line change
@@ -15,20 +15,14 @@ import {
1515
attachLargeFileRemoteUrls,
1616
uploadLargeFilesToProvider,
1717
} from '@/providers/file-attachments.server'
18-
import {
19-
getReasoningEffortValuesForModel,
20-
getThinkingLevelsForModel,
21-
getVerbosityValuesForModel,
22-
isKnownModelId,
23-
} from '@/providers/models'
18+
import { isKnownModelId } from '@/providers/models'
2419
import { getProviderExecutor } from '@/providers/registry'
2520
import {
2621
type ProviderRuntimeContext,
2722
runWithProviderRuntimeContext,
2823
} from '@/providers/runtime-context'
2924
import type { ProviderId, ProviderRequest, ProviderResponse } from '@/providers/types'
3025
import {
31-
describeModelLevel,
3226
generateStructuredOutputInstructions,
3327
sumToolCosts,
3428
supportsPromptCaching,
@@ -59,60 +53,6 @@ function normalizeModelLevel(value: string | undefined): string | undefined {
5953
return normalized || undefined
6054
}
6155

62-
const MODEL_LEVEL_SENTINELS = new Set(['auto', 'none'])
63-
64-
type ModelLevelField = 'reasoningEffort' | 'verbosity' | 'thinkingLevel'
65-
66-
/**
67-
* Clears a level whose resolved model does not accept the field at all.
68-
*
69-
* Dropping is the safe default — a provider that has no such parameter rejects the whole
70-
* request — but the discard is reported because the model can be bound to a variable or block
71-
* reference and is therefore only known at execution time. Without this, a run whose reference
72-
* resolved to a model that does not take the field would quietly fall back to that model's
73-
* default while the caller believed the level applied.
74-
*/
75-
function dropUnsupportedLevel(
76-
field: ModelLevelField,
77-
model: string,
78-
value: string | undefined
79-
): undefined {
80-
if (value) {
81-
logger.warn('Model does not support this level; dropping it from the request', {
82-
field,
83-
model,
84-
value: describeModelLevel(value),
85-
})
86-
}
87-
return undefined
88-
}
89-
90-
/**
91-
* Logs a level that the model accepts as a field but does not list as a value.
92-
*
93-
* Deliberately does not drop the value. Sim's per-model level lists exist to populate the
94-
* pickers and can lag a provider that has started accepting a new level, so rejecting on them
95-
* would refuse values the API would have taken. Forwarding instead surfaces the provider's own
96-
* error, which names the field and the values it accepts — the loud failure an eval sweeping
97-
* levels needs, where silently substituting the model default would corrupt the results.
98-
*/
99-
function warnOnUnrecognizedLevel(
100-
field: ModelLevelField,
101-
model: string | undefined,
102-
value: string | undefined,
103-
declaredValues: string[] | null
104-
): void {
105-
if (!model || !value || MODEL_LEVEL_SENTINELS.has(value)) return
106-
if (!declaredValues || declaredValues.includes(value)) return
107-
108-
logger.warn('Model level is not one this model declares; forwarding to the provider', {
109-
field,
110-
model,
111-
value: describeModelLevel(value),
112-
declaredValues,
113-
})
114-
}
115-
11656
function sanitizeRequest(request: ProviderRequest): ProviderRequest {
11757
const sanitizedRequest = { ...request }
11858
const model = sanitizedRequest.model
@@ -126,61 +66,31 @@ function sanitizeRequest(request: ProviderRequest): ProviderRequest {
12666
}
12767

12868
/**
129-
* A model absent from the catalogue is unknown, not known-incapable. Since the model can be
130-
* bound to a reference, that is exactly how a newly released model arrives before Sim has
131-
* catalogued it — so its levels are forwarded and the provider decides, rather than being
132-
* discarded on the strength of a list that has not caught up. Models the catalogue does
133-
* know, and every dynamic-provider id, keep the protective drop.
69+
* A model absent from the catalogue is unknown, not known-incapable. The model field is an
70+
* editable combobox, so a model newer than `models.ts` reaches this point routed by pattern
71+
* and executing normally — discarding its levels on the strength of a list that has not
72+
* caught up loses a setting the provider would have honoured. Those levels are forwarded and
73+
* the provider decides. Models the catalogue does know, and every dynamic-provider id, keep
74+
* the protective drop.
13475
*/
13576
const isCatalogued = Boolean(model) && isKnownModelId(model)
13677

13778
if (model && isCatalogued && !supportsReasoningEffort(model)) {
138-
sanitizedRequest.reasoningEffort = dropUnsupportedLevel(
139-
'reasoningEffort',
140-
model,
141-
sanitizedRequest.reasoningEffort
142-
)
79+
sanitizedRequest.reasoningEffort = undefined
14380
}
14481

14582
if (model && isCatalogued && !supportsVerbosity(model)) {
146-
sanitizedRequest.verbosity = dropUnsupportedLevel(
147-
'verbosity',
148-
model,
149-
sanitizedRequest.verbosity
150-
)
83+
sanitizedRequest.verbosity = undefined
15184
}
15285

15386
if (model && isCatalogued && !supportsThinking(model)) {
154-
sanitizedRequest.thinkingLevel = dropUnsupportedLevel(
155-
'thinkingLevel',
156-
model,
157-
sanitizedRequest.thinkingLevel
158-
)
87+
sanitizedRequest.thinkingLevel = undefined
15988
}
16089

16190
if (model && !supportsPromptCaching(model)) {
16291
sanitizedRequest.promptCaching = undefined
16392
}
16493

165-
warnOnUnrecognizedLevel(
166-
'reasoningEffort',
167-
model,
168-
sanitizedRequest.reasoningEffort,
169-
model ? getReasoningEffortValuesForModel(model) : null
170-
)
171-
warnOnUnrecognizedLevel(
172-
'verbosity',
173-
model,
174-
sanitizedRequest.verbosity,
175-
model ? getVerbosityValuesForModel(model) : null
176-
)
177-
warnOnUnrecognizedLevel(
178-
'thinkingLevel',
179-
model,
180-
sanitizedRequest.thinkingLevel,
181-
model ? getThinkingLevelsForModel(model) : null
182-
)
183-
18494
return sanitizedRequest
18595
}
18696

0 commit comments

Comments
 (0)