From 9d3ab6ca5c2be3aa9d9d0f385f1f37a93ddc4248 Mon Sep 17 00:00:00 2001 From: Dan van der Merwe Date: Thu, 10 Sep 2026 15:59:13 -0700 Subject: [PATCH 1/2] feat(ai): allow schedulers to defer tool parameter validation --- .../deferred-tool-parameter-validation.md | 5 + .../openai/test/OpenAiLanguageModel.test.ts | 78 ++++++++ .../effect/src/unstable/ai/LanguageModel.ts | 42 ++++- .../test/unstable/ai/LanguageModel.test.ts | 173 ++++++++++++++++++ .../typetest/unstable/ai/LanguageModel.tst.ts | 44 +++++ 5 files changed, 335 insertions(+), 7 deletions(-) create mode 100644 .changeset/deferred-tool-parameter-validation.md diff --git a/.changeset/deferred-tool-parameter-validation.md b/.changeset/deferred-tool-parameter-validation.md new file mode 100644 index 00000000000..debe540373e --- /dev/null +++ b/.changeset/deferred-tool-parameter-validation.md @@ -0,0 +1,5 @@ +--- +"effect": patch +--- + +Allow external tool schedulers to defer application parameter validation with `toolCallValidation: "deferred"` when tool resolution is disabled. Preserve unknown arguments and provider results for corrective feedback while retaining strict response and provider-executed parameter validation. diff --git a/packages/ai/openai/test/OpenAiLanguageModel.test.ts b/packages/ai/openai/test/OpenAiLanguageModel.test.ts index eb02c7516d3..a664d1464a4 100644 --- a/packages/ai/openai/test/OpenAiLanguageModel.test.ts +++ b/packages/ai/openai/test/OpenAiLanguageModel.test.ts @@ -17,6 +17,84 @@ const fileSearchOutcomes = [ ] as const describe("OpenAiLanguageModel", () => { + for (const method of ["generateText", "streamText"] as const) { + it.effect(`${method} defers refined application arguments while retaining completed web search and usage`, () => + Effect.gen(function*() { + const Inspect = Tool.make("Inspect", { + parameters: Schema.Struct({ + focus: Schema.String.check(Schema.isMaxLength(240)).annotate({ description: "What to inspect" }) + }), + success: Schema.String + }) + const params = { focus: "x".repeat(241) } + const call = makeFunctionCall("Inspect", params) + const completed = makeDefaultResponse({ + output: [makeWebSearchCall(), call], + usage: { + input_tokens: 20, + output_tokens: 10, + total_tokens: 30, + input_tokens_details: { cached_tokens: 0 }, + output_tokens_details: { reasoning_tokens: 0 } + } + }) + const options = { + prompt: "Inspect the result", + toolkit: Toolkit.make(Inspect, OpenAiTool.WebSearch({})), + disableToolCallResolution: true, + toolCallValidation: "deferred" + } as const + const request = method === "generateText" + ? LanguageModel.generateText(options).pipe( + Effect.map((response) => response.content), + Effect.provide(OpenAiLanguageModel.model("gpt-4o-mini")), + Effect.provide(makeTestLayer({ body: completed })) + ) + : LanguageModel.streamText(options).pipe( + Stream.runCollect, + Effect.provide(OpenAiLanguageModel.model("gpt-4o-mini")), + Effect.provide(makeStreamTestLayer([ + { + type: "response.created", + sequence_number: 1, + response: makeDefaultResponse({ status: "in_progress" }) + }, + { type: "response.output_item.done", sequence_number: 2, output_index: 0, item: makeWebSearchCall() }, + { + type: "response.output_item.added", + sequence_number: 3, + output_index: 1, + item: { ...call, arguments: "", status: "in_progress" } + }, + { + type: "response.function_call_arguments.done", + sequence_number: 4, + output_index: 1, + item_id: "fc_123", + name: "Inspect", + arguments: call.arguments + }, + { type: "response.completed", sequence_number: 5, response: completed } + ])) + ) + const parts = yield* request + const invalid = parts.find((part) => part.type === "tool-call" && part.name === "Inspect") + assert.isDefined(invalid) + if (invalid?.type === "tool-call") { + strictEqual(invalid.id, "call_123") + deepStrictEqual(invalid.params, params) + } + const results = parts.filter((part) => part.type === "tool-result") + strictEqual(results.length, 1) + strictEqual(results[0].id, "ws_123") + strictEqual(results[0].isFailure, false) + const finish = parts.find((part) => part.type === "finish") + assert.isDefined(finish) + strictEqual(finish?.usage.inputTokens.total, 20) + strictEqual(finish?.usage.outputTokens.total, 10) + })) + } + describe("make", () => { it.effect("sends correct model in request", () => Effect.gen(function*() { diff --git a/packages/effect/src/unstable/ai/LanguageModel.ts b/packages/effect/src/unstable/ai/LanguageModel.ts index a99da9d86d4..d3bb88da5be 100644 --- a/packages/effect/src/unstable/ai/LanguageModel.ts +++ b/packages/effect/src/unstable/ai/LanguageModel.ts @@ -297,6 +297,24 @@ export interface GenerateTextOptions> { * resolver execution yourself. */ readonly disableToolCallResolution?: boolean | undefined + + /** + * Validation of application tool parameters when `disableToolCallResolution` + * is `true`. Defaults to `"strict"`, which validates the encoded parameters. + * + * **Details** + * + * Use `"deferred"` when an external tool scheduler owns validation and needs + * to return corrective feedback for invalid arguments. Tool call parameters + * are then typed as `unknown`; the scheduler must validate them against the + * tool's parameter schema before executing a handler. Tool definitions sent + * to the provider are unchanged, and provider-executed calls and the rest of + * the response are still validated. + * + * This option has no effect when tool call resolution is enabled: `Toolkit` + * validates application parameters according to each tool's `failureMode`. + */ + readonly toolCallValidation?: "strict" | "deferred" | undefined } type GenerateTextOptionsWithoutToolkit = Omit, "toolkit"> & { @@ -567,15 +585,16 @@ export type ExtractTools = Options extends { : {} /** - * Resolves to `"encoded"` when tool call resolution is - * disabled, otherwise `"opaque"`. + * Resolves to `"encoded"` when tool call resolution is disabled and parameter + * validation is strict, otherwise `"opaque"`. * * @category utility types * @since 4.0.0 */ export type ExtractToolParametersMode = Options extends { readonly disableToolCallResolution: true -} ? "encoded" +} ? "toolCallValidation" extends keyof Options ? "deferred" extends Options["toolCallValidation"] ? "opaque" : "encoded" + : "encoded" : "opaque" type ExtractErrorFromToolkitOption = ToolkitValue extends @@ -1209,7 +1228,7 @@ export const make: (params: { } const ResponseSchema = Schema.mutable(Schema.Array(Response.Part( - options.disableToolCallResolution === true + options.disableToolCallResolution === true && options.toolCallValidation !== "deferred" ? makeToolkitWithEncodedParameters(toolkit) : makeToolkitWithOpaqueParameters(toolkit) ))) @@ -1219,6 +1238,9 @@ export const make: (params: { if (options.disableToolCallResolution === true) { const rawContent = yield* generateWithNonIncrementalFallback() const content = yield* Schema.decodeEffect(ResponseSchema)(rawContent) + if (options.toolCallValidation === "deferred") { + yield* validateProviderExecutedToolCalls(toolkit, rawContent, "encoded") + } if (tracker) { const responseMetadata = content.find((part) => part.type === "response-metadata") if (Predicate.isNotUndefined(responseMetadata) && Predicate.isNotUndefined(responseMetadata.id)) { @@ -1511,7 +1533,7 @@ export const make: (params: { } const ResponseSchema = Schema.NonEmptyArray(Response.StreamPart( - options.disableToolCallResolution === true + options.disableToolCallResolution === true && options.toolCallValidation !== "deferred" ? makeToolkitWithEncodedParameters(toolkit) : makeToolkitWithOpaqueParameters(toolkit) )) @@ -1523,6 +1545,9 @@ export const make: (params: { return streamWithNonIncrementalFallback().pipe( Stream.mapArrayEffect((parts) => decodeParts(parts).pipe( + options.toolCallValidation === "deferred" + ? Effect.tap(() => validateProviderExecutedToolCalls(toolkit, parts, "encoded")) + : identity, tracker ? Effect.tap((decodedParts) => { for (const part of decodedParts) { @@ -2391,7 +2416,8 @@ const makeToolkitWithOpaqueParameters = > // Provider-executed tools bypass Toolkit, so validate their parameters here. const validateProviderExecutedToolCalls = >( toolkit: Toolkit.WithHandler, - parts: ReadonlyArray + parts: ReadonlyArray, + parametersMode: "decoded" | "encoded" = "decoded" ): Effect.Effect => Effect.forEach( parts, @@ -2404,7 +2430,9 @@ const validateProviderExecutedToolCalls = }, { discard: true } diff --git a/packages/effect/test/unstable/ai/LanguageModel.test.ts b/packages/effect/test/unstable/ai/LanguageModel.test.ts index c7ff045b2e0..bc32f1eb774 100644 --- a/packages/effect/test/unstable/ai/LanguageModel.test.ts +++ b/packages/effect/test/unstable/ai/LanguageModel.test.ts @@ -69,6 +69,179 @@ describe("LanguageModel", () => { response: undefined } + describe("deferred tool parameter validation", () => { + for (const method of ["generateText", "streamText"] as const) { + it.effect(`${method} preserves invalid arguments, provider results, and usage without running handlers`, () => + Effect.gen(function*() { + let calls = 0 + const toolkit = Toolkit.make(ReturnModeTool, MyTool) + const handlers = toolkit.toLayer({ + ReturnModeTool: () => + Effect.sync(() => { + calls++ + return { testSuccess: "unexpected" } + }), + MyTool: () => + Effect.sync(() => { + calls++ + return { testSuccess: "unexpected" } + }) + }) + const raw: Array = + [ + { + type: "tool-call", + id: "hosted", + name: "MyTool", + params: { testParam: "valid" }, + providerExecuted: true + }, + { + type: "tool-result", + id: "hosted", + name: "MyTool", + result: { testSuccess: "retained" }, + isFailure: false, + providerExecuted: true + }, + { type: "tool-call", id: "invalid", name: "ReturnModeTool", params: { testParam: 123 } }, + { ...finishPart, reason: "tool-calls" } + ] + const options = { + prompt: [], + toolkit, + disableToolCallResolution: true, + toolCallValidation: "deferred" + } as const + const request: Effect.Effect< + ReadonlyArray< + | Response.Part, "opaque"> + | Response.StreamPart, "opaque"> + >, + AiError.AiError, + LanguageModel.LanguageModel + > = method === "generateText" + ? LanguageModel.generateText(options).pipe(Effect.map((response) => response.content)) + : LanguageModel.streamText(options).pipe(Stream.runCollect) + const parts = yield* request.pipe( + TestUtils.withLanguageModel({ + generateText: (options) => { + strictEqual(options.tools[0], ReturnModeTool) + return raw + }, + streamText: (options) => { + strictEqual(options.tools[0], ReturnModeTool) + return raw + } + }), + Effect.provide(handlers) + ) + const invalid = parts.find((part) => part.type === "tool-call" && part.id === "invalid") + assertDefined(invalid) + assertTrue(invalid.type === "tool-call") + deepStrictEqual(invalid.params, { testParam: 123 }) + const results = parts.filter((part) => part.type === "tool-result") + strictEqual(results.length, 1) + strictEqual(results[0].id, "hosted") + deepStrictEqual(results[0].result, { testSuccess: "retained" }) + const finish = parts.find((part) => part.type === "finish") + assertDefined(finish) + strictEqual(finish.usage.inputTokens.total, 5) + strictEqual(finish.usage.outputTokens.total, 5) + strictEqual(calls, 0) + })) + + for ( + const call of [ + { + type: "tool-call", + id: "hosted-invalid", + name: "MyTool", + params: { testParam: 123 }, + providerExecuted: true + }, + { type: "tool-call", id: "unknown", name: "UnknownTool", params: {} } + ] satisfies Array + ) { + it.effect(`${method} still rejects ${call.id}`, () => + Effect.gen(function*() { + const options = { + prompt: [], + toolkit: MyToolkit, + disableToolCallResolution: true, + toolCallValidation: "deferred" + } as const + const request = method === "generateText" + ? LanguageModel.generateText(options).pipe(Effect.asVoid) + : LanguageModel.streamText(options).pipe(Stream.runDrain) + const error = yield* request.pipe( + TestUtils.withLanguageModel({ generateText: [call, finishPart], streamText: [call, finishPart] }), + Effect.flip + ) + strictEqual(error.reason._tag, "InvalidOutputError") + })) + } + } + + it.effect("keeps transformed parameters encoded for both application and provider calls", () => + Effect.gen(function*() { + const parts = yield* LanguageModel.streamText({ + prompt: [], + toolkit: TransformToolkit, + disableToolCallResolution: true, + toolCallValidation: "deferred" + }).pipe( + Stream.runCollect, + TestUtils.withLanguageModel({ + streamText: [ + { type: "tool-call", id: "application", name: "TransformTool", params: "21" }, + { type: "tool-call", id: "provider", name: "TransformTool", params: "22", providerExecuted: true }, + finishPart + ] + }) + ) + const calls = parts.filter((part) => part.type === "tool-call") + deepStrictEqual(calls.map((call) => call.params), ["21", "22"]) + })) + + it.effect("preserves provider failure and finalization after an invalid application call", () => + Effect.gen(function*() { + let finalized = false + const failure = AiError.make({ + module: "Test", + method: "streamText", + reason: new AiError.InvalidRequestError({ description: "provider failed" }) + }) + const error = yield* LanguageModel.streamText({ + prompt: [], + toolkit: MyToolkit, + disableToolCallResolution: true, + toolCallValidation: "deferred" + }).pipe( + Stream.runDrain, + TestUtils.withLanguageModel({ + streamText: () => + Stream.succeed( + { + type: "tool-call", + id: "invalid", + name: "MyTool", + params: { testParam: 123 } + } satisfies Response.StreamPartEncoded + ).pipe( + Stream.concat(Stream.fail(failure)), + Stream.ensuring(Effect.sync(() => { + finalized = true + })) + ) + }), + Effect.flip + ) + strictEqual(error, failure) + strictEqual(finalized, true) + })) + }) + describe("generateText", () => { it.effect("does not resolve tool calls after an incomplete finish", () => Effect.gen(function*() { diff --git a/packages/effect/typetest/unstable/ai/LanguageModel.tst.ts b/packages/effect/typetest/unstable/ai/LanguageModel.tst.ts index 894d60e9302..048af855f82 100644 --- a/packages/effect/typetest/unstable/ai/LanguageModel.tst.ts +++ b/packages/effect/typetest/unstable/ai/LanguageModel.tst.ts @@ -55,6 +55,50 @@ const AsymmetricParamsTool = Tool.make("AsymmetricParamsTool", { }) describe("LanguageModel", () => { + it("keeps deferred parameters unknown without handler requirements or errors", () => { + const toolkit = Toolkit.make(FailureModeErrorTool, ToolWithRequestContext, AsymmetricParamsTool) + const options = { + prompt: "hello", + toolkit, + disableToolCallResolution: true, + toolCallValidation: "deferred" + } as const + const program = LanguageModel.generateText(options) + const stream = LanguageModel.streamText(options) + const object = LanguageModel.generateObject({ ...options, schema: Schema.Struct({ answer: Schema.String }) }) + const chat = null as unknown as Chat.Chat + const chatProgram = chat.generateText(options) + + type Output = Effect.Success + type Part = Stream.Success + expect().type.toBe() + expect["params"]>().type.toBe() + expect["toolCalls"][number]["params"]>().type.toBe() + expect["toolCalls"][number]["params"]>().type.toBe() + expect>().type.toBe() + expect>().type.toBe() + expect>().type.toBe() + expect>().type.toBe() + }) + + it("keeps parameters unknown when validation may be deferred", () => { + const toolCallValidation = null as unknown as "strict" | "deferred" | undefined + const program = LanguageModel.generateText({ + prompt: "hello", + toolkit: Toolkit.make(TransformTool), + disableToolCallResolution: true, + toolCallValidation + }) + expect["toolCalls"][number]["params"]>().type.toBe() + const strict = LanguageModel.generateText({ + prompt: "hello", + toolkit: Toolkit.make(TransformTool), + disableToolCallResolution: true, + toolCallValidation: "strict" + }) + expect["toolCalls"][number]["params"]>().type.toBe() + }) + describe("generateText", () => { it("uses encoded tool parameters when tool call resolution is disabled", () => { const toolkit = Toolkit.make(TransformTool) From 71a8887a4cddc4f4c3c07ec5f90e2f64bbbff005 Mon Sep 17 00:00:00 2001 From: Dan van der Merwe Date: Thu, 10 Sep 2026 16:33:27 -0700 Subject: [PATCH 2/2] refactor(ai): focus deferred validation contracts and tests --- .../deferred-tool-parameter-validation.md | 2 +- .../openai/test/OpenAiLanguageModel.test.ts | 17 ++- .../effect/src/unstable/ai/LanguageModel.ts | 20 ++-- .../test/unstable/ai/LanguageModel.test.ts | 112 ++++++------------ .../typetest/unstable/ai/LanguageModel.tst.ts | 8 ++ 5 files changed, 62 insertions(+), 97 deletions(-) diff --git a/.changeset/deferred-tool-parameter-validation.md b/.changeset/deferred-tool-parameter-validation.md index debe540373e..265cdacdfb9 100644 --- a/.changeset/deferred-tool-parameter-validation.md +++ b/.changeset/deferred-tool-parameter-validation.md @@ -2,4 +2,4 @@ "effect": patch --- -Allow external tool schedulers to defer application parameter validation with `toolCallValidation: "deferred"` when tool resolution is disabled. Preserve unknown arguments and provider results for corrective feedback while retaining strict response and provider-executed parameter validation. +Allow external tool schedulers to defer application parameter validation with `toolCallValidation: "deferred"` when tool resolution is disabled, preserving invalid arguments for corrective feedback. diff --git a/packages/ai/openai/test/OpenAiLanguageModel.test.ts b/packages/ai/openai/test/OpenAiLanguageModel.test.ts index a664d1464a4..c41ed8ca503 100644 --- a/packages/ai/openai/test/OpenAiLanguageModel.test.ts +++ b/packages/ai/openai/test/OpenAiLanguageModel.test.ts @@ -54,27 +54,22 @@ describe("OpenAiLanguageModel", () => { Stream.runCollect, Effect.provide(OpenAiLanguageModel.model("gpt-4o-mini")), Effect.provide(makeStreamTestLayer([ - { - type: "response.created", - sequence_number: 1, - response: makeDefaultResponse({ status: "in_progress" }) - }, - { type: "response.output_item.done", sequence_number: 2, output_index: 0, item: makeWebSearchCall() }, + { type: "response.output_item.done", sequence_number: 1, output_index: 0, item: makeWebSearchCall() }, { type: "response.output_item.added", - sequence_number: 3, + sequence_number: 2, output_index: 1, item: { ...call, arguments: "", status: "in_progress" } }, { type: "response.function_call_arguments.done", - sequence_number: 4, + sequence_number: 3, output_index: 1, item_id: "fc_123", name: "Inspect", arguments: call.arguments }, - { type: "response.completed", sequence_number: 5, response: completed } + { type: "response.completed", sequence_number: 4, response: completed } ])) ) const parts = yield* request @@ -88,6 +83,10 @@ describe("OpenAiLanguageModel", () => { strictEqual(results.length, 1) strictEqual(results[0].id, "ws_123") strictEqual(results[0].isFailure, false) + deepStrictEqual(results[0].result, { + action: { type: "search", query: "Effect TypeScript" }, + status: "completed" + }) const finish = parts.find((part) => part.type === "finish") assert.isDefined(finish) strictEqual(finish?.usage.inputTokens.total, 20) diff --git a/packages/effect/src/unstable/ai/LanguageModel.ts b/packages/effect/src/unstable/ai/LanguageModel.ts index d3bb88da5be..8f1c27be657 100644 --- a/packages/effect/src/unstable/ai/LanguageModel.ts +++ b/packages/effect/src/unstable/ai/LanguageModel.ts @@ -299,17 +299,15 @@ export interface GenerateTextOptions> { readonly disableToolCallResolution?: boolean | undefined /** - * Validation of application tool parameters when `disableToolCallResolution` - * is `true`. Defaults to `"strict"`, which validates the encoded parameters. + * Controls application tool parameter validation when + * `disableToolCallResolution` is `true`. Defaults to `"strict"`. * * **Details** * - * Use `"deferred"` when an external tool scheduler owns validation and needs - * to return corrective feedback for invalid arguments. Tool call parameters - * are then typed as `unknown`; the scheduler must validate them against the - * tool's parameter schema before executing a handler. Tool definitions sent - * to the provider are unchanged, and provider-executed calls and the rest of - * the response are still validated. + * `"strict"` validates encoded parameters. `"deferred"` returns parameters as + * `unknown` so an external scheduler can validate them before execution and + * return corrective feedback. Tool definitions, provider-executed parameter + * validation, and validation of the rest of the response are unchanged. * * This option has no effect when tool call resolution is enabled: `Toolkit` * validates application parameters according to each tool's `failureMode`. @@ -593,9 +591,9 @@ export type ExtractTools = Options extends { */ export type ExtractToolParametersMode = Options extends { readonly disableToolCallResolution: true -} ? "toolCallValidation" extends keyof Options ? "deferred" extends Options["toolCallValidation"] ? "opaque" : "encoded" - : "encoded" - : "opaque" + readonly toolCallValidation?: "strict" | undefined +} ? "encoded" : + "opaque" type ExtractErrorFromToolkitOption = ToolkitValue extends Toolkit.WithHandler ? diff --git a/packages/effect/test/unstable/ai/LanguageModel.test.ts b/packages/effect/test/unstable/ai/LanguageModel.test.ts index bc32f1eb774..fe4b61f5631 100644 --- a/packages/effect/test/unstable/ai/LanguageModel.test.ts +++ b/packages/effect/test/unstable/ai/LanguageModel.test.ts @@ -71,99 +71,59 @@ describe("LanguageModel", () => { describe("deferred tool parameter validation", () => { for (const method of ["generateText", "streamText"] as const) { - it.effect(`${method} preserves invalid arguments, provider results, and usage without running handlers`, () => + it.effect(`${method} defers application parameters without changing definitions or running handlers`, () => Effect.gen(function*() { - let calls = 0 - const toolkit = Toolkit.make(ReturnModeTool, MyTool) - const handlers = toolkit.toLayer({ - ReturnModeTool: () => - Effect.sync(() => { - calls++ - return { testSuccess: "unexpected" } - }), - MyTool: () => - Effect.sync(() => { - calls++ - return { testSuccess: "unexpected" } - }) + const handlers = ReturnModeToolkit.toLayer({ + ReturnModeTool: () => Effect.die("handler must not run") }) - const raw: Array = - [ - { - type: "tool-call", - id: "hosted", - name: "MyTool", - params: { testParam: "valid" }, - providerExecuted: true - }, - { - type: "tool-result", - id: "hosted", - name: "MyTool", - result: { testSuccess: "retained" }, - isFailure: false, - providerExecuted: true - }, + const respond = (options: LanguageModel.ProviderOptions) => { + strictEqual(options.tools[0], ReturnModeTool) + return [ { type: "tool-call", id: "invalid", name: "ReturnModeTool", params: { testParam: 123 } }, - { ...finishPart, reason: "tool-calls" } - ] + { type: "tool-call", id: "valid", name: "ReturnModeTool", params: { testParam: "valid" } }, + finishPart + ] satisfies Array + } const options = { prompt: [], - toolkit, + toolkit: ReturnModeToolkit, disableToolCallResolution: true, toolCallValidation: "deferred" } as const - const request: Effect.Effect< - ReadonlyArray< - | Response.Part, "opaque"> - | Response.StreamPart, "opaque"> - >, - AiError.AiError, - LanguageModel.LanguageModel - > = method === "generateText" - ? LanguageModel.generateText(options).pipe(Effect.map((response) => response.content)) - : LanguageModel.streamText(options).pipe(Stream.runCollect) - const parts = yield* request.pipe( - TestUtils.withLanguageModel({ - generateText: (options) => { - strictEqual(options.tools[0], ReturnModeTool) - return raw - }, - streamText: (options) => { - strictEqual(options.tools[0], ReturnModeTool) - return raw - } - }), + const request = method === "generateText" + ? LanguageModel.generateText(options).pipe(Effect.map((response) => response.toolCalls)) + : LanguageModel.streamText(options).pipe( + Stream.filter((part) => part.type === "tool-call"), + Stream.runCollect + ) + const calls = yield* request.pipe( + TestUtils.withLanguageModel({ generateText: respond, streamText: respond }), Effect.provide(handlers) ) - const invalid = parts.find((part) => part.type === "tool-call" && part.id === "invalid") - assertDefined(invalid) - assertTrue(invalid.type === "tool-call") - deepStrictEqual(invalid.params, { testParam: 123 }) - const results = parts.filter((part) => part.type === "tool-result") - strictEqual(results.length, 1) - strictEqual(results[0].id, "hosted") - deepStrictEqual(results[0].result, { testSuccess: "retained" }) - const finish = parts.find((part) => part.type === "finish") - assertDefined(finish) - strictEqual(finish.usage.inputTokens.total, 5) - strictEqual(finish.usage.outputTokens.total, 5) - strictEqual(calls, 0) + deepStrictEqual(calls.map((call) => call.params), [{ testParam: 123 }, { testParam: "valid" }]) })) for ( - const call of [ - { + const [label, part] of [ + ["invalid provider parameters", { type: "tool-call", - id: "hosted-invalid", + id: "hosted", name: "MyTool", params: { testParam: 123 }, providerExecuted: true - }, - { type: "tool-call", id: "unknown", name: "UnknownTool", params: {} } - ] satisfies Array + }], + ["unknown tool names", { type: "tool-call", id: "unknown", name: "UnknownTool", params: {} }], + ["invalid provider results", { + type: "tool-result", + id: "hosted", + name: "MyTool", + result: { testSuccess: 123 }, + isFailure: false, + providerExecuted: true + }] + ] satisfies Array<[string, Response.ToolCallPartEncoded | Response.ToolResultPartEncoded]> ) { - it.effect(`${method} still rejects ${call.id}`, () => + it.effect(`${method} still rejects ${label}`, () => Effect.gen(function*() { const options = { prompt: [], @@ -175,7 +135,7 @@ describe("LanguageModel", () => { ? LanguageModel.generateText(options).pipe(Effect.asVoid) : LanguageModel.streamText(options).pipe(Stream.runDrain) const error = yield* request.pipe( - TestUtils.withLanguageModel({ generateText: [call, finishPart], streamText: [call, finishPart] }), + TestUtils.withLanguageModel({ generateText: [part, finishPart], streamText: [part, finishPart] }), Effect.flip ) strictEqual(error.reason._tag, "InvalidOutputError") diff --git a/packages/effect/typetest/unstable/ai/LanguageModel.tst.ts b/packages/effect/typetest/unstable/ai/LanguageModel.tst.ts index 048af855f82..7e633a7a438 100644 --- a/packages/effect/typetest/unstable/ai/LanguageModel.tst.ts +++ b/packages/effect/typetest/unstable/ai/LanguageModel.tst.ts @@ -90,6 +90,14 @@ describe("LanguageModel", () => { toolCallValidation }) expect["toolCalls"][number]["params"]>().type.toBe() + const options: { readonly toolCallValidation?: "strict" | "deferred" } = {} + const optional = LanguageModel.generateText({ + ...options, + prompt: "hello", + toolkit: Toolkit.make(TransformTool), + disableToolCallResolution: true + }) + expect["toolCalls"][number]["params"]>().type.toBe() const strict = LanguageModel.generateText({ prompt: "hello", toolkit: Toolkit.make(TransformTool),