diff --git a/.changeset/deferred-tool-parameter-validation.md b/.changeset/deferred-tool-parameter-validation.md new file mode 100644 index 00000000000..265cdacdfb9 --- /dev/null +++ b/.changeset/deferred-tool-parameter-validation.md @@ -0,0 +1,5 @@ +--- +"effect": patch +--- + +Allow external tool schedulers to defer application parameter validation with `toolCallValidation: "deferred"` when tool resolution is disabled, preserving invalid arguments for corrective feedback. diff --git a/packages/ai/openai/test/OpenAiLanguageModel.test.ts b/packages/ai/openai/test/OpenAiLanguageModel.test.ts index eb02c7516d3..c41ed8ca503 100644 --- a/packages/ai/openai/test/OpenAiLanguageModel.test.ts +++ b/packages/ai/openai/test/OpenAiLanguageModel.test.ts @@ -17,6 +17,83 @@ const fileSearchOutcomes = [ ] as const describe("OpenAiLanguageModel", () => { + for (const method of ["generateText", "streamText"] as const) { + it.effect(`${method} defers refined application arguments while retaining completed web search and usage`, () => + Effect.gen(function*() { + const Inspect = Tool.make("Inspect", { + parameters: Schema.Struct({ + focus: Schema.String.check(Schema.isMaxLength(240)).annotate({ description: "What to inspect" }) + }), + success: Schema.String + }) + const params = { focus: "x".repeat(241) } + const call = makeFunctionCall("Inspect", params) + const completed = makeDefaultResponse({ + output: [makeWebSearchCall(), call], + usage: { + input_tokens: 20, + output_tokens: 10, + total_tokens: 30, + input_tokens_details: { cached_tokens: 0 }, + output_tokens_details: { reasoning_tokens: 0 } + } + }) + const options = { + prompt: "Inspect the result", + toolkit: Toolkit.make(Inspect, OpenAiTool.WebSearch({})), + disableToolCallResolution: true, + toolCallValidation: "deferred" + } as const + const request = method === "generateText" + ? LanguageModel.generateText(options).pipe( + Effect.map((response) => response.content), + Effect.provide(OpenAiLanguageModel.model("gpt-4o-mini")), + Effect.provide(makeTestLayer({ body: completed })) + ) + : LanguageModel.streamText(options).pipe( + Stream.runCollect, + Effect.provide(OpenAiLanguageModel.model("gpt-4o-mini")), + Effect.provide(makeStreamTestLayer([ + { type: "response.output_item.done", sequence_number: 1, output_index: 0, item: makeWebSearchCall() }, + { + type: "response.output_item.added", + sequence_number: 2, + output_index: 1, + item: { ...call, arguments: "", status: "in_progress" } + }, + { + type: "response.function_call_arguments.done", + sequence_number: 3, + output_index: 1, + item_id: "fc_123", + name: "Inspect", + arguments: call.arguments + }, + { type: "response.completed", sequence_number: 4, response: completed } + ])) + ) + const parts = yield* request + const invalid = parts.find((part) => part.type === "tool-call" && part.name === "Inspect") + assert.isDefined(invalid) + if (invalid?.type === "tool-call") { + strictEqual(invalid.id, "call_123") + deepStrictEqual(invalid.params, params) + } + const results = parts.filter((part) => part.type === "tool-result") + strictEqual(results.length, 1) + strictEqual(results[0].id, "ws_123") + strictEqual(results[0].isFailure, false) + deepStrictEqual(results[0].result, { + action: { type: "search", query: "Effect TypeScript" }, + status: "completed" + }) + const finish = parts.find((part) => part.type === "finish") + assert.isDefined(finish) + strictEqual(finish?.usage.inputTokens.total, 20) + strictEqual(finish?.usage.outputTokens.total, 10) + })) + } + describe("make", () => { it.effect("sends correct model in request", () => Effect.gen(function*() { diff --git a/packages/effect/src/unstable/ai/LanguageModel.ts b/packages/effect/src/unstable/ai/LanguageModel.ts index a99da9d86d4..8f1c27be657 100644 --- a/packages/effect/src/unstable/ai/LanguageModel.ts +++ b/packages/effect/src/unstable/ai/LanguageModel.ts @@ -297,6 +297,22 @@ export interface GenerateTextOptions> { * resolver execution yourself. */ readonly disableToolCallResolution?: boolean | undefined + + /** + * Controls application tool parameter validation when + * `disableToolCallResolution` is `true`. Defaults to `"strict"`. + * + * **Details** + * + * `"strict"` validates encoded parameters. `"deferred"` returns parameters as + * `unknown` so an external scheduler can validate them before execution and + * return corrective feedback. Tool definitions, provider-executed parameter + * validation, and validation of the rest of the response are unchanged. + * + * This option has no effect when tool call resolution is enabled: `Toolkit` + * validates application parameters according to each tool's `failureMode`. + */ + readonly toolCallValidation?: "strict" | "deferred" | undefined } type GenerateTextOptionsWithoutToolkit = Omit, "toolkit"> & { @@ -567,16 +583,17 @@ export type ExtractTools = Options extends { : {} /** - * Resolves to `"encoded"` when tool call resolution is - * disabled, otherwise `"opaque"`. + * Resolves to `"encoded"` when tool call resolution is disabled and parameter + * validation is strict, otherwise `"opaque"`. * * @category utility types * @since 4.0.0 */ export type ExtractToolParametersMode = Options extends { readonly disableToolCallResolution: true -} ? "encoded" - : "opaque" + readonly toolCallValidation?: "strict" | undefined +} ? "encoded" : + "opaque" type ExtractErrorFromToolkitOption = ToolkitValue extends Toolkit.WithHandler ? @@ -1209,7 +1226,7 @@ export const make: (params: { } const ResponseSchema = Schema.mutable(Schema.Array(Response.Part( - options.disableToolCallResolution === true + options.disableToolCallResolution === true && options.toolCallValidation !== "deferred" ? makeToolkitWithEncodedParameters(toolkit) : makeToolkitWithOpaqueParameters(toolkit) ))) @@ -1219,6 +1236,9 @@ export const make: (params: { if (options.disableToolCallResolution === true) { const rawContent = yield* generateWithNonIncrementalFallback() const content = yield* Schema.decodeEffect(ResponseSchema)(rawContent) + if (options.toolCallValidation === "deferred") { + yield* validateProviderExecutedToolCalls(toolkit, rawContent, "encoded") + } if (tracker) { const responseMetadata = content.find((part) => part.type === "response-metadata") if (Predicate.isNotUndefined(responseMetadata) && Predicate.isNotUndefined(responseMetadata.id)) { @@ -1511,7 +1531,7 @@ export const make: (params: { } const ResponseSchema = Schema.NonEmptyArray(Response.StreamPart( - options.disableToolCallResolution === true + options.disableToolCallResolution === true && options.toolCallValidation !== "deferred" ? makeToolkitWithEncodedParameters(toolkit) : makeToolkitWithOpaqueParameters(toolkit) )) @@ -1523,6 +1543,9 @@ export const make: (params: { return streamWithNonIncrementalFallback().pipe( Stream.mapArrayEffect((parts) => decodeParts(parts).pipe( + options.toolCallValidation === "deferred" + ? Effect.tap(() => validateProviderExecutedToolCalls(toolkit, parts, "encoded")) + : identity, tracker ? Effect.tap((decodedParts) => { for (const part of decodedParts) { @@ -2391,7 +2414,8 @@ const makeToolkitWithOpaqueParameters = > // Provider-executed tools bypass Toolkit, so validate their parameters here. const validateProviderExecutedToolCalls = >( toolkit: Toolkit.WithHandler, - parts: ReadonlyArray + parts: ReadonlyArray, + parametersMode: "decoded" | "encoded" = "decoded" ): Effect.Effect => Effect.forEach( parts, @@ -2404,7 +2428,9 @@ const validateProviderExecutedToolCalls = }, { discard: true } diff --git a/packages/effect/test/unstable/ai/LanguageModel.test.ts b/packages/effect/test/unstable/ai/LanguageModel.test.ts index c7ff045b2e0..fe4b61f5631 100644 --- a/packages/effect/test/unstable/ai/LanguageModel.test.ts +++ b/packages/effect/test/unstable/ai/LanguageModel.test.ts @@ -69,6 +69,139 @@ describe("LanguageModel", () => { response: undefined } + describe("deferred tool parameter validation", () => { + for (const method of ["generateText", "streamText"] as const) { + it.effect(`${method} defers application parameters without changing definitions or running handlers`, () => + Effect.gen(function*() { + const handlers = ReturnModeToolkit.toLayer({ + ReturnModeTool: () => Effect.die("handler must not run") + }) + const respond = (options: LanguageModel.ProviderOptions) => { + strictEqual(options.tools[0], ReturnModeTool) + return [ + { type: "tool-call", id: "invalid", name: "ReturnModeTool", params: { testParam: 123 } }, + { type: "tool-call", id: "valid", name: "ReturnModeTool", params: { testParam: "valid" } }, + finishPart + ] satisfies Array + } + const options = { + prompt: [], + toolkit: ReturnModeToolkit, + disableToolCallResolution: true, + toolCallValidation: "deferred" + } as const + const request = method === "generateText" + ? LanguageModel.generateText(options).pipe(Effect.map((response) => response.toolCalls)) + : LanguageModel.streamText(options).pipe( + Stream.filter((part) => part.type === "tool-call"), + Stream.runCollect + ) + const calls = yield* request.pipe( + TestUtils.withLanguageModel({ generateText: respond, streamText: respond }), + Effect.provide(handlers) + ) + deepStrictEqual(calls.map((call) => call.params), [{ testParam: 123 }, { testParam: "valid" }]) + })) + + for ( + const [label, part] of [ + ["invalid provider parameters", { + type: "tool-call", + id: "hosted", + name: "MyTool", + params: { testParam: 123 }, + providerExecuted: true + }], + ["unknown tool names", { type: "tool-call", id: "unknown", name: "UnknownTool", params: {} }], + ["invalid provider results", { + type: "tool-result", + id: "hosted", + name: "MyTool", + result: { testSuccess: 123 }, + isFailure: false, + providerExecuted: true + }] + ] satisfies Array<[string, Response.ToolCallPartEncoded | Response.ToolResultPartEncoded]> + ) { + it.effect(`${method} still rejects ${label}`, () => + Effect.gen(function*() { + const options = { + prompt: [], + toolkit: MyToolkit, + disableToolCallResolution: true, + toolCallValidation: "deferred" + } as const + const request = method === "generateText" + ? LanguageModel.generateText(options).pipe(Effect.asVoid) + : LanguageModel.streamText(options).pipe(Stream.runDrain) + const error = yield* request.pipe( + TestUtils.withLanguageModel({ generateText: [part, finishPart], streamText: [part, finishPart] }), + Effect.flip + ) + strictEqual(error.reason._tag, "InvalidOutputError") + })) + } + } + + it.effect("keeps transformed parameters encoded for both application and provider calls", () => + Effect.gen(function*() { + const parts = yield* LanguageModel.streamText({ + prompt: [], + toolkit: TransformToolkit, + disableToolCallResolution: true, + toolCallValidation: "deferred" + }).pipe( + Stream.runCollect, + TestUtils.withLanguageModel({ + streamText: [ + { type: "tool-call", id: "application", name: "TransformTool", params: "21" }, + { type: "tool-call", id: "provider", name: "TransformTool", params: "22", providerExecuted: true }, + finishPart + ] + }) + ) + const calls = parts.filter((part) => part.type === "tool-call") + deepStrictEqual(calls.map((call) => call.params), ["21", "22"]) + })) + + it.effect("preserves provider failure and finalization after an invalid application call", () => + Effect.gen(function*() { + let finalized = false + const failure = AiError.make({ + module: "Test", + method: "streamText", + reason: new AiError.InvalidRequestError({ description: "provider failed" }) + }) + const error = yield* LanguageModel.streamText({ + prompt: [], + toolkit: MyToolkit, + disableToolCallResolution: true, + toolCallValidation: "deferred" + }).pipe( + Stream.runDrain, + TestUtils.withLanguageModel({ + streamText: () => + Stream.succeed( + { + type: "tool-call", + id: "invalid", + name: "MyTool", + params: { testParam: 123 } + } satisfies Response.StreamPartEncoded + ).pipe( + Stream.concat(Stream.fail(failure)), + Stream.ensuring(Effect.sync(() => { + finalized = true + })) + ) + }), + Effect.flip + ) + strictEqual(error, failure) + strictEqual(finalized, true) + })) + }) + describe("generateText", () => { it.effect("does not resolve tool calls after an incomplete finish", () => Effect.gen(function*() { diff --git a/packages/effect/typetest/unstable/ai/LanguageModel.tst.ts b/packages/effect/typetest/unstable/ai/LanguageModel.tst.ts index 894d60e9302..7e633a7a438 100644 --- a/packages/effect/typetest/unstable/ai/LanguageModel.tst.ts +++ b/packages/effect/typetest/unstable/ai/LanguageModel.tst.ts @@ -55,6 +55,58 @@ const AsymmetricParamsTool = Tool.make("AsymmetricParamsTool", { }) describe("LanguageModel", () => { + it("keeps deferred parameters unknown without handler requirements or errors", () => { + const toolkit = Toolkit.make(FailureModeErrorTool, ToolWithRequestContext, AsymmetricParamsTool) + const options = { + prompt: "hello", + toolkit, + disableToolCallResolution: true, + toolCallValidation: "deferred" + } as const + const program = LanguageModel.generateText(options) + const stream = LanguageModel.streamText(options) + const object = LanguageModel.generateObject({ ...options, schema: Schema.Struct({ answer: Schema.String }) }) + const chat = null as unknown as Chat.Chat + const chatProgram = chat.generateText(options) + + type Output = Effect.Success + type Part = Stream.Success + expect().type.toBe() + expect["params"]>().type.toBe() + expect["toolCalls"][number]["params"]>().type.toBe() + expect["toolCalls"][number]["params"]>().type.toBe() + expect>().type.toBe() + expect>().type.toBe() + expect>().type.toBe() + expect>().type.toBe() + }) + + it("keeps parameters unknown when validation may be deferred", () => { + const toolCallValidation = null as unknown as "strict" | "deferred" | undefined + const program = LanguageModel.generateText({ + prompt: "hello", + toolkit: Toolkit.make(TransformTool), + disableToolCallResolution: true, + toolCallValidation + }) + expect["toolCalls"][number]["params"]>().type.toBe() + const options: { readonly toolCallValidation?: "strict" | "deferred" } = {} + const optional = LanguageModel.generateText({ + ...options, + prompt: "hello", + toolkit: Toolkit.make(TransformTool), + disableToolCallResolution: true + }) + expect["toolCalls"][number]["params"]>().type.toBe() + const strict = LanguageModel.generateText({ + prompt: "hello", + toolkit: Toolkit.make(TransformTool), + disableToolCallResolution: true, + toolCallValidation: "strict" + }) + expect["toolCalls"][number]["params"]>().type.toBe() + }) + describe("generateText", () => { it("uses encoded tool parameters when tool call resolution is disabled", () => { const toolkit = Toolkit.make(TransformTool)