From 1b9f09691082f223b9609f26a5a599023d09fbb2 Mon Sep 17 00:00:00 2001 From: Andrew Lee Date: Tue, 1 Sep 2026 22:37:01 -0600 Subject: [PATCH] unstable/ai: recover tool calls which Toolkit never sees A tool call naming a tool which is not in the toolkit fails the whole operation today: `Response.Part(toolkit)` has no member for it, and even if it decoded, `resolveToolCalls` would skip it and leave it unanswered, which the next request rejects. For an agent whose toolkit changes between turns that is an ordinary event rather than a hallucination. `generateText` and `streamText` accept `unknownToolCalls`. With `"return"` such a call comes back as a `tool-call-error` part carrying the original call, its JSON parameters, and a `ToolNotFoundError`, and `Prompt.fromResponseParts` adds it to history as the call plus a failed tool result so the model can correct it. The default is unchanged, and the part is absent from the response type unless the option is set. An operation with no toolkit, or an empty one, treats every tool call as unknown, so the option applies there too. A call which names a tool in the toolkit is untouched: `Toolkit` routes a failure of its parameters through that tool's `failureMode`, and with resolution disabled it fails the operation as before. Recovery is driven by the decode which already validates the response: parts are decoded one at a time, and a failure is recoverable only when the part is a call to a tool which is not in the toolkit. Parameters are never validated twice, and any other decode failure fails the operation as before. `HttpRequestDetails` and `HttpResponseDetails` move to a leaf module so `Response` can reference `AiError` without a cycle. Co-Authored-By: Claude Opus 5 (1M context) Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_019wFhhXHNc4Fnkye1orWQqr --- .changeset/return-invalid-tool-calls.md | 27 + packages/effect/src/unstable/ai/AiError.ts | 2 +- packages/effect/src/unstable/ai/Chat.ts | 25 +- .../effect/src/unstable/ai/LanguageModel.ts | 266 +++++++-- packages/effect/src/unstable/ai/Prompt.ts | 25 + packages/effect/src/unstable/ai/Response.ts | 242 +++++++-- .../src/unstable/ai/internal/ai-error.ts | 14 + .../src/unstable/ai/internal/http-details.ts | 28 + .../effect/src/unstable/ai/internal/json.ts | 23 + .../test/unstable/ai/LanguageModel.test.ts | 510 ++++++++++++++++++ .../effect/test/unstable/ai/Response.test.ts | 29 +- .../typetest/unstable/ai/LanguageModel.tst.ts | 30 ++ 12 files changed, 1117 insertions(+), 104 deletions(-) create mode 100644 .changeset/return-invalid-tool-calls.md create mode 100644 packages/effect/src/unstable/ai/internal/ai-error.ts create mode 100644 packages/effect/src/unstable/ai/internal/http-details.ts create mode 100644 packages/effect/src/unstable/ai/internal/json.ts diff --git a/.changeset/return-invalid-tool-calls.md b/.changeset/return-invalid-tool-calls.md new file mode 100644 index 00000000000..82a41bfedd0 --- /dev/null +++ b/.changeset/return-invalid-tool-calls.md @@ -0,0 +1,27 @@ +--- +"effect": patch +--- + +Recover tool calls which `Toolkit` never sees + +A tool call naming a tool which is not in the toolkit fails the whole +operation today: the response schema has no member for it, and even if it +decoded, `Toolkit` would skip it and leave it unanswered. + +`generateText` and `streamText` now accept `unknownToolCalls`. With `"return"`, +such a call comes back as a `tool-call-error` response part, and +`Prompt.fromResponseParts` adds it to history as the original tool call with a +failed tool result, so the model can correct the call on the next turn. The +default, `"error"`, is unchanged. A call which names a tool in the toolkit is +unaffected: `Toolkit` still routes a failure of its parameters through that +tool's `failureMode`. + +``` +const response = yield* LanguageModel.generateText({ + prompt, + toolkit, + unknownToolCalls: "return" +}) + +response.toolCallErrors // tool-call-error parts: { name, params, error } +``` diff --git a/packages/effect/src/unstable/ai/AiError.ts b/packages/effect/src/unstable/ai/AiError.ts index dc3d8cfe913..5ff7a1944ff 100644 --- a/packages/effect/src/unstable/ai/AiError.ts +++ b/packages/effect/src/unstable/ai/AiError.ts @@ -20,7 +20,7 @@ import { redact } from "../../Redactable.ts" import * as Redacted from "../../Redacted.ts" import * as Schema from "../../Schema.ts" import type * as HttpClientError from "../http/HttpClientError.ts" -import { HttpRequestDetails, HttpResponseDetails } from "./Response.ts" +import { HttpRequestDetails, HttpResponseDetails } from "./internal/http-details.ts" const ReasonTypeId = "~effect/ai/AiError/Reason" as const diff --git a/packages/effect/src/unstable/ai/Chat.ts b/packages/effect/src/unstable/ai/Chat.ts index 120664ebba3..0bed3f6f9ea 100644 --- a/packages/effect/src/unstable/ai/Chat.ts +++ b/packages/effect/src/unstable/ai/Chat.ts @@ -225,7 +225,7 @@ export interface Service { , Options>>( options: Options & { readonly toolkit?: undefined } & LanguageModel.GenerateTextOptions<{}> ): Effect.Effect< - LanguageModel.GenerateTextResponse<{}>, + LanguageModel.GenerateTextResponse<{}, "decoded", LanguageModel.ExtractUnknownToolCalls>, LanguageModel.ExtractError, LanguageModel.LanguageModel | LanguageModel.ExtractServices > @@ -240,7 +240,11 @@ export interface Service { readonly toolkit: LanguageModel.ToolkitInput } ): Effect.Effect< - LanguageModel.GenerateTextResponse>, + LanguageModel.GenerateTextResponse< + Tools, + LanguageModel.ExtractToolParametersMode, + LanguageModel.ExtractUnknownToolCalls + >, LanguageModel.ExtractError, LanguageModel.LanguageModel | LanguageModel.ExtractServices > @@ -255,7 +259,8 @@ export interface Service { ): Effect.Effect< LanguageModel.GenerateTextResponse< LanguageModel.ExtractTools, - LanguageModel.ExtractToolParametersMode + LanguageModel.ExtractToolParametersMode, + LanguageModel.ExtractUnknownToolCalls >, LanguageModel.ExtractError, LanguageModel.LanguageModel | LanguageModel.ExtractServices @@ -307,7 +312,7 @@ export interface Service { , Options>>( options: Options & { readonly toolkit?: undefined } & LanguageModel.GenerateTextOptions<{}> ): Stream.Stream< - Response.StreamPart<{}>, + Response.StreamPart<{}, "decoded", LanguageModel.ExtractUnknownToolCalls>, LanguageModel.ExtractError, LanguageModel.LanguageModel | LanguageModel.ExtractServices > @@ -322,7 +327,11 @@ export interface Service { readonly toolkit: LanguageModel.ToolkitInput } ): Stream.Stream< - Response.StreamPart>, + Response.StreamPart< + Tools, + LanguageModel.ExtractToolParametersMode, + LanguageModel.ExtractUnknownToolCalls + >, LanguageModel.ExtractError, LanguageModel.LanguageModel | LanguageModel.ExtractServices > @@ -337,7 +346,8 @@ export interface Service { ): Stream.Stream< Response.StreamPart< LanguageModel.ExtractTools, - LanguageModel.ExtractToolParametersMode + LanguageModel.ExtractToolParametersMode, + LanguageModel.ExtractUnknownToolCalls >, LanguageModel.ExtractError, LanguageModel.LanguageModel | LanguageModel.ExtractServices @@ -399,7 +409,8 @@ export interface Service { LanguageModel.GenerateObjectResponse< LanguageModel.ExtractTools, ObjectSchema["Type"], - LanguageModel.ExtractToolParametersMode + LanguageModel.ExtractToolParametersMode, + LanguageModel.ExtractUnknownToolCalls >, LanguageModel.ExtractError, LanguageModel.ExtractServices | ObjectSchema["DecodingServices"] | LanguageModel.LanguageModel diff --git a/packages/effect/src/unstable/ai/LanguageModel.ts b/packages/effect/src/unstable/ai/LanguageModel.ts index 97a8393ed8e..4697b67a0ab 100644 --- a/packages/effect/src/unstable/ai/LanguageModel.ts +++ b/packages/effect/src/unstable/ai/LanguageModel.ts @@ -20,6 +20,7 @@ import type * as JsonSchema from "../../JsonSchema.ts" import * as Option from "../../Option.ts" import * as Predicate from "../../Predicate.ts" import * as Queue from "../../Queue.ts" +import * as Result from "../../Result.ts" import * as Schema from "../../Schema.ts" import * as SchemaAST from "../../SchemaAST.ts" import * as Semaphore from "../../Semaphore.ts" @@ -29,7 +30,9 @@ import type { Span } from "../../Tracer.ts" import type { Concurrency, Mutable, NoExcessProperties } from "../../Types.ts" import * as AiError from "./AiError.ts" import { defaultIdGenerator, IdGenerator } from "./IdGenerator.ts" +import { encodeAiError } from "./internal/ai-error.ts" import * as InternalCodecTransformer from "./internal/codec-transformer.ts" +import { toJson } from "./internal/json.ts" import * as Prompt from "./Prompt.ts" import * as Response from "./Response.ts" import * as ResponseIdTracker from "./ResponseIdTracker.ts" @@ -101,7 +104,7 @@ export interface Service { >( options: Options & GenerateTextOptionsWithoutToolkit ): Effect.Effect< - GenerateTextResponse<{}>, + GenerateTextResponse<{}, "decoded", ExtractUnknownToolCalls>, ExtractError, ExtractServices > @@ -115,7 +118,7 @@ export interface Service { >( options: Options & GenerateTextOptions & { readonly toolkit: ToolkitInput } ): Effect.Effect< - GenerateTextResponse>, + GenerateTextResponse, ExtractUnknownToolCalls>, ExtractError, ExtractServices > @@ -127,7 +130,11 @@ export interface Service { >( options: Options & GenerateTextOptions> & { readonly toolkit: Options["toolkit"] } ): Effect.Effect< - GenerateTextResponse, ExtractToolParametersMode>, + GenerateTextResponse< + ExtractTools, + ExtractToolParametersMode, + ExtractUnknownToolCalls + >, ExtractError, ExtractServices > @@ -147,7 +154,12 @@ export interface Service { >( options: Options & GenerateObjectOptions ) => Effect.Effect< - GenerateObjectResponse>, + GenerateObjectResponse< + Tools, + StructuredOutputSchema["Type"], + ExtractToolParametersMode, + ExtractUnknownToolCalls + >, ExtractError, ExtractServices | StructuredOutputSchema["DecodingServices"] > @@ -160,7 +172,7 @@ export interface Service { >( options: Options & GenerateTextOptionsWithoutToolkit ): Stream.Stream< - Response.StreamPart<{}>, + Response.StreamPart<{}, "decoded", ExtractUnknownToolCalls>, ExtractError, ExtractServices > @@ -174,7 +186,7 @@ export interface Service { >( options: Options & GenerateTextOptions & { readonly toolkit: ToolkitInput } ): Stream.Stream< - Response.StreamPart>, + Response.StreamPart, ExtractUnknownToolCalls>, ExtractError, ExtractServices > @@ -186,7 +198,11 @@ export interface Service { >( options: Options & GenerateTextOptions> & { readonly toolkit: Options["toolkit"] } ): Stream.Stream< - Response.StreamPart, ExtractToolParametersMode>, + Response.StreamPart< + ExtractTools, + ExtractToolParametersMode, + ExtractUnknownToolCalls + >, ExtractError, ExtractServices > @@ -288,6 +304,31 @@ export interface GenerateTextOptions> { * resolver execution yourself. */ readonly disableToolCallResolution?: boolean | undefined + + /** + * How a tool call to a tool which is not in the toolkit is handled. + * + * **Details** + * + * With `"error"` (the default), such a call fails the operation with an + * `AiError` whose reason is `ToolNotFoundError`. + * + * With `"return"`, the call is returned as a `tool-call-error` response part + * carrying the original call, its JSON parameters, and that error. No + * handler runs for it, and `Prompt.fromResponseParts` adds it to history as + * a failed tool result, so the model can correct and retry the call. + * + * Use it when the toolkit changes between turns, so that a model calling a + * tool which has since been removed can be told to try again rather than + * failing the turn. An operation with no toolkit, or an empty one, treats + * every tool call as such a call. + * + * A call which does name a tool in the toolkit is not governed by this + * option. `Toolkit` routes a failure of its parameters through that tool's + * own `failureMode`, and with `disableToolCallResolution: true` such a call + * fails the operation as it does today. + */ + readonly unknownToolCalls?: "error" | "return" | undefined } type GenerateTextOptionsWithoutToolkit = Omit, "toolkit"> & { @@ -366,11 +407,12 @@ export type ToolChoice = */ export class GenerateTextResponse< Tools extends Record, - ParametersMode extends Response.ToolParametersMode = "decoded" + ParametersMode extends Response.ToolParametersMode = "decoded", + UnknownToolCalls extends "error" | "return" = "error" > { - readonly content: Array> + readonly content: Array> - constructor(content: Array>) { + constructor(content: Array>) { this.content = content } @@ -414,6 +456,19 @@ export class GenerateTextResponse< return this.content.filter((part) => part.type === "tool-call") } + /** + * Returns all tool call error parts from the response. + * + * **Details** + * + * Tool call errors are only produced when the operation was run with + * `unknownToolCalls: "return"` and the model called a tool which is not in + * the toolkit. + */ + get toolCallErrors(): Array> { + return this.content.filter((part) => part.type === "tool-call-error") + } + /** * Returns all tool result parts from the response. */ @@ -476,14 +531,15 @@ export class GenerateTextResponse< export class GenerateObjectResponse< Tools extends Record, A, - ParametersMode extends Response.ToolParametersMode = "decoded" -> extends GenerateTextResponse { + ParametersMode extends Response.ToolParametersMode = "decoded", + UnknownToolCalls extends "error" | "return" = "error" +> extends GenerateTextResponse { /** * The parsed structured object that conforms to the provided schema. */ readonly value: A - constructor(value: A, content: Array>) { + constructor(value: A, content: Array>) { super(content) this.value = value } @@ -569,6 +625,20 @@ export type ExtractToolParametersMode = Options extends { } ? "encoded" : "opaque" +/** + * Utility type that determines how the `unknownToolCalls` option of a + * language model operation was set. An option which is not statically known + * keeps both modes, so the response type admits the part the operation may + * produce. + * + * @category utility types + * @since 4.0.0 + */ +export type ExtractUnknownToolCalls = Options extends { + readonly unknownToolCalls: infer Mode +} ? [Exclude] extends [never] ? "error" : Extract + : "error" + type ExtractErrorFromToolkitOption = ToolkitValue extends Toolkit.WithHandler ? | AiError.AiError @@ -877,7 +947,12 @@ export const make: (params: { >( options: Options & GenerateObjectOptions ): Effect.Effect< - GenerateObjectResponse>, + GenerateObjectResponse< + Tools, + StructuredOutputSchema["Type"], + ExtractToolParametersMode, + ExtractUnknownToolCalls + >, ExtractError, ExtractServices | StructuredOutputSchema["DecodingServices"] > => { @@ -1065,6 +1140,17 @@ export const make: (params: { ) const hasPendingApprovals = approved.length > 0 || denied.length > 0 + const unknownToolCalls = options.unknownToolCalls ?? "error" + // Decodes a response against `toolkit`, recovering the tool calls which + // can never reach a handler + const makeContentDecoder = (toolkit: Toolkit.Any | Toolkit.WithHandler) => { + const decodePart = Schema.decodeUnknownEffect(Response.Part(toolkit, { unknownToolCalls })) as ( + part: unknown + ) => Effect.Effect, Schema.SchemaError> + return (parts: ReadonlyArray) => + decodeRecoveringToolCalls(decodePart, parts, toolkit, "generateText", unknownToolCalls) + } + // If there is no toolkit, the generated content can be returned immediately if (Predicate.isUndefined(options.toolkit)) { // But first check if we have pending approvals that require a toolkit @@ -1086,11 +1172,8 @@ export const make: (params: { providerOptions.incrementalPrompt = prepared.value.prompt } } - const ResponseSchema = Schema.mutable( - Schema.Array(Response.Part(Toolkit.empty)) - ) const rawContent = yield* generateWithNonIncrementalFallback() - const content = yield* Schema.decodeEffect(ResponseSchema)(rawContent) + const content = yield* makeContentDecoder(Toolkit.empty)(rawContent) if (tracker) { const responseMetadata = content.find((part) => part.type === "response-metadata") if (Predicate.isNotUndefined(responseMetadata) && Predicate.isNotUndefined(responseMetadata.id)) { @@ -1124,11 +1207,8 @@ export const make: (params: { providerOptions.incrementalPrompt = prepared.value.prompt } } - const ResponseSchema = Schema.mutable( - Schema.Array(Response.Part(Toolkit.empty)) - ) const rawContent = yield* generateWithNonIncrementalFallback() - const content = yield* Schema.decodeEffect(ResponseSchema)(rawContent) + const content = yield* makeContentDecoder(Toolkit.empty)(rawContent) if (tracker) { const responseMetadata = content.find((part) => part.type === "response-metadata") if (Predicate.isNotUndefined(responseMetadata) && Predicate.isNotUndefined(responseMetadata.id)) { @@ -1199,17 +1279,17 @@ export const make: (params: { } } - const ResponseSchema = Schema.mutable(Schema.Array(Response.Part( + const decodeContent = makeContentDecoder( options.disableToolCallResolution === true ? makeToolkitWithEncodedParameters(toolkit) : makeToolkitWithOpaqueParameters(toolkit) - ))) + ) // If tool call resolution is disabled, return the response without // resolving the tool calls that were generated if (options.disableToolCallResolution === true) { const rawContent = yield* generateWithNonIncrementalFallback() - const content = yield* Schema.decodeEffect(ResponseSchema)(rawContent) + const content = yield* decodeContent(rawContent) if (tracker) { const responseMetadata = content.find((part) => part.type === "response-metadata") if (Predicate.isNotUndefined(responseMetadata) && Predicate.isNotUndefined(responseMetadata.id)) { @@ -1222,7 +1302,7 @@ export const make: (params: { const rawContent = yield* generateWithNonIncrementalFallback() // Validate before running tool handlers. - const content = yield* Schema.decodeEffect(ResponseSchema)(rawContent) + const content = yield* decodeContent(rawContent) yield* validateProviderExecutedToolCalls(toolkit, rawContent) // Resolve the generated tool calls. When the finish reason indicates an @@ -1328,6 +1408,17 @@ export const make: (params: { }) const hasPendingApprovals = pendingApproved.length > 0 || pendingDenied.length > 0 + const unknownToolCalls = options.unknownToolCalls ?? "error" + // Decodes a chunk of the stream against `toolkit`, recovering the tool + // calls which can never reach a handler + const makePartsDecoder = (toolkit: Toolkit.Any | Toolkit.WithHandler) => { + const decodePart = Schema.decodeUnknownEffect(Response.StreamPart(toolkit, { unknownToolCalls })) as ( + part: unknown + ) => Effect.Effect, Schema.SchemaError> + return (parts: readonly [Response.StreamPartEncoded, ...Array]) => + decodeRecoveringToolCalls(decodePart, parts, toolkit, "streamText", unknownToolCalls) + } + // If there is no toolkit, return immediately if (Predicate.isUndefined(options.toolkit)) { // But first check if we have pending approvals that require a toolkit @@ -1349,8 +1440,7 @@ export const make: (params: { providerOptions.incrementalPrompt = prepared.value.prompt } } - const schema = Schema.NonEmptyArray(Response.StreamPart(Toolkit.empty)) - const decodeParts = Schema.decodeEffect(schema) + const decodeParts = makePartsDecoder(Toolkit.empty) return pipe( streamWithNonIncrementalFallback(), Stream.mapArrayEffect((parts) => @@ -1398,8 +1488,7 @@ export const make: (params: { providerOptions.incrementalPrompt = prepared.value.prompt } } - const schema = Schema.NonEmptyArray(Response.StreamPart(Toolkit.empty)) - const decodeParts = Schema.decodeEffect(schema) + const decodeParts = makePartsDecoder(Toolkit.empty) return pipe( streamWithNonIncrementalFallback(), Stream.mapArrayEffect((parts) => @@ -1501,12 +1590,11 @@ export const make: (params: { } } - const ResponseSchema = Schema.NonEmptyArray(Response.StreamPart( + const decodeParts = makePartsDecoder( options.disableToolCallResolution === true ? makeToolkitWithEncodedParameters(toolkit) : makeToolkitWithOpaqueParameters(toolkit) - )) - const decodeParts = Schema.decodeEffect(ResponseSchema) + ) // If tool call resolution is disabled, return the response without // resolving the tool calls that were generated @@ -1772,7 +1860,7 @@ export const generateText: { >( options: Options & GenerateTextOptionsWithoutToolkit ): Effect.Effect< - GenerateTextResponse<{}>, + GenerateTextResponse<{}, "decoded", ExtractUnknownToolCalls>, ExtractError, LanguageModel | ExtractServices > @@ -1783,7 +1871,7 @@ export const generateText: { >( options: Options & GenerateTextOptions & { readonly toolkit: ToolkitInput } ): Effect.Effect< - GenerateTextResponse>, + GenerateTextResponse, ExtractUnknownToolCalls>, ExtractError, LanguageModel | ExtractServices > @@ -1795,7 +1883,11 @@ export const generateText: { >( options: Options & GenerateTextOptions> & { readonly toolkit: Options["toolkit"] } ): Effect.Effect< - GenerateTextResponse, ExtractToolParametersMode>, + GenerateTextResponse< + ExtractTools, + ExtractToolParametersMode, + ExtractUnknownToolCalls + >, ExtractError, ExtractServices | LanguageModel > @@ -1864,7 +1956,8 @@ export const generateObject = < GenerateObjectResponse< ExtractTools, StructuredOutputSchema["Type"], - ExtractToolParametersMode + ExtractToolParametersMode, + ExtractUnknownToolCalls >, ExtractError, ExtractServices | StructuredOutputSchema["DecodingServices"] | LanguageModel @@ -1920,7 +2013,7 @@ export const streamText: { >( options: Options & GenerateTextOptionsWithoutToolkit ): Stream.Stream< - Response.StreamPart<{}>, + Response.StreamPart<{}, "decoded", ExtractUnknownToolCalls>, ExtractError, ExtractServices | LanguageModel > @@ -1931,7 +2024,7 @@ export const streamText: { >( options: Options & GenerateTextOptions & { readonly toolkit: ToolkitInput } ): Stream.Stream< - Response.StreamPart>, + Response.StreamPart, ExtractUnknownToolCalls>, ExtractError, ExtractServices | LanguageModel > @@ -1943,7 +2036,11 @@ export const streamText: { >( options: Options & GenerateTextOptions> & { readonly toolkit: Options["toolkit"] } ): Stream.Stream< - Response.StreamPart, ExtractToolParametersMode>, + Response.StreamPart< + ExtractTools, + ExtractToolParametersMode, + ExtractUnknownToolCalls + >, ExtractError, ExtractServices | LanguageModel > @@ -2378,6 +2475,95 @@ const makeToolkitWithOpaqueParameters = > ...Object.values(toolkit.tools).map((tool) => tool.setParameters(Schema.Unknown)) ) +// Decodes response parts one at a time, so that a decode failure can be traced +// to the part which caused it. A failure is recoverable only when that part is +// a call to a tool which is not in the toolkit: such a call can never reach a +// handler, so no `failureMode` speaks for it. A call which names a tool in the +// toolkit is left to `Toolkit`, which routes a parameter failure through the +// tool's own `failureMode`, so parameters are never validated twice. Any other +// decode failure fails the operation exactly as it does by default. Every part +// decodes to exactly one part, so a non-empty chunk stays non-empty. +function decodeRecoveringToolCalls< + A, + Part extends Response.PartEncoded | Response.StreamPartEncoded +>( + decodePart: (part: unknown) => Effect.Effect, + parts: readonly [Part, ...Array], + toolkit: Toolkit.Any | Toolkit.WithHandler, + method: "generateText" | "streamText", + unknownToolCalls: "error" | "return" +): Effect.Effect<[A, ...Array], Schema.SchemaError> +function decodeRecoveringToolCalls< + A, + Part extends Response.PartEncoded | Response.StreamPartEncoded +>( + decodePart: (part: unknown) => Effect.Effect, + parts: ReadonlyArray, + toolkit: Toolkit.Any | Toolkit.WithHandler, + method: "generateText" | "streamText", + unknownToolCalls: "error" | "return" +): Effect.Effect, Schema.SchemaError> +function decodeRecoveringToolCalls< + A, + Part extends Response.PartEncoded | Response.StreamPartEncoded +>( + decodePart: (part: unknown) => Effect.Effect, + parts: ReadonlyArray, + toolkit: Toolkit.Any | Toolkit.WithHandler, + method: "generateText" | "streamText", + unknownToolCalls: "error" | "return" +): Effect.Effect, Schema.SchemaError> { + return Effect.forEach(parts, (part) => + Effect.flatMap(Effect.result(decodePart(part)), (decoded) => { + if (Result.isSuccess(decoded)) { + return Effect.succeed(decoded.success) + } + // Provider-executed calls already ran, so they are validated as response + // output rather than returned to the model + if (part.type !== "tool-call" || part.providerExecuted === true) { + return Effect.fail(decoded.failure) + } + // A recovered call is described by its `id` and `name`, and `AiError` + // reasons validate their own fields. A call which does not carry both as + // strings is malformed rather than unresolvable, and keeps failing. + if (!Predicate.isString(part.name) || !Predicate.isString(part.id)) { + return Effect.fail(decoded.failure) + } + // A call which names a tool in the toolkit belongs to that tool, and + // `Toolkit` routes its parameter failure through the tool's own + // `failureMode` when it resolves the call. When resolution is disabled + // the caller owns the call, so it fails here exactly as it does today. + if (Object.hasOwn(toolkit.tools, part.name)) { + return Effect.fail(decoded.failure) + } + // Only a call which names no tool at all has no declaration to decide + // for it, so it is the operation which opts in. + if (unknownToolCalls !== "return") { + return Effect.fail(decoded.failure) + } + const reason = new AiError.ToolNotFoundError({ + toolName: part.name, + availableTools: Object.keys(toolkit.tools) + }) + // The recovered part is decoded rather than constructed: the call which + // failed to decode may be malformed in a way which has nothing to do + // with the tool - a non-string `id`, invalid metadata - and such a part + // must keep failing the operation instead of being returned to the model + // under a misleading error. + return Effect.catchCause( + decodePart({ + type: "tool-call-error", + id: part.id, + name: part.name, + params: toJson(part.params), + error: encodeAiError(AiError.make({ module: "LanguageModel", method, reason })), + metadata: part.metadata + }), + () => Effect.fail(decoded.failure) + ) + })) +} + // Provider-executed tools bypass Toolkit, so validate their parameters here. const validateProviderExecutedToolCalls = >( toolkit: Toolkit.WithHandler, diff --git a/packages/effect/src/unstable/ai/Prompt.ts b/packages/effect/src/unstable/ai/Prompt.ts index 04a76862614..b59efd022d9 100644 --- a/packages/effect/src/unstable/ai/Prompt.ts +++ b/packages/effect/src/unstable/ai/Prompt.ts @@ -19,6 +19,7 @@ import * as Schema from "../../Schema.ts" import * as SchemaIssue from "../../SchemaIssue.ts" import * as SchemaParser from "../../SchemaParser.ts" import * as SchemaTransformation from "../../SchemaTransformation.ts" +import { encodeAiError } from "./internal/ai-error.ts" import type * as Response from "./Response.ts" // ============================================================================= @@ -2152,6 +2153,30 @@ export const fromResponseParts = (parts: ReadonlyArray): Promp break } + // Tool Call Error Parts + // + // The original call is preserved as an assistant tool call and the + // validation error becomes its failed result, so the model can correct + // the call on the next turn + case "tool-call-error": { + assistantParts.push(makePart("tool-call", { + id: part.id, + name: part.name, + params: part.params, + providerExecuted: false, + options: part.metadata + })) + toolParts.push(makePart("tool-result", { + id: part.id, + name: part.name, + isFailure: true, + result: encodeAiError(part.error), + providerExecuted: false, + options: part.metadata + })) + break + } + // Tool Result Parts (skip preliminary results) case "tool-result": { if (part.preliminary !== true) { diff --git a/packages/effect/src/unstable/ai/Response.ts b/packages/effect/src/unstable/ai/Response.ts index 4bfabb6255a..3f53651d14b 100644 --- a/packages/effect/src/unstable/ai/Response.ts +++ b/packages/effect/src/unstable/ai/Response.ts @@ -16,6 +16,8 @@ import { identity } from "../../Function.ts" import * as Predicate from "../../Predicate.ts" import * as Schema from "../../Schema.ts" import * as SchemaTransformation from "../../SchemaTransformation.ts" +import * as AiError from "./AiError.ts" +import * as HttpDetails from "./internal/http-details.ts" import type * as Tool from "./Tool.ts" import type * as Toolkit from "./Toolkit.ts" @@ -52,6 +54,7 @@ export type AnyPart = | ToolParamsDeltaPart | ToolParamsEndPart | ToolCallPart + | ToolCallErrorPart | ToolResultPart | ToolApprovalRequestPart | FilePart @@ -80,6 +83,7 @@ export type AnyPartEncoded = | ToolParamsDeltaPartEncoded | ToolParamsEndPartEncoded | ToolCallPartEncoded + | ToolCallErrorPartEncoded | ToolResultPartEncoded | ToolApprovalRequestPartEncoded | FilePartEncoded @@ -108,6 +112,7 @@ export type AllParts> = | ToolParamsDeltaPart | ToolParamsEndPart | ToolCallParts + | ToolCallErrorPart | ToolResultParts | ToolApprovalRequestPart | FilePart @@ -136,6 +141,7 @@ export type AllPartsEncoded = | ToolParamsDeltaPartEncoded | ToolParamsEndPartEncoded | ToolCallPartEncoded + | ToolCallErrorPartEncoded | ToolResultPartEncoded | ToolApprovalRequestPartEncoded | FilePartEncoded @@ -181,14 +187,7 @@ export const AllParts = >( Tool.ResultDecodingServices[keyof Toolkit.Tools]>, Tool.ResultEncodingServices[keyof Toolkit.Tools]> > => { - const toolCalls: Array = [] - const toolResults: Array = [] - for (const tool of Object.values(toolkit.tools as Record)) { - const toolCall = ToolCallPart(tool.name, tool.parametersSchema) - const toolResult = ToolResultPart(tool.name, tool.successSchema, tool.failureSchema) - toolCalls.push(toolCall) - toolResults.push(toolResult) - } + const { toolCalls, toolResults } = makeToolPartSchemas(toolkit) return Schema.Union([ TextPart, TextStartPart, @@ -209,6 +208,7 @@ export const AllParts = >( FinishPart, ErrorPart, ...toolCalls, + AnyToolCallErrorPart, ...toolResults ]) as any } @@ -226,11 +226,13 @@ export const AllParts = >( */ export type Part< Tools extends Record, - ParametersMode extends ToolParametersMode = "decoded" + ParametersMode extends ToolParametersMode = "decoded", + UnknownToolCalls extends "error" | "return" = "error" > = | TextPart | ReasoningPart | ToolCallParts + | ToolCallErrorParts | ToolResultParts | ToolApprovalRequestPart | FilePart @@ -251,6 +253,7 @@ export type PartEncoded = | ReasoningDeltaPartEncoded | ReasoningEndPartEncoded | ToolCallPartEncoded + | ToolCallErrorPartEncoded | ToolResultPartEncoded | ToolApprovalRequestPartEncoded | FilePartEncoded @@ -265,22 +268,20 @@ export type PartEncoded = * @category schemas * @since 4.0.0 */ -export const Part = >( - toolkit: T +export const Part = < + T extends Toolkit.Any | Toolkit.WithHandler, + const Options extends PartSchemaOptions = {} +>( + toolkit: T, + options?: Options ): Schema.Codec< - Part : Toolkit.WithHandlerTools>, + Part : Toolkit.WithHandlerTools, "decoded", UnknownToolCallsOf>, PartEncoded, Tool.ResultDecodingServices[keyof Toolkit.Tools]>, Tool.ResultEncodingServices[keyof Toolkit.Tools]> > => { - const toolCalls: Array = [] - const toolResults: Array = [] - for (const tool of Object.values(toolkit.tools as Record)) { - const toolCall = ToolCallPart(tool.name, tool.parametersSchema) - const toolResult = ToolResultPart(tool.name, tool.successSchema, tool.failureSchema) - toolCalls.push(toolCall) - toolResults.push(toolResult) - } + const { toolCalls, toolResults } = makeToolPartSchemas(toolkit) + const toolCallErrors = toolCallErrorSchemas(options) return Schema.Union([ TextPart, ReasoningPart, @@ -291,6 +292,7 @@ export const Part = >( ResponseMetadataPart, FinishPart, ...toolCalls, + ...toolCallErrors, ...toolResults ]) as any } @@ -307,7 +309,8 @@ export const Part = >( */ export type StreamPart< Tools extends Record, - ParametersMode extends ToolParametersMode = "decoded" + ParametersMode extends ToolParametersMode = "decoded", + UnknownToolCalls extends "error" | "return" = "error" > = | TextStartPart | TextDeltaPart @@ -319,6 +322,7 @@ export type StreamPart< | ToolParamsDeltaPart | ToolParamsEndPart | ToolCallParts + | ToolCallErrorParts | ToolResultParts | ToolApprovalRequestPart | FilePart @@ -345,6 +349,7 @@ export type StreamPartEncoded = | ToolParamsDeltaPartEncoded | ToolParamsEndPartEncoded | ToolCallPartEncoded + | ToolCallErrorPartEncoded | ToolResultPartEncoded | ToolApprovalRequestPartEncoded | FilePartEncoded @@ -360,22 +365,24 @@ export type StreamPartEncoded = * @category schemas * @since 4.0.0 */ -export const StreamPart = >( - toolkit: T +export const StreamPart = < + T extends Toolkit.Any | Toolkit.WithHandler, + const Options extends PartSchemaOptions = {} +>( + toolkit: T, + options?: Options ): Schema.Codec< - StreamPart : Toolkit.WithHandlerTools>, + StreamPart< + T extends Toolkit.Any ? Toolkit.Tools : Toolkit.WithHandlerTools, + "decoded", + UnknownToolCallsOf + >, StreamPartEncoded, Tool.ResultDecodingServices[keyof Toolkit.Tools]>, Tool.ResultEncodingServices[keyof Toolkit.Tools]> > => { - const toolCalls: Array = [] - const toolResults: Array = [] - for (const tool of Object.values(toolkit.tools as Record)) { - const toolCall = ToolCallPart(tool.name, tool.parametersSchema) - const toolResult = ToolResultPart(tool.name, tool.successSchema, tool.failureSchema) - toolCalls.push(toolCall) - toolResults.push(toolResult) - } + const { toolCalls, toolResults } = makeToolPartSchemas(toolkit) + const toolCallErrors = toolCallErrorSchemas(options) return Schema.Union([ TextStartPart, TextDeltaPart, @@ -394,10 +401,45 @@ export const StreamPart = >( FinishPart, ErrorPart, ...toolCalls, + ...toolCallErrors, ...toolResults ]) as any } +/** + * Options for the response part schema constructors. + * + * @category models + * @since 4.0.0 + */ +export interface PartSchemaOptions { + /** + * The `unknownToolCalls` option of the language model operation whose + * response is being decoded. With `"return"`, the schema also admits a + * `tool-call-error` part for any tool name. + */ + readonly unknownToolCalls?: "error" | "return" | undefined +} + +type UnknownToolCallsOf = Options extends { readonly unknownToolCalls: infer Mode } ? + [Exclude] extends [never] ? "error" : Extract + : "error" + +const makeToolPartSchemas = (toolkit: Toolkit.Any | Toolkit.WithHandler) => { + const toolCalls: Array = [] + const toolResults: Array = [] + for (const tool of Object.values(toolkit.tools as Record)) { + toolCalls.push(ToolCallPart(tool.name, tool.parametersSchema)) + toolResults.push(ToolResultPart(tool.name, tool.successSchema, tool.failureSchema)) + } + return { toolCalls, toolResults } as const +} + +// A call to a tool which is not in the toolkit is only returned to the model +// as a part when the operation opted in +const toolCallErrorSchemas = (options: PartSchemaOptions | undefined): Array => + options?.unknownToolCalls === "return" ? [AnyToolCallErrorPart] : [] + // ============================================================================= // utility types // ============================================================================= @@ -433,6 +475,22 @@ type ToolCallPartForName< > : never +/** + * Utility type that extracts tool call error parts for a language model + * operation. + * + * **Details** + * + * A `tool-call-error` part is only produced when the operation was run with + * `unknownToolCalls: "return"`, so with the default `"error"` this is `never`. + * + * @category utility types + * @since 4.0.0 + */ +export type ToolCallErrorParts = UnknownToolCalls extends + "return" ? ToolCallErrorPart + : never + /** * Utility type that extracts tool result parts from a set of tools. * @@ -1453,6 +1511,101 @@ export const toolCallPart = ( params: ConstructorParams> ): ToolCallPart => makePart("tool-call", params) +// ============================================================================= +// Tool Call Error Part +// ============================================================================= + +/** + * Response part representing a tool call generated by the model which names a + * tool that is not in the toolkit. + * + * **Details** + * + * This part is only produced when the language model operation was run with + * `unknownToolCalls: "return"`. The original call is preserved so it can be + * added to the conversation history together with the error as a failed tool + * result, which allows the model to correct and retry the call. No tool + * handler is executed for a tool call error. + * + * A call which names a tool in the toolkit is never returned as this part: + * `Toolkit` routes a failure of its parameters through that tool's + * `failureMode` instead. + * + * **Example** (Inspecting a tool call error) + * + * ```ts import.meta.vitest + * import { AiError, Response } from "effect/unstable/ai" + * + * const part = Response.makePart("tool-call-error", { + * id: "call_123", + * name: "get_weather", + * params: { city: "Paris" }, + * error: AiError.make({ + * module: "LanguageModel", + * method: "generateText", + * reason: new AiError.ToolNotFoundError({ + * toolName: "get_weather", + * availableTools: ["get_forecast"] + * }) + * }) + * }) + * + * const result = [part.name, part.error.reason._tag] // => ["get_weather", "ToolNotFoundError"] + * ``` + * + * @category models + * @since 4.0.0 + */ +export interface ToolCallErrorPart extends BasePart<"tool-call-error", ToolCallPartMetadata> { + /** + * Unique identifier for this tool call. + */ + readonly id: string + /** + * Name of the tool the model attempted to call. + */ + readonly name: string + /** + * The parameters generated by the model, as JSON. + */ + readonly params: Schema.Json + /** + * The reason the tool call could not be resolved. + */ + readonly error: AiError.AiError +} + +/** + * Encoded representation of tool call error parts for serialization. + * + * @category models + * @since 4.0.0 + */ +export interface ToolCallErrorPartEncoded extends BasePartEncoded<"tool-call-error", ToolCallPartMetadata> { + readonly id: string + readonly name: string + readonly params: Schema.Json + readonly error: AiError.AiErrorEncoded +} + +/** + * Schema for a tool call error part, which carries any tool name because the + * call it describes may name a tool which is not in the toolkit. + * + * @internal + */ +export const AnyToolCallErrorPart: Schema.Codec< + ToolCallErrorPart, + ToolCallErrorPartEncoded +> = Schema.Struct({ + ...BasePart.fields, + type: Schema.Literal("tool-call-error"), + id: Schema.String, + name: Schema.String, + params: Schema.Json, + error: AiError.AiError +}).annotate({ identifier: "ToolCallErrorPart" }) as any + // ============================================================================= // Tool Call Result Part // ============================================================================= @@ -2168,19 +2321,7 @@ export const UrlSourcePart: Schema.Struct<{ * @category schemas * @since 4.0.0 */ -export const HttpRequestDetails = Schema.Struct({ - method: Schema.Literals(["GET", "POST", "PATCH", "PUT", "DELETE", "HEAD", "OPTIONS", "TRACE"]), - url: Schema.String, - urlParams: Schema.Array(Schema.Tuple([Schema.String, Schema.String])), - hash: Schema.optional(Schema.String), - headers: Schema.Record( - Schema.String, - Schema.Union([ - Schema.String, - Schema.Redacted(Schema.String) - ]) - ) -}).annotate({ identifier: "HttpRequestDetails" }) +export const HttpRequestDetails = HttpDetails.HttpRequestDetails /** * Schema for HTTP response details associated with an AI response. @@ -2209,16 +2350,7 @@ export const HttpRequestDetails = Schema.Struct({ * @category schemas * @since 4.0.0 */ -export const HttpResponseDetails = Schema.Struct({ - status: Schema.Int, - headers: Schema.Record( - Schema.String, - Schema.Union([ - Schema.String, - Schema.Redacted(Schema.String) - ]) - ) -}).annotate({ identifier: "HttpResponseDetails" }) +export const HttpResponseDetails = HttpDetails.HttpResponseDetails // ============================================================================= // Response Metadata Part diff --git a/packages/effect/src/unstable/ai/internal/ai-error.ts b/packages/effect/src/unstable/ai/internal/ai-error.ts new file mode 100644 index 00000000000..dd02e57519d --- /dev/null +++ b/packages/effect/src/unstable/ai/internal/ai-error.ts @@ -0,0 +1,14 @@ +import * as Schema from "../../../Schema.ts" +import * as AiError from "../AiError.ts" + +/** + * Encodes an `AiError` for a serialized response or prompt part. + * + * `AiError` is a self-referential `Schema.Error` class, so its service types + * widen to `any`; the cast mirrors how `Toolkit` encodes the same schema. + * + * @internal + */ +export const encodeAiError = Schema.encodeSync(AiError.AiError as any) as ( + error: AiError.AiError +) => AiError.AiErrorEncoded diff --git a/packages/effect/src/unstable/ai/internal/http-details.ts b/packages/effect/src/unstable/ai/internal/http-details.ts new file mode 100644 index 00000000000..e2e8a72e065 --- /dev/null +++ b/packages/effect/src/unstable/ai/internal/http-details.ts @@ -0,0 +1,28 @@ +import * as Schema from "../../../Schema.ts" + +// Kept in a leaf module because both AiError and Response expose these schemas. + +export const HttpRequestDetails = Schema.Struct({ + method: Schema.Literals(["GET", "POST", "PATCH", "PUT", "DELETE", "HEAD", "OPTIONS", "TRACE"]), + url: Schema.String, + urlParams: Schema.Array(Schema.Tuple([Schema.String, Schema.String])), + hash: Schema.optional(Schema.String), + headers: Schema.Record( + Schema.String, + Schema.Union([ + Schema.String, + Schema.Redacted(Schema.String) + ]) + ) +}).annotate({ identifier: "HttpRequestDetails" }) + +export const HttpResponseDetails = Schema.Struct({ + status: Schema.Int, + headers: Schema.Record( + Schema.String, + Schema.Union([ + Schema.String, + Schema.Redacted(Schema.String) + ]) + ) +}).annotate({ identifier: "HttpResponseDetails" }) diff --git a/packages/effect/src/unstable/ai/internal/json.ts b/packages/effect/src/unstable/ai/internal/json.ts new file mode 100644 index 00000000000..16d238c5fa8 --- /dev/null +++ b/packages/effect/src/unstable/ai/internal/json.ts @@ -0,0 +1,23 @@ +import { format, formatJson } from "../../../Formatter.ts" +import type * as Schema from "../../../Schema.ts" + +/** + * Normalizes an arbitrary runtime value into JSON. + * + * Tool call parameters are JSON when they come from a provider, but a custom + * `LanguageModel` implementation can produce any value. A `tool-call-error` + * part carries the parameters the model sent, so they must stay JSON-safe; + * anything else is normalized with Effect's JSON formatter (the same fallback + * `Schema.Defect` uses for non-error values), which handles bigints, circular + * references, and redacted values, and falls back to a formatted string. + * + * @internal + */ +export const toJson = (value: unknown): Schema.Json => { + try { + const json = formatJson(value) + return json === undefined ? format(value) : JSON.parse(json) + } catch { + return format(value) + } +} diff --git a/packages/effect/test/unstable/ai/LanguageModel.test.ts b/packages/effect/test/unstable/ai/LanguageModel.test.ts index e89309edf76..b258cf98bbb 100644 --- a/packages/effect/test/unstable/ai/LanguageModel.test.ts +++ b/packages/effect/test/unstable/ai/LanguageModel.test.ts @@ -3,6 +3,7 @@ import { assertDefined, assertTrue, deepStrictEqual, strictEqual } from "@effect import { type Cause, Effect, Fiber, Latch, Option, Queue, Ref, Schema, Stream } from "effect" import { TestClock } from "effect/testing" import { AiError, LanguageModel, Prompt, Response, ResponseIdTracker, Tool, Toolkit } from "effect/unstable/ai" +import * as fc from "fast-check" import * as TestUtils from "./utils.ts" const MyTool = Tool.make("MyTool", { @@ -2001,6 +2002,515 @@ describe("LanguageModel", () => { })) }) + describe("invalid tool calls", () => { + const unknownToolCall: Response.ToolCallPartEncoded = { + type: "tool-call", + id: "call-unknown", + name: "NotATool", + params: { testParam: "test-param" } + } + + it.effect("fails the operation by default when the model calls an unknown tool", () => + Effect.gen(function*() { + const error = yield* LanguageModel.generateText({ + prompt: [], + toolkit: MyToolkit + }).pipe( + TestUtils.withLanguageModel({ + generateText: [unknownToolCall, finishPart] + }), + Effect.provide(MyToolkitLayer), + Effect.flip + ) + + assertDefined(error) + })) + + it.effect("returns an unknown tool call to the model when the operation opts in", () => + Effect.gen(function*() { + const calls = yield* Ref.make(0) + const handlers = MyToolkit.toLayer({ + MyTool: () => + Ref.update(calls, (n) => n + 1).pipe( + Effect.as({ testSuccess: "test-success" }) + ) + }) + + const response = yield* LanguageModel.generateText({ + prompt: [], + toolkit: MyToolkit, + unknownToolCalls: "return" + }).pipe( + TestUtils.withLanguageModel({ + generateText: [unknownToolCall, finishPart] + }), + Effect.provide(handlers) + ) + + strictEqual(response.toolCallErrors.length, 1) + strictEqual(response.toolResults.length, 0) + strictEqual(yield* Ref.get(calls), 0) + + const toolCallError = response.toolCallErrors[0]! + strictEqual(toolCallError.name, "NotATool") + deepStrictEqual(toolCallError.params, { testParam: "test-param" }) + assertTrue(AiError.isAiError(toolCallError.error)) + strictEqual(toolCallError.error.reason._tag, "ToolNotFoundError") + })) + + it.effect("keeps the stream alive when an unknown tool call is returned", () => + Effect.gen(function*() { + const parts = yield* LanguageModel.streamText({ + prompt: [], + toolkit: MyToolkit, + unknownToolCalls: "return" + }).pipe( + Stream.runCollect, + TestUtils.withLanguageModel({ + streamText: [ + unknownToolCall, + { type: "tool-call", id: "call-known", name: "MyTool", params: { testParam: "test-param" } }, + finishPart + ] + }), + Effect.provide(MyToolkit.toLayer({ + MyTool: () => Effect.succeed({ testSuccess: "test-success" }) + })) + ) + + const toolCallErrors = parts.filter((part) => part.type === "tool-call-error") + const toolResults = parts.filter((part) => part.type === "tool-result") + + strictEqual(toolCallErrors.length, 1) + strictEqual(toolCallErrors[0]!.name, "NotATool") + // The known tool call still resolves normally + strictEqual(toolResults.length, 1) + deepStrictEqual(toolResults[0]!.result, { testSuccess: "test-success" }) + })) + + it.effect("returns an unknown tool call when the operation has no toolkit", () => + Effect.gen(function*() { + // Without a toolkit every tool call is unknown, so the option still + // decides what happens to it + const response = yield* LanguageModel.generateText({ + prompt: [], + unknownToolCalls: "return" + }).pipe( + TestUtils.withLanguageModel({ + generateText: [unknownToolCall, finishPart] + }) + ) + + strictEqual(response.toolCallErrors.length, 1) + strictEqual(response.toolCallErrors[0]!.name, "NotATool") + strictEqual(response.toolCallErrors[0]!.error.reason._tag, "ToolNotFoundError") + })) + + it.effect("returns an unknown tool call when the toolkit is empty", () => + Effect.gen(function*() { + const response = yield* LanguageModel.generateText({ + prompt: [], + toolkit: Toolkit.empty, + unknownToolCalls: "return" + }).pipe( + TestUtils.withLanguageModel({ + generateText: [unknownToolCall, finishPart] + }) + ) + + strictEqual(response.toolCallErrors.length, 1) + strictEqual(response.toolCallErrors[0]!.name, "NotATool") + })) + + it.effect("still fails by default when the operation has no toolkit", () => + Effect.gen(function*() { + const error = yield* LanguageModel.generateText({ prompt: [] }).pipe( + TestUtils.withLanguageModel({ + generateText: [unknownToolCall, finishPart] + }), + Effect.flip + ) + + assertDefined(error) + })) + + it.effect("returns an unknown tool call in a stream without a toolkit", () => + Effect.gen(function*() { + const parts = yield* LanguageModel.streamText({ + prompt: [], + unknownToolCalls: "return" + }).pipe( + Stream.runCollect, + TestUtils.withLanguageModel({ + streamText: [unknownToolCall, finishPart] + }) + ) + + const toolCallErrors = parts.filter((part) => part.type === "tool-call-error") + strictEqual(toolCallErrors.length, 1) + strictEqual(toolCallErrors[0]!.name, "NotATool") + })) + + it.effect("returns an unknown tool call in a stream with an empty toolkit", () => + Effect.gen(function*() { + const parts = yield* LanguageModel.streamText({ + prompt: [], + toolkit: Toolkit.empty, + unknownToolCalls: "return" + }).pipe( + Stream.runCollect, + TestUtils.withLanguageModel({ + streamText: [unknownToolCall, finishPart] + }) + ) + + const toolCallErrors = parts.filter((part) => part.type === "tool-call-error") + strictEqual(toolCallErrors.length, 1) + strictEqual(toolCallErrors[0]!.name, "NotATool") + })) + + it.effect("still fails invalid parameters for a known tool when resolution is disabled", () => + Effect.gen(function*() { + // The option governs unknown tools only. A call which names a tool in + // the toolkit belongs to that tool, and with resolution disabled the + // caller owns it, so neither opt-in changes what happens to it + const error = yield* LanguageModel.generateText({ + prompt: [], + toolkit: ReturnModeToolkit, + disableToolCallResolution: true, + unknownToolCalls: "return" + }).pipe( + TestUtils.withLanguageModel({ + generateText: [ + { + type: "tool-call", + id: "call-invalid", + name: "ReturnModeTool", + params: { testParam: { nested: "not-a-string" } } + }, + finishPart + ] + }), + Effect.provide(ReturnModeToolkit.toLayer({ + ReturnModeTool: () => Effect.succeed({ testSuccess: "test-success" }) + })), + Effect.flip + ) + + strictEqual(error.reason._tag, "InvalidOutputError") + })) + + it.effect("still fails invalid parameters for a known tool in a stream with resolution disabled", () => + Effect.gen(function*() { + const error = yield* LanguageModel.streamText({ + prompt: [], + toolkit: ReturnModeToolkit, + disableToolCallResolution: true, + unknownToolCalls: "return" + }).pipe( + Stream.runCollect, + TestUtils.withLanguageModel({ + streamText: [ + { + type: "tool-call", + id: "call-invalid", + name: "ReturnModeTool", + params: { testParam: { nested: "not-a-string" } } + }, + unknownToolCall, + finishPart + ] + }), + Effect.provide(ReturnModeToolkit.toLayer({ + ReturnModeTool: () => Effect.succeed({ testSuccess: "test-success" }) + })), + Effect.flip + ) + + assertDefined(error) + })) + + it.effect("leaves a handler-reachable parameter failure to the tool's failureMode", () => + Effect.gen(function*() { + const response = yield* LanguageModel.generateText({ + prompt: [], + toolkit: ReturnModeToolkit, + unknownToolCalls: "return" + }).pipe( + TestUtils.withLanguageModel({ + generateText: [ + { + type: "tool-call", + id: "call-invalid", + name: "ReturnModeTool", + params: { testParam: { nested: "not-a-string" } } + }, + finishPart + ] + }), + Effect.provide(ReturnModeToolkit.toLayer({ + ReturnModeTool: () => Effect.succeed({ testSuccess: "test-success" }) + })) + ) + + // `Toolkit` owns calls which reach a handler, so this is a failed tool + // result rather than a returned tool call error + strictEqual(response.toolCallErrors.length, 0) + strictEqual(response.toolResults.length, 1) + strictEqual(response.toolResults[0]!.isFailure, true) + })) + + it.effect("returns invalid tool calls from generateObject", () => + Effect.gen(function*() { + const response = yield* LanguageModel.generateObject({ + prompt: [], + schema: Schema.Struct({ count: Schema.Number }), + toolkit: MyToolkit, + unknownToolCalls: "return" + }).pipe( + TestUtils.withLanguageModel({ + generateText: [ + unknownToolCall, + { type: "text", text: "{\"count\":1}" }, + finishPart + ] + }), + Effect.provide(MyToolkitLayer) + ) + + deepStrictEqual(response.value, { count: 1 }) + strictEqual(response.toolCallErrors.length, 1) + strictEqual(response.toolCallErrors[0]!.name, "NotATool") + })) + + it.effect("honours an unknownToolCalls value which is not statically known", () => + Effect.gen(function*() { + const unknownToolCalls = "return" as "error" | "return" + + const response = yield* LanguageModel.generateText({ + prompt: [], + toolkit: MyToolkit, + unknownToolCalls + }).pipe( + TestUtils.withLanguageModel({ + generateText: [unknownToolCall, finishPart] + }), + Effect.provide(MyToolkitLayer) + ) + + strictEqual(response.toolCallErrors.length, 1) + strictEqual(response.toolCallErrors[0]!.error.reason._tag, "ToolNotFoundError") + })) + + it.effect("keeps failing a tool call which is malformed beyond its parameters", () => + Effect.gen(function*() { + // Valid parameters, but a non-string `id`: the call is not recoverable, + // so it must keep failing rather than come back under a parameter error + const error = yield* LanguageModel.generateText({ + prompt: [], + toolkit: MyToolkit, + disableToolCallResolution: true, + unknownToolCalls: "return" + }).pipe( + TestUtils.withLanguageModel({ + generateText: [ + { type: "tool-call", id: 42 as any, name: "MyTool", params: { testParam: "test-param" } }, + finishPart + ] + }), + Effect.provide(MyToolkitLayer), + Effect.flip + ) + + strictEqual(error.reason._tag, "InvalidOutputError") + })) + + it.effect("keeps failing a malformed part which is not a tool call", () => + Effect.gen(function*() { + const error = yield* LanguageModel.generateText({ + prompt: [], + toolkit: MyToolkit, + unknownToolCalls: "return" + }).pipe( + TestUtils.withLanguageModel({ + generateText: [{ type: "text", text: 99 as any }, finishPart] + }), + Effect.provide(MyToolkitLayer), + Effect.flip + ) + + strictEqual(error.reason._tag, "InvalidOutputError") + })) + + it.effect("does not return a provider-executed call to an unknown tool", () => + Effect.gen(function*() { + // Provider-executed calls already ran, so they stay response output + const error = yield* LanguageModel.generateText({ + prompt: [], + toolkit: MyToolkit, + unknownToolCalls: "return" + }).pipe( + TestUtils.withLanguageModel({ + generateText: [{ ...unknownToolCall, providerExecuted: true }, finishPart] + }), + Effect.provide(MyToolkitLayer), + Effect.flip + ) + + strictEqual(error.reason._tag, "InvalidOutputError") + })) + + // The hand-written cases above only cover calls which are malformed in ways + // that were anticipated. This asserts the invariant itself against + // arbitrary junk: a returned part is always a part the schema accepts. + it("only ever returns parts which satisfy the response schema", () => { + const junk = fc.oneof( + fc.string(), + fc.integer(), + fc.boolean(), + fc.constant(null), + fc.object(), + fc.array(fc.string()) + ) + const toolCall = fc.record({ + type: fc.constant("tool-call"), + id: junk, + name: fc.oneof(fc.constantFrom("MyTool", "NotATool"), junk), + params: junk + }) + + return fc.assert( + fc.asyncProperty(fc.array(toolCall, { maxLength: 4 }), async (parts) => { + const result = await Effect.runPromise( + LanguageModel.generateText({ + prompt: [], + toolkit: MyToolkit, + disableToolCallResolution: true, + unknownToolCalls: "return" + }).pipe( + TestUtils.withLanguageModel({ + generateText: [...parts, finishPart] as any + }), + Effect.provide(MyToolkitLayer), + Effect.result + ) + ) + + if (result._tag === "Failure") return true + + // Nothing unvalidated may reach the caller + for (const part of result.success.content) { + if (part.type === "tool-call-error") { + strictEqual(typeof part.id, "string") + strictEqual(typeof part.name, "string") + assertTrue(AiError.isAiError(part.error)) + } + if (part.type === "tool-call") { + strictEqual(typeof part.id, "string") + strictEqual(typeof part.name, "string") + } + } + return true + }), + { numRuns: 300 } + ) + }) + + it.effect("runs no handler when another call in the response is unrecoverable", () => + Effect.gen(function*() { + const calls = yield* Ref.make(0) + const handlers = MyToolkit.toLayer({ + MyTool: () => Ref.update(calls, (n) => n + 1).pipe(Effect.as({ testSuccess: "test-success" })) + }) + + // The malformed call fails the response, so the valid one must not run: + // a partially executed response would leave real side effects behind + yield* LanguageModel.generateText({ + prompt: [], + toolkit: MyToolkit, + unknownToolCalls: "return" + }).pipe( + TestUtils.withLanguageModel({ + generateText: [ + unknownToolCall, + { type: "tool-call", id: 42 as any, name: "MyTool", params: { testParam: "test-param" } }, + finishPart + ] + }), + Effect.provide(handlers), + Effect.flip + ) + + strictEqual(yield* Ref.get(calls), 0) + })) + + it.effect("leaves no tool call unanswered in the resulting history", () => + Effect.gen(function*() { + const response = yield* LanguageModel.generateText({ + prompt: [], + toolkit: MyToolkit, + unknownToolCalls: "return" + }).pipe( + TestUtils.withLanguageModel({ + generateText: [ + unknownToolCall, + { type: "tool-call", id: "call-known", name: "MyTool", params: { testParam: "test-param" } }, + { ...finishPart, reason: "tool-calls" } + ] + }), + Effect.provide(MyToolkit.toLayer({ + MyTool: () => Effect.succeed({ testSuccess: "test-success" }) + })) + ) + + // A tool call left without a result is rejected by providers on the + // next request, so a returned tool call error must still be answered + const callIds: Array = [] + const resultIds: Array = [] + for (const message of Prompt.fromResponseParts(response.content).content) { + for (const part of message.content as ReadonlyArray<{ readonly type: string; readonly id: string }>) { + if (part.type === "tool-call") callIds.push(part.id) + if (part.type === "tool-result") resultIds.push(part.id) + } + } + + deepStrictEqual(callIds.slice().sort(), ["call-known", "call-unknown"]) + deepStrictEqual(callIds.filter((id) => !resultIds.includes(id)), []) + // And a result with no call is rejected just the same, so recovery + // must never drop the call it answers + deepStrictEqual(resultIds.filter((id) => !callIds.includes(id)), []) + })) + + it.effect("adds a returned tool call error to history as a failed tool result", () => + Effect.gen(function*() { + const response = yield* LanguageModel.generateText({ + prompt: [], + toolkit: MyToolkit, + unknownToolCalls: "return" + }).pipe( + TestUtils.withLanguageModel({ + generateText: [unknownToolCall, finishPart] + }), + Effect.provide(MyToolkitLayer) + ) + + const prompt = Prompt.fromResponseParts(response.content) + const assistant = prompt.content.find((message) => message.role === "assistant") + const tool = prompt.content.find((message) => message.role === "tool") + + assertDefined(assistant) + assertDefined(tool) + + const toolCall = assistant.content.find((part) => part.type === "tool-call") + assertDefined(toolCall) + strictEqual(toolCall.name, "NotATool") + + const toolResult = tool.content.find((part) => part.type === "tool-result") + assertDefined(toolResult) + strictEqual(toolResult.isFailure, true) + })) + }) + describe("tool approval", () => { it.effect("emits tool-approval-request when tool has needsApproval: true", () => Effect.gen(function*() { diff --git a/packages/effect/test/unstable/ai/Response.test.ts b/packages/effect/test/unstable/ai/Response.test.ts index 46641349688..1ccb8f1db43 100644 --- a/packages/effect/test/unstable/ai/Response.test.ts +++ b/packages/effect/test/unstable/ai/Response.test.ts @@ -1,7 +1,7 @@ import { describe, it } from "@effect/vitest" import { deepStrictEqual } from "@effect/vitest/utils" import { Effect, Schema } from "effect" -import { Response } from "effect/unstable/ai" +import { AiError, Response, Toolkit } from "effect/unstable/ai" describe("Response", () => { it.effect("decodes response metadata with omitted optional fields", () => @@ -108,4 +108,31 @@ describe("Response", () => { "decoded part" ) })) + + it.effect("round-trips a tool call error through the schema for all parts", () => + Effect.gen(function*() { + // A tool call error names a tool which is not in the toolkit, so the + // schema for every part admits it regardless of the toolkit + const schema = Response.AllParts(Toolkit.empty) + const part = Response.makePart("tool-call-error", { + id: "call-unknown", + name: "NotATool", + params: { city: "Paris" }, + error: AiError.make({ + module: "LanguageModel", + method: "generateText", + reason: new AiError.ToolNotFoundError({ toolName: "NotATool", availableTools: [] }) + }) + }) + + const encoded = yield* Schema.encodeEffect(schema)(part) + const decoded = yield* Schema.decodeEffect(schema)(encoded) + + deepStrictEqual(decoded.type, "tool-call-error") + if (decoded.type === "tool-call-error") { + deepStrictEqual(decoded.name, "NotATool") + deepStrictEqual(decoded.params, { city: "Paris" }) + deepStrictEqual(decoded.error.reason._tag, "ToolNotFoundError") + } + })) }) diff --git a/packages/effect/typetest/unstable/ai/LanguageModel.tst.ts b/packages/effect/typetest/unstable/ai/LanguageModel.tst.ts index 482c645390b..a753c580298 100644 --- a/packages/effect/typetest/unstable/ai/LanguageModel.tst.ts +++ b/packages/effect/typetest/unstable/ai/LanguageModel.tst.ts @@ -56,6 +56,36 @@ const AsymmetricParamsTool = Tool.make("AsymmetricParamsTool", { describe("LanguageModel", () => { describe("generateText", () => { + it("exposes tool call errors only when the operation returns invalid tool calls", () => { + const toolkit = Toolkit.make(FailureModeErrorTool) + + const strict = LanguageModel.generateText({ prompt: "hello", toolkit }) + expect["toolCallErrors"]>().type.toBe>() + + const explicit = LanguageModel.generateText({ prompt: "hello", toolkit, unknownToolCalls: "error" }) + expect["toolCallErrors"]>().type.toBe>() + + const returned = LanguageModel.generateText({ prompt: "hello", toolkit, unknownToolCalls: "return" }) + expect["toolCallErrors"]>().type.toBe>() + + // An option which is not statically known keeps the part in the type + const unknownToolCalls = null as unknown as "error" | "return" + const dynamic = LanguageModel.generateText({ prompt: "hello", toolkit, unknownToolCalls }) + expect["toolCallErrors"]>().type.toBe>() + + // An operation without a toolkit can still opt in + const withoutToolkit = LanguageModel.generateText({ prompt: "hello", unknownToolCalls: "return" }) + expect["toolCallErrors"]>().type.toBe>() + + const streamed = LanguageModel.streamText({ prompt: "hello", toolkit, unknownToolCalls: "return" }) + type StreamedPart = typeof streamed extends Stream.Stream ? A : never + expect<"tool-call-error">().type.toBeAssignableTo() + + const strictStream = LanguageModel.streamText({ prompt: "hello", toolkit }) + type StrictPart = typeof strictStream extends Stream.Stream ? A : never + expect<"tool-call-error">().type.not.toBeAssignableTo() + }) + it("uses encoded tool parameters when tool call resolution is disabled", () => { const toolkit = Toolkit.make(TransformTool) const program = LanguageModel.generateText({