feat(ai): allow schedulers to defer tool parameter validation

.changeset/deferred-tool-parameter-validation.md

@@ -0,0 +1,5 @@
+---
+"effect": patch
+---
+
+Allow external tool schedulers to defer application parameter validation with `toolCallValidation: "deferred"` when tool resolution is disabled, preserving invalid arguments for corrective feedback.

packages/ai/openai/test/OpenAiLanguageModel.test.ts

@@ -17,6 +17,83 @@ const fileSearchOutcomes = [
 ] as const
 
 describe("OpenAiLanguageModel", () => {
+  for (const method of ["generateText", "streamText"] as const) {
+    it.effect(`${method} defers refined application arguments while retaining completed web search and usage`, () =>
+      Effect.gen(function*() {
+        const Inspect = Tool.make("Inspect", {
+          parameters: Schema.Struct({
+            focus: Schema.String.check(Schema.isMaxLength(240)).annotate({ description: "What to inspect" })
+          }),
+          success: Schema.String
+        })
+        const params = { focus: "x".repeat(241) }
+        const call = makeFunctionCall("Inspect", params)
+        const completed = makeDefaultResponse({
+          output: [makeWebSearchCall(), call],
+          usage: {
+            input_tokens: 20,
+            output_tokens: 10,
+            total_tokens: 30,
+            input_tokens_details: { cached_tokens: 0 },
+            output_tokens_details: { reasoning_tokens: 0 }
+          }
+        })
+        const options = {
+          prompt: "Inspect the result",
+          toolkit: Toolkit.make(Inspect, OpenAiTool.WebSearch({})),
+          disableToolCallResolution: true,
+          toolCallValidation: "deferred"
+        } as const
+        const request = method === "generateText"
+          ? LanguageModel.generateText(options).pipe(
+            Effect.map((response) => response.content),
+            Effect.provide(OpenAiLanguageModel.model("gpt-4o-mini")),
+            Effect.provide(makeTestLayer({ body: completed }))
+          )
+          : LanguageModel.streamText(options).pipe(
+            Stream.runCollect,
+            Effect.provide(OpenAiLanguageModel.model("gpt-4o-mini")),
+            Effect.provide(makeStreamTestLayer([
+              { type: "response.output_item.done", sequence_number: 1, output_index: 0, item: makeWebSearchCall() },
+              {
+                type: "response.output_item.added",
+                sequence_number: 2,
+                output_index: 1,
+                item: { ...call, arguments: "", status: "in_progress" }
+              },
+              {
+                type: "response.function_call_arguments.done",
+                sequence_number: 3,
+                output_index: 1,
+                item_id: "fc_123",
+                name: "Inspect",
+                arguments: call.arguments
+              },
+              { type: "response.completed", sequence_number: 4, response: completed }
+            ]))
+          )
+        const parts = yield* request
+        const invalid = parts.find((part) => part.type === "tool-call" && part.name === "Inspect")
+        assert.isDefined(invalid)
+        if (invalid?.type === "tool-call") {
+          strictEqual(invalid.id, "call_123")
+          deepStrictEqual(invalid.params, params)
+        }
+        const results = parts.filter((part) => part.type === "tool-result")
+        strictEqual(results.length, 1)
+        strictEqual(results[0].id, "ws_123")
+        strictEqual(results[0].isFailure, false)
+        deepStrictEqual(results[0].result, {
+          action: { type: "search", query: "Effect TypeScript" },
+          status: "completed"
+        })
+        const finish = parts.find((part) => part.type === "finish")
+        assert.isDefined(finish)
+        strictEqual(finish?.usage.inputTokens.total, 20)
+        strictEqual(finish?.usage.outputTokens.total, 10)
+      }))
+  }
+
   describe("make", () => {
     it.effect("sends correct model in request", () =>
       Effect.gen(function*() {

packages/effect/src/unstable/ai/LanguageModel.ts

@@ -297,6 +297,22 @@ export interface GenerateTextOptions<Tools extends Record<string, Tool.Any>> {
    * resolver execution yourself.
    */
   readonly disableToolCallResolution?: boolean | undefined
+
+  /**
+   * Controls application tool parameter validation when
+   * `disableToolCallResolution` is `true`. Defaults to `"strict"`.
+   *
+   * **Details**
+   *
+   * `"strict"` validates encoded parameters. `"deferred"` returns parameters as
+   * `unknown` so an external scheduler can validate them before execution and
+   * return corrective feedback. Tool definitions, provider-executed parameter
+   * validation, and validation of the rest of the response are unchanged.
+   *
+   * This option has no effect when tool call resolution is enabled: `Toolkit`
+   * validates application parameters according to each tool's `failureMode`.
+   */
+  readonly toolCallValidation?: "strict" | "deferred" | undefined
 }
 
 type GenerateTextOptionsWithoutToolkit = Omit<GenerateTextOptions<{}>, "toolkit"> & {
@@ -567,16 +583,17 @@ export type ExtractTools<Options> = Options extends {
   : {}
 
 /**
- * Resolves to `"encoded"` when tool call resolution is
- * disabled, otherwise `"opaque"`.
+ * Resolves to `"encoded"` when tool call resolution is disabled and parameter
+ * validation is strict, otherwise `"opaque"`.
  *
  * @category utility types
  * @since 4.0.0
  */
 export type ExtractToolParametersMode<Options> = Options extends {
   readonly disableToolCallResolution: true
-} ? "encoded"
-  : "opaque"
+  readonly toolCallValidation?: "strict" | undefined
+} ? "encoded" :
+  "opaque"
 
 type ExtractErrorFromToolkitOption<ToolkitValue, DisableToolCallResolution extends boolean> = ToolkitValue extends
   Toolkit.WithHandler<infer Tools> ?
@@ -1209,7 +1226,7 @@ export const make: (params: {
     }
 
     const ResponseSchema = Schema.mutable(Schema.Array(Response.Part(
-      options.disableToolCallResolution === true
+      options.disableToolCallResolution === true && options.toolCallValidation !== "deferred"
         ? makeToolkitWithEncodedParameters(toolkit)
         : makeToolkitWithOpaqueParameters(toolkit)
     )))
@@ -1219,6 +1236,9 @@ export const make: (params: {
     if (options.disableToolCallResolution === true) {
       const rawContent = yield* generateWithNonIncrementalFallback()
       const content = yield* Schema.decodeEffect(ResponseSchema)(rawContent)
+      if (options.toolCallValidation === "deferred") {
+        yield* validateProviderExecutedToolCalls(toolkit, rawContent, "encoded")
+      }
       if (tracker) {
         const responseMetadata = content.find((part) => part.type === "response-metadata")
         if (Predicate.isNotUndefined(responseMetadata) && Predicate.isNotUndefined(responseMetadata.id)) {
@@ -1511,7 +1531,7 @@ export const make: (params: {
     }
 
     const ResponseSchema = Schema.NonEmptyArray(Response.StreamPart(
-      options.disableToolCallResolution === true
+      options.disableToolCallResolution === true && options.toolCallValidation !== "deferred"
         ? makeToolkitWithEncodedParameters(toolkit)
         : makeToolkitWithOpaqueParameters(toolkit)
     ))
@@ -1523,6 +1543,9 @@ export const make: (params: {
       return streamWithNonIncrementalFallback().pipe(
         Stream.mapArrayEffect((parts) =>
           decodeParts(parts).pipe(
+            options.toolCallValidation === "deferred"
+              ? Effect.tap(() => validateProviderExecutedToolCalls(toolkit, parts, "encoded"))
+              : identity,
             tracker ?
               Effect.tap((decodedParts) => {
                 for (const part of decodedParts) {
@@ -2391,7 +2414,8 @@ const makeToolkitWithOpaqueParameters = <Tools extends Record<string, Tool.Any>>
 // Provider-executed tools bypass Toolkit, so validate their parameters here.
 const validateProviderExecutedToolCalls = <Tools extends Record<string, Tool.Any>>(
   toolkit: Toolkit.WithHandler<Tools>,
-  parts: ReadonlyArray<Response.PartEncoded | Response.StreamPartEncoded>
+  parts: ReadonlyArray<Response.PartEncoded | Response.StreamPartEncoded>,
+  parametersMode: "decoded" | "encoded" = "decoded"
 ): Effect.Effect<void, Schema.SchemaError> =>
   Effect.forEach(
     parts,
@@ -2404,7 +2428,9 @@ const validateProviderExecutedToolCalls = <Tools extends Record<string, Tool.Any
         return Effect.void
       }
       return Effect.asVoid(
-        Schema.decodeUnknownEffect(tool.parametersSchema)(part.params)
+        Schema.decodeUnknownEffect(
+          parametersMode === "encoded" ? Schema.toEncoded(tool.parametersSchema) : tool.parametersSchema
+        )(part.params)
       ) as Effect.Effect<void, Schema.SchemaError>
     },
     { discard: true }

packages/effect/test/unstable/ai/LanguageModel.test.ts

@@ -69,6 +69,139 @@ describe("LanguageModel", () => {
     response: undefined
   }
 
+  describe("deferred tool parameter validation", () => {
+    for (const method of ["generateText", "streamText"] as const) {
+      it.effect(`${method} defers application parameters without changing definitions or running handlers`, () =>
+        Effect.gen(function*() {
+          const handlers = ReturnModeToolkit.toLayer({
+            ReturnModeTool: () => Effect.die("handler must not run")
+          })
+          const respond = (options: LanguageModel.ProviderOptions) => {
+            strictEqual(options.tools[0], ReturnModeTool)
+            return [
+              { type: "tool-call", id: "invalid", name: "ReturnModeTool", params: { testParam: 123 } },
+              { type: "tool-call", id: "valid", name: "ReturnModeTool", params: { testParam: "valid" } },
+              finishPart
+            ] satisfies Array<Response.PartEncoded>
+          }
+          const options = {
+            prompt: [],
+            toolkit: ReturnModeToolkit,
+            disableToolCallResolution: true,
+            toolCallValidation: "deferred"
+          } as const
+          const request = method === "generateText"
+            ? LanguageModel.generateText(options).pipe(Effect.map((response) => response.toolCalls))
+            : LanguageModel.streamText(options).pipe(
+              Stream.filter((part) => part.type === "tool-call"),
+              Stream.runCollect
+            )
+          const calls = yield* request.pipe(
+            TestUtils.withLanguageModel({ generateText: respond, streamText: respond }),
+            Effect.provide(handlers)
+          )
+          deepStrictEqual(calls.map((call) => call.params), [{ testParam: 123 }, { testParam: "valid" }])
+        }))
+
+      for (
+        const [label, part] of [
+          ["invalid provider parameters", {
+            type: "tool-call",
+            id: "hosted",
+            name: "MyTool",
+            params: { testParam: 123 },
+            providerExecuted: true
+          }],
+          ["unknown tool names", { type: "tool-call", id: "unknown", name: "UnknownTool", params: {} }],
+          ["invalid provider results", {
+            type: "tool-result",
+            id: "hosted",
+            name: "MyTool",
+            result: { testSuccess: 123 },
+            isFailure: false,
+            providerExecuted: true
+          }]
+        ] satisfies Array<[string, Response.ToolCallPartEncoded | Response.ToolResultPartEncoded]>
+      ) {
+        it.effect(`${method} still rejects ${label}`, () =>
+          Effect.gen(function*() {
+            const options = {
+              prompt: [],
+              toolkit: MyToolkit,
+              disableToolCallResolution: true,
+              toolCallValidation: "deferred"
+            } as const
+            const request = method === "generateText"
+              ? LanguageModel.generateText(options).pipe(Effect.asVoid)
+              : LanguageModel.streamText(options).pipe(Stream.runDrain)
+            const error = yield* request.pipe(
+              TestUtils.withLanguageModel({ generateText: [part, finishPart], streamText: [part, finishPart] }),
+              Effect.flip
+            )
+            strictEqual(error.reason._tag, "InvalidOutputError")
+          }))
+      }
+    }
+
+    it.effect("keeps transformed parameters encoded for both application and provider calls", () =>
+      Effect.gen(function*() {
+        const parts = yield* LanguageModel.streamText({
+          prompt: [],
+          toolkit: TransformToolkit,
+          disableToolCallResolution: true,
+          toolCallValidation: "deferred"
+        }).pipe(
+          Stream.runCollect,
+          TestUtils.withLanguageModel({
+            streamText: [
+              { type: "tool-call", id: "application", name: "TransformTool", params: "21" },
+              { type: "tool-call", id: "provider", name: "TransformTool", params: "22", providerExecuted: true },
+              finishPart
+            ]
+          })
+        )
+        const calls = parts.filter((part) => part.type === "tool-call")
+        deepStrictEqual(calls.map((call) => call.params), ["21", "22"])
+      }))
+
+    it.effect("preserves provider failure and finalization after an invalid application call", () =>
+      Effect.gen(function*() {
+        let finalized = false
+        const failure = AiError.make({
+          module: "Test",
+          method: "streamText",
+          reason: new AiError.InvalidRequestError({ description: "provider failed" })
+        })
+        const error = yield* LanguageModel.streamText({
+          prompt: [],
+          toolkit: MyToolkit,
+          disableToolCallResolution: true,
+          toolCallValidation: "deferred"
+        }).pipe(
+          Stream.runDrain,
+          TestUtils.withLanguageModel({
+            streamText: () =>
+              Stream.succeed(
+                {
+                  type: "tool-call",
+                  id: "invalid",
+                  name: "MyTool",
+                  params: { testParam: 123 }
+                } satisfies Response.StreamPartEncoded
+              ).pipe(
+                Stream.concat(Stream.fail(failure)),
+                Stream.ensuring(Effect.sync(() => {
+                  finalized = true
+                }))
+              )
+          }),
+          Effect.flip
+        )
+        strictEqual(error, failure)
+        strictEqual(finalized, true)
+      }))
+  })
+
   describe("generateText", () => {
     it.effect("does not resolve tool calls after an incomplete finish", () =>
       Effect.gen(function*() {

packages/effect/typetest/unstable/ai/LanguageModel.tst.ts

@@ -55,6 +55,58 @@ const AsymmetricParamsTool = Tool.make("AsymmetricParamsTool", {
 })
 
 describe("LanguageModel", () => {
+  it("keeps deferred parameters unknown without handler requirements or errors", () => {
+    const toolkit = Toolkit.make(FailureModeErrorTool, ToolWithRequestContext, AsymmetricParamsTool)
+    const options = {
+      prompt: "hello",
+      toolkit,
+      disableToolCallResolution: true,
+      toolCallValidation: "deferred"
+    } as const
+    const program = LanguageModel.generateText(options)
+    const stream = LanguageModel.streamText(options)
+    const object = LanguageModel.generateObject({ ...options, schema: Schema.Struct({ answer: Schema.String }) })
+    const chat = null as unknown as Chat.Chat
+    const chatProgram = chat.generateText(options)
+
+    type Output = Effect.Success<typeof program>
+    type Part = Stream.Success<typeof stream>
+    expect<Output["toolCalls"][number]["params"]>().type.toBe<unknown>()
+    expect<Extract<Part, { readonly type: "tool-call" }>["params"]>().type.toBe<unknown>()
+    expect<Effect.Success<typeof object>["toolCalls"][number]["params"]>().type.toBe<unknown>()
+    expect<Effect.Success<typeof chatProgram>["toolCalls"][number]["params"]>().type.toBe<unknown>()
+    expect<Effect.Error<typeof program>>().type.toBe<AiError.AiError>()
+    expect<Stream.Error<typeof stream>>().type.toBe<AiError.AiError>()
+    expect<Effect.Services<typeof program>>().type.toBe<LanguageModel.LanguageModel | ParamEncodeService>()
+    expect<Stream.Services<typeof stream>>().type.toBe<LanguageModel.LanguageModel | ParamEncodeService>()
+  })
+
+  it("keeps parameters unknown when validation may be deferred", () => {
+    const toolCallValidation = null as unknown as "strict" | "deferred" | undefined
+    const program = LanguageModel.generateText({
+      prompt: "hello",
+      toolkit: Toolkit.make(TransformTool),
+      disableToolCallResolution: true,
+      toolCallValidation
+    })
+    expect<Effect.Success<typeof program>["toolCalls"][number]["params"]>().type.toBe<unknown>()
+    const options: { readonly toolCallValidation?: "strict" | "deferred" } = {}
+    const optional = LanguageModel.generateText({
+      ...options,
+      prompt: "hello",
+      toolkit: Toolkit.make(TransformTool),
+      disableToolCallResolution: true
+    })
+    expect<Effect.Success<typeof optional>["toolCalls"][number]["params"]>().type.toBe<unknown>()
+    const strict = LanguageModel.generateText({
+      prompt: "hello",
+      toolkit: Toolkit.make(TransformTool),
+      disableToolCallResolution: true,
+      toolCallValidation: "strict"
+    })
+    expect<Effect.Success<typeof strict>["toolCalls"][number]["params"]>().type.toBe<string>()
+  })
+
   describe("generateText", () => {
     it("uses encoded tool parameters when tool call resolution is disabled", () => {
       const toolkit = Toolkit.make(TransformTool)