From 8712cde1596ac2267791a53387e2485e1e931217 Mon Sep 17 00:00:00 2001 From: mikemikimike <13286568797@163.com> Date: Wed, 14 Oct 2026 16:47:03 +0800 Subject: [PATCH] fix: preflight TypeScript schemas before model calls --- typescript/README.md | 8 +++ typescript/src/ts/validate.ts | 79 ++++++++++++++++++++++-- typescript/src/typechat.ts | 10 +++ typescript/test/validate.test.ts | 101 +++++++++++++++++++++++++++++++ typescript/test/zod.test.ts | 22 +++++++ 5 files changed, 215 insertions(+), 5 deletions(-) diff --git a/typescript/README.md b/typescript/README.md index 4ed66add..36cc8c7f 100644 --- a/typescript/README.md +++ b/typescript/README.md @@ -36,6 +36,14 @@ npm run build Run the test suite with `npm test`. The Zod schema tests also compile their generated TypeScript schema text to check for syntax errors and unresolved type references. +The TypeScript-backed JSON validator checks its schema and target export before a +translator makes its first model request. Schema configuration errors are returned +without asking the model to repair them. This preflight uses the same minimal +TypeScript library as response validation and is cached for the validator's lifetime. +Once a schema passes preflight, invalid model responses continue through the normal +validation and repair flow. Zod and custom validators without schema-only checks keep +their existing instance-validation behavior. + To see TypeChat in action, we recommend exploring the [TypeChat example projects](https://github.com/microsoft/TypeChat/tree/main/typescript/examples). You can try them on your local machine or in a GitHub Codespace. To learn more about TypeChat, visit the [documentation](https://microsoft.github.io/TypeChat) which includes more information on TypeChat and how to get started. diff --git a/typescript/src/ts/validate.ts b/typescript/src/ts/validate.ts index b2b0b9c5..61add389 100644 --- a/typescript/src/ts/validate.ts +++ b/typescript/src/ts/validate.ts @@ -33,11 +33,16 @@ const configFileName = "/tsconfig.json"; const libFileName = "/lib.d.ts"; const schemaFileName = "/schema.ts"; const jsonFileName = "/json.ts"; +const schemaValidationFileName = "/schema-validation.ts"; /** * Represents an object that can validate JSON strings according to a given TypeScript schema. */ export interface TypeScriptJsonValidator extends TypeChatJsonValidator { + /** + * Validate the TypeScript schema without requiring a JSON instance. + */ + validateSchema(): Result; /** * Transform JSON into TypeScript code for validation. Returns a `Success` object if the conversion is * successful, or an `Error` object if the JSON can't be transformed. The returned TypeScript source code is @@ -57,7 +62,7 @@ export interface TypeScriptJsonValidator extends TypeChatJsonV * the TypeScript compiler. The specified type argument `T` must be the same type as `typeName` in the given `schema`. * @param schema A string containing the TypeScript source code for the JSON schema. * @param typeName The name of the JSON target type in the schema. - * @returns A `TypeChatJsonValidator` instance. + * @returns A `TypeScriptJsonValidator` instance. */ export function createTypeScriptJsonValidator(schema: string, typeName: string): TypeScriptJsonValidator { const fileSystem = createVirtualFileSystem({ @@ -68,33 +73,78 @@ export function createTypeScriptJsonValidator(schema: noLib: true, types: [] }, - files: [libFileName, schemaFileName, jsonFileName] + files: [libFileName, schemaFileName, jsonFileName, schemaValidationFileName] }), [libFileName]: libText, [schemaFileName]: schema, - [jsonFileName]: "" + [jsonFileName]: "", + [schemaValidationFileName]: "" }); const api = new API({ fs: fileSystem, cwd: "/" }); api.updateSnapshot({ openProjects: [configFileName] }); if (!fileSystem.writeFile) { throw new Error("The TypeScript virtual file system doesn't support writing files."); } - const writeJsonFile = fileSystem.writeFile; + const writeFile = fileSystem.writeFile; + let schemaValidationResult: Result | undefined; const validator: TypeScriptJsonValidator = { getSchemaText: () => schema, getTypeName: () => typeName, + validateSchema, createModuleTextFromJson, validate, close: () => api.close() }; return validator; + function validateSchema(): Result { + if (schemaValidationResult) { + return schemaValidationResult; + } + + writeFile( + schemaValidationFileName, + `type __TypeChatSchemaTarget = import('./schema').${typeName};\n` + ); + const snapshot = api.updateSnapshot({ + fileChanges: { changed: [schemaValidationFileName] } + }); + const project = snapshot.getProject(configFileName); + if (!project) { + schemaValidationResult = error( + `The TypeScript schema project '${configFileName}' couldn't be loaded.` + ); + return schemaValidationResult; + } + + const program = project.program; + const syntacticDiagnostics = program.getSyntacticDiagnostics(); + const diagnostics = syntacticDiagnostics.length + ? syntacticDiagnostics + : program.getSemanticDiagnostics(); + schemaValidationResult = diagnostics.length + ? error( + diagnostics + .map(diagnostic => + formatSchemaDiagnostic( + diagnostic, + diagnostic.fileName === undefined + ? undefined + : program.getSourceFile(diagnostic.fileName) + ) + ) + .join("\n") + ) + : success(undefined); + return schemaValidationResult; + } + function validate(jsonObject: object) { const moduleResult = validator.createModuleTextFromJson(jsonObject); if (!moduleResult.success) { return moduleResult; } - writeJsonFile(jsonFileName, moduleResult.data); + writeFile(jsonFileName, moduleResult.data); const snapshot = api.updateSnapshot({ fileChanges: { changed: [jsonFileName] } }); const project = snapshot.getProject(configFileName); if (!project) { @@ -136,6 +186,25 @@ export function createTypeScriptJsonValidator(schema: } } + function formatSchemaDiagnostic( + diagnostic: Diagnostic, + sourceFile: SourceFile | undefined + ): string { + const message = flattenDiagnosticMessageText(diagnostic, "\n"); + if (!sourceFile || diagnostic.pos < 0) { + return message; + } + const { line, character } = sourceFile.getLineAndCharacterOfPosition( + diagnostic.pos + ); + const fileName = sourceFile.fileName === schemaFileName + ? "schema.ts" + : sourceFile.fileName === schemaValidationFileName + ? "schema validation" + : sourceFile.fileName; + return `${fileName}(${line + 1},${character + 1}): ${message}`; + } + /** * For TypeScript error 2740 (missing required properties, truncated with "and N more"), * uses the type checker to compute the full list of missing required properties from the diff --git a/typescript/src/typechat.ts b/typescript/src/typechat.ts index 74c67331..5bee9dad 100644 --- a/typescript/src/typechat.ts +++ b/typescript/src/typechat.ts @@ -75,6 +75,12 @@ export interface TypeChatJsonValidator { * Return the name of the JSON object target type in the schema. */ getTypeName(): string; + /** + * Optionally validates schema configuration without a JSON instance. Translators call this before + * the first model request. Validators whose schemas are checked during instance validation may omit it. + * @returns A success result when the schema is valid, or an error describing the schema problem. + */ + validateSchema?(): Result; /** * Validates the given JSON object according to the associated TypeScript schema. Returns a * `Success` object containing the JSON object if validation was successful. Otherwise, returns @@ -122,6 +128,10 @@ export function createJsonTranslator(model: TypeChatLanguageMo } async function translate(request: string, promptPreamble?: string | PromptSection[]) { + const schemaValidation = validator.validateSchema?.(); + if (schemaValidation && !schemaValidation.success) { + return error(`Schema validation failed before model request: ${schemaValidation.message}`); + } const preamble: PromptSection[] = typeof promptPreamble === "string" ? [{ role: "user", content: promptPreamble }] : promptPreamble ?? []; let prompt: PromptSection[] = [...preamble, { role: "user", content: typeChat.createRequestPrompt(request) }]; let attemptRepair = typeChat.attemptRepair; diff --git a/typescript/test/validate.test.ts b/typescript/test/validate.test.ts index 31ef89fa..6a90e51e 100644 --- a/typescript/test/validate.test.ts +++ b/typescript/test/validate.test.ts @@ -1,5 +1,6 @@ import { describe, it } from "node:test"; import assert from "node:assert/strict"; +import { createJsonTranslator } from "../dist/index.js"; import { createTypeScriptJsonValidator } from "../dist/ts/index.js"; // --------------------------------------------------------------------------- @@ -53,6 +54,106 @@ function validatorFor(schema: string, typeName: string) { describe("createTypeScriptJsonValidator", () => { + describe("schema-only validation", () => { + const invalidSchemas = [ + { + name: "unsupported built-in types", + schema: "export interface Event { dateTime?: Date; }", + typeName: "Event", + diagnostic: /Date/ + }, + { + name: "unsupported library generics", + schema: "export interface Event { values: ReadonlyArray; }", + typeName: "Event", + diagnostic: /ReadonlyArray/ + }, + { + name: "syntax errors", + schema: "export interface Event { dateTime: string;", + typeName: "Event", + diagnostic: /schema\.ts\(\d+,\d+\):/ + }, + { + name: "unresolved references", + schema: "export interface Event { dateTime: MissingType; }", + typeName: "Event", + diagnostic: /MissingType/ + }, + { + name: "missing target exports", + schema: "export interface Event { dateTime: string; }", + typeName: "MissingEvent", + diagnostic: /MissingEvent/ + } + ]; + + for (const { name, schema, typeName, diagnostic } of invalidSchemas) { + for (const attemptRepair of [true, false]) { + it(`rejects ${name} before a model call when repair is ${attemptRepair ? "enabled" : "disabled"}`, async () => { + const validator = validatorFor(schema, typeName); + let modelCalls = 0; + const translator = createJsonTranslator({ + async complete() { + modelCalls++; + return { success: true as const, data: "{}" }; + } + }, validator); + translator.attemptRepair = attemptRepair; + + try { + const result = await translator.translate("test request"); + assert.equal(modelCalls, 0); + assert.equal(result.success, false); + assert.match(result.message, /schema validation failed before model request/i); + assert.match(result.message, diagnostic); + } finally { + validator.close(); + } + }); + } + } + + it("accepts a valid schema with required properties and caches the result", () => { + const validator = validatorFor( + "export interface Event { required: string; }", + "Event" + ); + try { + const firstResult = validator.validateSchema(); + assert.ok(firstResult.success, "schema validation must not require a JSON instance"); + assert.strictEqual(validator.validateSchema(), firstResult); + } finally { + validator.close(); + } + }); + + it("keeps response repair for a valid schema", async () => { + const validator = validatorFor(smallSchema, "Point"); + let modelCalls = 0; + const translator = createJsonTranslator({ + async complete() { + modelCalls++; + return { + success: true as const, + data: modelCalls === 1 + ? '{"x":"not-a-number","y":2}' + : '{"x":1,"y":2}' + }; + } + }, validator); + + try { + const result = await translator.translate("point"); + assert.ok(result.success); + assert.deepEqual(result.data, { x: 1, y: 2 }); + assert.equal(modelCalls, 2); + } finally { + validator.close(); + } + }); + }); + describe("valid objects", () => { it("accepts an object that satisfies all required properties", () => { const v = validatorFor(largeSchema, "Doc"); diff --git a/typescript/test/zod.test.ts b/typescript/test/zod.test.ts index f77d1648..e22068c9 100644 --- a/typescript/test/zod.test.ts +++ b/typescript/test/zod.test.ts @@ -5,6 +5,7 @@ import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { dirname, join } from "node:path"; import { z } from "zod"; +import { createJsonTranslator } from "../dist/index.js"; import { createZodJsonValidator, getZodSchemaAsTypeScript } from "../dist/zod/index.js"; // --------------------------------------------------------------------------- @@ -525,4 +526,25 @@ describe("createZodJsonValidator", () => { }); + describe("translator compatibility", () => { + it("keeps instance validation for validators without schema preflight", async () => { + const validator = createZodJsonValidator(SentimentSchema, "SentimentResponse"); + generatedSchemas.push(validator.getSchemaText()); + assert.equal(validator.validateSchema, undefined); + + let modelCalls = 0; + const translator = createJsonTranslator({ + async complete() { + modelCalls++; + return { success: true as const, data: '{"sentiment":"positive"}' }; + } + }, validator); + + const result = await translator.translate("positive sentiment"); + assert.ok(result.success); + assert.deepStrictEqual(result.data, { sentiment: "positive" }); + assert.equal(modelCalls, 1); + }); + }); + });