Preflight TypeScript schemas before model requests

typescript/README.md

@@ -36,6 +36,14 @@ npm run build
 Run the test suite with `npm test`. The Zod schema tests also compile their generated
 TypeScript schema text to check for syntax errors and unresolved type references.
 
+The TypeScript-backed JSON validator checks its schema and target export before a
+translator makes its first model request. Schema configuration errors are returned
+without asking the model to repair them. This preflight uses the same minimal
+TypeScript library as response validation and is cached for the validator's lifetime.
+Once a schema passes preflight, invalid model responses continue through the normal
+validation and repair flow. Zod and custom validators without schema-only checks keep
+their existing instance-validation behavior.
+
 To see TypeChat in action, we recommend exploring the [TypeChat example projects](https://github.com/microsoft/TypeChat/tree/main/typescript/examples). You can try them on your local machine or in a GitHub Codespace.
 
 To learn more about TypeChat, visit the [documentation](https://microsoft.github.io/TypeChat) which includes more information on TypeChat and how to get started.

typescript/src/ts/validate.ts

@@ -33,11 +33,16 @@ const configFileName = "/tsconfig.json";
 const libFileName = "/lib.d.ts";
 const schemaFileName = "/schema.ts";
 const jsonFileName = "/json.ts";
+const schemaValidationFileName = "/schema-validation.ts";
 
 /**
  * Represents an object that can validate JSON strings according to a given TypeScript schema.
  */
 export interface TypeScriptJsonValidator<T extends object> extends TypeChatJsonValidator<T> {
+    /**
+     * Validate the TypeScript schema without requiring a JSON instance.
+     */
+    validateSchema(): Result<void>;
     /**
      * Transform JSON into TypeScript code for validation. Returns a `Success<string>` object if the conversion is
      * successful, or an `Error` object if the JSON can't be transformed. The returned TypeScript source code is
@@ -57,7 +62,7 @@ export interface TypeScriptJsonValidator<T extends object> extends TypeChatJsonV
  * the TypeScript compiler. The specified type argument `T` must be the same type as `typeName` in the given `schema`.
  * @param schema A string containing the TypeScript source code for the JSON schema.
  * @param typeName The name of the JSON target type in the schema.
- * @returns A `TypeChatJsonValidator<T>` instance.
+ * @returns A `TypeScriptJsonValidator<T>` instance.
  */
 export function createTypeScriptJsonValidator<T extends object = object>(schema: string, typeName: string): TypeScriptJsonValidator<T> {
     const fileSystem = createVirtualFileSystem({
@@ -68,27 +73,72 @@ export function createTypeScriptJsonValidator<T extends object = object>(schema:
                 noLib: true,
                 types: []
             },
-            files: [libFileName, schemaFileName, jsonFileName]
+            files: [libFileName, schemaFileName, jsonFileName, schemaValidationFileName]
         }),
         [libFileName]: libText,
         [schemaFileName]: schema,
-        [jsonFileName]: ""
+        [jsonFileName]: "",
+        [schemaValidationFileName]: ""
     });
     const api = new API({ fs: fileSystem, cwd: "/" });
     api.updateSnapshot({ openProjects: [configFileName] });
     if (!fileSystem.writeFile) {
         throw new Error("The TypeScript virtual file system doesn't support writing files.");
     }
-    const writeJsonFile = fileSystem.writeFile;
+    const writeFile = fileSystem.writeFile;
+    let schemaValidationResult: Result<void> | undefined;
     const validator: TypeScriptJsonValidator<T> = {
         getSchemaText: () => schema,
         getTypeName: () => typeName,
+        validateSchema,
         createModuleTextFromJson,
         validate,
         close: () => api.close()
     };
     return validator;
 
+    function validateSchema(): Result<void> {
+        if (schemaValidationResult) {
+            return schemaValidationResult;
+        }
+
+        writeFile(
+            schemaValidationFileName,
+            `type __TypeChatSchemaTarget = import('./schema').${typeName};\n`
+        );
+        const snapshot = api.updateSnapshot({
+            fileChanges: { changed: [schemaValidationFileName] }
+        });
+        const project = snapshot.getProject(configFileName);
+        if (!project) {
+            schemaValidationResult = error(
+                `The TypeScript schema project '${configFileName}' couldn't be loaded.`
+            );
+            return schemaValidationResult;
+        }
+
+        const program = project.program;
+        const syntacticDiagnostics = program.getSyntacticDiagnostics();
+        const diagnostics = syntacticDiagnostics.length
+            ? syntacticDiagnostics
+            : program.getSemanticDiagnostics();
+        schemaValidationResult = diagnostics.length
+            ? error(
+                diagnostics
+                    .map(diagnostic =>
+                        formatSchemaDiagnostic(
+                            diagnostic,
+                            diagnostic.fileName === undefined
+                                ? undefined
+                                : program.getSourceFile(diagnostic.fileName)
+                        )
+                    )
+                    .join("\n")
+            )
+            : success(undefined);
+        return schemaValidationResult;
+    }
+
     function validate(jsonObject: object): Result<T> {
         const moduleResult = validator.createModuleTextFromJson(jsonObject);
         if (!moduleResult.success) {
@@ -99,7 +149,7 @@ export function createTypeScriptJsonValidator<T extends object = object>(schema:
                 data: jsonObject
             };
         }
-        writeJsonFile(jsonFileName, moduleResult.data);
+        writeFile(jsonFileName, moduleResult.data);
         const snapshot = api.updateSnapshot({ fileChanges: { changed: [jsonFileName] } });
         const project = snapshot.getProject(configFileName);
         if (!project) {
@@ -141,6 +191,25 @@ export function createTypeScriptJsonValidator<T extends object = object>(schema:
         }
     }
 
+    function formatSchemaDiagnostic(
+        diagnostic: Diagnostic,
+        sourceFile: SourceFile | undefined
+    ): string {
+        const message = flattenDiagnosticMessageText(diagnostic, "\n");
+        if (!sourceFile || diagnostic.pos < 0) {
+            return message;
+        }
+        const { line, character } = sourceFile.getLineAndCharacterOfPosition(
+            diagnostic.pos
+        );
+        const fileName = sourceFile.fileName === schemaFileName
+            ? "schema.ts"
+            : sourceFile.fileName === schemaValidationFileName
+                ? "schema validation"
+                : sourceFile.fileName;
+        return `${fileName}(${line + 1},${character + 1}): ${message}`;
+    }
+
     /**
      * For TypeScript error 2740 (missing required properties, truncated with "and N more"),
      * uses the type checker to compute the full list of missing required properties from the

typescript/src/typechat.ts

@@ -77,6 +77,12 @@ export interface TypeChatJsonValidator<T extends object> {
      * Return the name of the JSON object target type in the schema.
      */
     getTypeName(): string;
+    /**
+     * Optionally validates schema configuration without a JSON instance. Translators call this before
+     * the first model request. Validators whose schemas are checked during instance validation may omit it.
+     * @returns A success result when the schema is valid, or an error describing the schema problem.
+     */
+    validateSchema?(): Result<void>;
     /**
      * Validates the given JSON object according to the associated TypeScript schema. Returns a
      * `Success<T>` object containing the JSON object if validation was successful. Otherwise, returns
@@ -124,6 +130,10 @@ export function createJsonTranslator<T extends object>(model: TypeChatLanguageMo
     }
 
     async function translate(request: string, promptPreamble?: string | PromptSection[]): Promise<Result<T>> {
+        const preflight = validator.validateSchema?.();
+        if (preflight && !preflight.success) {
+            return error(`Schema validation failed before model request: ${preflight.message}`);
+        }
         const preamble: PromptSection[] = typeof promptPreamble === "string" ? [{ role: "user", content: promptPreamble }] : promptPreamble ?? [];
         let prompt: PromptSection[] = [...preamble, { role: "user", content: typeChat.createRequestPrompt(request) }];
         let attemptRepair = typeChat.attemptRepair;

typescript/test/validate.test.ts

@@ -1,5 +1,6 @@
 import { describe, it } from "node:test";
 import assert from "node:assert/strict";
+import { createJsonTranslator } from "../dist/index.js";
 import { createTypeScriptJsonValidator } from "../dist/ts/index.js";
 
 // ---------------------------------------------------------------------------
@@ -53,6 +54,106 @@ function validatorFor<T extends object>(schema: string, typeName: string) {
 
 describe("createTypeScriptJsonValidator", () => {
 
+    describe("schema-only validation", () => {
+        const invalidSchemas = [
+            {
+                name: "unsupported built-in types",
+                schema: "export interface Event { dateTime?: Date; }",
+                typeName: "Event",
+                diagnostic: /Date/
+            },
+            {
+                name: "unsupported library generics",
+                schema: "export interface Event { values: ReadonlyArray<string>; }",
+                typeName: "Event",
+                diagnostic: /ReadonlyArray/
+            },
+            {
+                name: "syntax errors",
+                schema: "export interface Event { dateTime: string;",
+                typeName: "Event",
+                diagnostic: /schema\.ts\(\d+,\d+\):/
+            },
+            {
+                name: "unresolved references",
+                schema: "export interface Event { dateTime: MissingType; }",
+                typeName: "Event",
+                diagnostic: /MissingType/
+            },
+            {
+                name: "missing target exports",
+                schema: "export interface Event { dateTime: string; }",
+                typeName: "MissingEvent",
+                diagnostic: /MissingEvent/
+            }
+        ];
+
+        for (const { name, schema, typeName, diagnostic } of invalidSchemas) {
+            for (const attemptRepair of [true, false]) {
+                it(`rejects ${name} before a model call when repair is ${attemptRepair ? "enabled" : "disabled"}`, async () => {
+                    const validator = validatorFor(schema, typeName);
+                    let modelCalls = 0;
+                    const translator = createJsonTranslator({
+                        async complete() {
+                            modelCalls++;
+                            return { success: true as const, data: "{}" };
+                        }
+                    }, validator);
+                    translator.attemptRepair = attemptRepair;
+
+                    try {
+                        const result = await translator.translate("test request");
+                        assert.equal(modelCalls, 0);
+                        assert.equal(result.success, false);
+                        assert.match(result.message, /schema validation failed before model request/i);
+                        assert.match(result.message, diagnostic);
+                    } finally {
+                        validator.close();
+                    }
+                });
+            }
+        }
+
+        it("accepts a valid schema with required properties and caches the result", () => {
+            const validator = validatorFor(
+                "export interface Event { required: string; }",
+                "Event"
+            );
+            try {
+                const firstResult = validator.validateSchema();
+                assert.ok(firstResult.success, "schema validation must not require a JSON instance");
+                assert.strictEqual(validator.validateSchema(), firstResult);
+            } finally {
+                validator.close();
+            }
+        });
+
+        it("keeps response repair for a valid schema", async () => {
+            const validator = validatorFor(smallSchema, "Point");
+            let modelCalls = 0;
+            const translator = createJsonTranslator({
+                async complete() {
+                    modelCalls++;
+                    return {
+                        success: true as const,
+                        data: modelCalls === 1
+                            ? '{"x":"not-a-number","y":2}'
+                            : '{"x":1,"y":2}'
+                    };
+                }
+            }, validator);
+
+            try {
+                const result = await translator.translate("point");
+                assert.ok(result.success);
+                assert.deepEqual(result.data, { x: 1, y: 2 });
+                assert.equal(modelCalls, 2);
+            } finally {
+                validator.close();
+            }
+        });
+    });
+
     describe("valid objects", () => {
         it("accepts an object that satisfies all required properties", () => {
             const v = validatorFor(largeSchema, "Doc");

typescript/test/zod.test.ts

@@ -5,6 +5,7 @@ import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
 import { tmpdir } from "node:os";
 import { dirname, join } from "node:path";
 import { z } from "zod";
+import { createJsonTranslator } from "../dist/index.js";
 import { createZodJsonValidator, getZodSchemaAsTypeScript } from "../dist/zod/index.js";
 
 // ---------------------------------------------------------------------------
@@ -525,4 +526,25 @@ describe("createZodJsonValidator", () => {
 
     });
 
+    describe("translator compatibility", () => {
+        it("keeps instance validation for validators without schema preflight", async () => {
+            const validator = createZodJsonValidator(SentimentSchema, "SentimentResponse");
+            generatedSchemas.push(validator.getSchemaText());
+            assert.equal(validator.validateSchema, undefined);
+
+            let modelCalls = 0;
+            const translator = createJsonTranslator({
+                async complete() {
+                    modelCalls++;
+                    return { success: true as const, data: '{"sentiment":"positive"}' };
+                }
+            }, validator);
+
+            const result = await translator.translate("positive sentiment");
+            assert.ok(result.success);
+            assert.deepStrictEqual(result.data, { sentiment: "positive" });
+            assert.equal(modelCalls, 1);
+        });
+    });
+
 });