From 96a313ded6009fd72ad92b77d690ed633261e093 Mon Sep 17 00:00:00 2001 From: Robert Gruen Date: Mon, 12 Oct 2026 19:10:11 -0700 Subject: [PATCH 1/4] Retain rejected output and repair failure context Add untyped FailureInfo to validator results and translation exceptions, preserve custom metadata and the latest useful candidate across repairs, and retain strict parser and cancellation behavior. Add focused regression coverage and document the additive failure-context API. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- README.md | 48 ++ src/typechat.program/ProgramValidator.cs | 8 +- src/typechat/ConstraintsValidator.cs | 2 +- src/typechat/FailureInfo.cs | 90 ++++ src/typechat/IConstraintsValidator.cs | 2 +- src/typechat/IJsonTypeValidator.cs | 3 +- src/typechat/JsonSerializerTypeValidator.cs | 9 +- src/typechat/JsonTranslator.cs | 34 +- src/typechat/Result.cs | 29 +- src/typechat/TypeChatException.cs | 19 +- src/typechat/TypeValidator.cs | 2 +- .../TypeChat.UnitTests/TestFailureContext.cs | 498 ++++++++++++++++++ 12 files changed, 730 insertions(+), 14 deletions(-) create mode 100644 src/typechat/FailureInfo.cs create mode 100644 tests/TypeChat.UnitTests/TestFailureContext.cs diff --git a/README.md b/README.md index e487857b..b45efe8c 100644 --- a/README.md +++ b/README.md @@ -44,6 +44,54 @@ public class Milks } ``` +### Accessing failed output + +Failed validator results expose optional `FailureInfo` through `Result.Failure`. +`FailureInfo.Value` is an unvalidated `object`, not a value guaranteed to conform to `T`. +Check `HasValue` to distinguish unavailable data from explicit `null`, `false`, `0`, or empty strings. +JSON validators retain a cloned `JsonElement` when parsing succeeds, plus the original `InputText` +even when parsing fails. Their comment, trailing-comma, and depth-limit options are respected when +collecting failure data. Constraint and program validators retain the rejected instance. + +Translation still returns a typed value on success and throws `TypeChatException` on final validation +failure. The exception now exposes the same context through `Failure`, alongside its existing +`Response`: + +```cs +try +{ + CalendarActions actions = await translator.TranslateAsync(requestText); +} +catch (TypeChatException error) +{ + FailureInfo? failure = error.Failure; + if (failure?.HasValue == true) + { + object? rejected = failure.Value; + // Validate or repair the rejected value before treating it as CalendarActions. + } + string? originalResponse = failure?.ResponseText; +} +``` + +To manage repairs yourself, set `translator.MaxRepairAttempts = 0`. +On final failure, the context retains the latest rejected candidate and available model response text, +not a complete attempt history. These can come from different attempts if the latest repair response +cannot be parsed. Existing repair limits, diagnostics, events, typed success values, and cancellation +behavior are unchanged. + +HTTP, I/O, JSON response-decoding, and nested TypeChat exceptions during a repair are rethrown unchanged, +with context available as `error.Data[FailureInfo.ExceptionDataKey]`. +An initial transport failure has no invented candidate, and cancellation and programming exceptions +are not intercepted. Custom validators can return +`Result.Error(message).WithFailure(new FailureInfo { Value = rejected, ResponseText = text })`; +existing failure values and metadata take precedence over fallback context, without mutating the +original result. The existing `Error(value, message)` overload still retains its typed `Value` and +also exposes it as unvalidated failure context. + +Use `Success` to distinguish outcomes, not the presence of a value. Rejected data and response text +may contain sensitive output; do not log or serialize them automatically. + ## Microsoft.TypeChat.Program ## TypeChat.Program translates natural language requests into simple programs (***Plans***), represented as JSON. diff --git a/src/typechat.program/ProgramValidator.cs b/src/typechat.program/ProgramValidator.cs index 33ae030c..0c48594c 100644 --- a/src/typechat.program/ProgramValidator.cs +++ b/src/typechat.program/ProgramValidator.cs @@ -40,7 +40,8 @@ public Result Validate(string json) if (!result.Value.HasNotTranslated) { // Now validate the full program - return ValidateProgram(result.Value); + result = ValidateProgram(result.Value); + return result.Success ? result : result.WithFailure(new FailureInfo { InputText = json }); } } return result; @@ -50,7 +51,8 @@ public virtual Result ValidateProgram(Program program) { if (_programValidator is not null) { - return _programValidator.ValidateProgram(program); + Result result = _programValidator.ValidateProgram(program); + return result.Success ? result : result.WithFailure(new FailureInfo { Value = program }); } return program; } @@ -85,7 +87,7 @@ public override Result ValidateProgram(Program program) } catch (Exception ex) { - return Result.Error(ex.Message); + return Result.Error(ex.Message).WithFailure(new FailureInfo { Value = program }); } } } diff --git a/src/typechat/ConstraintsValidator.cs b/src/typechat/ConstraintsValidator.cs index f57d3832..b7a0b270 100644 --- a/src/typechat/ConstraintsValidator.cs +++ b/src/typechat/ConstraintsValidator.cs @@ -63,7 +63,7 @@ public Result Validate(T value) return new Result(value); } - return Result.Error(result.ErrorMessage); + return Result.Error(result.ErrorMessage).WithFailure(new FailureInfo { Value = value }); } private string ToErrorString(List validationResults) diff --git a/src/typechat/FailureInfo.cs b/src/typechat/FailureInfo.cs new file mode 100644 index 00000000..e51ef94e --- /dev/null +++ b/src/typechat/FailureInfo.cs @@ -0,0 +1,90 @@ +// Copyright (c) Microsoft. All rights reserved. + +namespace Microsoft.TypeChat; + +/// +/// Unvalidated output associated with a failure. Retains bounded context, not an attempt history. +/// +public class FailureInfo +{ + /// + /// Key used to attach failure context to transport exceptions without changing their type. + /// + public const string ExceptionDataKey = "Microsoft.TypeChat.FailureInfo"; + + private object? _value; + + /// + /// Rejected value, when available. Check HasValue before using it, and validate it before + /// treating it as the target type. Parsed JSON is retained as an independent JsonElement. + /// + public object? Value + { + get => _value; + set + { + _value = value; + HasValue = true; + } + } + + /// + /// Distinguishes unavailable data from an explicitly supplied value, including null. + /// + public bool HasValue { get; private set; } + + /// + /// JSON input supplied to a validator, when available. If supplied without a value, + /// the translator respects the validator's decision not to provide parsed data. + /// + public string? InputText { get; set; } + + /// + /// Latest model response text, when available. This can come from a later repair attempt + /// than Value if that response couldn't be parsed. + /// + public string? ResponseText { get; set; } + + internal FailureInfo WithFallback(FailureInfo? fallback) + { + var result = (FailureInfo)MemberwiseClone(); + if (!result.HasValue && fallback?.HasValue == true) + { + result.Value = fallback.Value; + } + result.InputText ??= fallback?.InputText; + result.ResponseText ??= fallback?.ResponseText; + return result; + } + + internal void AttachTo(Exception exception) + { + exception.Data[ExceptionDataKey] = exception.Data[ExceptionDataKey] is FailureInfo existing ? + existing.WithFallback(this) : + WithFallback(null); + } + + internal static FailureInfo FromJson(string? json, JsonSerializerOptions options) + { + var result = new FailureInfo { InputText = json }; + if (json is null) + { + return result; + } + try + { + using JsonDocument document = JsonDocument.Parse(json, new JsonDocumentOptions + { + AllowTrailingCommas = options.AllowTrailingCommas, + CommentHandling = options.ReadCommentHandling, + MaxDepth = options.MaxDepth + }); + result.Value = document.RootElement.Clone(); + } + catch (JsonException) + { + // The original validation error reports malformed JSON; only input text is available. + } + return result; + } +} diff --git a/src/typechat/IConstraintsValidator.cs b/src/typechat/IConstraintsValidator.cs index 6e32d7bb..685a2701 100644 --- a/src/typechat/IConstraintsValidator.cs +++ b/src/typechat/IConstraintsValidator.cs @@ -9,7 +9,7 @@ namespace Microsoft.TypeChat; public interface IConstraintsValidator { /// - /// Validate the value T + /// Validate the value T. Failures can include FailureInfo retaining the rejected instance. /// /// check constraints for this value /// result of the validation diff --git a/src/typechat/IJsonTypeValidator.cs b/src/typechat/IJsonTypeValidator.cs index 3c1e0586..4fdffb40 100644 --- a/src/typechat/IJsonTypeValidator.cs +++ b/src/typechat/IJsonTypeValidator.cs @@ -17,7 +17,8 @@ public interface IJsonTypeValidator /// /// Parses and validates the given JSON string according to the associated schema. Returns a /// Result containing the JSON object if validation was successful.Otherwise, returns - /// an Result with a message property that contains validation diagnostics. + /// a Result with a message containing validation diagnostics and optional FailureInfo + /// retaining the rejected value and input text. /// /// /// diff --git a/src/typechat/JsonSerializerTypeValidator.cs b/src/typechat/JsonSerializerTypeValidator.cs index cca26b99..81d6feeb 100644 --- a/src/typechat/JsonSerializerTypeValidator.cs +++ b/src/typechat/JsonSerializerTypeValidator.cs @@ -57,14 +57,19 @@ public JsonSerializerTypeValidator(JsonSerializerOptions? options = null) } catch (JsonException jex) { - return Result.Error(ToErrorString(json, jex)); + return CreateError(json, ToErrorString(json, jex)); } catch (Exception ex) { - return Result.Error(ex.Message); + return CreateError(json, ex.Message); } } + private Result CreateError(string json, string message) + { + return Result.Error(message).WithFailure(FailureInfo.FromJson(json, _options)); + } + private string ToErrorString(string json, JsonException error) { StringBuilder sb = new StringBuilder(); diff --git a/src/typechat/JsonTranslator.cs b/src/typechat/JsonTranslator.cs index 3a7fe71c..b9cf8f5e 100644 --- a/src/typechat/JsonTranslator.cs +++ b/src/typechat/JsonTranslator.cs @@ -177,6 +177,7 @@ public async Task TranslateToObjectAsync(string request, CancellationTok /// /// Translate a natural language request into an object of type 'T' + /// Final validation failures retain unvalidated output in TypeChatException.Failure. /// /// text request /// optional cancel token @@ -208,9 +209,20 @@ public async Task TranslateAsync( requestSettings ??= _translationSettings; Prompt prompt = CreateRequestPrompt(request, preamble); int repairAttempts = 0; + FailureInfo? lastFailure = null; while (true) { - string responseText = await GetResponseAsync(prompt, requestSettings, cancelToken).ConfigureAwait(false); + string responseText; + try + { + responseText = await GetResponseAsync(prompt, requestSettings, cancelToken).ConfigureAwait(false); + } + catch (Exception ex) when (lastFailure is not null && + ex is HttpRequestException or IOException or JsonException or TypeChatException) + { + lastFailure.AttachTo(ex); + throw; + } JsonResponse jsonResponse = JsonResponse.Parse(responseText); Result validationResult; @@ -232,11 +244,18 @@ public async Task TranslateAsync( validationResult = Result.Error(TypeChatException.NoJson(jsonResponse)); } + validationResult = validationResult.WithFailure(new FailureInfo { ResponseText = responseText }); + if (lastFailure is not null) + { + validationResult = validationResult.WithFailure(lastFailure); + } + lastFailure = validationResult.Failure; + // Attempt to repair the Json that was returned ++repairAttempts; if (repairAttempts > _maxRepairAttempts) { - TypeChatException.ThrowJsonValidation(request, jsonResponse, validationResult.Message); + TypeChatException.ThrowJsonValidation(request, jsonResponse, validationResult.Message, lastFailure); } NotifyEvent(AttemptingRepair, validationResult.Message); @@ -275,15 +294,26 @@ protected virtual PromptSection CreateRepairPrompt(string responseText, Result ValidateJson(string json) { var result = Validator.Validate(json); + if (!result.Success) + { + result = result.WithFailure(result.Failure?.HasValue == true || result.Failure?.InputText is not null ? + new FailureInfo { InputText = json } : + FailureInfo.FromJson(json, JsonSerializerTypeValidator.DefaultOptions())); + } if (!OnValidationComplete(result)) { return result; } if (result.Success) { + T instance = result.Value; result = (_constraintsValidator is not null) ? _constraintsValidator.Validate(result.Value) : result; + if (!result.Success) + { + result = result.WithFailure(new FailureInfo { Value = instance, InputText = json }); + } } return result; } diff --git a/src/typechat/Result.cs b/src/typechat/Result.cs index 5431d4ed..e1236e24 100644 --- a/src/typechat/Result.cs +++ b/src/typechat/Result.cs @@ -29,8 +29,9 @@ public Result(T value, string? message = null) public Result(Result value) { Success = value.Success; - Value = (T)value.Value; + Value = value.Success || value.Value is T ? (T)value.Value : default; Message = value.Message; + Failure = value.Failure; } internal Result(bool success, string? message) @@ -55,6 +56,31 @@ internal Result(bool success, string? message) /// public string? Message { get; set; } + /// + /// Optional unvalidated output associated with a failed result. Value remains the typed + /// result; use Failure.Value for rejected data that does not conform to T. + /// + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public FailureInfo? Failure { get; private set; } + + /// + /// Returns a shallow copy of this failed result, retaining existing failure fields and + /// filling unavailable fields from the supplied context. Custom result metadata is preserved. + /// + /// Fallback failure context. + /// A failed result with the combined context. + public Result WithFailure(FailureInfo failure) + { + ArgumentVerify.ThrowIfNull(failure, nameof(failure)); + if (Success) + { + throw new InvalidOperationException("Failure context cannot be added to a successful result."); + } + var result = (Result)MemberwiseClone(); + result.Failure = Failure is null ? failure.WithFallback(null) : Failure.WithFallback(failure); + return result; + } + public static implicit operator Result(T value) { return new Result(value); @@ -84,6 +110,7 @@ public static Result Error(T value, string message) { var result = new Result(false, message); result.Value = value; + result.Failure = new FailureInfo { Value = value }; return result; } } diff --git a/src/typechat/TypeChatException.cs b/src/typechat/TypeChatException.cs index 7d8e4652..9c6f649d 100644 --- a/src/typechat/TypeChatException.cs +++ b/src/typechat/TypeChatException.cs @@ -48,6 +48,15 @@ public TypeChatException(ErrorCode code, string request, JsonResponse? response _response = response; } + /// + /// Creates an exception retaining the rejected output and available model response text. + /// + public TypeChatException(ErrorCode code, string request, JsonResponse? response, string? message, FailureInfo? failure) + : this(code, request, response, message) + { + failure?.AttachTo(this); + } + /// /// Error code /// @@ -63,6 +72,11 @@ public TypeChatException(ErrorCode code, string request, JsonResponse? response /// public JsonResponse? Response => _response; + /// + /// Optional unvalidated output associated with the failure. + /// + public FailureInfo? Failure => Data[FailureInfo.ExceptionDataKey] as FailureInfo; + public override string ToString() { return $"Error: {Code}\n{Message}\n\nRequest:\n{Request}\n\nResponse:\n{Response.ResponseText}"; @@ -78,13 +92,14 @@ internal static void ThrowNoJson(string request, JsonResponse response) ); } - internal static void ThrowJsonValidation(string request, JsonResponse response, string message) + internal static void ThrowJsonValidation(string request, JsonResponse response, string message, FailureInfo? failure = null) { throw new TypeChatException( TypeChatException.ErrorCode.JsonValidation, request, response, - message + message, + failure ); } diff --git a/src/typechat/TypeValidator.cs b/src/typechat/TypeValidator.cs index abb2a8d0..3ee2a31a 100644 --- a/src/typechat/TypeValidator.cs +++ b/src/typechat/TypeValidator.cs @@ -47,7 +47,7 @@ public TypeValidator(TypescriptSchema schema) /// /// Validate the given Json string. /// If Result.Success: includes a value of type T - /// If Result.False: includes a diagnostic message + /// If Result.False: includes a diagnostic message and FailureInfo for the rejected JSON /// /// /// diff --git a/tests/TypeChat.UnitTests/TestFailureContext.cs b/tests/TypeChat.UnitTests/TestFailureContext.cs new file mode 100644 index 00000000..41bfbe95 --- /dev/null +++ b/tests/TypeChat.UnitTests/TestFailureContext.cs @@ -0,0 +1,498 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Net.Http; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; + +namespace Microsoft.TypeChat.Tests; + +public class TestFailureContext +{ + public class Example + { + public int Value { get; set; } + } + + private sealed class FixedModel : ILanguageModel + { + private readonly Queue _responses; + + public FixedModel(params object[] responses) + { + _responses = new Queue(responses); + } + + public ModelInfo ModelInfo => "Mock"; + + public List Prompts { get; } = new List(); + + public Task CompleteAsync(Prompt prompt, TranslationSettings? settings, CancellationToken cancelToken) + { + Prompts.Add(prompt.Select(section => section.GetText()).ToArray()); + object response = _responses.Dequeue(); + return response is Exception error ? + Task.FromException(error) : + Task.FromResult((string)response); + } + } + + private sealed class FixedValidator : IJsonTypeValidator + { + private readonly Result _result; + + public FixedValidator(Result result) + { + _result = result; + } + + public TypeSchema Schema { get; } = ExampleSchema(); + + public Result Validate(string json) => _result; + } + + private sealed class FixedConstraints : IConstraintsValidator + { + private readonly Result _result; + + public FixedConstraints(Result result) + { + _result = result; + } + + public Result Validate(Example value) => _result; + } + + private sealed class CustomFailureInfo : FailureInfo + { + public string Code { get; set; } = "CUSTOM"; + } + + private sealed class CustomResult : Result + { + public CustomResult() + : base(Result.Error("custom")) + { + } + + public string Code { get; } = "CUSTOM"; + } + + private static JsonTranslator Translator(FixedModel model, int repairs = 1) + { + return new JsonTranslator(model, new FixedValidator(Result.Error("invalid value"))) + { + MaxRepairAttempts = repairs + }; + } + + private static TypeSchema ExampleSchema() + { + return new TypeSchema(typeof(Example), + new SchemaText("export interface Example { Value: number; }", SchemaText.Languages.Typescript)); + } + + [Fact] + public void MessageOnlyErrorsHaveNoFailureContext() + { + Result result = Result.Error("failed"); + Assert.False(result.Success); + Assert.Equal(0, result.Value); + Assert.Null(result.Failure); + } + + [Theory] + [InlineData(null)] + [InlineData(false)] + [InlineData(0)] + [InlineData("")] + public void ExplicitFailureValuesAreRetained(object? value) + { + Result result = Result.Error(value, "failed"); + Assert.False(result.Success); + Assert.Same(value, result.Value); + Assert.True(result.Failure.HasValue); + Assert.Same(value, result.Failure.Value); + } + + [Fact] + public void CloningFailuresDoesNotCastRejectedDataToTheSuccessType() + { + var source = Result.Error("rejected", "failed"); + var clone = new Result(source); + Assert.False(clone.Success); + Assert.Equal(0, clone.Value); + Assert.Equal("rejected", clone.Failure.Value); + } + + [Fact] + public void AddingContextDoesNotMutateExistingFailures() + { + Result original = Result.Error("failed"); + Result enriched = original.WithFailure(new FailureInfo { Value = null, ResponseText = "" }); + Assert.Null(original.Failure); + Assert.True(enriched.Failure.HasValue); + Assert.Null(enriched.Failure.Value); + Assert.Equal("", enriched.Failure.ResponseText); + } + + [Fact] + public void AddingContextPreservesCustomResultSubclasses() + { + var original = new CustomResult(); + var enriched = Assert.IsType(original.WithFailure(new FailureInfo { Value = 0 })); + Assert.Equal("CUSTOM", enriched.Code); + Assert.Equal(0, enriched.Failure.Value); + Assert.Null(original.Failure); + } + + [Fact] + public void ExistingFailureMetadataTakesPrecedenceOverFallbacks() + { + var metadata = new CustomFailureInfo { Value = null, ResponseText = "" }; + Result original = Result.Error("custom").WithFailure(metadata); + Result enriched = original.WithFailure(new FailureInfo { Value = 42, ResponseText = "fallback" }); + var failure = Assert.IsType(enriched.Failure); + Assert.Equal("CUSTOM", failure.Code); + Assert.True(failure.HasValue); + Assert.Null(failure.Value); + Assert.Equal("", failure.ResponseText); + Assert.Null(metadata.InputText); + } + + [Fact] + public void SuccessfulResultsKeepTheirTypedValueAndNoFailureContext() + { + Example value = new Example { Value = 42 }; + var result = new Result(value); + Assert.True(result.Success); + Assert.Same(value, result.Value); + Assert.Null(result.Failure); + Assert.Throws(() => result.WithFailure(new FailureInfo())); + } + + [Fact] + public void JsonValidatorsRetainRejectedJsonWithAnIndependentLifetime() + { + var validator = new TypeValidator(); + string json = "{\"Value\":\"invalid\"}"; + Result result = validator.Validate(json); + Assert.False(result.Success); + Assert.True(result.Failure.HasValue); + JsonElement rejected = Assert.IsType(result.Failure.Value); + Assert.Equal("invalid", rejected.GetProperty("Value").GetString()); + Assert.Equal(json, result.Failure.InputText); + Assert.Null(result.Failure.ResponseText); + } + + [Theory] + [InlineData("{\"Value\":}")] + [InlineData("")] + public void JsonValidatorsRetainMalformedInputWithoutInventingAValue(string json) + { + Result result = new TypeValidator().Validate(json); + Assert.False(result.Success); + Assert.False(result.Failure.HasValue); + Assert.Equal(json, result.Failure.InputText); + } + + [Fact] + public void ValueTypeValidationFailuresRemainFailuresInsteadOfThrowingDuringConversion() + { + var schema = new TypeSchema(typeof(int), new SchemaText("type Number = number;", SchemaText.Languages.Typescript)); + var validator = new JsonSerializerTypeValidator(schema); + Result result = validator.Validate("\"invalid\""); + Assert.False(result.Success); + Assert.Equal(0, result.Value); + Assert.Equal("invalid", Assert.IsType(result.Failure.Value).GetString()); + } + + [Fact] + public void FailureParsingHonorsValidatorJsonOptions() + { + var options = JsonSerializerTypeValidator.DefaultOptions(); + options.MaxDepth = 8; + var validator = new JsonSerializerTypeValidator(ExampleSchema(), options); + Result result = validator.Validate("{/* comment */\"Value\":\"invalid\",}"); + Assert.False(result.Success); + Assert.True(result.Failure.HasValue); + Assert.Equal("invalid", Assert.IsType(result.Failure.Value).GetProperty("Value").GetString()); + } + + [Fact] + public async Task TranslationFailureContextDoesNotBypassTheValidatorDepthLimit() + { + var options = JsonSerializerTypeValidator.DefaultOptions(); + options.MaxDepth = 1; + var validator = new JsonSerializerTypeValidator(ExampleSchema(), options); + string response = "{\"Value\":{\"nested\":1}}"; + var translator = new JsonTranslator(new FixedModel(response), validator) { MaxRepairAttempts = 0 }; + TypeChatException error = await Assert.ThrowsAsync(() => translator.TranslateAsync("request")); + Assert.False(error.Failure.HasValue); + Assert.Equal(response, error.Failure.InputText); + Assert.Equal(response, error.Failure.ResponseText); + } + + [Theory] + [InlineData("{/* comment */\"Value\":42}")] + [InlineData("{\"Value\":42,}")] + public async Task TranslationFailureContextDoesNotReparseRejectedJsonWithLooserOptions(string response) + { + var validator = new JsonSerializerTypeValidator(ExampleSchema(), new JsonSerializerOptions()); + var translator = new JsonTranslator(new FixedModel(response), validator) { MaxRepairAttempts = 0 }; + TypeChatException error = await Assert.ThrowsAsync(() => translator.TranslateAsync("request")); + Assert.False(error.Failure.HasValue); + Assert.Equal(response, error.Failure.InputText); + Assert.Equal(response, error.Failure.ResponseText); + } + + [Fact] + public void ConstraintValidatorsRetainTheRejectedInstance() + { + Person person = new Person + { + Name = new Name { FirstName = "First", LastName = "Last" }, + Age = 125 + }; + Result result = new ConstraintsValidator().Validate(person); + Assert.False(result.Success); + Assert.Same(person, result.Failure.Value); + Assert.Null(result.Value); + } + + [Fact] + public void ProgramValidationRetainsTheRejectedProgramAndInput() + { + string json = "{\"@steps\":[{\"@func\":\"missing\"}]}"; + Result result = new ProgramValidator(MathAPI.Default).Validate(json); + Assert.False(result.Success); + Assert.IsType(result.Failure.Value); + Assert.Equal(json, result.Failure.InputText); + Assert.Null(result.Value); + } + + [Fact] + public async Task TranslationFailuresRetainRejectedJsonAndFullModelText() + { + string response = "before {\"Value\":\"invalid\"} after"; + var model = new FixedModel(response); + var translator = new JsonTranslator(model, new TypeValidator()) { MaxRepairAttempts = 0 }; + TypeChatException error = await Assert.ThrowsAsync(() => translator.TranslateAsync("request")); + Assert.Equal(TypeChatException.ErrorCode.JsonValidation, error.Code); + Assert.Equal(response, error.Response.ResponseText); + Assert.Equal(response, error.Failure.ResponseText); + Assert.Equal("{\"Value\":\"invalid\"}", error.Failure.InputText); + Assert.Equal("invalid", Assert.IsType(error.Failure.Value).GetProperty("Value").GetString()); + Assert.Single(model.Prompts); + } + + [Theory] + [InlineData("not JSON")] + [InlineData("{\"Value\":")] + [InlineData("{\"Value\":}")] + [InlineData("")] + public async Task UnparseableOutputHasTextButNoInventedCandidate(string response) + { + var model = new FixedModel(response); + TypeChatException error = await Assert.ThrowsAsync(() => Translator(model, 0).TranslateAsync("request")); + Assert.False(error.Failure.HasValue); + Assert.Equal(response, error.Failure.ResponseText); + } + + [Fact] + public async Task SuccessfulRepairAndDiagnosticEventsRemainUnchanged() + { + string first = "{\"Value\":\"invalid\"}"; + var model = new FixedModel(first, "{\"Value\":42}"); + var translator = new JsonTranslator(model, new TypeValidator()); + var responses = new List(); + var repairs = new List(); + translator.CompletionReceived += responses.Add; + translator.AttemptingRepair += repairs.Add; + Example result = await translator.TranslateAsync("request"); + Assert.Equal(42, result.Value); + Assert.Equal(2, model.Prompts.Count); + Assert.Contains(first, model.Prompts[1]); + Assert.Equal(2, responses.Count); + Assert.Single(repairs); + } + + [Fact] + public async Task FailedRepairReturnsTheLatestRejectedCandidate() + { + string second = "{\"Value\":\"second\"}"; + var model = new FixedModel("{\"Value\":\"first\"}", second); + TypeChatException error = await Assert.ThrowsAsync(() => Translator(model).TranslateAsync("request")); + Assert.Equal("second", Assert.IsType(error.Failure.Value).GetProperty("Value").GetString()); + Assert.Equal(second, error.Failure.ResponseText); + Assert.Equal(2, model.Prompts.Count); + } + + [Theory] + [InlineData("not JSON")] + [InlineData("{\"Value\":")] + [InlineData("{\"Value\":}")] + public async Task UnparseableRepairRetainsTheEarlierCandidateAndLatestText(string response) + { + var model = new FixedModel("{\"Value\":\"first\"}", response); + TypeChatException error = await Assert.ThrowsAsync(() => Translator(model).TranslateAsync("request")); + Assert.Equal("first", Assert.IsType(error.Failure.Value).GetProperty("Value").GetString()); + Assert.Equal(response, error.Failure.ResponseText); + Assert.Equal(response, error.Response.ResponseText); + Assert.Equal(2, model.Prompts.Count); + } + + [Fact] + public async Task AdditionalValidationRetainsTheTransformedInstance() + { + var transformed = new Example { Value = 99 }; + var translator = new JsonTranslator( + new FixedModel("{\"Value\":\"input\"}"), new FixedValidator(new Result(transformed))) + { + ConstraintsValidator = new FixedConstraints(Result.Error("constraint failure")), + MaxRepairAttempts = 0 + }; + TypeChatException error = await Assert.ThrowsAsync(() => translator.TranslateAsync("request")); + Assert.Same(transformed, error.Failure.Value); + Assert.Equal("constraint failure", error.Message); + } + + [Theory] + [InlineData(null)] + [InlineData(false)] + [InlineData(0)] + [InlineData("")] + public async Task CustomFailureValuesAndMetadataArePreservedWithoutMutation(object? value) + { + var metadata = new CustomFailureInfo { Value = value, ResponseText = "" }; + Result original = Result.Error("custom").WithFailure(metadata); + var translator = new JsonTranslator( + new FixedModel("{\"Value\":42}"), new FixedValidator(original)) + { MaxRepairAttempts = 0 }; + TypeChatException error = await Assert.ThrowsAsync(() => translator.TranslateAsync("request")); + var failure = Assert.IsType(error.Failure); + Assert.Equal("CUSTOM", failure.Code); + Assert.True(failure.HasValue); + Assert.Same(value, failure.Value); + Assert.Equal("", failure.ResponseText); + Assert.Null(original.Failure.InputText); + Assert.Null(metadata.InputText); + } + + [Theory] + [InlineData("not JSON")] + [InlineData("{\"Value\":")] + [InlineData("{\"Value\":}")] + public async Task MalformedInitialOutputStillAllowsRepair(string response) + { + var model = new FixedModel(response, "{\"Value\":42}"); + var translator = new JsonTranslator(model, new TypeValidator()); + Assert.Equal(42, (await translator.TranslateAsync("request")).Value); + Assert.Equal(2, model.Prompts.Count); + } + + [Fact] + public async Task MultipleRepairsRetainOnlyTheLatestCandidateAndExistingPromptWindow() + { + var model = new FixedModel("{\"Value\":\"first\"}", "{\"Value\":\"second\"}", "{\"Value\":\"third\"}"); + TypeChatException error = await Assert.ThrowsAsync(() => Translator(model, 2).TranslateAsync("request")); + Assert.Equal("third", Assert.IsType(error.Failure.Value).GetProperty("Value").GetString()); + Assert.Equal(3, model.Prompts.Count); + Assert.Equal(model.Prompts[0].Length + 2, model.Prompts[1].Length); + Assert.Equal(model.Prompts[1].Length, model.Prompts[2].Length); + Assert.DoesNotContain("{\"Value\":\"first\"}", model.Prompts[2]); + } + + [Fact] + public async Task RepairHttpFailuresKeepTheirOriginalExceptionAndCandidateContext() + { + var original = new HttpRequestException("network failure"); + var model = new FixedModel("{\"Value\":\"invalid\"}", original); + HttpRequestException error = await Assert.ThrowsAsync(() => Translator(model).TranslateAsync("request")); + Assert.Same(original, error); + var failure = Assert.IsType(error.Data[FailureInfo.ExceptionDataKey]); + Assert.Equal("invalid", Assert.IsType(failure.Value).GetProperty("Value").GetString()); + Assert.Equal("{\"Value\":\"invalid\"}", failure.ResponseText); + } + + [Fact] + public async Task InitialHttpFailuresHaveNoInventedCandidateContext() + { + var original = new HttpRequestException("network failure"); + HttpRequestException error = await Assert.ThrowsAsync(() => + Translator(new FixedModel(original)).TranslateAsync("request")); + Assert.Same(original, error); + Assert.False(error.Data.Contains(FailureInfo.ExceptionDataKey)); + } + + [Theory] + [InlineData("io")] + [InlineData("json")] + [InlineData("typechat")] + public async Task OtherExpectedRepairExceptionsRetainContextWithoutWrapping(string kind) + { + Exception original = kind switch + { + "io" => new IOException("response failure"), + "json" => new JsonException("response failure"), + "typechat" => new TypeChatException(TypeChatException.ErrorCode.Unknown, "request"), + _ => throw new ArgumentException("Unexpected exception kind", nameof(kind)) + }; + var model = new FixedModel("{\"Value\":\"invalid\"}", original); + Exception error = await Assert.ThrowsAnyAsync(() => Translator(model).TranslateAsync("request")); + Assert.Same(original, error); + var failure = Assert.IsType(error.Data[FailureInfo.ExceptionDataKey]); + Assert.Equal("invalid", Assert.IsType(failure.Value).GetProperty("Value").GetString()); + } + + [Fact] + public async Task ProgrammingExceptionsAreNotIntercepted() + { + var original = new InvalidOperationException("model bug"); + var model = new FixedModel("{\"Value\":\"invalid\"}", original); + InvalidOperationException error = await Assert.ThrowsAsync(() => + Translator(model).TranslateAsync("request")); + Assert.Same(original, error); + Assert.False(error.Data.Contains(FailureInfo.ExceptionDataKey)); + } + + [Fact] + public async Task ExistingTransportFailureMetadataIsPreserved() + { + var original = new HttpRequestException("network failure"); + original.Data[FailureInfo.ExceptionDataKey] = new CustomFailureInfo { Value = null, ResponseText = "" }; + var model = new FixedModel("{\"Value\":\"invalid\"}", original); + HttpRequestException error = await Assert.ThrowsAsync(() => Translator(model).TranslateAsync("request")); + var failure = Assert.IsType(error.Data[FailureInfo.ExceptionDataKey]); + Assert.Equal("CUSTOM", failure.Code); + Assert.True(failure.HasValue); + Assert.Null(failure.Value); + Assert.Equal("", failure.ResponseText); + } + + [Fact] + public async Task CancellationRemainsTheOriginalCancellationException() + { + using var cancellation = new CancellationTokenSource(); + cancellation.Cancel(); + var original = new OperationCanceledException(cancellation.Token); + var model = new FixedModel("{\"Value\":\"invalid\"}", original); + OperationCanceledException error = await Assert.ThrowsAsync(() => + Translator(model).TranslateAsync("request", cancellation.Token)); + Assert.Same(original, error); + Assert.Equal(cancellation.Token, error.CancellationToken); + Assert.False(error.Data.Contains(FailureInfo.ExceptionDataKey)); + } + + [Fact] + public async Task FailureContextDoesNotLeakIntoLaterTranslations() + { + var second = new HttpRequestException("second request"); + var model = new FixedModel("{\"Value\":\"invalid\"}", new HttpRequestException("repair"), second); + JsonTranslator translator = Translator(model); + await Assert.ThrowsAsync(() => translator.TranslateAsync("first")); + HttpRequestException error = await Assert.ThrowsAsync(() => translator.TranslateAsync("second")); + Assert.Same(second, error); + Assert.False(error.Data.Contains(FailureInfo.ExceptionDataKey)); + } +} From d1221af51078a8e6d4090d33091f6f435494d668 Mon Sep 17 00:00:00 2001 From: Robert Gruen Date: Mon, 12 Oct 2026 19:14:08 -0700 Subject: [PATCH 2/4] Use spell-check-compatible malformed-output test names Rename the two test methods flagged by the repository spelling check without changing their coverage or behavior. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- tests/TypeChat.UnitTests/TestFailureContext.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/TypeChat.UnitTests/TestFailureContext.cs b/tests/TypeChat.UnitTests/TestFailureContext.cs index 41bfbe95..be26e097 100644 --- a/tests/TypeChat.UnitTests/TestFailureContext.cs +++ b/tests/TypeChat.UnitTests/TestFailureContext.cs @@ -291,7 +291,7 @@ public async Task TranslationFailuresRetainRejectedJsonAndFullModelText() [InlineData("{\"Value\":")] [InlineData("{\"Value\":}")] [InlineData("")] - public async Task UnparseableOutputHasTextButNoInventedCandidate(string response) + public async Task MalformedOutputHasTextButNoInventedCandidate(string response) { var model = new FixedModel(response); TypeChatException error = await Assert.ThrowsAsync(() => Translator(model, 0).TranslateAsync("request")); @@ -332,7 +332,7 @@ public async Task FailedRepairReturnsTheLatestRejectedCandidate() [InlineData("not JSON")] [InlineData("{\"Value\":")] [InlineData("{\"Value\":}")] - public async Task UnparseableRepairRetainsTheEarlierCandidateAndLatestText(string response) + public async Task MalformedRepairRetainsTheEarlierCandidateAndLatestText(string response) { var model = new FixedModel("{\"Value\":\"first\"}", response); TypeChatException error = await Assert.ThrowsAsync(() => Translator(model).TranslateAsync("request")); From 0f1828375199f00bc8e0accb723343e0062c7ba2 Mon Sep 17 00:00:00 2001 From: Robert Gruen Date: Tue, 13 Oct 2026 18:33:25 -0700 Subject: [PATCH 3/4] Use serviced SDK SourceLink instead of vulnerable legacy packages Remove the redundant SourceLink.GitHub 8.0.0 reference that overrides SDK-integrated SourceLink and pulls in Microsoft.Build.Tasks.Git 8.0.0. Document the fixed SDK requirement, retain transitive NuGet auditing, and preserve embedded source-debugging metadata. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- Directory.Packages.props | 1 - README.md | 6 ++++++ src/package/nuget.props | 7 +------ 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/Directory.Packages.props b/Directory.Packages.props index d0b055ab..8ac71779 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -31,7 +31,6 @@ - + true snupkg @@ -28,12 +29,6 @@ bin\$(Configuration)\$(TargetFramework)\$(AssemblyName).xml - - - - - - From 05ec8f758ced3c5b002398d76b69a1487ad278d0 Mon Sep 17 00:00:00 2001 From: Robert Gruen Date: Wed, 14 Oct 2026 13:55:18 -0700 Subject: [PATCH 4/4] Preserve repair failure metadata and resolve README conflict Retain custom fallback metadata without replacing current values or text, add regression coverage, and reconcile SDK guidance with main. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- README.md | 9 +- src/typechat/FailureInfo.cs | 10 +- .../TypeChat.UnitTests/TestFailureContext.cs | 96 +++++++++++++++++++ 3 files changed, 109 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 38dcade6..199c3660 100644 --- a/README.md +++ b/README.md @@ -89,6 +89,10 @@ existing failure values and metadata take precedence over fallback context, with original result. The existing `Error(value, message)` overload still retains its typed `Value` and also exposes it as unvalidated failure context. +When a malformed repair reuses an earlier candidate, its custom failure metadata is retained alongside +the latest available input and response text. A new rejected candidate does not inherit the previous +candidate's custom metadata. + Use `Success` to distinguish outcomes, not the presence of a value. Rejected data and response text may contain sensitive output; do not log or serialize them automatically. @@ -137,8 +141,6 @@ Prior versions of TypeChat.NET used the pre-release programming model of Semanti Use a serviced .NET SDK 8 or newer, including when building from Visual Studio. [Source Link is included in .NET SDK 8+](https://github.com/dotnet/sourcelink#using-source-link-in-net-projects), so the projects use its SDK-provided implementation rather than overriding it with legacy NuGet packages. -For [CVE-2026-62900](https://github.com/advisories/GHSA-23fw-v26w-5fgq), use a fixed .NET 8 SDK -(8.0.130 or 8.0.424, or later servicing releases), or a patched .NET 9/10 SDK. * Visual Studio 2022. * Load **typechat.sln** from the root directory. @@ -149,6 +151,9 @@ For [CVE-2026-62900](https://github.com/advisories/GHSA-23fw-v26w-5fgq), use a f * Go to the root directory of the project * dotnet build Typechat.sln +For [CVE-2026-62900](https://github.com/advisories/GHSA-23fw-v26w-5fgq), use a fixed .NET 8 SDK +(8.0.130 or 8.0.424, or later servicing releases), or a patched .NET 9/10 SDK. + ## Nuget Packages * Microsoft.Typechat [![](https://img.shields.io/nuget/dt/Microsoft.TypeChat.svg?label=downloads&color=007edf&logo=nuget)](https://www.nuget.org/packages/Microsoft.TypeChat) diff --git a/src/typechat/FailureInfo.cs b/src/typechat/FailureInfo.cs index e51ef94e..67babb02 100644 --- a/src/typechat/FailureInfo.cs +++ b/src/typechat/FailureInfo.cs @@ -1,4 +1,4 @@ -// Copyright (c) Microsoft. All rights reserved. +// Copyright (c) Microsoft. All rights reserved. namespace Microsoft.TypeChat; @@ -47,13 +47,15 @@ public object? Value internal FailureInfo WithFallback(FailureInfo? fallback) { - var result = (FailureInfo)MemberwiseClone(); + // Preserve fallback metadata unless this context has its own value or custom metadata. + FailureInfo source = !HasValue && GetType() == typeof(FailureInfo) && fallback is not null ? fallback : this; + var result = (FailureInfo)source.MemberwiseClone(); if (!result.HasValue && fallback?.HasValue == true) { result.Value = fallback.Value; } - result.InputText ??= fallback?.InputText; - result.ResponseText ??= fallback?.ResponseText; + result.InputText = InputText ?? fallback?.InputText; + result.ResponseText = ResponseText ?? fallback?.ResponseText; return result; } diff --git a/tests/TypeChat.UnitTests/TestFailureContext.cs b/tests/TypeChat.UnitTests/TestFailureContext.cs index be26e097..7d9db797 100644 --- a/tests/TypeChat.UnitTests/TestFailureContext.cs +++ b/tests/TypeChat.UnitTests/TestFailureContext.cs @@ -160,6 +160,70 @@ public void ExistingFailureMetadataTakesPrecedenceOverFallbacks() Assert.Null(metadata.InputText); } + [Theory] + [InlineData(false, null)] + [InlineData(true, null)] + [InlineData(true, false)] + [InlineData(true, 0)] + [InlineData(true, "")] + public void FallbackFailureMetadataIsPreservedWithoutOverwritingLatestText(bool hasValue, object? value) + { + var metadata = new CustomFailureInfo { InputText = "earlier input", ResponseText = "earlier response" }; + if (hasValue) + { + metadata.Value = value; + } + Result original = Result.Error("latest failure") + .WithFailure(new FailureInfo { InputText = "", ResponseText = "" }); + + Result enriched = original.WithFailure(metadata); + var failure = Assert.IsType(enriched.Failure); + Assert.NotSame(metadata, failure); + Assert.Equal("CUSTOM", failure.Code); + Assert.Equal(hasValue, failure.HasValue); + Assert.Same(value, failure.Value); + Assert.Equal("", failure.InputText); + Assert.Equal("", failure.ResponseText); + Assert.IsType(original.Failure); + Assert.False(original.Failure.HasValue); + Assert.Equal("earlier input", metadata.InputText); + Assert.Equal("earlier response", metadata.ResponseText); + } + + [Fact] + public void CurrentCustomMetadataTakesPrecedenceWhenFallingBackToAValue() + { + var metadata = new CustomFailureInfo { Code = "LATEST", ResponseText = "" }; + Result original = Result.Error("latest failure").WithFailure(metadata); + var earlier = new CustomFailureInfo { Value = 42, Code = "EARLIER", ResponseText = "earlier response" }; + + Result enriched = original.WithFailure(earlier); + var failure = Assert.IsType(enriched.Failure); + Assert.Equal("LATEST", failure.Code); + Assert.True(failure.HasValue); + Assert.Equal(42, failure.Value); + Assert.Equal("", failure.ResponseText); + Assert.False(original.Failure.HasValue); + Assert.False(metadata.HasValue); + Assert.Equal("EARLIER", earlier.Code); + } + + [Theory] + [InlineData(null)] + [InlineData(false)] + [InlineData(0)] + [InlineData("")] + public void CurrentFailureValuesDoNotInheritEarlierCustomMetadata(object? value) + { + Result original = Result.Error("latest failure").WithFailure(new FailureInfo { Value = value }); + var earlier = new CustomFailureInfo { Value = 42 }; + + Result enriched = original.WithFailure(earlier); + var failure = Assert.IsType(enriched.Failure); + Assert.True(failure.HasValue); + Assert.Same(value, failure.Value); + } + [Fact] public void SuccessfulResultsKeepTheirTypedValueAndNoFailureContext() { @@ -342,6 +406,38 @@ public async Task MalformedRepairRetainsTheEarlierCandidateAndLatestText(string Assert.Equal(2, model.Prompts.Count); } + [Theory] + [InlineData("not JSON", "{\"Value\":42}")] + [InlineData("{\"Value\":", "{\"Value\":42}")] + [InlineData("{\"Value\":}", "{\"Value\":}")] + [InlineData("", "{\"Value\":42}")] + public async Task MalformedRepairRetainsCustomFailureMetadataAndLatestText(string response, string inputText) + { + var rejected = new Example { Value = 42 }; + var metadata = new CustomFailureInfo { Value = rejected }; + Result original = Result.Error("constraint failure").WithFailure(metadata); + var model = new FixedModel("before {\"Value\":42} after", response); + var translator = new JsonTranslator(model, new TypeValidator()) + { + ConstraintsValidator = new FixedConstraints(original) + }; + + TypeChatException error = await Assert.ThrowsAsync(() => translator.TranslateAsync("request")); + var failure = Assert.IsType(error.Failure); + Assert.Equal("CUSTOM", failure.Code); + Assert.True(failure.HasValue); + Assert.Same(rejected, failure.Value); + Assert.Equal(inputText, failure.InputText); + Assert.Equal(response, failure.ResponseText); + Assert.Equal(response, error.Response.ResponseText); + Assert.NotSame(original.Failure, failure); + Assert.Null(original.Failure.InputText); + Assert.Null(original.Failure.ResponseText); + Assert.Null(metadata.InputText); + Assert.Null(metadata.ResponseText); + Assert.Equal(2, model.Prompts.Count); + } + [Fact] public async Task AdditionalValidationRetainsTheTransformedInstance() {