diff --git a/README.md b/README.md index 662e10d..199c366 100644 --- a/README.md +++ b/README.md @@ -44,6 +44,58 @@ public class Milks } ``` +### Accessing failed output + +Failed validator results expose optional `FailureInfo` through `Result.Failure`. +`FailureInfo.Value` is an unvalidated `object`, not a value guaranteed to conform to `T`. +Check `HasValue` to distinguish unavailable data from explicit `null`, `false`, `0`, or empty strings. +JSON validators retain a cloned `JsonElement` when parsing succeeds, plus the original `InputText` +even when parsing fails. Their comment, trailing-comma, and depth-limit options are respected when +collecting failure data. Constraint and program validators retain the rejected instance. + +Translation still returns a typed value on success and throws `TypeChatException` on final validation +failure. The exception now exposes the same context through `Failure`, alongside its existing +`Response`: + +```cs +try +{ + CalendarActions actions = await translator.TranslateAsync(requestText); +} +catch (TypeChatException error) +{ + FailureInfo? failure = error.Failure; + if (failure?.HasValue == true) + { + object? rejected = failure.Value; + // Validate or repair the rejected value before treating it as CalendarActions. + } + string? originalResponse = failure?.ResponseText; +} +``` + +To manage repairs yourself, set `translator.MaxRepairAttempts = 0`. +On final failure, the context retains the latest rejected candidate and available model response text, +not a complete attempt history. These can come from different attempts if the latest repair response +cannot be parsed. Existing repair limits, diagnostics, events, typed success values, and cancellation +behavior are unchanged. + +HTTP, I/O, JSON response-decoding, and nested TypeChat exceptions during a repair are rethrown unchanged, +with context available as `error.Data[FailureInfo.ExceptionDataKey]`. +An initial transport failure has no invented candidate, and cancellation and programming exceptions +are not intercepted. Custom validators can return +`Result.Error(message).WithFailure(new FailureInfo { Value = rejected, ResponseText = text })`; +existing failure values and metadata take precedence over fallback context, without mutating the +original result. The existing `Error(value, message)` overload still retains its typed `Value` and +also exposes it as unvalidated failure context. + +When a malformed repair reuses an earlier candidate, its custom failure metadata is retained alongside +the latest available input and response text. A new rejected candidate does not inherit the previous +candidate's custom metadata. + +Use `Success` to distinguish outcomes, not the presence of a value. Rejected data and response text +may contain sensitive output; do not log or serialize them automatically. + ## Microsoft.TypeChat.Program ## TypeChat.Program translates natural language requests into simple programs (***Plans***), represented as JSON. @@ -99,6 +151,9 @@ so the projects use its SDK-provided implementation rather than overriding it wi * Go to the root directory of the project * dotnet build Typechat.sln +For [CVE-2026-62900](https://github.com/advisories/GHSA-23fw-v26w-5fgq), use a fixed .NET 8 SDK +(8.0.130 or 8.0.424, or later servicing releases), or a patched .NET 9/10 SDK. + ## Nuget Packages * Microsoft.Typechat [![](https://img.shields.io/nuget/dt/Microsoft.TypeChat.svg?label=downloads&color=007edf&logo=nuget)](https://www.nuget.org/packages/Microsoft.TypeChat) diff --git a/src/typechat.program/ProgramValidator.cs b/src/typechat.program/ProgramValidator.cs index 33ae030..0c48594 100644 --- a/src/typechat.program/ProgramValidator.cs +++ b/src/typechat.program/ProgramValidator.cs @@ -40,7 +40,8 @@ public Result Validate(string json) if (!result.Value.HasNotTranslated) { // Now validate the full program - return ValidateProgram(result.Value); + result = ValidateProgram(result.Value); + return result.Success ? result : result.WithFailure(new FailureInfo { InputText = json }); } } return result; @@ -50,7 +51,8 @@ public virtual Result ValidateProgram(Program program) { if (_programValidator is not null) { - return _programValidator.ValidateProgram(program); + Result result = _programValidator.ValidateProgram(program); + return result.Success ? result : result.WithFailure(new FailureInfo { Value = program }); } return program; } @@ -85,7 +87,7 @@ public override Result ValidateProgram(Program program) } catch (Exception ex) { - return Result.Error(ex.Message); + return Result.Error(ex.Message).WithFailure(new FailureInfo { Value = program }); } } } diff --git a/src/typechat/ConstraintsValidator.cs b/src/typechat/ConstraintsValidator.cs index f57d383..b7a0b27 100644 --- a/src/typechat/ConstraintsValidator.cs +++ b/src/typechat/ConstraintsValidator.cs @@ -63,7 +63,7 @@ public Result Validate(T value) return new Result(value); } - return Result.Error(result.ErrorMessage); + return Result.Error(result.ErrorMessage).WithFailure(new FailureInfo { Value = value }); } private string ToErrorString(List validationResults) diff --git a/src/typechat/FailureInfo.cs b/src/typechat/FailureInfo.cs new file mode 100644 index 0000000..67babb0 --- /dev/null +++ b/src/typechat/FailureInfo.cs @@ -0,0 +1,92 @@ +// Copyright (c) Microsoft. All rights reserved. + +namespace Microsoft.TypeChat; + +/// +/// Unvalidated output associated with a failure. Retains bounded context, not an attempt history. +/// +public class FailureInfo +{ + /// + /// Key used to attach failure context to transport exceptions without changing their type. + /// + public const string ExceptionDataKey = "Microsoft.TypeChat.FailureInfo"; + + private object? _value; + + /// + /// Rejected value, when available. Check HasValue before using it, and validate it before + /// treating it as the target type. Parsed JSON is retained as an independent JsonElement. + /// + public object? Value + { + get => _value; + set + { + _value = value; + HasValue = true; + } + } + + /// + /// Distinguishes unavailable data from an explicitly supplied value, including null. + /// + public bool HasValue { get; private set; } + + /// + /// JSON input supplied to a validator, when available. If supplied without a value, + /// the translator respects the validator's decision not to provide parsed data. + /// + public string? InputText { get; set; } + + /// + /// Latest model response text, when available. This can come from a later repair attempt + /// than Value if that response couldn't be parsed. + /// + public string? ResponseText { get; set; } + + internal FailureInfo WithFallback(FailureInfo? fallback) + { + // Preserve fallback metadata unless this context has its own value or custom metadata. + FailureInfo source = !HasValue && GetType() == typeof(FailureInfo) && fallback is not null ? fallback : this; + var result = (FailureInfo)source.MemberwiseClone(); + if (!result.HasValue && fallback?.HasValue == true) + { + result.Value = fallback.Value; + } + result.InputText = InputText ?? fallback?.InputText; + result.ResponseText = ResponseText ?? fallback?.ResponseText; + return result; + } + + internal void AttachTo(Exception exception) + { + exception.Data[ExceptionDataKey] = exception.Data[ExceptionDataKey] is FailureInfo existing ? + existing.WithFallback(this) : + WithFallback(null); + } + + internal static FailureInfo FromJson(string? json, JsonSerializerOptions options) + { + var result = new FailureInfo { InputText = json }; + if (json is null) + { + return result; + } + try + { + using JsonDocument document = JsonDocument.Parse(json, new JsonDocumentOptions + { + AllowTrailingCommas = options.AllowTrailingCommas, + CommentHandling = options.ReadCommentHandling, + MaxDepth = options.MaxDepth + }); + result.Value = document.RootElement.Clone(); + } + catch (JsonException) + { + // The original validation error reports malformed JSON; only input text is available. + } + return result; + } +} diff --git a/src/typechat/IConstraintsValidator.cs b/src/typechat/IConstraintsValidator.cs index 6e32d7b..685a270 100644 --- a/src/typechat/IConstraintsValidator.cs +++ b/src/typechat/IConstraintsValidator.cs @@ -9,7 +9,7 @@ namespace Microsoft.TypeChat; public interface IConstraintsValidator { /// - /// Validate the value T + /// Validate the value T. Failures can include FailureInfo retaining the rejected instance. /// /// check constraints for this value /// result of the validation diff --git a/src/typechat/IJsonTypeValidator.cs b/src/typechat/IJsonTypeValidator.cs index 3c1e058..4fdffb4 100644 --- a/src/typechat/IJsonTypeValidator.cs +++ b/src/typechat/IJsonTypeValidator.cs @@ -17,7 +17,8 @@ public interface IJsonTypeValidator /// /// Parses and validates the given JSON string according to the associated schema. Returns a /// Result containing the JSON object if validation was successful.Otherwise, returns - /// an Result with a message property that contains validation diagnostics. + /// a Result with a message containing validation diagnostics and optional FailureInfo + /// retaining the rejected value and input text. /// /// /// diff --git a/src/typechat/JsonSerializerTypeValidator.cs b/src/typechat/JsonSerializerTypeValidator.cs index cca26b9..81d6fee 100644 --- a/src/typechat/JsonSerializerTypeValidator.cs +++ b/src/typechat/JsonSerializerTypeValidator.cs @@ -57,14 +57,19 @@ public JsonSerializerTypeValidator(JsonSerializerOptions? options = null) } catch (JsonException jex) { - return Result.Error(ToErrorString(json, jex)); + return CreateError(json, ToErrorString(json, jex)); } catch (Exception ex) { - return Result.Error(ex.Message); + return CreateError(json, ex.Message); } } + private Result CreateError(string json, string message) + { + return Result.Error(message).WithFailure(FailureInfo.FromJson(json, _options)); + } + private string ToErrorString(string json, JsonException error) { StringBuilder sb = new StringBuilder(); diff --git a/src/typechat/JsonTranslator.cs b/src/typechat/JsonTranslator.cs index 3a7fe71..b9cf8f5 100644 --- a/src/typechat/JsonTranslator.cs +++ b/src/typechat/JsonTranslator.cs @@ -177,6 +177,7 @@ public async Task TranslateToObjectAsync(string request, CancellationTok /// /// Translate a natural language request into an object of type 'T' + /// Final validation failures retain unvalidated output in TypeChatException.Failure. /// /// text request /// optional cancel token @@ -208,9 +209,20 @@ public async Task TranslateAsync( requestSettings ??= _translationSettings; Prompt prompt = CreateRequestPrompt(request, preamble); int repairAttempts = 0; + FailureInfo? lastFailure = null; while (true) { - string responseText = await GetResponseAsync(prompt, requestSettings, cancelToken).ConfigureAwait(false); + string responseText; + try + { + responseText = await GetResponseAsync(prompt, requestSettings, cancelToken).ConfigureAwait(false); + } + catch (Exception ex) when (lastFailure is not null && + ex is HttpRequestException or IOException or JsonException or TypeChatException) + { + lastFailure.AttachTo(ex); + throw; + } JsonResponse jsonResponse = JsonResponse.Parse(responseText); Result validationResult; @@ -232,11 +244,18 @@ public async Task TranslateAsync( validationResult = Result.Error(TypeChatException.NoJson(jsonResponse)); } + validationResult = validationResult.WithFailure(new FailureInfo { ResponseText = responseText }); + if (lastFailure is not null) + { + validationResult = validationResult.WithFailure(lastFailure); + } + lastFailure = validationResult.Failure; + // Attempt to repair the Json that was returned ++repairAttempts; if (repairAttempts > _maxRepairAttempts) { - TypeChatException.ThrowJsonValidation(request, jsonResponse, validationResult.Message); + TypeChatException.ThrowJsonValidation(request, jsonResponse, validationResult.Message, lastFailure); } NotifyEvent(AttemptingRepair, validationResult.Message); @@ -275,15 +294,26 @@ protected virtual PromptSection CreateRepairPrompt(string responseText, Result ValidateJson(string json) { var result = Validator.Validate(json); + if (!result.Success) + { + result = result.WithFailure(result.Failure?.HasValue == true || result.Failure?.InputText is not null ? + new FailureInfo { InputText = json } : + FailureInfo.FromJson(json, JsonSerializerTypeValidator.DefaultOptions())); + } if (!OnValidationComplete(result)) { return result; } if (result.Success) { + T instance = result.Value; result = (_constraintsValidator is not null) ? _constraintsValidator.Validate(result.Value) : result; + if (!result.Success) + { + result = result.WithFailure(new FailureInfo { Value = instance, InputText = json }); + } } return result; } diff --git a/src/typechat/Result.cs b/src/typechat/Result.cs index 5431d4e..e1236e2 100644 --- a/src/typechat/Result.cs +++ b/src/typechat/Result.cs @@ -29,8 +29,9 @@ public Result(T value, string? message = null) public Result(Result value) { Success = value.Success; - Value = (T)value.Value; + Value = value.Success || value.Value is T ? (T)value.Value : default; Message = value.Message; + Failure = value.Failure; } internal Result(bool success, string? message) @@ -55,6 +56,31 @@ internal Result(bool success, string? message) /// public string? Message { get; set; } + /// + /// Optional unvalidated output associated with a failed result. Value remains the typed + /// result; use Failure.Value for rejected data that does not conform to T. + /// + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public FailureInfo? Failure { get; private set; } + + /// + /// Returns a shallow copy of this failed result, retaining existing failure fields and + /// filling unavailable fields from the supplied context. Custom result metadata is preserved. + /// + /// Fallback failure context. + /// A failed result with the combined context. + public Result WithFailure(FailureInfo failure) + { + ArgumentVerify.ThrowIfNull(failure, nameof(failure)); + if (Success) + { + throw new InvalidOperationException("Failure context cannot be added to a successful result."); + } + var result = (Result)MemberwiseClone(); + result.Failure = Failure is null ? failure.WithFallback(null) : Failure.WithFallback(failure); + return result; + } + public static implicit operator Result(T value) { return new Result(value); @@ -84,6 +110,7 @@ public static Result Error(T value, string message) { var result = new Result(false, message); result.Value = value; + result.Failure = new FailureInfo { Value = value }; return result; } } diff --git a/src/typechat/TypeChatException.cs b/src/typechat/TypeChatException.cs index 7d8e465..9c6f649 100644 --- a/src/typechat/TypeChatException.cs +++ b/src/typechat/TypeChatException.cs @@ -48,6 +48,15 @@ public TypeChatException(ErrorCode code, string request, JsonResponse? response _response = response; } + /// + /// Creates an exception retaining the rejected output and available model response text. + /// + public TypeChatException(ErrorCode code, string request, JsonResponse? response, string? message, FailureInfo? failure) + : this(code, request, response, message) + { + failure?.AttachTo(this); + } + /// /// Error code /// @@ -63,6 +72,11 @@ public TypeChatException(ErrorCode code, string request, JsonResponse? response /// public JsonResponse? Response => _response; + /// + /// Optional unvalidated output associated with the failure. + /// + public FailureInfo? Failure => Data[FailureInfo.ExceptionDataKey] as FailureInfo; + public override string ToString() { return $"Error: {Code}\n{Message}\n\nRequest:\n{Request}\n\nResponse:\n{Response.ResponseText}"; @@ -78,13 +92,14 @@ internal static void ThrowNoJson(string request, JsonResponse response) ); } - internal static void ThrowJsonValidation(string request, JsonResponse response, string message) + internal static void ThrowJsonValidation(string request, JsonResponse response, string message, FailureInfo? failure = null) { throw new TypeChatException( TypeChatException.ErrorCode.JsonValidation, request, response, - message + message, + failure ); } diff --git a/src/typechat/TypeValidator.cs b/src/typechat/TypeValidator.cs index abb2a8d..3ee2a31 100644 --- a/src/typechat/TypeValidator.cs +++ b/src/typechat/TypeValidator.cs @@ -47,7 +47,7 @@ public TypeValidator(TypescriptSchema schema) /// /// Validate the given Json string. /// If Result.Success: includes a value of type T - /// If Result.False: includes a diagnostic message + /// If Result.False: includes a diagnostic message and FailureInfo for the rejected JSON /// /// /// diff --git a/tests/TypeChat.UnitTests/TestFailureContext.cs b/tests/TypeChat.UnitTests/TestFailureContext.cs new file mode 100644 index 0000000..7d9db79 --- /dev/null +++ b/tests/TypeChat.UnitTests/TestFailureContext.cs @@ -0,0 +1,594 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Net.Http; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; + +namespace Microsoft.TypeChat.Tests; + +public class TestFailureContext +{ + public class Example + { + public int Value { get; set; } + } + + private sealed class FixedModel : ILanguageModel + { + private readonly Queue _responses; + + public FixedModel(params object[] responses) + { + _responses = new Queue(responses); + } + + public ModelInfo ModelInfo => "Mock"; + + public List Prompts { get; } = new List(); + + public Task CompleteAsync(Prompt prompt, TranslationSettings? settings, CancellationToken cancelToken) + { + Prompts.Add(prompt.Select(section => section.GetText()).ToArray()); + object response = _responses.Dequeue(); + return response is Exception error ? + Task.FromException(error) : + Task.FromResult((string)response); + } + } + + private sealed class FixedValidator : IJsonTypeValidator + { + private readonly Result _result; + + public FixedValidator(Result result) + { + _result = result; + } + + public TypeSchema Schema { get; } = ExampleSchema(); + + public Result Validate(string json) => _result; + } + + private sealed class FixedConstraints : IConstraintsValidator + { + private readonly Result _result; + + public FixedConstraints(Result result) + { + _result = result; + } + + public Result Validate(Example value) => _result; + } + + private sealed class CustomFailureInfo : FailureInfo + { + public string Code { get; set; } = "CUSTOM"; + } + + private sealed class CustomResult : Result + { + public CustomResult() + : base(Result.Error("custom")) + { + } + + public string Code { get; } = "CUSTOM"; + } + + private static JsonTranslator Translator(FixedModel model, int repairs = 1) + { + return new JsonTranslator(model, new FixedValidator(Result.Error("invalid value"))) + { + MaxRepairAttempts = repairs + }; + } + + private static TypeSchema ExampleSchema() + { + return new TypeSchema(typeof(Example), + new SchemaText("export interface Example { Value: number; }", SchemaText.Languages.Typescript)); + } + + [Fact] + public void MessageOnlyErrorsHaveNoFailureContext() + { + Result result = Result.Error("failed"); + Assert.False(result.Success); + Assert.Equal(0, result.Value); + Assert.Null(result.Failure); + } + + [Theory] + [InlineData(null)] + [InlineData(false)] + [InlineData(0)] + [InlineData("")] + public void ExplicitFailureValuesAreRetained(object? value) + { + Result result = Result.Error(value, "failed"); + Assert.False(result.Success); + Assert.Same(value, result.Value); + Assert.True(result.Failure.HasValue); + Assert.Same(value, result.Failure.Value); + } + + [Fact] + public void CloningFailuresDoesNotCastRejectedDataToTheSuccessType() + { + var source = Result.Error("rejected", "failed"); + var clone = new Result(source); + Assert.False(clone.Success); + Assert.Equal(0, clone.Value); + Assert.Equal("rejected", clone.Failure.Value); + } + + [Fact] + public void AddingContextDoesNotMutateExistingFailures() + { + Result original = Result.Error("failed"); + Result enriched = original.WithFailure(new FailureInfo { Value = null, ResponseText = "" }); + Assert.Null(original.Failure); + Assert.True(enriched.Failure.HasValue); + Assert.Null(enriched.Failure.Value); + Assert.Equal("", enriched.Failure.ResponseText); + } + + [Fact] + public void AddingContextPreservesCustomResultSubclasses() + { + var original = new CustomResult(); + var enriched = Assert.IsType(original.WithFailure(new FailureInfo { Value = 0 })); + Assert.Equal("CUSTOM", enriched.Code); + Assert.Equal(0, enriched.Failure.Value); + Assert.Null(original.Failure); + } + + [Fact] + public void ExistingFailureMetadataTakesPrecedenceOverFallbacks() + { + var metadata = new CustomFailureInfo { Value = null, ResponseText = "" }; + Result original = Result.Error("custom").WithFailure(metadata); + Result enriched = original.WithFailure(new FailureInfo { Value = 42, ResponseText = "fallback" }); + var failure = Assert.IsType(enriched.Failure); + Assert.Equal("CUSTOM", failure.Code); + Assert.True(failure.HasValue); + Assert.Null(failure.Value); + Assert.Equal("", failure.ResponseText); + Assert.Null(metadata.InputText); + } + + [Theory] + [InlineData(false, null)] + [InlineData(true, null)] + [InlineData(true, false)] + [InlineData(true, 0)] + [InlineData(true, "")] + public void FallbackFailureMetadataIsPreservedWithoutOverwritingLatestText(bool hasValue, object? value) + { + var metadata = new CustomFailureInfo { InputText = "earlier input", ResponseText = "earlier response" }; + if (hasValue) + { + metadata.Value = value; + } + Result original = Result.Error("latest failure") + .WithFailure(new FailureInfo { InputText = "", ResponseText = "" }); + + Result enriched = original.WithFailure(metadata); + var failure = Assert.IsType(enriched.Failure); + Assert.NotSame(metadata, failure); + Assert.Equal("CUSTOM", failure.Code); + Assert.Equal(hasValue, failure.HasValue); + Assert.Same(value, failure.Value); + Assert.Equal("", failure.InputText); + Assert.Equal("", failure.ResponseText); + Assert.IsType(original.Failure); + Assert.False(original.Failure.HasValue); + Assert.Equal("earlier input", metadata.InputText); + Assert.Equal("earlier response", metadata.ResponseText); + } + + [Fact] + public void CurrentCustomMetadataTakesPrecedenceWhenFallingBackToAValue() + { + var metadata = new CustomFailureInfo { Code = "LATEST", ResponseText = "" }; + Result original = Result.Error("latest failure").WithFailure(metadata); + var earlier = new CustomFailureInfo { Value = 42, Code = "EARLIER", ResponseText = "earlier response" }; + + Result enriched = original.WithFailure(earlier); + var failure = Assert.IsType(enriched.Failure); + Assert.Equal("LATEST", failure.Code); + Assert.True(failure.HasValue); + Assert.Equal(42, failure.Value); + Assert.Equal("", failure.ResponseText); + Assert.False(original.Failure.HasValue); + Assert.False(metadata.HasValue); + Assert.Equal("EARLIER", earlier.Code); + } + + [Theory] + [InlineData(null)] + [InlineData(false)] + [InlineData(0)] + [InlineData("")] + public void CurrentFailureValuesDoNotInheritEarlierCustomMetadata(object? value) + { + Result original = Result.Error("latest failure").WithFailure(new FailureInfo { Value = value }); + var earlier = new CustomFailureInfo { Value = 42 }; + + Result enriched = original.WithFailure(earlier); + var failure = Assert.IsType(enriched.Failure); + Assert.True(failure.HasValue); + Assert.Same(value, failure.Value); + } + + [Fact] + public void SuccessfulResultsKeepTheirTypedValueAndNoFailureContext() + { + Example value = new Example { Value = 42 }; + var result = new Result(value); + Assert.True(result.Success); + Assert.Same(value, result.Value); + Assert.Null(result.Failure); + Assert.Throws(() => result.WithFailure(new FailureInfo())); + } + + [Fact] + public void JsonValidatorsRetainRejectedJsonWithAnIndependentLifetime() + { + var validator = new TypeValidator(); + string json = "{\"Value\":\"invalid\"}"; + Result result = validator.Validate(json); + Assert.False(result.Success); + Assert.True(result.Failure.HasValue); + JsonElement rejected = Assert.IsType(result.Failure.Value); + Assert.Equal("invalid", rejected.GetProperty("Value").GetString()); + Assert.Equal(json, result.Failure.InputText); + Assert.Null(result.Failure.ResponseText); + } + + [Theory] + [InlineData("{\"Value\":}")] + [InlineData("")] + public void JsonValidatorsRetainMalformedInputWithoutInventingAValue(string json) + { + Result result = new TypeValidator().Validate(json); + Assert.False(result.Success); + Assert.False(result.Failure.HasValue); + Assert.Equal(json, result.Failure.InputText); + } + + [Fact] + public void ValueTypeValidationFailuresRemainFailuresInsteadOfThrowingDuringConversion() + { + var schema = new TypeSchema(typeof(int), new SchemaText("type Number = number;", SchemaText.Languages.Typescript)); + var validator = new JsonSerializerTypeValidator(schema); + Result result = validator.Validate("\"invalid\""); + Assert.False(result.Success); + Assert.Equal(0, result.Value); + Assert.Equal("invalid", Assert.IsType(result.Failure.Value).GetString()); + } + + [Fact] + public void FailureParsingHonorsValidatorJsonOptions() + { + var options = JsonSerializerTypeValidator.DefaultOptions(); + options.MaxDepth = 8; + var validator = new JsonSerializerTypeValidator(ExampleSchema(), options); + Result result = validator.Validate("{/* comment */\"Value\":\"invalid\",}"); + Assert.False(result.Success); + Assert.True(result.Failure.HasValue); + Assert.Equal("invalid", Assert.IsType(result.Failure.Value).GetProperty("Value").GetString()); + } + + [Fact] + public async Task TranslationFailureContextDoesNotBypassTheValidatorDepthLimit() + { + var options = JsonSerializerTypeValidator.DefaultOptions(); + options.MaxDepth = 1; + var validator = new JsonSerializerTypeValidator(ExampleSchema(), options); + string response = "{\"Value\":{\"nested\":1}}"; + var translator = new JsonTranslator(new FixedModel(response), validator) { MaxRepairAttempts = 0 }; + TypeChatException error = await Assert.ThrowsAsync(() => translator.TranslateAsync("request")); + Assert.False(error.Failure.HasValue); + Assert.Equal(response, error.Failure.InputText); + Assert.Equal(response, error.Failure.ResponseText); + } + + [Theory] + [InlineData("{/* comment */\"Value\":42}")] + [InlineData("{\"Value\":42,}")] + public async Task TranslationFailureContextDoesNotReparseRejectedJsonWithLooserOptions(string response) + { + var validator = new JsonSerializerTypeValidator(ExampleSchema(), new JsonSerializerOptions()); + var translator = new JsonTranslator(new FixedModel(response), validator) { MaxRepairAttempts = 0 }; + TypeChatException error = await Assert.ThrowsAsync(() => translator.TranslateAsync("request")); + Assert.False(error.Failure.HasValue); + Assert.Equal(response, error.Failure.InputText); + Assert.Equal(response, error.Failure.ResponseText); + } + + [Fact] + public void ConstraintValidatorsRetainTheRejectedInstance() + { + Person person = new Person + { + Name = new Name { FirstName = "First", LastName = "Last" }, + Age = 125 + }; + Result result = new ConstraintsValidator().Validate(person); + Assert.False(result.Success); + Assert.Same(person, result.Failure.Value); + Assert.Null(result.Value); + } + + [Fact] + public void ProgramValidationRetainsTheRejectedProgramAndInput() + { + string json = "{\"@steps\":[{\"@func\":\"missing\"}]}"; + Result result = new ProgramValidator(MathAPI.Default).Validate(json); + Assert.False(result.Success); + Assert.IsType(result.Failure.Value); + Assert.Equal(json, result.Failure.InputText); + Assert.Null(result.Value); + } + + [Fact] + public async Task TranslationFailuresRetainRejectedJsonAndFullModelText() + { + string response = "before {\"Value\":\"invalid\"} after"; + var model = new FixedModel(response); + var translator = new JsonTranslator(model, new TypeValidator()) { MaxRepairAttempts = 0 }; + TypeChatException error = await Assert.ThrowsAsync(() => translator.TranslateAsync("request")); + Assert.Equal(TypeChatException.ErrorCode.JsonValidation, error.Code); + Assert.Equal(response, error.Response.ResponseText); + Assert.Equal(response, error.Failure.ResponseText); + Assert.Equal("{\"Value\":\"invalid\"}", error.Failure.InputText); + Assert.Equal("invalid", Assert.IsType(error.Failure.Value).GetProperty("Value").GetString()); + Assert.Single(model.Prompts); + } + + [Theory] + [InlineData("not JSON")] + [InlineData("{\"Value\":")] + [InlineData("{\"Value\":}")] + [InlineData("")] + public async Task MalformedOutputHasTextButNoInventedCandidate(string response) + { + var model = new FixedModel(response); + TypeChatException error = await Assert.ThrowsAsync(() => Translator(model, 0).TranslateAsync("request")); + Assert.False(error.Failure.HasValue); + Assert.Equal(response, error.Failure.ResponseText); + } + + [Fact] + public async Task SuccessfulRepairAndDiagnosticEventsRemainUnchanged() + { + string first = "{\"Value\":\"invalid\"}"; + var model = new FixedModel(first, "{\"Value\":42}"); + var translator = new JsonTranslator(model, new TypeValidator()); + var responses = new List(); + var repairs = new List(); + translator.CompletionReceived += responses.Add; + translator.AttemptingRepair += repairs.Add; + Example result = await translator.TranslateAsync("request"); + Assert.Equal(42, result.Value); + Assert.Equal(2, model.Prompts.Count); + Assert.Contains(first, model.Prompts[1]); + Assert.Equal(2, responses.Count); + Assert.Single(repairs); + } + + [Fact] + public async Task FailedRepairReturnsTheLatestRejectedCandidate() + { + string second = "{\"Value\":\"second\"}"; + var model = new FixedModel("{\"Value\":\"first\"}", second); + TypeChatException error = await Assert.ThrowsAsync(() => Translator(model).TranslateAsync("request")); + Assert.Equal("second", Assert.IsType(error.Failure.Value).GetProperty("Value").GetString()); + Assert.Equal(second, error.Failure.ResponseText); + Assert.Equal(2, model.Prompts.Count); + } + + [Theory] + [InlineData("not JSON")] + [InlineData("{\"Value\":")] + [InlineData("{\"Value\":}")] + public async Task MalformedRepairRetainsTheEarlierCandidateAndLatestText(string response) + { + var model = new FixedModel("{\"Value\":\"first\"}", response); + TypeChatException error = await Assert.ThrowsAsync(() => Translator(model).TranslateAsync("request")); + Assert.Equal("first", Assert.IsType(error.Failure.Value).GetProperty("Value").GetString()); + Assert.Equal(response, error.Failure.ResponseText); + Assert.Equal(response, error.Response.ResponseText); + Assert.Equal(2, model.Prompts.Count); + } + + [Theory] + [InlineData("not JSON", "{\"Value\":42}")] + [InlineData("{\"Value\":", "{\"Value\":42}")] + [InlineData("{\"Value\":}", "{\"Value\":}")] + [InlineData("", "{\"Value\":42}")] + public async Task MalformedRepairRetainsCustomFailureMetadataAndLatestText(string response, string inputText) + { + var rejected = new Example { Value = 42 }; + var metadata = new CustomFailureInfo { Value = rejected }; + Result original = Result.Error("constraint failure").WithFailure(metadata); + var model = new FixedModel("before {\"Value\":42} after", response); + var translator = new JsonTranslator(model, new TypeValidator()) + { + ConstraintsValidator = new FixedConstraints(original) + }; + + TypeChatException error = await Assert.ThrowsAsync(() => translator.TranslateAsync("request")); + var failure = Assert.IsType(error.Failure); + Assert.Equal("CUSTOM", failure.Code); + Assert.True(failure.HasValue); + Assert.Same(rejected, failure.Value); + Assert.Equal(inputText, failure.InputText); + Assert.Equal(response, failure.ResponseText); + Assert.Equal(response, error.Response.ResponseText); + Assert.NotSame(original.Failure, failure); + Assert.Null(original.Failure.InputText); + Assert.Null(original.Failure.ResponseText); + Assert.Null(metadata.InputText); + Assert.Null(metadata.ResponseText); + Assert.Equal(2, model.Prompts.Count); + } + + [Fact] + public async Task AdditionalValidationRetainsTheTransformedInstance() + { + var transformed = new Example { Value = 99 }; + var translator = new JsonTranslator( + new FixedModel("{\"Value\":\"input\"}"), new FixedValidator(new Result(transformed))) + { + ConstraintsValidator = new FixedConstraints(Result.Error("constraint failure")), + MaxRepairAttempts = 0 + }; + TypeChatException error = await Assert.ThrowsAsync(() => translator.TranslateAsync("request")); + Assert.Same(transformed, error.Failure.Value); + Assert.Equal("constraint failure", error.Message); + } + + [Theory] + [InlineData(null)] + [InlineData(false)] + [InlineData(0)] + [InlineData("")] + public async Task CustomFailureValuesAndMetadataArePreservedWithoutMutation(object? value) + { + var metadata = new CustomFailureInfo { Value = value, ResponseText = "" }; + Result original = Result.Error("custom").WithFailure(metadata); + var translator = new JsonTranslator( + new FixedModel("{\"Value\":42}"), new FixedValidator(original)) + { MaxRepairAttempts = 0 }; + TypeChatException error = await Assert.ThrowsAsync(() => translator.TranslateAsync("request")); + var failure = Assert.IsType(error.Failure); + Assert.Equal("CUSTOM", failure.Code); + Assert.True(failure.HasValue); + Assert.Same(value, failure.Value); + Assert.Equal("", failure.ResponseText); + Assert.Null(original.Failure.InputText); + Assert.Null(metadata.InputText); + } + + [Theory] + [InlineData("not JSON")] + [InlineData("{\"Value\":")] + [InlineData("{\"Value\":}")] + public async Task MalformedInitialOutputStillAllowsRepair(string response) + { + var model = new FixedModel(response, "{\"Value\":42}"); + var translator = new JsonTranslator(model, new TypeValidator()); + Assert.Equal(42, (await translator.TranslateAsync("request")).Value); + Assert.Equal(2, model.Prompts.Count); + } + + [Fact] + public async Task MultipleRepairsRetainOnlyTheLatestCandidateAndExistingPromptWindow() + { + var model = new FixedModel("{\"Value\":\"first\"}", "{\"Value\":\"second\"}", "{\"Value\":\"third\"}"); + TypeChatException error = await Assert.ThrowsAsync(() => Translator(model, 2).TranslateAsync("request")); + Assert.Equal("third", Assert.IsType(error.Failure.Value).GetProperty("Value").GetString()); + Assert.Equal(3, model.Prompts.Count); + Assert.Equal(model.Prompts[0].Length + 2, model.Prompts[1].Length); + Assert.Equal(model.Prompts[1].Length, model.Prompts[2].Length); + Assert.DoesNotContain("{\"Value\":\"first\"}", model.Prompts[2]); + } + + [Fact] + public async Task RepairHttpFailuresKeepTheirOriginalExceptionAndCandidateContext() + { + var original = new HttpRequestException("network failure"); + var model = new FixedModel("{\"Value\":\"invalid\"}", original); + HttpRequestException error = await Assert.ThrowsAsync(() => Translator(model).TranslateAsync("request")); + Assert.Same(original, error); + var failure = Assert.IsType(error.Data[FailureInfo.ExceptionDataKey]); + Assert.Equal("invalid", Assert.IsType(failure.Value).GetProperty("Value").GetString()); + Assert.Equal("{\"Value\":\"invalid\"}", failure.ResponseText); + } + + [Fact] + public async Task InitialHttpFailuresHaveNoInventedCandidateContext() + { + var original = new HttpRequestException("network failure"); + HttpRequestException error = await Assert.ThrowsAsync(() => + Translator(new FixedModel(original)).TranslateAsync("request")); + Assert.Same(original, error); + Assert.False(error.Data.Contains(FailureInfo.ExceptionDataKey)); + } + + [Theory] + [InlineData("io")] + [InlineData("json")] + [InlineData("typechat")] + public async Task OtherExpectedRepairExceptionsRetainContextWithoutWrapping(string kind) + { + Exception original = kind switch + { + "io" => new IOException("response failure"), + "json" => new JsonException("response failure"), + "typechat" => new TypeChatException(TypeChatException.ErrorCode.Unknown, "request"), + _ => throw new ArgumentException("Unexpected exception kind", nameof(kind)) + }; + var model = new FixedModel("{\"Value\":\"invalid\"}", original); + Exception error = await Assert.ThrowsAnyAsync(() => Translator(model).TranslateAsync("request")); + Assert.Same(original, error); + var failure = Assert.IsType(error.Data[FailureInfo.ExceptionDataKey]); + Assert.Equal("invalid", Assert.IsType(failure.Value).GetProperty("Value").GetString()); + } + + [Fact] + public async Task ProgrammingExceptionsAreNotIntercepted() + { + var original = new InvalidOperationException("model bug"); + var model = new FixedModel("{\"Value\":\"invalid\"}", original); + InvalidOperationException error = await Assert.ThrowsAsync(() => + Translator(model).TranslateAsync("request")); + Assert.Same(original, error); + Assert.False(error.Data.Contains(FailureInfo.ExceptionDataKey)); + } + + [Fact] + public async Task ExistingTransportFailureMetadataIsPreserved() + { + var original = new HttpRequestException("network failure"); + original.Data[FailureInfo.ExceptionDataKey] = new CustomFailureInfo { Value = null, ResponseText = "" }; + var model = new FixedModel("{\"Value\":\"invalid\"}", original); + HttpRequestException error = await Assert.ThrowsAsync(() => Translator(model).TranslateAsync("request")); + var failure = Assert.IsType(error.Data[FailureInfo.ExceptionDataKey]); + Assert.Equal("CUSTOM", failure.Code); + Assert.True(failure.HasValue); + Assert.Null(failure.Value); + Assert.Equal("", failure.ResponseText); + } + + [Fact] + public async Task CancellationRemainsTheOriginalCancellationException() + { + using var cancellation = new CancellationTokenSource(); + cancellation.Cancel(); + var original = new OperationCanceledException(cancellation.Token); + var model = new FixedModel("{\"Value\":\"invalid\"}", original); + OperationCanceledException error = await Assert.ThrowsAsync(() => + Translator(model).TranslateAsync("request", cancellation.Token)); + Assert.Same(original, error); + Assert.Equal(cancellation.Token, error.CancellationToken); + Assert.False(error.Data.Contains(FailureInfo.ExceptionDataKey)); + } + + [Fact] + public async Task FailureContextDoesNotLeakIntoLaterTranslations() + { + var second = new HttpRequestException("second request"); + var model = new FixedModel("{\"Value\":\"invalid\"}", new HttpRequestException("repair"), second); + JsonTranslator translator = Translator(model); + await Assert.ThrowsAsync(() => translator.TranslateAsync("first")); + HttpRequestException error = await Assert.ThrowsAsync(() => translator.TranslateAsync("second")); + Assert.Same(second, error); + Assert.False(error.Data.Contains(FailureInfo.ExceptionDataKey)); + } +}