From 2fd112eeb5ee6db6eb0150b64d4ab4ab373490d6 Mon Sep 17 00:00:00 2001 From: Garrett Beatty Date: Wed, 2 Sep 2026 20:41:15 +0000 Subject: [PATCH 01/12] test(DurableExecution): conformance handlers for per-item/result serdes (map 9-14, parallel 8-15, invoke 5-16) Add conformance handlers exercising the new per-operation serializer slots: - map/MapCustomSerdes (9-14): MapConfig.ItemSerializer wraps each item result - parallel/ParallelCustomSerdes (8-15): ParallelConfig.ItemSerializer wraps each branch result - invoke/InvokeCustomResultSerdes (5-16): InvokeConfig.Serializer uppercases the result on deserialize Remove the corresponding NotImplemented declarations and wire the resources in template_map/parallel/invoke.yaml. All three pass against real AWS via the runner. Packaging fix: add Conformance/Directory.Build.props supplying the SDK's runtime NuGet dependencies (AWSSDK.Lambda, Microsoft.Extensions.Logging.Abstractions) to every handler, since those transitive package assets do not flow into a handler's net8.0 framework-dependent publish (WrapAsyncCore threw FileNotFoundException at runtime). Drop the now-redundant per-handler AWSSDK.Lambda references. --- .../Conformance/Directory.Build.props | 19 +++++ .../invoke/InvokeBasic/InvokeBasic.csproj | 1 - .../InvokeComplexObject.csproj | 1 - .../InvokeCustomPayloadSerdes.csproj | 1 - .../InvokeCustomResultSerdes/Function.cs | 61 ++++++++++++++++ .../InvokeCustomResultSerdes.csproj | 19 +++++ .../InvokeInChildContext.csproj | 1 - .../InvokeLargePayload.csproj | 1 - .../invoke/InvokeNull/InvokeNull.csproj | 1 - .../InvokeReplayRethrows.csproj | 1 - .../InvokeReplaySkips.csproj | 1 - .../InvokeSequential/InvokeSequential.csproj | 1 - .../InvokeTargetFails.csproj | 1 - .../InvokeTargetFailsCaught.csproj | 1 - .../InvokeThenStep/InvokeThenStep.csproj | 1 - .../InvokeWithName/InvokeWithName.csproj | 1 - .../InvokeWithTenantId.csproj | 1 - .../StepThenInvoke/StepThenInvoke.csproj | 1 - .../map/MapCustomSerdes/Function.cs | 72 ++++++++++++++++++ .../MapCustomSerdes/MapCustomSerdes.csproj | 19 +++++ .../parallel/ParallelCustomSerdes/Function.cs | 73 +++++++++++++++++++ .../ParallelCustomSerdes.csproj | 19 +++++ .../Conformance/template_invoke.yaml | 25 ++++++- .../Conformance/template_map.yaml | 29 ++++++-- .../Conformance/template_parallel.yaml | 21 +++++- 25 files changed, 345 insertions(+), 27 deletions(-) create mode 100644 Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/Directory.Build.props create mode 100644 Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/invoke/InvokeCustomResultSerdes/Function.cs create mode 100644 Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/invoke/InvokeCustomResultSerdes/InvokeCustomResultSerdes.csproj create mode 100644 Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/map/MapCustomSerdes/Function.cs create mode 100644 Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/map/MapCustomSerdes/MapCustomSerdes.csproj create mode 100644 Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/parallel/ParallelCustomSerdes/Function.cs create mode 100644 Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/parallel/ParallelCustomSerdes/ParallelCustomSerdes.csproj diff --git a/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/Directory.Build.props b/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/Directory.Build.props new file mode 100644 index 000000000..09b9cb0db --- /dev/null +++ b/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/Directory.Build.props @@ -0,0 +1,19 @@ + + + + + + + + + diff --git a/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/invoke/InvokeBasic/InvokeBasic.csproj b/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/invoke/InvokeBasic/InvokeBasic.csproj index fede088c4..4b354e178 100644 --- a/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/invoke/InvokeBasic/InvokeBasic.csproj +++ b/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/invoke/InvokeBasic/InvokeBasic.csproj @@ -14,7 +14,6 @@ - diff --git a/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/invoke/InvokeComplexObject/InvokeComplexObject.csproj b/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/invoke/InvokeComplexObject/InvokeComplexObject.csproj index fede088c4..4b354e178 100644 --- a/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/invoke/InvokeComplexObject/InvokeComplexObject.csproj +++ b/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/invoke/InvokeComplexObject/InvokeComplexObject.csproj @@ -14,7 +14,6 @@ - diff --git a/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/invoke/InvokeCustomPayloadSerdes/InvokeCustomPayloadSerdes.csproj b/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/invoke/InvokeCustomPayloadSerdes/InvokeCustomPayloadSerdes.csproj index fede088c4..4b354e178 100644 --- a/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/invoke/InvokeCustomPayloadSerdes/InvokeCustomPayloadSerdes.csproj +++ b/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/invoke/InvokeCustomPayloadSerdes/InvokeCustomPayloadSerdes.csproj @@ -14,7 +14,6 @@ - diff --git a/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/invoke/InvokeCustomResultSerdes/Function.cs b/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/invoke/InvokeCustomResultSerdes/Function.cs new file mode 100644 index 000000000..fdd984b14 --- /dev/null +++ b/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/invoke/InvokeCustomResultSerdes/Function.cs @@ -0,0 +1,61 @@ +// 5-16: Invoke with custom result serdes (custom deserializer for the returned result) +// The target (echo) returns the input unchanged; the custom result serializer uppercases +// the deserialized result. Input is serialized normally (so the checkpointed input / +// ChainedInvokeSucceeded result stay "hello"), and the workflow returns the uppercased +// "HELLO" after the custom serde runs on replay. +using System.IO; +using Amazon.Lambda.Core; +using Amazon.Lambda.DurableExecution; +using Amazon.Lambda.RuntimeSupport; +using Amazon.Lambda.Serialization.SystemTextJson; + +namespace InvokeCustomResultSerdes; + +public class Function +{ + public static async Task Main(string[] args) + { + var handler = new Function(); + var serializer = new DefaultLambdaJsonSerializer(); + using var handlerWrapper = HandlerWrapper.GetHandlerWrapper(handler.Handler, serializer); + using var bootstrap = new LambdaBootstrap(handlerWrapper); + await bootstrap.RunAsync(); + } + + public Task Handler( + DurableExecutionInvocationInput input, ILambdaContext context) + => DurableFunction.WrapAsync(Workflow, input, context); + + private async Task Workflow(string input, IDurableContext context) + { + var targetFunctionName = Environment.GetEnvironmentVariable("TARGET_FUNCTION_NAME")!; + + var result = await context.InvokeAsync( + targetFunctionName, + input, + config: new InvokeConfig { Serializer = new UppercaseResultSerializer() }); + + return result; + } +} + +/// +/// Custom result serializer: serializes the outgoing payload normally, but uppercases +/// the raw serialized result on deserialize (the "custom result serdes" applied to the +/// chained invoke's serialized payload — e.g. "hello" becomes "HELLO", +/// so the JSON-decoded execution result is the string "HELLO"). +/// +public sealed class UppercaseResultSerializer : ILambdaSerializer +{ + private readonly DefaultLambdaJsonSerializer _inner = new(); + + public T Deserialize(Stream requestStream) + { + using var reader = new StreamReader(requestStream); + var raw = reader.ReadToEnd(); + return (T)(object)raw.ToUpperInvariant(); + } + + public void Serialize(T response, Stream responseStream) + => _inner.Serialize(response, responseStream); +} diff --git a/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/invoke/InvokeCustomResultSerdes/InvokeCustomResultSerdes.csproj b/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/invoke/InvokeCustomResultSerdes/InvokeCustomResultSerdes.csproj new file mode 100644 index 000000000..4b354e178 --- /dev/null +++ b/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/invoke/InvokeCustomResultSerdes/InvokeCustomResultSerdes.csproj @@ -0,0 +1,19 @@ + + + + net8.0 + Exe + true + bootstrap + enable + enable + + + + + + + + + + diff --git a/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/invoke/InvokeInChildContext/InvokeInChildContext.csproj b/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/invoke/InvokeInChildContext/InvokeInChildContext.csproj index fede088c4..4b354e178 100644 --- a/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/invoke/InvokeInChildContext/InvokeInChildContext.csproj +++ b/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/invoke/InvokeInChildContext/InvokeInChildContext.csproj @@ -14,7 +14,6 @@ - diff --git a/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/invoke/InvokeLargePayload/InvokeLargePayload.csproj b/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/invoke/InvokeLargePayload/InvokeLargePayload.csproj index fede088c4..4b354e178 100644 --- a/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/invoke/InvokeLargePayload/InvokeLargePayload.csproj +++ b/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/invoke/InvokeLargePayload/InvokeLargePayload.csproj @@ -14,7 +14,6 @@ - diff --git a/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/invoke/InvokeNull/InvokeNull.csproj b/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/invoke/InvokeNull/InvokeNull.csproj index fede088c4..4b354e178 100644 --- a/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/invoke/InvokeNull/InvokeNull.csproj +++ b/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/invoke/InvokeNull/InvokeNull.csproj @@ -14,7 +14,6 @@ - diff --git a/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/invoke/InvokeReplayRethrows/InvokeReplayRethrows.csproj b/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/invoke/InvokeReplayRethrows/InvokeReplayRethrows.csproj index fede088c4..4b354e178 100644 --- a/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/invoke/InvokeReplayRethrows/InvokeReplayRethrows.csproj +++ b/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/invoke/InvokeReplayRethrows/InvokeReplayRethrows.csproj @@ -14,7 +14,6 @@ - diff --git a/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/invoke/InvokeReplaySkips/InvokeReplaySkips.csproj b/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/invoke/InvokeReplaySkips/InvokeReplaySkips.csproj index fede088c4..4b354e178 100644 --- a/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/invoke/InvokeReplaySkips/InvokeReplaySkips.csproj +++ b/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/invoke/InvokeReplaySkips/InvokeReplaySkips.csproj @@ -14,7 +14,6 @@ - diff --git a/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/invoke/InvokeSequential/InvokeSequential.csproj b/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/invoke/InvokeSequential/InvokeSequential.csproj index fede088c4..4b354e178 100644 --- a/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/invoke/InvokeSequential/InvokeSequential.csproj +++ b/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/invoke/InvokeSequential/InvokeSequential.csproj @@ -14,7 +14,6 @@ - diff --git a/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/invoke/InvokeTargetFails/InvokeTargetFails.csproj b/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/invoke/InvokeTargetFails/InvokeTargetFails.csproj index fede088c4..4b354e178 100644 --- a/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/invoke/InvokeTargetFails/InvokeTargetFails.csproj +++ b/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/invoke/InvokeTargetFails/InvokeTargetFails.csproj @@ -14,7 +14,6 @@ - diff --git a/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/invoke/InvokeTargetFailsCaught/InvokeTargetFailsCaught.csproj b/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/invoke/InvokeTargetFailsCaught/InvokeTargetFailsCaught.csproj index fede088c4..4b354e178 100644 --- a/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/invoke/InvokeTargetFailsCaught/InvokeTargetFailsCaught.csproj +++ b/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/invoke/InvokeTargetFailsCaught/InvokeTargetFailsCaught.csproj @@ -14,7 +14,6 @@ - diff --git a/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/invoke/InvokeThenStep/InvokeThenStep.csproj b/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/invoke/InvokeThenStep/InvokeThenStep.csproj index fede088c4..4b354e178 100644 --- a/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/invoke/InvokeThenStep/InvokeThenStep.csproj +++ b/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/invoke/InvokeThenStep/InvokeThenStep.csproj @@ -14,7 +14,6 @@ - diff --git a/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/invoke/InvokeWithName/InvokeWithName.csproj b/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/invoke/InvokeWithName/InvokeWithName.csproj index fede088c4..4b354e178 100644 --- a/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/invoke/InvokeWithName/InvokeWithName.csproj +++ b/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/invoke/InvokeWithName/InvokeWithName.csproj @@ -14,7 +14,6 @@ - diff --git a/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/invoke/InvokeWithTenantId/InvokeWithTenantId.csproj b/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/invoke/InvokeWithTenantId/InvokeWithTenantId.csproj index fede088c4..4b354e178 100644 --- a/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/invoke/InvokeWithTenantId/InvokeWithTenantId.csproj +++ b/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/invoke/InvokeWithTenantId/InvokeWithTenantId.csproj @@ -14,7 +14,6 @@ - diff --git a/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/invoke/StepThenInvoke/StepThenInvoke.csproj b/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/invoke/StepThenInvoke/StepThenInvoke.csproj index fede088c4..4b354e178 100644 --- a/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/invoke/StepThenInvoke/StepThenInvoke.csproj +++ b/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/invoke/StepThenInvoke/StepThenInvoke.csproj @@ -14,7 +14,6 @@ - diff --git a/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/map/MapCustomSerdes/Function.cs b/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/map/MapCustomSerdes/Function.cs new file mode 100644 index 000000000..78dce1ec6 --- /dev/null +++ b/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/map/MapCustomSerdes/Function.cs @@ -0,0 +1,72 @@ +// 9-14: Map with a custom per-item serdes (ItemSerializer) +// Each iteration returns the uppercased item directly; the custom per-item +// serializer wraps the value as `wrapped:` on serialize (the checkpointed +// iteration payload) and unwraps it on deserialize, so the ordered results +// survive the round-trip. MaxConcurrency=1 gives a deterministic history. +using System.IO; +using System.Text; +using Amazon.Lambda.Core; +using Amazon.Lambda.DurableExecution; +using Amazon.Lambda.RuntimeSupport; +using Amazon.Lambda.Serialization.SystemTextJson; + +namespace MapCustomSerdes; + +public class Function +{ + public static async Task Main(string[] args) + { + var handler = new Function(); + var serializer = new DefaultLambdaJsonSerializer(); + using var handlerWrapper = HandlerWrapper.GetHandlerWrapper(handler.Handler, serializer); + using var bootstrap = new LambdaBootstrap(handlerWrapper); + await bootstrap.RunAsync(); + } + + public Task Handler( + DurableExecutionInvocationInput input, ILambdaContext context) + => DurableFunction.WrapAsync>(Workflow, input, context); + + private async Task> Workflow(object? input, IDurableContext context) + { + var result = await context.MapAsync( + new List { "x", "y" }, + async (ctx, item, index, all, ct) => item.ToUpperInvariant(), + name: "serdes", + config: new MapConfig + { + MaxConcurrency = 1, + ItemSerializer = new WrapSerializer(), + }); + + return result.GetResults().ToList(); + } +} + +/// +/// A real (non-identity) per-item serializer: serializes a string value v +/// as the raw text wrapped:v and deserializes wrapped:v back to +/// v. Used only for per-item results, so it only handles strings. +/// +public sealed class WrapSerializer : ILambdaSerializer +{ + private const string Prefix = "wrapped:"; + + public T Deserialize(Stream requestStream) + { + using var reader = new StreamReader(requestStream); + var text = reader.ReadToEnd(); + if (text.StartsWith(Prefix, StringComparison.Ordinal)) + { + text = text.Substring(Prefix.Length); + } + return (T)(object)text; + } + + public void Serialize(T response, Stream responseStream) + { + var text = response as string ?? response?.ToString() ?? string.Empty; + var bytes = Encoding.UTF8.GetBytes(Prefix + text); + responseStream.Write(bytes, 0, bytes.Length); + } +} diff --git a/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/map/MapCustomSerdes/MapCustomSerdes.csproj b/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/map/MapCustomSerdes/MapCustomSerdes.csproj new file mode 100644 index 000000000..4b354e178 --- /dev/null +++ b/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/map/MapCustomSerdes/MapCustomSerdes.csproj @@ -0,0 +1,19 @@ + + + + net8.0 + Exe + true + bootstrap + enable + enable + + + + + + + + + + diff --git a/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/parallel/ParallelCustomSerdes/Function.cs b/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/parallel/ParallelCustomSerdes/Function.cs new file mode 100644 index 000000000..13eb6987d --- /dev/null +++ b/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/parallel/ParallelCustomSerdes/Function.cs @@ -0,0 +1,73 @@ +// 8-15: Parallel with a custom per-branch serde (ItemSerializer) +// Each branch returns a string directly ("x", "y"); the custom per-branch serializer +// wraps each value as the envelope {"wrapped": v} on serialize (the checkpointed branch +// payload) and unwraps it on deserialize, so the ordered results round-trip back to +// ["x", "y"]. MaxConcurrency=1 gives a deterministic history. +using System.IO; +using System.Text; +using System.Text.Json; +using Amazon.Lambda.Core; +using Amazon.Lambda.DurableExecution; +using Amazon.Lambda.RuntimeSupport; +using Amazon.Lambda.Serialization.SystemTextJson; + +namespace ParallelCustomSerdes; + +public class Function +{ + public static async Task Main(string[] args) + { + var handler = new Function(); + var serializer = new DefaultLambdaJsonSerializer(); + using var handlerWrapper = HandlerWrapper.GetHandlerWrapper(handler.Handler, serializer); + using var bootstrap = new LambdaBootstrap(handlerWrapper); + await bootstrap.RunAsync(); + } + + public Task Handler( + DurableExecutionInvocationInput input, ILambdaContext context) + => DurableFunction.WrapAsync>(Workflow, input, context); + + private async Task> Workflow(object? input, IDurableContext context) + { + var result = await context.ParallelAsync( + new Func>[] + { + async (_, _) => { await Task.CompletedTask; return "x"; }, + async (_, _) => { await Task.CompletedTask; return "y"; }, + }, + name: "serde", + config: new ParallelConfig + { + MaxConcurrency = 1, + ItemSerializer = new WrapJsonSerializer(), + }); + + return result.GetResults().ToList(); + } +} + +/// +/// A symmetric per-branch serializer: serializes a string value v as the JSON +/// envelope {"wrapped":"v"} and deserializes that envelope back to v. +/// Used only for per-branch results, so it only handles strings. +/// +public sealed class WrapJsonSerializer : ILambdaSerializer +{ + public T Deserialize(Stream requestStream) + { + using var reader = new StreamReader(requestStream); + var text = reader.ReadToEnd(); + using var doc = JsonDocument.Parse(text); + var value = doc.RootElement.GetProperty("wrapped").GetString() ?? string.Empty; + return (T)(object)value; + } + + public void Serialize(T response, Stream responseStream) + { + var value = response as string ?? response?.ToString() ?? string.Empty; + var json = "{\"wrapped\":" + JsonSerializer.Serialize(value) + "}"; + var bytes = Encoding.UTF8.GetBytes(json); + responseStream.Write(bytes, 0, bytes.Length); + } +} diff --git a/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/parallel/ParallelCustomSerdes/ParallelCustomSerdes.csproj b/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/parallel/ParallelCustomSerdes/ParallelCustomSerdes.csproj new file mode 100644 index 000000000..4b354e178 --- /dev/null +++ b/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/parallel/ParallelCustomSerdes/ParallelCustomSerdes.csproj @@ -0,0 +1,19 @@ + + + + net8.0 + Exe + true + bootstrap + enable + enable + + + + + + + + + + diff --git a/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/template_invoke.yaml b/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/template_invoke.yaml index 747dbe078..074d2a342 100644 --- a/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/template_invoke.yaml +++ b/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/template_invoke.yaml @@ -69,9 +69,6 @@ Resources: Type: AWS::Serverless::Function TestingMetadata: TestDescription: ["5-1"] - NotImplemented: - - id: "5-16" - reason: "Invoke with custom result serdes: .NET has no per-operation serdes slot; the invoke result is deserialized via the one registered ILambdaSerializer (matches the Java approach)." Metadata: BuildMethod: makefile Properties: @@ -415,3 +412,25 @@ Resources: Variables: TARGET_FUNCTION_NAME: !Sub "${InvokeEchoTarget.Arn}:$LATEST" + + InvokeCustomResultSerdes: + Type: AWS::Serverless::Function + TestingMetadata: + TestDescription: ["5-16"] + Metadata: + BuildMethod: makefile + Properties: + CodeUri: publish/InvokeCustomResultSerdes/ + Handler: bootstrap + Description: Invoke with a custom result serdes (uppercases the deserialized result) + Role: + Fn::GetAtt: + - DurableFunctionRole + - Arn + DurableConfig: + RetentionPeriodInDays: 7 + ExecutionTimeout: 300 + Environment: + Variables: + TARGET_FUNCTION_NAME: + !Sub "${InvokeEchoTarget.Arn}:$LATEST" diff --git a/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/template_map.yaml b/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/template_map.yaml index 2c74104f3..550f2edb9 100644 --- a/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/template_map.yaml +++ b/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/template_map.yaml @@ -8,10 +8,11 @@ Globals: MemorySize: 512 # Coverage note: .NET's MapConfig exposes MaxConcurrency, CompletionConfig, -# NestingType, and ItemNamer, but no item-level or whole-result serdes slots -# (per-item checkpoint payloads use the registered ILambdaSerializer). Tests -# 9-14 (per-item serdes), 9-19 and 9-20 (operation-level serdes) therefore have -# no .NET example and are left uncovered by design, matching the Java approach. +# NestingType, ItemNamer, and ItemSerializer (per-item result serializer). The +# per-item serdes test (9-14) uses ItemSerializer. The whole-result serdes tests +# 9-19 and 9-20 have no .NET example — MapConfig has no operation-level (aggregate) +# serializer slot; the aggregate BatchResult envelope is SDK-internal — so they are +# left uncovered by design, matching the Java approach. Resources: DurableFunctionRole: @@ -42,8 +43,6 @@ Resources: TestingMetadata: TestDescription: ["9-1"] NotImplemented: - - id: "9-14" - reason: "Per-item serdes: .NET MapConfig has no item-level serializer slot; all payloads use the one registered ILambdaSerializer (matches the Java approach)." - id: "9-19" reason: "Operation-level serdes: .NET MapConfig has no whole-result serializer slot; all payloads use the one registered ILambdaSerializer." - id: "9-20" @@ -349,3 +348,21 @@ Resources: DurableConfig: RetentionPeriodInDays: 7 ExecutionTimeout: 300 + + MapCustomSerdes: + Type: AWS::Serverless::Function + TestingMetadata: + TestDescription: ["9-14"] + Metadata: + BuildMethod: makefile + Properties: + CodeUri: publish/MapCustomSerdes/ + Handler: bootstrap + Description: Map with a custom per-item serializer (ItemSerializer round-trips each result) + Role: + Fn::GetAtt: + - DurableFunctionRole + - Arn + DurableConfig: + RetentionPeriodInDays: 7 + ExecutionTimeout: 300 diff --git a/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/template_parallel.yaml b/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/template_parallel.yaml index 589382ca8..3313371e0 100644 --- a/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/template_parallel.yaml +++ b/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/template_parallel.yaml @@ -36,9 +36,6 @@ Resources: Type: AWS::Serverless::Function TestingMetadata: TestDescription: ["8-1"] - NotImplemented: - - id: "8-15" - reason: "Parallel with custom per-item serdes: .NET ParallelConfig has no item serializer slot; all branch payloads use the one registered ILambdaSerializer (matches the Java approach)." Metadata: BuildMethod: makefile Properties: @@ -412,3 +409,21 @@ Resources: DurableConfig: RetentionPeriodInDays: 7 ExecutionTimeout: 300 + + ParallelCustomSerdes: + Type: AWS::Serverless::Function + TestingMetadata: + TestDescription: ["8-15"] + Metadata: + BuildMethod: makefile + Properties: + CodeUri: publish/ParallelCustomSerdes/ + Handler: bootstrap + Description: Parallel with a custom per-branch serde (ItemSerializer wraps/unwraps each branch result) + Role: + Fn::GetAtt: + - DurableFunctionRole + - Arn + DurableConfig: + RetentionPeriodInDays: 7 + ExecutionTimeout: 300 From ff4b9b719bd1ff8ae0d14f8a18dc7d7bcb3d2b07 Mon Sep 17 00:00:00 2001 From: Garrett Beatty Date: Wed, 2 Sep 2026 20:50:46 +0000 Subject: [PATCH 02/12] feat(DurableExecution): round-trip step/child results through the serializer on fresh success MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On a fresh (non-replay) success, StepOperation and ChildContextOperation now return the value deserialized from the just-written checkpoint instead of the original in-memory object, matching replay semantics. This makes a custom per-operation serializer's transform observable in the operation result on the first execution, not only on replay (a non-round-tripping serializer previously had no effect on the fresh result). Overflow (replay-children) child results are unaffected — the payload is stripped and the value is recovered by re-execution. Behavior change documented in the AutoVer change file. --- .../12a4a1f7-d59f-4544-aa1f-6db30289485e.json | 11 +++++++++++ .../Internal/ChildContextOperation.cs | 13 +++++++++++++ .../Internal/StepOperation.cs | 12 ++++++++++-- 3 files changed, 34 insertions(+), 2 deletions(-) create mode 100644 .autover/changes/12a4a1f7-d59f-4544-aa1f-6db30289485e.json diff --git a/.autover/changes/12a4a1f7-d59f-4544-aa1f-6db30289485e.json b/.autover/changes/12a4a1f7-d59f-4544-aa1f-6db30289485e.json new file mode 100644 index 000000000..c2b4327cc --- /dev/null +++ b/.autover/changes/12a4a1f7-d59f-4544-aa1f-6db30289485e.json @@ -0,0 +1,11 @@ +{ + "Projects": [ + { + "Name": "Amazon.Lambda.DurableExecution", + "Type": "Minor", + "ChangelogMessages": [ + "Behavior change: steps and child contexts now round-trip their result through the configured serializer on a fresh (non-replay) success, deserializing the just-written checkpoint before returning \u2014 matching replay semantics. A custom per-operation serializer\u0027s transform is now reflected in the operation result on the first execution, not only on replay. The returned value is a fresh deserialized instance rather than the exact object the step/child body produced. Overflow (replay-children) results are unaffected." + ] + } + ] +} \ No newline at end of file diff --git a/Libraries/src/Amazon.Lambda.DurableExecution/Internal/ChildContextOperation.cs b/Libraries/src/Amazon.Lambda.DurableExecution/Internal/ChildContextOperation.cs index 5dd4671b1..2cef36626 100644 --- a/Libraries/src/Amazon.Lambda.DurableExecution/Internal/ChildContextOperation.cs +++ b/Libraries/src/Amazon.Lambda.DurableExecution/Internal/ChildContextOperation.cs @@ -253,6 +253,19 @@ await EnqueueAsync(new SdkOperationUpdate ? new SdkContextOptions { ReplayChildren = true } : null }, cancellationToken); + + // Non-overflow: round-trip the just-written checkpoint so the value the + // workflow observes on this fresh execution matches replay (where the + // result is always deserialized from the checkpoint). This makes a + // custom (possibly non-round-tripping) ChildContextConfig.Serializer's + // transform visible in the child-context result on the first run. + // Behavior change: the returned object is no longer the same instance + // the child body produced. See the AutoVer change note. Overflow skips + // this (the payload was stripped; the value is recovered by replay). + if (!overflow) + { + return DeserializeResult(serialized); + } } return result; diff --git a/Libraries/src/Amazon.Lambda.DurableExecution/Internal/StepOperation.cs b/Libraries/src/Amazon.Lambda.DurableExecution/Internal/StepOperation.cs index 76780957a..bb580fbfd 100644 --- a/Libraries/src/Amazon.Lambda.DurableExecution/Internal/StepOperation.cs +++ b/Libraries/src/Amazon.Lambda.DurableExecution/Internal/StepOperation.cs @@ -234,6 +234,7 @@ private async Task ExecuteFunc(int attemptNumber, CancellationToken cancellat result = await _func(stepContext, linked.Token); } + var serialized = SerializeResult(result); await EnqueueAsync(new SdkOperationUpdate { Id = OperationId, @@ -242,10 +243,17 @@ await EnqueueAsync(new SdkOperationUpdate Action = OperationAction.SUCCEED, SubType = OperationSubTypes.Step, Name = Name, - Payload = SerializeResult(result) + Payload = serialized }, cancellationToken); - return result; + // Round-trip the just-written checkpoint so the value the workflow + // observes on this fresh execution is the deserialized-from-checkpoint + // value, exactly as it would be on replay. This makes a custom + // (possibly non-round-tripping) StepConfig.Serializer's transform + // visible in the step result on the first run, not just on replay. + // Behavior change: the returned object is no longer the same instance + // the step body produced. See the AutoVer change note. + return DeserializeResult(serialized); } catch (OperationCanceledException) when (linked.IsCancellationRequested) { From 21e1e53f2ecf736dc17fc8ed44ca3ecf6878f671 Mon Sep 17 00:00:00 2001 From: Garrett Beatty Date: Wed, 2 Sep 2026 20:50:46 +0000 Subject: [PATCH 03/12] test(DurableExecution): real per-op serdes conformance handlers for step 1-6 and child 3-14 Refactor StepCustomSerdes (1-6) to use a real StepConfig.Serializer (uppercase-on-serialize) instead of transforming inside the step body, and add child/ChildCustomSerdes (3-14) using ChildContextConfig.Serializer. Both rely on the fresh-success round-trip so the serialize-side transform reaches the result. Remove the 3-14 NotImplemented and wire the resource in template_child.yaml. Both pass against real AWS via the runner. --- .../ChildCustomSerdes.csproj | 19 +++++ .../child/ChildCustomSerdes/Function.cs | 74 +++++++++++++++++++ .../step/StepCustomSerdes/Function.cs | 38 ++++++++-- .../Conformance/template_child.yaml | 21 +++++- 4 files changed, 142 insertions(+), 10 deletions(-) create mode 100644 Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/child/ChildCustomSerdes/ChildCustomSerdes.csproj create mode 100644 Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/child/ChildCustomSerdes/Function.cs diff --git a/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/child/ChildCustomSerdes/ChildCustomSerdes.csproj b/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/child/ChildCustomSerdes/ChildCustomSerdes.csproj new file mode 100644 index 000000000..4b354e178 --- /dev/null +++ b/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/child/ChildCustomSerdes/ChildCustomSerdes.csproj @@ -0,0 +1,19 @@ + + + + net8.0 + Exe + true + bootstrap + enable + enable + + + + + + + + + + diff --git a/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/child/ChildCustomSerdes/Function.cs b/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/child/ChildCustomSerdes/Function.cs new file mode 100644 index 000000000..dec0ffe46 --- /dev/null +++ b/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/child/ChildCustomSerdes/Function.cs @@ -0,0 +1,74 @@ +// 3-14: Child context with custom serdes (succeed) +// A child context whose custom serializer uppercases the child result on serialize. +// The inner step returns the input via the global serializer (so its checkpoint stays +// "hello child"); the child context serializes its result with the custom serdes +// (raw uppercased "HELLO CHILD"). Thanks to the SDK's fresh-success round-trip, the +// child context returns the uppercased value, so the execution result is uppercased. +using System.IO; +using System.Text; +using Amazon.Lambda.Core; +using Amazon.Lambda.DurableExecution; +using Amazon.Lambda.RuntimeSupport; +using Amazon.Lambda.Serialization.SystemTextJson; + +namespace ChildCustomSerdes; + +public class Function +{ + public static async Task Main(string[] args) + { + var handler = new Function(); + var serializer = new DefaultLambdaJsonSerializer(); + using var handlerWrapper = HandlerWrapper.GetHandlerWrapper(handler.Handler, serializer); + using var bootstrap = new LambdaBootstrap(handlerWrapper); + await bootstrap.RunAsync(); + } + + public Task Handler( + DurableExecutionInvocationInput input, ILambdaContext context) + => DurableFunction.WrapAsync(Workflow, input, context); + + private async Task Workflow(string input, IDurableContext context) + { + var result = await context.RunInChildContextAsync( + async (childContext, _ct) => + { + var stepResult = await childContext.StepAsync( + async (_, _s) => + { + await Task.CompletedTask; + return input; + }); + + return stepResult; + }, + name: "child-serdes", + config: new ChildContextConfig + { + SubType = "RunInChildContext", + Serializer = new UppercaseSerializer(), + }); + + return result; + } +} + +/// +/// A custom serializer that uppercases the result on serialize (writing the raw +/// transformed text) and reads it back verbatim on deserialize. +/// +public sealed class UppercaseSerializer : ILambdaSerializer +{ + public T Deserialize(Stream requestStream) + { + using var reader = new StreamReader(requestStream); + return (T)(object)reader.ReadToEnd(); + } + + public void Serialize(T response, Stream responseStream) + { + var value = (response as string ?? response?.ToString() ?? string.Empty).ToUpperInvariant(); + var bytes = Encoding.UTF8.GetBytes(value); + responseStream.Write(bytes, 0, bytes.Length); + } +} diff --git a/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/step/StepCustomSerdes/Function.cs b/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/step/StepCustomSerdes/Function.cs index 7f043ef18..d3774fe24 100644 --- a/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/step/StepCustomSerdes/Function.cs +++ b/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/step/StepCustomSerdes/Function.cs @@ -1,7 +1,11 @@ -// 1-6: Custom serdes (per-step) - transforms string to uppercase -// Note: The .NET SDK does not have a per-step serdes API like the JS SDK. -// Instead, we achieve the same effect by transforming the value within the step -// function itself, since the step result is what gets checkpointed. +// 1-6: Custom serdes (per-step) via StepConfig.Serializer +// The step returns the input unchanged; a custom per-step serializer transforms the +// result to uppercase on serialize. Thanks to the SDK's fresh-success round-trip +// (the result is deserialized from the just-written checkpoint before being returned), +// the uppercased value is what the workflow observes and returns — so the execution +// result is the uppercased input. +using System.IO; +using System.Text; using Amazon.Lambda.Core; using Amazon.Lambda.DurableExecution; using Amazon.Lambda.RuntimeSupport; @@ -30,10 +34,30 @@ private async Task Workflow(string input, IDurableContext context) async (_, _ct) => { await Task.CompletedTask; - // Simulate custom serdes by transforming to uppercase - return input.ToUpperInvariant(); - }); + return input; + }, + config: new StepConfig { Serializer = new UppercaseSerializer() }); return result; } } + +/// +/// A custom per-step serializer that uppercases the result on serialize (writing the +/// raw transformed text) and reads it back verbatim on deserialize. +/// +public sealed class UppercaseSerializer : ILambdaSerializer +{ + public T Deserialize(Stream requestStream) + { + using var reader = new StreamReader(requestStream); + return (T)(object)reader.ReadToEnd(); + } + + public void Serialize(T response, Stream responseStream) + { + var value = (response as string ?? response?.ToString() ?? string.Empty).ToUpperInvariant(); + var bytes = Encoding.UTF8.GetBytes(value); + responseStream.Write(bytes, 0, bytes.Length); + } +} diff --git a/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/template_child.yaml b/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/template_child.yaml index 4bb91c2db..f985be4d8 100644 --- a/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/template_child.yaml +++ b/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/template_child.yaml @@ -60,9 +60,6 @@ Resources: Type: AWS::Serverless::Function TestingMetadata: TestDescription: ["3-1"] - NotImplemented: - - id: "3-14" - reason: "Child context with custom serdes: .NET has no per-operation serdes slot; all payloads use the one registered ILambdaSerializer (matches the Java approach)." Metadata: BuildMethod: makefile Properties: @@ -372,3 +369,21 @@ Resources: DurableConfig: RetentionPeriodInDays: 7 ExecutionTimeout: 300 + + ChildCustomSerdes: + Type: AWS::Serverless::Function + TestingMetadata: + TestDescription: ["3-14"] + Metadata: + BuildMethod: makefile + Properties: + CodeUri: publish/ChildCustomSerdes/ + Handler: bootstrap + Description: Child context with a custom serdes (uppercases the child result on serialize) + Role: + Fn::GetAtt: + - DurableFunctionRole + - Arn + DurableConfig: + RetentionPeriodInDays: 7 + ExecutionTimeout: 300 From 545df80d55d2cc40da47ce2846460a55a60d4624 Mon Sep 17 00:00:00 2001 From: Garrett Beatty Date: Thu, 3 Sep 2026 00:11:56 +0000 Subject: [PATCH 04/12] address review: harden conformance serde handlers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Custom serializers now guard typeof(T) == typeof(string) and throw a clear NotSupportedException for unsupported result types (instead of an opaque InvalidCastException from the (T)(object) cast). - Read with an explicit UTF-8 StreamReader for symmetry with the UTF-8 bytes written. - MapCustomSerdes: drop the async-without-await iteration lambda (CS1998); return Task.FromResult(...) so no per-item async state machine is allocated. - InvokeCustomResultSerdes: correct the header comment — the serializer serializes the outbound request payload normally on the initial execution and applies the uppercase transform on deserialize (which, for a chained invoke, happens on replay). The handlers still transform the raw serialized payload, which is the behavior the conformance requirements assert (e.g. invoke 5-16 ExpectedResult '"HELLO"'); the string path is byte-identical, so the suite results are unchanged. --- .../child/ChildCustomSerdes/Function.cs | 5 ++++- .../invoke/InvokeCustomResultSerdes/Function.cs | 15 ++++++++++----- .../Conformance/map/MapCustomSerdes/Function.cs | 7 +++++-- .../parallel/ParallelCustomSerdes/Function.cs | 5 ++++- .../Conformance/step/StepCustomSerdes/Function.cs | 5 ++++- 5 files changed, 27 insertions(+), 10 deletions(-) diff --git a/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/child/ChildCustomSerdes/Function.cs b/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/child/ChildCustomSerdes/Function.cs index dec0ffe46..d32c1248b 100644 --- a/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/child/ChildCustomSerdes/Function.cs +++ b/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/child/ChildCustomSerdes/Function.cs @@ -61,7 +61,10 @@ public sealed class UppercaseSerializer : ILambdaSerializer { public T Deserialize(Stream requestStream) { - using var reader = new StreamReader(requestStream); + if (typeof(T) != typeof(string)) + throw new NotSupportedException( + $"{nameof(UppercaseSerializer)} only supports string results; got {typeof(T)}."); + using var reader = new StreamReader(requestStream, Encoding.UTF8); return (T)(object)reader.ReadToEnd(); } diff --git a/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/invoke/InvokeCustomResultSerdes/Function.cs b/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/invoke/InvokeCustomResultSerdes/Function.cs index fdd984b14..70fe3470a 100644 --- a/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/invoke/InvokeCustomResultSerdes/Function.cs +++ b/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/invoke/InvokeCustomResultSerdes/Function.cs @@ -1,9 +1,11 @@ // 5-16: Invoke with custom result serdes (custom deserializer for the returned result) -// The target (echo) returns the input unchanged; the custom result serializer uppercases -// the deserialized result. Input is serialized normally (so the checkpointed input / -// ChainedInvokeSucceeded result stay "hello"), and the workflow returns the uppercased -// "HELLO" after the custom serde runs on replay. +// The custom serializer serializes the OUTBOUND request payload normally on the initial +// execution (its Serialize delegates to the default JSON serializer, so the checkpointed +// input and the ChainedInvokeSucceeded result both stay "hello"). It uppercases the value +// when the result is deserialized — which, for a chained invoke, happens on the replay +// that observes the completed invocation — so the workflow returns the uppercased "HELLO". using System.IO; +using System.Text; using Amazon.Lambda.Core; using Amazon.Lambda.DurableExecution; using Amazon.Lambda.RuntimeSupport; @@ -51,7 +53,10 @@ public sealed class UppercaseResultSerializer : ILambdaSerializer public T Deserialize(Stream requestStream) { - using var reader = new StreamReader(requestStream); + if (typeof(T) != typeof(string)) + throw new NotSupportedException( + $"{nameof(UppercaseResultSerializer)} only supports string results; got {typeof(T)}."); + using var reader = new StreamReader(requestStream, Encoding.UTF8); var raw = reader.ReadToEnd(); return (T)(object)raw.ToUpperInvariant(); } diff --git a/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/map/MapCustomSerdes/Function.cs b/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/map/MapCustomSerdes/Function.cs index 78dce1ec6..80d77927e 100644 --- a/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/map/MapCustomSerdes/Function.cs +++ b/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/map/MapCustomSerdes/Function.cs @@ -31,7 +31,7 @@ private async Task> Workflow(object? input, IDurableContext context { var result = await context.MapAsync( new List { "x", "y" }, - async (ctx, item, index, all, ct) => item.ToUpperInvariant(), + (ctx, item, index, all, ct) => Task.FromResult(item.ToUpperInvariant()), name: "serdes", config: new MapConfig { @@ -54,7 +54,10 @@ public sealed class WrapSerializer : ILambdaSerializer public T Deserialize(Stream requestStream) { - using var reader = new StreamReader(requestStream); + if (typeof(T) != typeof(string)) + throw new NotSupportedException( + $"{nameof(WrapSerializer)} only supports string results; got {typeof(T)}."); + using var reader = new StreamReader(requestStream, Encoding.UTF8); var text = reader.ReadToEnd(); if (text.StartsWith(Prefix, StringComparison.Ordinal)) { diff --git a/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/parallel/ParallelCustomSerdes/Function.cs b/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/parallel/ParallelCustomSerdes/Function.cs index 13eb6987d..4a7e455d6 100644 --- a/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/parallel/ParallelCustomSerdes/Function.cs +++ b/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/parallel/ParallelCustomSerdes/Function.cs @@ -56,7 +56,10 @@ public sealed class WrapJsonSerializer : ILambdaSerializer { public T Deserialize(Stream requestStream) { - using var reader = new StreamReader(requestStream); + if (typeof(T) != typeof(string)) + throw new NotSupportedException( + $"{nameof(WrapJsonSerializer)} only supports string results; got {typeof(T)}."); + using var reader = new StreamReader(requestStream, Encoding.UTF8); var text = reader.ReadToEnd(); using var doc = JsonDocument.Parse(text); var value = doc.RootElement.GetProperty("wrapped").GetString() ?? string.Empty; diff --git a/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/step/StepCustomSerdes/Function.cs b/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/step/StepCustomSerdes/Function.cs index d3774fe24..6fa4233f6 100644 --- a/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/step/StepCustomSerdes/Function.cs +++ b/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/step/StepCustomSerdes/Function.cs @@ -50,7 +50,10 @@ public sealed class UppercaseSerializer : ILambdaSerializer { public T Deserialize(Stream requestStream) { - using var reader = new StreamReader(requestStream); + if (typeof(T) != typeof(string)) + throw new NotSupportedException( + $"{nameof(UppercaseSerializer)} only supports string results; got {typeof(T)}."); + using var reader = new StreamReader(requestStream, Encoding.UTF8); return (T)(object)reader.ReadToEnd(); } From f0bf542ff2d91165e3543d35cfcd31f9860b1757 Mon Sep 17 00:00:00 2001 From: Garrett Beatty Date: Tue, 1 Sep 2026 22:32:06 +0000 Subject: [PATCH 05/12] feat(DurableExecution): incremental, heterogeneous Parallel API (#2519) Adds an additive, branch-oriented parallel API alongside the existing homogeneous ParallelAsync overloads: await using var parallel = ctx.CreateParallel(name: "process-order"); IParallelBranch inv = parallel.BranchAsync("inventory", ...); IParallelBranch pay = parallel.BranchAsync("payment", ...); IBatchResult summary = await parallel.CompleteAsync(); InventoryReservation r = await inv; // own type, no shared base/cast/envelope Each branch declares its own result type (heterogeneous) and returns an awaitable typed handle; branches are registered incrementally and start executing on registration (gated by MaxConcurrency); CompleteAsync seals, awaits per CompletionConfig, and checkpoints the aggregate. Implementation reuses the existing machinery so replay is identical to the batch API: each branch runs as a ChildContextOperation with the same deterministic child op id (hash("{parentId}-{index}")) and the same parent CONTEXT/Parallel BatchSummary checkpoint shape. Terminal-parent replay reconstructs branch outcomes from the frozen inline summary (re-running only overflow-stripped branches); DisposeAsync auto-completes so `await using` always writes the terminal checkpoint. MaxConcurrency, CompletionConfig, NestingType, cancellation, and ILambdaSerializer are honored unchanged. Also factors the BatchSummary (de)serialization + overflow handling out of ConcurrentOperation into a shared BatchSummaryCodec so the batch and incremental parallel paths cannot diverge on the wire format. New public API: - IDurableContext.CreateParallel(name?, config?) - IDurableParallel (BranchAsync, CompleteAsync, IAsyncDisposable) - IParallelBranch (Name/Index/Status, awaitable) Tests: 16 unit tests (IncrementalParallelOperationTests) covering fresh happy path, heterogeneous types, deterministic ids, MaxConcurrency, completion short-circuit/skip, failure surfacing, empty, replay reconstruct, name-drift, and STARTED-parent replay. Two integration tests (heterogeneous end-to-end and replay determinism across the Run and Terminal-reconstruct paths), both verified green against the durable execution service. All 428 unit tests pass; docs/core/parallel.md documents the new API. --- .../DurableContext.cs | 13 + .../IDurableContext.cs | 39 + .../IDurableParallel.cs | 94 +++ .../IParallelBranch.cs | 64 ++ .../Internal/BatchSummaryCodec.cs | 86 ++ .../Internal/ConcurrentOperation.cs | 70 +- .../Internal/IncrementalParallelOperation.cs | 791 ++++++++++++++++++ .../docs/core/parallel.md | 34 + .../IncrementalParallelHeterogeneousTest.cs | 74 ++ .../IncrementalParallelReplayTest.cs | 111 +++ .../Function.cs | 91 ++ ...mentalParallelHeterogeneousFunction.csproj | 18 + .../Function.cs | 90 ++ .../IncrementalParallelReplayFunction.csproj | 18 + .../IncrementalParallelOperationTests.cs | 529 ++++++++++++ 15 files changed, 2063 insertions(+), 59 deletions(-) create mode 100644 Libraries/src/Amazon.Lambda.DurableExecution/IDurableParallel.cs create mode 100644 Libraries/src/Amazon.Lambda.DurableExecution/IParallelBranch.cs create mode 100644 Libraries/src/Amazon.Lambda.DurableExecution/Internal/BatchSummaryCodec.cs create mode 100644 Libraries/src/Amazon.Lambda.DurableExecution/Internal/IncrementalParallelOperation.cs create mode 100644 Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/IncrementalParallelHeterogeneousTest.cs create mode 100644 Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/IncrementalParallelReplayTest.cs create mode 100644 Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/TestFunctions/IncrementalParallelHeterogeneousFunction/Function.cs create mode 100644 Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/TestFunctions/IncrementalParallelHeterogeneousFunction/IncrementalParallelHeterogeneousFunction.csproj create mode 100644 Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/TestFunctions/IncrementalParallelReplayFunction/Function.cs create mode 100644 Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/TestFunctions/IncrementalParallelReplayFunction/IncrementalParallelReplayFunction.csproj create mode 100644 Libraries/test/Amazon.Lambda.DurableExecution.Tests/IncrementalParallelOperationTests.cs diff --git a/Libraries/src/Amazon.Lambda.DurableExecution/DurableContext.cs b/Libraries/src/Amazon.Lambda.DurableExecution/DurableContext.cs index 1144c6bf1..1de3f84ee 100644 --- a/Libraries/src/Amazon.Lambda.DurableExecution/DurableContext.cs +++ b/Libraries/src/Amazon.Lambda.DurableExecution/DurableContext.cs @@ -192,6 +192,19 @@ private Task> RunCallback( return op.ExecuteAsync(cancellationToken); } + public IDurableParallel CreateParallel( + string? name = null, + ParallelConfig? config = null) + { + var effectiveConfig = config ?? new ParallelConfig(); + var serializer = LambdaSerializerHelper.GetRequired(LambdaContext); + + var operationId = _idGenerator.NextId(); + return new Internal.IncrementalParallelOperation( + operationId, name, _idGenerator.ParentId, effectiveConfig, serializer, MakeChildFactory(), + _state, _terminationManager, _workflowCancellation, _durableExecutionArn, _batcher); + } + public Task> ParallelAsync( IReadOnlyList>> branches, string? name = null, diff --git a/Libraries/src/Amazon.Lambda.DurableExecution/IDurableContext.cs b/Libraries/src/Amazon.Lambda.DurableExecution/IDurableContext.cs index 9d536f5d3..3f1465f72 100644 --- a/Libraries/src/Amazon.Lambda.DurableExecution/IDurableContext.cs +++ b/Libraries/src/Amazon.Lambda.DurableExecution/IDurableContext.cs @@ -367,6 +367,45 @@ Task WaitForConditionAsync( string? name = null, CancellationToken cancellationToken = default); + /// + /// Create an incremental, branch-oriented parallel operation. Unlike the + /// + /// overloads — which take a complete branch list up front and share one result + /// type — the returned lets you register branches + /// one at a time via + /// , + /// each with its own result type (heterogeneous), starting each branch as it is + /// registered. Call + /// to seal registration and obtain the aggregate . + /// + /// + /// Use await using so the operation is sealed and its terminal checkpoint + /// written even if + /// is not called. Branch identity is positional and deterministic across + /// replays, so register the same branches in the same order every invocation — + /// derive any dynamic branch set from a checkpointed step. Per-branch results + /// are serialized via the registered on + /// . Honors the same + /// , + /// , and + /// as the homogeneous API. + /// + /// + /// An optional name for the parallel operation, used for observability and to + /// derive the deterministic operation ID. Defaults to a name inferred from the + /// call site. + /// + /// + /// Optional parallel configuration. Defaults are used when null. + /// + /// + /// An for registering branches and awaiting the + /// aggregate result. + /// + IDurableParallel CreateParallel( + string? name = null, + ParallelConfig? config = null); + /// /// Execute multiple branches concurrently. Each branch runs inside its own /// child context; per-branch results are aggregated into an diff --git a/Libraries/src/Amazon.Lambda.DurableExecution/IDurableParallel.cs b/Libraries/src/Amazon.Lambda.DurableExecution/IDurableParallel.cs new file mode 100644 index 000000000..7c5118deb --- /dev/null +++ b/Libraries/src/Amazon.Lambda.DurableExecution/IDurableParallel.cs @@ -0,0 +1,94 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +namespace Amazon.Lambda.DurableExecution; + +/// +/// An incremental, branch-oriented parallel operation created by +/// . Branches +/// are registered one at a time via +/// , +/// each with its own result type (heterogeneous), and each begins executing +/// immediately (subject to ). Call +/// to seal +/// registration, await the branches according to the +/// , and obtain the aggregate result. +/// +/// +/// This is an additive alternative to the homogeneous +/// +/// overloads, which accept a complete branch list up front and share one result +/// type. Use CreateParallel when branches return unrelated types, or when +/// branches are discovered incrementally (for example, tool calls derived from a +/// checkpointed plan) and earlier branches should start before later ones are known. +/// +/// Deterministic replay. Branch identity is positional: the n-th +/// +/// call reuses the n-th deterministic operation ID. Workflow code must therefore +/// register the same branches in the same order across invocations — produce any +/// dynamic branch list inside a checkpointed +/// so replay sees the same set. +/// +/// +/// Disposal. seals and +/// completes the operation if +/// was not called, +/// so an await using block always writes the parallel's terminal checkpoint. +/// Calling +/// explicitly is recommended so you can capture the aggregate result. +/// +/// +public interface IDurableParallel : IAsyncDisposable +{ + /// + /// Registers a branch and immediately begins executing it (respecting + /// ). Returns a typed + /// handle for retrieving the branch's result. + /// + /// + /// The branch runs inside its own child context with a deterministic + /// operation-ID space; its result is serialized to a checkpoint via the + /// registered on + /// . Per-branch + /// failures are captured on the handle and aggregated into the + /// result — a + /// branch failure never throws out of this method. + /// + /// The branch's result type. + /// + /// Human-readable branch name. Required; surfaces on + /// OperationUpdate.Name and must remain stable at a given branch index + /// across deployments (a drift is a non-deterministic-execution error). + /// + /// + /// The branch body. Receives its own and a + /// linking the SDK's + /// workflow-shutdown signal with the operation's completion-policy + /// short-circuit, and returns the branch's result. + /// + /// A typed handle for awaiting the branch's result. + /// + /// The operation has already been sealed by + /// or disposal. + /// + IParallelBranch BranchAsync( + string name, + Func> func); + + /// + /// Seals registration (no further branches may be added), awaits the + /// registered branches according to the + /// , checkpoints the aggregate + /// outcome, and returns it. Idempotent — repeated calls return the same result. + /// + /// + /// Like the homogeneous parallel API, this never throws on per-branch failure: + /// inspect / + /// , or await individual branch + /// handles, to observe failures. It does propagate workflow-level errors (for + /// example ) and cancellation. + /// + /// A token to observe for cancellation. + /// The aggregate summarizing branch outcomes. + Task CompleteAsync(CancellationToken cancellationToken = default); +} diff --git a/Libraries/src/Amazon.Lambda.DurableExecution/IParallelBranch.cs b/Libraries/src/Amazon.Lambda.DurableExecution/IParallelBranch.cs new file mode 100644 index 000000000..7a282d518 --- /dev/null +++ b/Libraries/src/Amazon.Lambda.DurableExecution/IParallelBranch.cs @@ -0,0 +1,64 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +using System.Runtime.CompilerServices; + +namespace Amazon.Lambda.DurableExecution; + +/// +/// A typed handle to a single branch registered on an +/// via . +/// Unlike the homogeneous +/// API — where every branch shares one result type T — each branch on an +/// declares its own result type, so a single +/// parallel operation can mix, for example, an InventoryReservation branch +/// with a PaymentAuthorization branch. +/// +/// +/// The handle is await-able: await branch yields the branch's typed +/// result once it succeeds, or rethrows the branch's failure (a +/// ) if it failed. Awaiting a branch that was +/// skipped by the operation's short-circuit (its +/// is ) throws a +/// — inspect before +/// awaiting when a completion policy may skip branches. +/// +/// Typically you await the handle after +/// +/// has sealed and resolved the operation, mirroring the Java SDK's +/// future.get() after the try-with-resources block. A branch may +/// still be awaited earlier; the await simply completes when the branch does. +/// +/// +/// The branch's result type. +public interface IParallelBranch +{ + /// + /// The branch name supplied at registration. Surfaces on the wire + /// OperationUpdate.Name field and in execution traces. + /// + string Name { get; } + + /// + /// Zero-based registration order of this branch within its parallel + /// operation. Stable across replays and used to derive the branch's + /// deterministic operation ID. + /// + int Index { get; } + + /// + /// The branch's outcome. until the + /// branch settles (and permanently for a branch skipped by a completion-policy + /// short-circuit), then or + /// . + /// + BatchItemStatus Status { get; } + + /// + /// Enables await branch. Yields the branch's typed result on success, + /// rethrows its on failure, or throws a + /// if the branch was skipped. + /// + /// An awaiter over the branch's result. + TaskAwaiter GetAwaiter(); +} diff --git a/Libraries/src/Amazon.Lambda.DurableExecution/Internal/BatchSummaryCodec.cs b/Libraries/src/Amazon.Lambda.DurableExecution/Internal/BatchSummaryCodec.cs new file mode 100644 index 000000000..032c421f5 --- /dev/null +++ b/Libraries/src/Amazon.Lambda.DurableExecution/Internal/BatchSummaryCodec.cs @@ -0,0 +1,86 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +using System.Text; +using System.Text.Json; + +namespace Amazon.Lambda.DurableExecution.Internal; + +/// +/// Shared (de)serialization primitives for the payload +/// stored on a concurrent operation's parent CONTEXT checkpoint. Centralising the +/// wire-string mappings and the payload serialize / overflow check here keeps the +/// batch () and incremental +/// () parallel implementations in exact +/// agreement on the on-the-wire format, so a checkpoint written by one is +/// reconstructable by the other. +/// +internal static class BatchSummaryCodec +{ + /// + /// Serializes the summary to its JSON payload using the source-generated + /// (trim/AOT safe). + /// + public static string ToPayload(BatchSummary summary) + => JsonSerializer.Serialize(summary, BatchJsonContext.Default.BatchSummary); + + /// + /// True when exceeds the per-operation checkpoint + /// byte limit and must be re-emitted stripped (statuses only) with + /// ReplayChildren=true. + /// + public static bool IsOverflow(string payload) + => Encoding.UTF8.GetByteCount(payload) > DurableConstants.MaxOperationCheckpointBytes; + + /// + /// Deserializes a from a checkpoint payload, + /// tolerating null/empty/corrupt payloads by returning null (callers + /// fall back to inferring per-unit status from child checkpoints). + /// + public static BatchSummary? ParseSummary(string? payload) + { + if (string.IsNullOrEmpty(payload)) return null; + try + { + return JsonSerializer.Deserialize(payload, BatchJsonContext.Default.BatchSummary); + } + catch (JsonException) + { + // Tolerate older / corrupted payloads — fall back to inferring status + // from per-unit checkpoints. + return null; + } + } + + public static string SerializeStatus(BatchItemStatus status) => status switch + { + BatchItemStatus.Succeeded => "SUCCEEDED", + BatchItemStatus.Failed => "FAILED", + BatchItemStatus.Started => "STARTED", + _ => throw new ArgumentOutOfRangeException(nameof(status)) + }; + + public static BatchItemStatus DeserializeStatus(string? wire) => wire switch + { + "SUCCEEDED" => BatchItemStatus.Succeeded, + "FAILED" => BatchItemStatus.Failed, + "STARTED" => BatchItemStatus.Started, + _ => BatchItemStatus.Started + }; + + public static string SerializeCompletionReason(CompletionReason reason) => reason switch + { + CompletionReason.AllCompleted => "ALL_COMPLETED", + CompletionReason.MinSuccessfulReached => "MIN_SUCCESSFUL_REACHED", + CompletionReason.FailureToleranceExceeded => "FAILURE_TOLERANCE_EXCEEDED", + _ => throw new ArgumentOutOfRangeException(nameof(reason)) + }; + + public static CompletionReason DeserializeCompletionReason(string? wire) => wire switch + { + "ALL_COMPLETED" => CompletionReason.AllCompleted, + "MIN_SUCCESSFUL_REACHED" => CompletionReason.MinSuccessfulReached, + "FAILURE_TOLERANCE_EXCEEDED" => CompletionReason.FailureToleranceExceeded, + _ => CompletionReason.AllCompleted + }; +} diff --git a/Libraries/src/Amazon.Lambda.DurableExecution/Internal/ConcurrentOperation.cs b/Libraries/src/Amazon.Lambda.DurableExecution/Internal/ConcurrentOperation.cs index 44658262f..b8000b077 100644 --- a/Libraries/src/Amazon.Lambda.DurableExecution/Internal/ConcurrentOperation.cs +++ b/Libraries/src/Amazon.Lambda.DurableExecution/Internal/ConcurrentOperation.cs @@ -3,7 +3,6 @@ using System.IO; using System.Text; -using System.Text.Json; using Amazon.Lambda; using Amazon.Lambda.Core; using SdkContextOptions = Amazon.Lambda.Model.ContextOptions; @@ -390,7 +389,7 @@ private async Task> ReplayChildrenAsync(Operation frozen, Cancel { cancellationToken.ThrowIfCancellationRequested(); - var summary = ParseSummary(frozen.ContextDetails?.Result); + var summary = BatchSummaryCodec.ParseSummary(frozen.ContextDetails?.Result); var unitCount = UnitCount; var items = new List>(unitCount); @@ -401,7 +400,7 @@ private async Task> ReplayChildrenAsync(Operation frozen, Cancel // Frozen per-unit status is authoritative. var status = summaryEntry != null - ? DeserializeStatus(summaryEntry.Status) + ? BatchSummaryCodec.DeserializeStatus(summaryEntry.Status) : BatchItemStatus.Started; // Same unit-name drift check as ReconstructFromCheckpoints: code must @@ -459,7 +458,7 @@ private async Task> ReplayChildrenAsync(Operation frozen, Cancel // Completion reason is pinned from the frozen summary; fall back to // recomputing only if the summary is absent/corrupt. var completionReason = summary != null - ? DeserializeCompletionReason(summary.CompletionReason) + ? BatchSummaryCodec.DeserializeCompletionReason(summary.CompletionReason) : ComputeCompletionReason(items, unitCount); // No re-checkpoint: the parent is already terminal in state. Return the @@ -707,7 +706,7 @@ BatchSummary BuildSummary(bool includeInline) var s = new BatchSummary { - CompletionReason = SerializeCompletionReason(completionReason), + CompletionReason = BatchSummaryCodec.SerializeCompletionReason(completionReason), Units = new List(UnitCount) }; for (var i = 0; i < UnitCount; i++) @@ -718,7 +717,7 @@ BatchSummary BuildSummary(bool includeInline) { Index = i, Name = item?.Name ?? unitName, - Status = SerializeStatus(item?.Status ?? BatchItemStatus.Started) + Status = BatchSummaryCodec.SerializeStatus(item?.Status ?? BatchItemStatus.Started) }; // Persist each unit's result/error inline on the parent summary — // for BOTH Nested and Flat units. The service collapses completed @@ -741,18 +740,18 @@ BatchSummary BuildSummary(bool includeInline) } var summary = BuildSummary(includeInline: true); - var payload = JsonSerializer.Serialize(summary, BatchJsonContext.Default.BatchSummary); + var payload = BatchSummaryCodec.ToPayload(summary); // Overflow: the inline per-unit results pushed the summary over the // checkpoint limit. Re-emit a stripped summary (statuses only) and flag // ReplayChildren so replay reconstructs the values by re-executing units. // Applies to both Nested and Flat now that both inline their results. var overflow = - Encoding.UTF8.GetByteCount(payload) > DurableConstants.MaxOperationCheckpointBytes; + BatchSummaryCodec.IsOverflow(payload); if (overflow) { summary = BuildSummary(includeInline: false); - payload = JsonSerializer.Serialize(summary, BatchJsonContext.Default.BatchSummary); + payload = BatchSummaryCodec.ToPayload(summary); } // Always checkpoint as SUCCEED — even when FailureToleranceExceeded. @@ -776,7 +775,7 @@ await EnqueueAsync(new SdkOperationUpdate private IBatchResult ReconstructFromCheckpoints(Operation parent) { - var summary = ParseSummary(parent.ContextDetails?.Result); + var summary = BatchSummaryCodec.ParseSummary(parent.ContextDetails?.Result); var items = new List>(UnitCount); for (var i = 0; i < UnitCount; i++) @@ -787,7 +786,7 @@ private IBatchResult ReconstructFromCheckpoints(Operation parent) var summaryEntry = summary?.Units.FirstOrDefault(b => b.Index == i); BatchItemStatus status = summaryEntry != null - ? DeserializeStatus(summaryEntry.Status) + ? BatchSummaryCodec.DeserializeStatus(summaryEntry.Status) : InferStatusFromChildOp(childOp); // Prefer the name that was checkpointed at the moment the batch @@ -866,7 +865,7 @@ private IBatchResult ReconstructFromCheckpoints(Operation parent) } var completionReason = summary != null - ? DeserializeCompletionReason(summary.CompletionReason) + ? BatchSummaryCodec.DeserializeCompletionReason(summary.CompletionReason) : ComputeCompletionReason(items, UnitCount); return new BatchResult(items, completionReason); @@ -883,53 +882,6 @@ private static BatchItemStatus InferStatusFromChildOp(Operation? childOp) }; } - private static BatchSummary? ParseSummary(string? payload) - { - if (string.IsNullOrEmpty(payload)) return null; - try - { - return JsonSerializer.Deserialize(payload, BatchJsonContext.Default.BatchSummary); - } - catch (JsonException) - { - // Tolerate older / corrupted payloads — fall back to inferring status - // from per-unit checkpoints. - return null; - } - } - - private static string SerializeStatus(BatchItemStatus status) => status switch - { - BatchItemStatus.Succeeded => "SUCCEEDED", - BatchItemStatus.Failed => "FAILED", - BatchItemStatus.Started => "STARTED", - _ => throw new ArgumentOutOfRangeException(nameof(status)) - }; - - private static BatchItemStatus DeserializeStatus(string? wire) => wire switch - { - "SUCCEEDED" => BatchItemStatus.Succeeded, - "FAILED" => BatchItemStatus.Failed, - "STARTED" => BatchItemStatus.Started, - _ => BatchItemStatus.Started - }; - - private static string SerializeCompletionReason(CompletionReason reason) => reason switch - { - CompletionReason.AllCompleted => "ALL_COMPLETED", - CompletionReason.MinSuccessfulReached => "MIN_SUCCESSFUL_REACHED", - CompletionReason.FailureToleranceExceeded => "FAILURE_TOLERANCE_EXCEEDED", - _ => throw new ArgumentOutOfRangeException(nameof(reason)) - }; - - private static CompletionReason DeserializeCompletionReason(string? wire) => wire switch - { - "ALL_COMPLETED" => CompletionReason.AllCompleted, - "MIN_SUCCESSFUL_REACHED" => CompletionReason.MinSuccessfulReached, - "FAILURE_TOLERANCE_EXCEEDED" => CompletionReason.FailureToleranceExceeded, - _ => CompletionReason.AllCompleted - }; - private T DeserializeResult(string serialized) { var bytes = Encoding.UTF8.GetBytes(serialized); diff --git a/Libraries/src/Amazon.Lambda.DurableExecution/Internal/IncrementalParallelOperation.cs b/Libraries/src/Amazon.Lambda.DurableExecution/Internal/IncrementalParallelOperation.cs new file mode 100644 index 000000000..58be79e7a --- /dev/null +++ b/Libraries/src/Amazon.Lambda.DurableExecution/Internal/IncrementalParallelOperation.cs @@ -0,0 +1,791 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +using System.IO; +using System.Runtime.CompilerServices; +using System.Text; +using Amazon.Lambda; +using Amazon.Lambda.Core; +using SdkContextOptions = Amazon.Lambda.Model.ContextOptions; +using SdkOperationUpdate = Amazon.Lambda.Model.OperationUpdate; + +namespace Amazon.Lambda.DurableExecution.Internal; + +/// +/// Which replay branch the operation is on, decided once from the parent CONTEXT +/// checkpoint at construction. +/// +internal enum ParallelExecutionMode +{ + /// No terminal parent checkpoint: run branches (fresh, or STARTED/PENDING + /// where each branch replays from its own checkpoint). The parent SUCCEED is + /// written by . + Run, + + /// Parent already terminal: reconstruct branch outcomes from the frozen + /// (re-running a branch only to recover a value that + /// was stripped on overflow). The parent is NOT re-checkpointed. + Terminal +} + +/// +/// Type-erased outcome of a single branch, gathered by the orchestrator to build +/// the parent without knowing each branch's T. +/// +internal readonly struct BranchOutcome +{ + public int Index { get; init; } + public string? Name { get; init; } + public BatchItemStatus Status { get; init; } + + /// Serialized branch result (succeeded branches only). + public string? SerializedResult { get; init; } + + /// Branch error (failed branches only). + public ErrorObject? Error { get; init; } + + public static BranchOutcome Success(int index, string? name, string? serialized) => + new() { Index = index, Name = name, Status = BatchItemStatus.Succeeded, SerializedResult = serialized }; + + public static BranchOutcome Failure(int index, string? name, ErrorObject error) => + new() { Index = index, Name = name, Status = BatchItemStatus.Failed, Error = error }; + + public static BranchOutcome Skipped(int index, string? name) => + new() { Index = index, Name = name, Status = BatchItemStatus.Started }; +} + +/// +/// Type-erased view the orchestrator holds over each branch handle, so it can +/// await settlement and read per-branch identity/status without the branch's +/// generic parameter. +/// +internal interface IParallelBranchController +{ + int Index { get; } + string Name { get; } + BatchItemStatus Status { get; } + + /// + /// Completes (never faults for a graceful per-branch failure) with the branch's + /// . Faults only for workflow-level errors + /// (e.g. ) or control-token + /// cancellation, which the orchestrator surfaces. + /// + Task Settlement { get; } +} + +/// +/// Typed, awaitable handle for a single branch of an +/// . Backs the public +/// ; also exposes the type-erased +/// the orchestrator uses to aggregate. +/// +internal sealed class IncrementalParallelBranch : IParallelBranch, IParallelBranchController +{ + private readonly TaskCompletionSource _result = + new(TaskCreationOptions.RunContinuationsAsynchronously); + private readonly ILambdaSerializer _serializer; + private readonly string _childSubType; + + // Frozen status wins over the live re-run outcome on the overflow-recovery path + // (Terminal mode): the checkpointed verdict is authoritative even if a + // non-deterministic body re-executes to a different result. + private BatchItemStatus? _frozenStatus; + private volatile int _status = (int)BatchItemStatus.Started; + + public IncrementalParallelBranch(int index, string name, ILambdaSerializer serializer, string childSubType) + { + Index = index; + Name = name; + _serializer = serializer; + _childSubType = childSubType; + } + + public string Name { get; } + public int Index { get; } + public BatchItemStatus Status => _frozenStatus ?? (BatchItemStatus)_status; + public Task Settlement { get; private set; } = null!; + + public TaskAwaiter GetAwaiter() => _result.Task.GetAwaiter(); + + /// + /// Run mode: wrap a live child-context execution. + /// is set only on the overflow-recovery path, where the checkpointed status is + /// authoritative and the run merely recovers the stripped value. + /// + public void Launch( + Func> run, + CancellationToken shortCircuitToken, + CancellationToken controlToken, + BatchItemStatus? frozenStatus = null) + { + _frozenStatus = frozenStatus; + Settlement = ExecuteAsync(run, shortCircuitToken, controlToken); + } + + /// + /// Terminal mode: resolve the branch directly from the frozen summary without + /// running it (the common, non-overflow reconstruct path). + /// + public void ResolveFromInline(BatchItemStatus status, string? serializedResult, ErrorObject? error) + { + _frozenStatus = status; + switch (status) + { + case BatchItemStatus.Succeeded: + _result.TrySetResult(Deserialize(serializedResult)); + Settlement = Task.FromResult(BranchOutcome.Success(Index, Name, serializedResult)); + break; + case BatchItemStatus.Failed: + _result.TrySetException(BuildError(error)); + Settlement = Task.FromResult(BranchOutcome.Failure(Index, Name, error ?? new ErrorObject { ErrorMessage = "Branch failed" })); + break; + default: + _result.TrySetException(SkippedError()); + Settlement = Task.FromResult(BranchOutcome.Skipped(Index, Name)); + break; + } + } + + private async Task ExecuteAsync( + Func> run, + CancellationToken shortCircuitToken, + CancellationToken controlToken) + { + try + { + var value = await run().ConfigureAwait(false); + if (_frozenStatus is null) _status = (int)BatchItemStatus.Succeeded; + _result.TrySetResult(value); + return BranchOutcome.Success(Index, Name, Serialize(value)); + } + catch (ChildContextException ex) + { + if (_frozenStatus is null) _status = (int)BatchItemStatus.Failed; + _result.TrySetException(ex); + return BranchOutcome.Failure(Index, Name, ErrorObject.FromException(ex)); + } + catch (DurableExecutionException) + { + // Workflow-level error (e.g. NonDeterministicExecutionException): not a + // graceful per-branch failure. Fault the settlement so the orchestrator + // surfaces it out of CompleteAsync. + throw; + } + catch (OperationCanceledException) + when (shortCircuitToken.IsCancellationRequested && !controlToken.IsCancellationRequested) + { + // Cooperative bail: a sibling satisfied the CompletionConfig before this + // branch acquired its concurrency slot (or its body honored the bail + // token). Record it as skipped — never a failure. + if (_frozenStatus is null) _status = (int)BatchItemStatus.Started; + _result.TrySetException(SkippedError()); + return BranchOutcome.Skipped(Index, Name); + } + catch (OperationCanceledException) when (controlToken.IsCancellationRequested) + { + // Caller-cancel or workflow shutdown: propagate. + _result.TrySetCanceled(); + throw; + } + catch (OperationCanceledException ex) + { + var wrapped = Wrap(ex); + if (_frozenStatus is null) _status = (int)BatchItemStatus.Failed; + _result.TrySetException(wrapped); + return BranchOutcome.Failure(Index, Name, ErrorObject.FromException(wrapped)); + } + catch (Exception ex) + { + var wrapped = Wrap(ex); + if (_frozenStatus is null) _status = (int)BatchItemStatus.Failed; + _result.TrySetException(wrapped); + return BranchOutcome.Failure(Index, Name, ErrorObject.FromException(wrapped)); + } + } + + private ChildContextException Wrap(Exception ex) => new(ex.Message, ex) + { + SubType = _childSubType, + ErrorType = ex.GetType().FullName + }; + + private ChildContextException BuildError(ErrorObject? error) => + new(error?.ErrorMessage ?? "Branch failed") + { + SubType = _childSubType, + ErrorType = error?.ErrorType, + ErrorData = error?.ErrorData, + OriginalStackTrace = error?.StackTrace + }; + + private DurableExecutionException SkippedError() => new( + $"Parallel branch '{Name}' (index {Index}) did not execute: the parallel " + + $"operation completed before it started (completion-policy short-circuit). " + + $"Inspect the branch's Status before awaiting it."); + + private string Serialize(T value) + { + using var ms = new MemoryStream(); + _serializer.Serialize(value, ms); + return Encoding.UTF8.GetString(ms.ToArray()); + } + + private T Deserialize(string? serialized) + { + if (serialized == null) return default!; + var bytes = Encoding.UTF8.GetBytes(serialized); + using var ms = new MemoryStream(bytes); + return _serializer.Deserialize(ms); + } +} + +/// +/// Incremental, heterogeneous parallel orchestrator implementing +/// . Each branch runs as a +/// under the SAME deterministic child +/// operation-ID scheme (hash("{parentId}-{index}")) and the SAME parent +/// checkpoint shape as the batch +/// , so a checkpoint written by one is +/// reconstructable by the other. Branch identity is positional: register the same +/// branches in the same order across replays. +/// +internal sealed class IncrementalParallelOperation : IDurableParallel +{ + private readonly string _operationId; + private readonly string? _name; + private readonly string? _parentId; + private readonly CompletionPolicy _policy; + private readonly int? _maxConcurrency; + private readonly bool _isVirtual; + private readonly ILambdaSerializer _serializer; + private readonly Func _childContextFactory; + private readonly ExecutionState _state; + private readonly TerminationManager _termination; + private readonly WorkflowCancellation _workflowCancellation; + private readonly string _durableExecutionArn; + private readonly CheckpointBatcher? _batcher; + + private readonly object _lock = new(); + private readonly List _branches = new(); + private readonly SemaphoreSlim? _semaphore; + private readonly CancellationTokenSource _shortCircuitCts = new(); + private readonly CancellationTokenSource _dispatchCts; + + private readonly ParallelExecutionMode _mode; + private readonly BatchSummary? _frozenSummary; + private readonly Task _startTask; + + private int _succeeded; + private int _failed; + private int _registeredCount; + private bool _sealed; + private bool _completed; + private bool _disposed; + private IBatchResult? _cachedResult; + + public IncrementalParallelOperation( + string operationId, + string? name, + string? parentId, + ParallelConfig config, + ILambdaSerializer serializer, + Func childContextFactory, + ExecutionState state, + TerminationManager termination, + WorkflowCancellation workflowCancellation, + string durableExecutionArn, + CheckpointBatcher? batcher = null) + { + _operationId = operationId; + _name = name; + _parentId = parentId; + _policy = new CompletionPolicy(config.CompletionConfig); + _maxConcurrency = config.MaxConcurrency; + _isVirtual = config.NestingType == NestingType.Flat; + _serializer = serializer; + _childContextFactory = childContextFactory; + _state = state; + _termination = termination; + _workflowCancellation = workflowCancellation; + _durableExecutionArn = durableExecutionArn; + _batcher = batcher; + + _semaphore = _maxConcurrency is { } mc ? new SemaphoreSlim(mc, mc) : null; + _dispatchCts = CancellationTokenSource.CreateLinkedTokenSource( + _shortCircuitCts.Token, workflowCancellation.Token); + + // The parent operation position has been reached — mirror the base + // DurableOperation.ExecuteAsync bookkeeping for the parent CONTEXT op. + _state.ValidateReplayConsistency(_operationId, OperationTypes.Context, _name); + _state.TrackReplay(_operationId); + + var existing = _state.GetOperation(_operationId); + var terminal = existing != null && + (existing.Status == OperationStatuses.Succeeded || existing.Status == OperationStatuses.Failed); + + if (terminal) + { + _mode = ParallelExecutionMode.Terminal; + _frozenSummary = BatchSummaryCodec.ParseSummary(existing!.ContextDetails?.Result); + _startTask = Task.CompletedTask; + } + else + { + _mode = ParallelExecutionMode.Run; + // Fresh (no checkpoint) emits the parent CONTEXT START so the service + // has a parent record if a branch suspends. STARTED/PENDING replay does + // not re-emit it (the original is authoritative). Enqueued once here so + // it is ordered before any branch's child START. + _startTask = existing == null + ? EnqueueAsync(new SdkOperationUpdate + { + Id = _operationId, + ParentId = _parentId, + Type = OperationTypes.Context, + Action = OperationAction.START, + SubType = OperationSubTypes.Parallel, + Name = _name + }) + : Task.CompletedTask; + } + } + + public IParallelBranch BranchAsync( + string name, + Func> func) + { + if (name == null) throw new ArgumentNullException(nameof(name)); + if (func == null) throw new ArgumentNullException(nameof(func)); + + lock (_lock) + { + if (_disposed) throw new ObjectDisposedException(nameof(IDurableParallel)); + if (_sealed) + throw new InvalidOperationException( + "Cannot register a branch after the parallel operation has been sealed by CompleteAsync() or disposal."); + + var index = _branches.Count; // zero-based branch index + var childOpId = OperationIdGenerator.HashOperationId($"{_operationId}-{index + 1}"); + var handle = new IncrementalParallelBranch(index, name, _serializer, OperationSubTypes.ParallelBranch); + + var summaryEntry = FindSummaryUnit(index); + + // Strict name-drift check: a branch's name must be stable at its index + // across deployments (matches the batch Parallel reconstruct check). + if (summaryEntry?.Name != null && summaryEntry.Name != name) + { + throw new NonDeterministicExecutionException( + $"Non-deterministic execution detected for parallel branch {index} of operation " + + $"'{_name ?? _operationId}': expected name '{name}' but found '{summaryEntry.Name}' " + + $"from a previous invocation. Code must not change the order or name of branches " + + $"between deployments."); + } + + if (_mode == ParallelExecutionMode.Terminal) + { + ResolveTerminalBranch(handle, name, childOpId, func, summaryEntry); + } + else + { + LaunchRunBranch(handle, name, childOpId, func); + } + + _branches.Add(handle); + _registeredCount = _branches.Count; + return handle; + } + } + + public async Task CompleteAsync(CancellationToken cancellationToken = default) + { + lock (_lock) + { + if (_cachedResult != null) return _cachedResult; + _sealed = true; + } + + // Ensure the parent START is durably enqueued even for an empty operation. + await _startTask.ConfigureAwait(false); + + var controllers = SnapshotControllers(); + + // Await every branch settlement. Task.WhenAll surfaces only the first + // exception; swallow here and inspect each below so a workflow-level fault + // is surfaced deterministically and graceful failures aggregate. + if (controllers.Count > 0) + { + try { await Task.WhenAll(controllers.Select(c => c.Settlement)).ConfigureAwait(false); } + catch { /* inspected below */ } + } + + foreach (var c in controllers) + { + var s = c.Settlement; + if (s.IsFaulted && s.Exception is { } agg) + { + foreach (var inner in agg.InnerExceptions) + { + if (inner is DurableExecutionException dex && inner is not ChildContextException) + throw dex; + } + } + } + + // A torn-down operation propagates cancellation rather than a synthesized verdict. + _workflowCancellation.Token.ThrowIfCancellationRequested(); + cancellationToken.ThrowIfCancellationRequested(); + + IBatchResult result = _mode == ParallelExecutionMode.Terminal + ? BuildTerminalResult(controllers) + : await BuildAndCheckpointRunResultAsync(controllers, cancellationToken).ConfigureAwait(false); + + lock (_lock) + { + _completed = true; + _cachedResult = result; + } + return result; + } + + public async ValueTask DisposeAsync() + { + bool needComplete; + lock (_lock) + { + if (_disposed) return; + _disposed = true; + needComplete = !_completed; + } + + try + { + // Guarantee the parent's terminal checkpoint is written even if the + // caller forgot CompleteAsync — otherwise replay would see a STARTED + // parent forever and re-run the whole operation. + if (needComplete) + { + await CompleteAsync(CancellationToken.None).ConfigureAwait(false); + } + } + finally + { + _shortCircuitCts.Dispose(); + _dispatchCts.Dispose(); + _semaphore?.Dispose(); + } + } + + // ── Run mode ──────────────────────────────────────────────────────── + + private void LaunchRunBranch( + IncrementalParallelBranch handle, + string name, + string childOpId, + Func> func, + BatchItemStatus? frozenStatus = null) + { + async Task Run() + { + // Parent START must be enqueued before this branch's child START. + await _startTask.ConfigureAwait(false); + + if (_semaphore != null) + { + await _semaphore.WaitAsync(_dispatchCts.Token).ConfigureAwait(false); + } + + try + { + // A short-circuit may have fired while waiting on the semaphore. + _dispatchCts.Token.ThrowIfCancellationRequested(); + + var childOp = new ChildContextOperation( + childOpId, + name, + _operationId, + func, + new ChildContextConfig { SubType = OperationSubTypes.ParallelBranch }, + _serializer, + _childContextFactory, + _state, + _termination, + _workflowCancellation, + _durableExecutionArn, + _batcher, + _shortCircuitCts.Token, + isVirtual: _isVirtual); + + // Branch child ops receive CancellationToken.None here — they re-link + // workflow-shutdown and the cooperative-bail token internally, and + // their checkpoint writes must not observe shutdown mid-flush. + return await childOp.ExecuteAsync(CancellationToken.None).ConfigureAwait(false); + } + finally + { + _semaphore?.Release(); + } + } + + handle.Launch(Run, _shortCircuitCts.Token, _workflowCancellation.Token, frozenStatus); + ObserveSettlement(handle.Settlement); + } + + private void ObserveSettlement(Task settlement) + { + _ = settlement.ContinueWith( + OnBranchSettled, + CancellationToken.None, + TaskContinuationOptions.ExecuteSynchronously, + TaskScheduler.Default); + } + + private void OnBranchSettled(Task settlement) + { + if (settlement.Status != TaskStatus.RanToCompletion) return; + + switch (settlement.Result.Status) + { + case BatchItemStatus.Succeeded: Interlocked.Increment(ref _succeeded); break; + case BatchItemStatus.Failed: Interlocked.Increment(ref _failed); break; + } + + // The deciding completion usually lands after all currently-registered + // branches were dispatched, so re-check here and signal stragglers to bail. + if (ShouldStopDispatchingNow()) + { + try { _shortCircuitCts.Cancel(); } + catch (ObjectDisposedException) { } + } + } + + // During incremental registration the "total" is the number registered so far; + // percentage-based tolerance is evaluated against that running total. MinSuccessful + // and count-based tolerance don't depend on the total. + private bool ShouldStopDispatchingNow() => _policy.ShouldStopDispatching( + Volatile.Read(ref _succeeded), Volatile.Read(ref _failed), Volatile.Read(ref _registeredCount)); + + private async Task BuildAndCheckpointRunResultAsync( + IReadOnlyList controllers, + CancellationToken cancellationToken) + { + var outcomes = new List(controllers.Count); + foreach (var c in controllers) + { + var s = c.Settlement; + outcomes.Add(s.Status == TaskStatus.RanToCompletion + ? s.Result + : BranchOutcome.Skipped(c.Index, c.Name)); // defensive; faults handled above + } + + var reason = ComputeCompletionReason(outcomes); + await CheckpointParentSucceedAsync(outcomes, reason, cancellationToken).ConfigureAwait(false); + return BuildResult(outcomes, reason); + } + + private CompletionReason ComputeCompletionReason(IReadOnlyList outcomes) + { + var succeeded = 0; + var failed = 0; + foreach (var o in outcomes) + { + if (o.Status == BatchItemStatus.Succeeded) succeeded++; + else if (o.Status == BatchItemStatus.Failed) failed++; + } + + var total = outcomes.Count; + var started = total - succeeded - failed; + return _policy.Evaluate(succeeded, failed, started, total); + } + + private async Task CheckpointParentSucceedAsync( + IReadOnlyList outcomes, + CompletionReason reason, + CancellationToken cancellationToken) + { + BatchSummary Build(bool includeInline) + { + var s = new BatchSummary + { + CompletionReason = BatchSummaryCodec.SerializeCompletionReason(reason), + Units = new List(outcomes.Count) + }; + foreach (var o in outcomes) + { + var unit = new BatchUnitSummary + { + Index = o.Index, + Name = o.Name, + Status = BatchSummaryCodec.SerializeStatus(o.Status) + }; + if (includeInline) + { + if (o.Status == BatchItemStatus.Succeeded) unit.Result = o.SerializedResult; + else if (o.Status == BatchItemStatus.Failed) unit.Error = o.Error; + } + s.Units.Add(unit); + } + return s; + } + + var summary = Build(includeInline: true); + var payload = BatchSummaryCodec.ToPayload(summary); + + var overflow = BatchSummaryCodec.IsOverflow(payload); + if (overflow) + { + summary = Build(includeInline: false); + payload = BatchSummaryCodec.ToPayload(summary); + } + + await EnqueueAsync(new SdkOperationUpdate + { + Id = _operationId, + ParentId = _parentId, + Type = OperationTypes.Context, + Action = OperationAction.SUCCEED, + SubType = OperationSubTypes.Parallel, + Name = _name, + Payload = payload, + ContextOptions = overflow ? new SdkContextOptions { ReplayChildren = true } : null + }, cancellationToken).ConfigureAwait(false); + } + + // ── Terminal (reconstruct) mode ───────────────────────────────────── + + private void ResolveTerminalBranch( + IncrementalParallelBranch handle, + string name, + string childOpId, + Func> func, + BatchUnitSummary? summaryEntry) + { + // A branch registered now but absent from the frozen summary (registered + // after the original seal) never ran — surface it as skipped. + if (summaryEntry == null) + { + handle.ResolveFromInline(BatchItemStatus.Started, null, null); + return; + } + + var status = BatchSummaryCodec.DeserializeStatus(summaryEntry.Status); + + switch (status) + { + case BatchItemStatus.Succeeded when summaryEntry.Result != null: + handle.ResolveFromInline(BatchItemStatus.Succeeded, summaryEntry.Result, null); + break; + case BatchItemStatus.Failed when summaryEntry.Error != null: + handle.ResolveFromInline(BatchItemStatus.Failed, null, summaryEntry.Error); + break; + case BatchItemStatus.Succeeded: + case BatchItemStatus.Failed: + // Overflow: the inline value/error was stripped. Re-run the branch to + // recover it from the branch's own checkpoint; the frozen status stays + // authoritative. + LaunchRunBranch(handle, name, childOpId, func, frozenStatus: status); + break; + default: + handle.ResolveFromInline(BatchItemStatus.Started, null, null); + break; + } + } + + private IBatchResult BuildTerminalResult(IReadOnlyList controllers) + { + // Prefer the frozen summary (authoritative for status + completion reason). + // Fall back to the registered controllers when the payload is missing/corrupt. + if (_frozenSummary != null) + { + var succeeded = 0; + var failed = 0; + var started = 0; + foreach (var u in _frozenSummary.Units) + { + switch (BatchSummaryCodec.DeserializeStatus(u.Status)) + { + case BatchItemStatus.Succeeded: succeeded++; break; + case BatchItemStatus.Failed: failed++; break; + default: started++; break; + } + } + var reason = BatchSummaryCodec.DeserializeCompletionReason(_frozenSummary.CompletionReason); + return new IncrementalBatchResult(reason, succeeded, failed, started, _frozenSummary.Units.Count); + } + + var outcomes = new List(controllers.Count); + foreach (var c in controllers) + { + var s = c.Settlement; + outcomes.Add(s.Status == TaskStatus.RanToCompletion ? s.Result : BranchOutcome.Skipped(c.Index, c.Name)); + } + return BuildResult(outcomes, ComputeCompletionReason(outcomes)); + } + + // ── Shared helpers ────────────────────────────────────────────────── + + private static IBatchResult BuildResult(IReadOnlyList outcomes, CompletionReason reason) + { + var succeeded = 0; + var failed = 0; + var started = 0; + foreach (var o in outcomes) + { + switch (o.Status) + { + case BatchItemStatus.Succeeded: succeeded++; break; + case BatchItemStatus.Failed: failed++; break; + default: started++; break; + } + } + return new IncrementalBatchResult(reason, succeeded, failed, started, outcomes.Count); + } + + private BatchUnitSummary? FindSummaryUnit(int index) + { + if (_frozenSummary == null) return null; + foreach (var u in _frozenSummary.Units) + { + if (u.Index == index) return u; + } + return null; + } + + private IReadOnlyList SnapshotControllers() + { + lock (_lock) return _branches.ToArray(); + } + + private Task EnqueueAsync(SdkOperationUpdate update, CancellationToken cancellationToken = default) + => _batcher?.EnqueueAsync(update, cancellationToken) ?? Task.CompletedTask; +} + +/// +/// Non-generic returned by +/// . Per-branch typed +/// values are retrieved from the individual +/// handles; this type carries only the aggregate bookkeeping. +/// +internal sealed class IncrementalBatchResult : IBatchResult +{ + public IncrementalBatchResult( + CompletionReason completionReason, + int successCount, + int failureCount, + int startedCount, + int totalCount) + { + CompletionReason = completionReason; + SuccessCount = successCount; + FailureCount = failureCount; + StartedCount = startedCount; + TotalCount = totalCount; + } + + public CompletionReason CompletionReason { get; } + public bool HasFailure => FailureCount > 0; + public int SuccessCount { get; } + public int FailureCount { get; } + public int StartedCount { get; } + public int TotalCount { get; } +} diff --git a/Libraries/src/Amazon.Lambda.DurableExecution/docs/core/parallel.md b/Libraries/src/Amazon.Lambda.DurableExecution/docs/core/parallel.md index 7a258717a..d95b51ab4 100644 --- a/Libraries/src/Amazon.Lambda.DurableExecution/docs/core/parallel.md +++ b/Libraries/src/Amazon.Lambda.DurableExecution/docs/core/parallel.md @@ -24,6 +24,40 @@ Task> ParallelAsync( Each branch receives its own `IDurableContext` and a `CancellationToken` (linking the caller-supplied token with the SDK's workflow-shutdown signal — see [Cancellation](cancellation.md)), so a branch can itself use steps, waits, and nested durable operations. Branch results are serialized to per-branch checkpoints via the `ILambdaSerializer` registered on `ILambdaContext.Serializer`. The operation `name` is used for observability and to derive the deterministic operation ID, so keep it stable across deployments. +## Incremental, heterogeneous branches (`CreateParallel`) + +`ParallelAsync` takes a complete branch list up front and every branch shares one result type `T`. When branches return **unrelated types**, or are **discovered incrementally**, use `CreateParallel` instead. It returns an `IDurableParallel` you register branches on one at a time — each with its own result type — and each branch **begins executing as soon as it is registered** (subject to `MaxConcurrency`), so earlier independent work makes progress while later branches are still being assembled. + +```csharp +await using var parallel = ctx.CreateParallel(name: "process-order"); + +IParallelBranch inventory = parallel.BranchAsync( + "inventory", async (branch, ct) => await ReserveInventoryAsync(branch, ct)); + +IParallelBranch payment = parallel.BranchAsync( + "payment", async (branch, ct) => await AuthorizePaymentAsync(branch, ct)); + +if (plan.RequiresComplianceReview) +{ + // Branches can be added conditionally / incrementally. + _ = parallel.BranchAsync("compliance", + async (branch, ct) => await ReviewComplianceAsync(branch, ct)); +} + +// Seal registration, await the branches per CompletionConfig, checkpoint the aggregate. +IBatchResult summary = await parallel.CompleteAsync(); + +// Each handle yields its own concrete type — no shared base type, casts, or envelopes. +InventoryReservation reservedInventory = await inventory; +PaymentAuthorization authorizedPayment = await payment; +``` + +`BranchAsync` returns an awaitable `IParallelBranch` handle exposing `Name`, `Index`, and `Status`. `await handle` yields the branch's typed result, rethrows its `ChildContextException` on failure, or throws a `DurableExecutionException` if the branch was skipped by a completion-policy short-circuit (inspect `Status` first when that's possible). `CompleteAsync()` returns the non-generic aggregate `IBatchResult` (counts + `CompletionReason`); it is idempotent and never throws on per-branch failure. `DisposeAsync` (via `await using`) seals and completes the operation if you did not call `CompleteAsync`, so the parallel's terminal checkpoint is always written. + +> **Deterministic replay applies unchanged.** Branch identity is positional: the n-th `BranchAsync` call reuses the n-th deterministic operation ID, so workflow code must register the same branches in the same order across invocations (a name change at a given index throws `NonDeterministicExecutionException`). Produce any dynamic branch set inside a checkpointed `StepAsync` so replay sees the same branches. `MaxConcurrency`, `CompletionConfig`, `NestingType`, cancellation, and the checkpoint format are identical to `ParallelAsync` — `CreateParallel` writes the same `Parallel` / `ParallelBranch` checkpoints, so it is purely an additive, front-end alternative. + +The homogeneous `ParallelAsync` overloads remain the simplest choice for a fixed set of same-typed branches and convenient `GetResults()` usage. + ## Example Fan out three independent lookups and collect the results: diff --git a/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/IncrementalParallelHeterogeneousTest.cs b/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/IncrementalParallelHeterogeneousTest.cs new file mode 100644 index 000000000..901b4d489 --- /dev/null +++ b/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/IncrementalParallelHeterogeneousTest.cs @@ -0,0 +1,74 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +using System.Linq; +using System.Text; +using Amazon.Lambda.Model; +using Xunit; +using Xunit.Abstractions; + +namespace Amazon.Lambda.DurableExecution.IntegrationTests; + +public class IncrementalParallelHeterogeneousTest +{ + private readonly ITestOutputHelper _output; + public IncrementalParallelHeterogeneousTest(ITestOutputHelper output) => _output = output; + + /// + /// End-to-end incremental, heterogeneous parallel: three branches registered + /// one at a time via CreateParallel/BranchAsync return three + /// unrelated types (string, int, POCO), each retrieved through its own typed + /// handle. Validates the parent CONTEXT and per-branch CONTEXT checkpoints all + /// land in the service-side history with the correct names, and that the + /// heterogeneous per-branch values round-trip into the user-visible result. + /// + [Fact] + public async Task IncrementalParallel_HeterogeneousBranches_Succeed() + { + await using var deployment = await DurableFunctionDeployment.CreateAsync( + DurableFunctionDeployment.FindTestFunctionDir("IncrementalParallelHeterogeneousFunction"), + "iphetero", _output); + + var (invokeResponse, executionName) = await deployment.InvokeAsync("""{"orderId": "p1"}"""); + Assert.Equal(200, invokeResponse.StatusCode); + + var responsePayload = Encoding.UTF8.GetString(invokeResponse.Payload.ToArray()); + _output.WriteLine($"Response: {responsePayload}"); + + var arn = await deployment.FindDurableExecutionArnByNameAsync(executionName, TimeSpan.FromSeconds(60)); + Assert.NotNull(arn); + + var status = await deployment.PollForCompletionAsync(arn!, TimeSpan.FromSeconds(60)); + Assert.Equal("SUCCEEDED", status, ignoreCase: true); + + // Each heterogeneous branch's typed result surfaces in the user payload. + Assert.Contains("reserved-p1", responsePayload); // string branch + Assert.Contains("200", responsePayload); // int branch + Assert.Contains("USD:4200", responsePayload); // POCO branch + + // History is eventually consistent — wait until the parent CONTEXT and all + // three child CONTEXT checkpoints are visible. + var history = await deployment.WaitForHistoryAsync( + arn!, + h => (h.Events?.Count(e => e.EventType == EventType.ContextStarted) ?? 0) >= 4 + && (h.Events?.Count(e => e.EventType == EventType.ContextSucceeded) ?? 0) >= 4, + TimeSpan.FromSeconds(60)); + var events = history.Events ?? new List(); + + // Parent + 3 branches = 4 ContextStarted, 4 ContextSucceeded. + Assert.Equal(4, events.Count(e => e.EventType == EventType.ContextStarted)); + Assert.Equal(4, events.Count(e => e.EventType == EventType.ContextSucceeded)); + + var startedNames = events + .Where(e => e.EventType == EventType.ContextStarted) + .Select(e => e.Name) + .ToList(); + Assert.Contains("process-order", startedNames); + Assert.Contains("inventory", startedNames); + Assert.Contains("payment", startedNames); + Assert.Contains("shipping", startedNames); + + // No branch failed. + Assert.Empty(events.Where(e => e.EventType == EventType.ContextFailed)); + } +} diff --git a/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/IncrementalParallelReplayTest.cs b/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/IncrementalParallelReplayTest.cs new file mode 100644 index 000000000..7d20a338d --- /dev/null +++ b/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/IncrementalParallelReplayTest.cs @@ -0,0 +1,111 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +using System.Linq; +using System.Security.Cryptography; +using System.Text; +using Amazon.Lambda.Model; +using Xunit; +using Xunit.Abstractions; + +namespace Amazon.Lambda.DurableExecution.IntegrationTests; + +public class IncrementalParallelReplayTest +{ + private readonly ITestOutputHelper _output; + public IncrementalParallelReplayTest(ITestOutputHelper output) => _output = output; + + private static string HashOpId(string raw) + { + var bytes = Encoding.UTF8.GetBytes(raw); + var hash = SHA256.HashData(bytes); + var sb = new StringBuilder(hash.Length * 2); + foreach (var b in hash) sb.Append(b.ToString("x2")); + return sb.ToString(); + } + + /// + /// Deterministic replay of the incremental CreateParallel API across + /// both replay paths. Three branches each do a step (generating a GUID) then a + /// durable wait; a further wait runs after the parallel completes. This forces + /// (1) a Run-mode replay while the parent CONTEXT is still STARTED, and (2) a + /// terminal-reconstruct resume once the parent CONTEXT is SUCCEEDED. Verifies: + /// 1. Branch operation IDs match SHA-256("<parentId>-<n>"). + /// 2. Each branch's "generate" step succeeds EXACTLY once — proving neither + /// the STARTED-parent replay nor the terminal reconstruct re-executes a + /// branch body. + /// 3. The run spans multiple invocations (suspend/resume actually happened). + /// + [Fact] + public async Task IncrementalParallel_ReplayDeterminism_AcrossRunAndTerminalPaths() + { + await using var deployment = await DurableFunctionDeployment.CreateAsync( + DurableFunctionDeployment.FindTestFunctionDir("IncrementalParallelReplayFunction"), + "ipreplay", _output); + + var (invokeResponse, executionName) = await deployment.InvokeAsync("""{"orderId": "p6"}"""); + var responsePayload = Encoding.UTF8.GetString(invokeResponse.Payload.ToArray()); + _output.WriteLine($"Response: {responsePayload}"); + + var arn = await deployment.FindDurableExecutionArnByNameAsync(executionName, TimeSpan.FromSeconds(60)); + Assert.NotNull(arn); + + var status = await deployment.PollForCompletionAsync(arn!, TimeSpan.FromSeconds(180)); + Assert.Equal("SUCCEEDED", status, ignoreCase: true); + + // The parallel parent is the first root-level operation -> SHA256("1"). + var parentOpId = HashOpId("1"); + var expectedBranchIds = new[] + { + HashOpId($"{parentOpId}-1"), + HashOpId($"{parentOpId}-2"), + HashOpId($"{parentOpId}-3"), + }; + + var history = await deployment.WaitForHistoryAsync( + arn!, + h => + { + var events = h.Events ?? new List(); + // Parent + 3 branch CONTEXTs all succeeded. + if (events.Count(e => e.EventType == EventType.ContextSucceeded) < 4) return false; + // Each branch ran one step and one wait, plus the post-parallel wait. + if (events.Count(e => e.EventType == EventType.StepSucceeded) < 3) return false; + if (events.Count(e => e.EventType == EventType.WaitSucceeded) < 4) return false; + return true; + }, + TimeSpan.FromSeconds(120)); + var allEvents = history.Events ?? new List(); + + // 1. Branch operation IDs match the deterministic hash. + var observedBranchIds = allEvents + .Where(e => e.EventType == EventType.ContextStarted && e.Id != null && e.Id != parentOpId) + .Select(e => e.Id) + .Distinct() + .ToList(); + Assert.Equal(3, observedBranchIds.Count); + foreach (var expected in expectedBranchIds) + { + Assert.Contains(expected, observedBranchIds); + } + + // 2. Each branch's "generate" step succeeded exactly once — no branch body + // re-executed on either the STARTED-parent replay or the terminal resume. + var generateSucceeded = allEvents + .Where(e => e.EventType == EventType.StepSucceeded && e.Name == "generate") + .ToList(); + Assert.Equal(3, generateSucceeded.Count); + + // 3. Parent + 3 branches succeeded once each. + Assert.Equal(4, allEvents.Count(e => e.EventType == EventType.ContextSucceeded)); + + // 4. The run spans multiple invocations (branch waits + post-parallel wait). + var invocations = allEvents.Where(e => e.InvocationCompletedDetails != null).ToList(); + Assert.True( + invocations.Count >= 2, + $"Expected >= 2 InvocationCompleted events (suspend + resume), got {invocations.Count}"); + + // 5. The user-visible response carries the joined per-branch results. + Assert.Contains("completed", responsePayload, StringComparison.OrdinalIgnoreCase); + } +} diff --git a/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/TestFunctions/IncrementalParallelHeterogeneousFunction/Function.cs b/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/TestFunctions/IncrementalParallelHeterogeneousFunction/Function.cs new file mode 100644 index 000000000..c66681b10 --- /dev/null +++ b/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/TestFunctions/IncrementalParallelHeterogeneousFunction/Function.cs @@ -0,0 +1,91 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +using Amazon.Lambda.Core; +using Amazon.Lambda.DurableExecution; +using Amazon.Lambda.RuntimeSupport; +using Amazon.Lambda.Serialization.SystemTextJson; + +namespace DurableExecutionTestFunction; + +/// +/// Deployed entry point exercising the incremental, heterogeneous branch API +/// (). Three branches return three +/// unrelated types (a string, an int, and a POCO); each is retrieved through its +/// own typed handle with no shared base type, +/// cast, or envelope. Validates that heterogeneous per-branch results round-trip +/// through the service checkpoint history end-to-end. +/// +public class Function +{ + public static async Task Main(string[] args) + { + var handler = new Function(); + var serializer = new DefaultLambdaJsonSerializer(); + using var handlerWrapper = HandlerWrapper.GetHandlerWrapper(handler.Handler, serializer); + using var bootstrap = new LambdaBootstrap(handlerWrapper); + await bootstrap.RunAsync(); + } + + public Task Handler( + DurableExecutionInvocationInput input, ILambdaContext context) + => DurableFunction.WrapAsync(Workflow, input, context); + + private static async Task Workflow(OrderRequest input, IDurableContext context) + { + var orderId = input?.OrderId ?? "unknown"; + + await using var parallel = context.CreateParallel(name: "process-order"); + + // Each branch declares its own result type — string, int, and Money. + IParallelBranch inventory = parallel.BranchAsync( + "inventory", + async (branch, ct) => await branch.StepAsync( + (_, _) => Task.FromResult($"reserved-{orderId}"), name: "reserve")); + + IParallelBranch payment = parallel.BranchAsync( + "payment", + async (branch, ct) => await branch.StepAsync( + (_, _) => Task.FromResult(200), name: "charge")); + + IParallelBranch shipping = parallel.BranchAsync( + "shipping", + async (branch, ct) => await branch.StepAsync( + (_, _) => Task.FromResult(new Money { Currency = "USD", Amount = 4200 }), name: "quote")); + + IBatchResult summary = await parallel.CompleteAsync(); + + var reservedInventory = await inventory; + var authorizedPayment = await payment; + var shippingQuote = await shipping; + + return new OrderResult + { + Inventory = reservedInventory, + Payment = authorizedPayment, + Shipping = $"{shippingQuote.Currency}:{shippingQuote.Amount}", + SuccessCount = summary.SuccessCount, + TotalCount = summary.TotalCount, + }; + } +} + +public class OrderRequest +{ + public string? OrderId { get; set; } +} + +public class OrderResult +{ + public string Inventory { get; set; } = ""; + public int Payment { get; set; } + public string Shipping { get; set; } = ""; + public int SuccessCount { get; set; } + public int TotalCount { get; set; } +} + +public class Money +{ + public string Currency { get; set; } = ""; + public int Amount { get; set; } +} diff --git a/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/TestFunctions/IncrementalParallelHeterogeneousFunction/IncrementalParallelHeterogeneousFunction.csproj b/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/TestFunctions/IncrementalParallelHeterogeneousFunction/IncrementalParallelHeterogeneousFunction.csproj new file mode 100644 index 000000000..f8bf7fd0c --- /dev/null +++ b/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/TestFunctions/IncrementalParallelHeterogeneousFunction/IncrementalParallelHeterogeneousFunction.csproj @@ -0,0 +1,18 @@ + + + + net10.0 + Exe + true + bootstrap + enable + enable + + + + + + + + + diff --git a/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/TestFunctions/IncrementalParallelReplayFunction/Function.cs b/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/TestFunctions/IncrementalParallelReplayFunction/Function.cs new file mode 100644 index 000000000..e7b3a3ecd --- /dev/null +++ b/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/TestFunctions/IncrementalParallelReplayFunction/Function.cs @@ -0,0 +1,90 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +using Amazon.Lambda.Core; +using Amazon.Lambda.DurableExecution; +using Amazon.Lambda.RuntimeSupport; +using Amazon.Lambda.Serialization.SystemTextJson; + +namespace DurableExecutionTestFunction; + +/// +/// Deployed entry point exercising deterministic replay of the incremental +/// () API across two distinct replay +/// paths: +/// +/// Each branch does a step (generating a GUID) then a durable wait. The +/// wait suspends the whole invocation, so the parallel re-runs with its +/// parent CONTEXT still STARTED — branches replay from their own checkpoints +/// and the cached GUID must survive. +/// After the parallel completes, a second durable wait suspends again. On +/// that resume the parent CONTEXT is already SUCCEEDED, so the incremental +/// operation takes the terminal-reconstruct path: branch handles resolve +/// from the frozen inline summary WITHOUT re-running, and the aggregate is +/// rebuilt from the checkpoint. +/// +/// If replay determinism were broken, the per-branch GUIDs would change between +/// invocations, or a branch step would re-execute (surfacing as duplicate +/// StepSucceeded events). +/// +public class Function +{ + public static async Task Main(string[] args) + { + var handler = new Function(); + var serializer = new DefaultLambdaJsonSerializer(); + using var handlerWrapper = HandlerWrapper.GetHandlerWrapper(handler.Handler, serializer); + using var bootstrap = new LambdaBootstrap(handlerWrapper); + await bootstrap.RunAsync(); + } + + public Task Handler( + DurableExecutionInvocationInput input, ILambdaContext context) + => DurableFunction.WrapAsync(Workflow, input, context); + + private static async Task Workflow(TestEvent input, IDurableContext context) + { + await using var parallel = context.CreateParallel(name: "fanout"); + + IParallelBranch a = parallel.BranchAsync("a", BranchAsync); + IParallelBranch b = parallel.BranchAsync("b", BranchAsync); + IParallelBranch c = parallel.BranchAsync("c", BranchAsync); + + var summary = await parallel.CompleteAsync(); + + // Retrieve each branch's typed result through its own handle. + var joined = string.Join(",", await a, await b, await c); + + // Force a resume where the parallel is ALREADY terminal, so CreateParallel + // takes the terminal-reconstruct path on the next invocation. + await context.WaitAsync(TimeSpan.FromSeconds(2), name: "post-boundary"); + + return new TestResult + { + Status = "completed", + Data = joined, + SuccessCount = summary.SuccessCount + }; + } + + private static async Task BranchAsync(IDurableContext ctx, CancellationToken cancellationToken) + { + var generatedId = await ctx.StepAsync( + async (_, _) => { await Task.CompletedTask; return Guid.NewGuid().ToString(); }, + name: "generate"); + + // Suspend/resume cycle so the parallel replays with its parent still STARTED. + await ctx.WaitAsync(TimeSpan.FromSeconds(2), name: "boundary"); + + return generatedId; + } +} + +public class TestEvent { public string? OrderId { get; set; } } + +public class TestResult +{ + public string? Status { get; set; } + public string? Data { get; set; } + public int SuccessCount { get; set; } +} diff --git a/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/TestFunctions/IncrementalParallelReplayFunction/IncrementalParallelReplayFunction.csproj b/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/TestFunctions/IncrementalParallelReplayFunction/IncrementalParallelReplayFunction.csproj new file mode 100644 index 000000000..f8bf7fd0c --- /dev/null +++ b/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/TestFunctions/IncrementalParallelReplayFunction/IncrementalParallelReplayFunction.csproj @@ -0,0 +1,18 @@ + + + + net10.0 + Exe + true + bootstrap + enable + enable + + + + + + + + + diff --git a/Libraries/test/Amazon.Lambda.DurableExecution.Tests/IncrementalParallelOperationTests.cs b/Libraries/test/Amazon.Lambda.DurableExecution.Tests/IncrementalParallelOperationTests.cs new file mode 100644 index 000000000..924c6aa2c --- /dev/null +++ b/Libraries/test/Amazon.Lambda.DurableExecution.Tests/IncrementalParallelOperationTests.cs @@ -0,0 +1,529 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +using Amazon.Lambda.DurableExecution; +using Amazon.Lambda.DurableExecution.Internal; +using Amazon.Lambda.Serialization.SystemTextJson; +using Amazon.Lambda.TestUtilities; +using Xunit; + +namespace Amazon.Lambda.DurableExecution.Tests; + +/// +/// Tests for the incremental, heterogeneous parallel API +/// ( / ). +/// The checkpoint shape mirrors the batch ParallelOperation<T>, so these +/// reuse the same IdAt/ChildIdAt/CreateContext harness as +/// . +/// +public class IncrementalParallelOperationTests +{ + /// Reproduces the Id that emits for the n-th root-level operation. + private static string IdAt(int position) => OperationIdGenerator.HashOperationId(position.ToString()); + + /// The hashed ID of the n-th child operation under . + private static string ChildIdAt(string parentOpId, int position) => + OperationIdGenerator.HashOperationId($"{parentOpId}-{position}"); + + private static (DurableContext context, RecordingBatcher recorder, TerminationManager tm, ExecutionState state) + CreateContext(InitialExecutionState? initialState = null) + { + var state = new ExecutionState(); + state.LoadFromCheckpoint(initialState); + var tm = new TerminationManager(); + var idGen = new OperationIdGenerator(); + var lambdaContext = new TestLambdaContext { Serializer = new DefaultLambdaJsonSerializer() }; + var recorder = new RecordingBatcher(); + var context = new DurableContext(state, tm, new WorkflowCancellation(tm), idGen, "arn:test", lambdaContext, recorder.Batcher); + return (context, recorder, tm, state); + } + + public sealed class Money + { + public string Currency { get; set; } = ""; + public int Amount { get; set; } + } + + // ────────────────────────────────────────────────────────────────────── + // Fresh execution — happy paths + // ────────────────────────────────────────────────────────────────────── + + [Fact] + public async Task CreateParallel_FreshExecution_HeterogeneousBranches_ResolveTypedResults() + { + var (context, recorder, tm, _) = CreateContext(); + + IParallelBranch inventory; + IParallelBranch payment; + IParallelBranch shipping; + + await using (var parallel = context.CreateParallel(name: "process-order")) + { + inventory = parallel.BranchAsync("inventory", async (_, _) => { await Task.Yield(); return "reserved"; }); + payment = parallel.BranchAsync("payment", async (_, _) => { await Task.Yield(); return 200; }); + shipping = parallel.BranchAsync("shipping", async (_, _) => + { + await Task.Yield(); + return new Money { Currency = "USD", Amount = 4200 }; + }); + + var summary = await parallel.CompleteAsync(); + + Assert.False(tm.IsTerminated); + Assert.Equal(3, summary.TotalCount); + Assert.Equal(3, summary.SuccessCount); + Assert.Equal(0, summary.FailureCount); + Assert.False(summary.HasFailure); + Assert.Equal(CompletionReason.AllCompleted, summary.CompletionReason); + } + + // Each branch handle yields its own concrete type — no shared T, no casts. + Assert.Equal("reserved", await inventory); + Assert.Equal(200, await payment); + var ship = await shipping; + Assert.Equal("USD", ship.Currency); + Assert.Equal(4200, ship.Amount); + + Assert.Equal(BatchItemStatus.Succeeded, inventory.Status); + Assert.Equal(0, inventory.Index); + Assert.Equal(2, shipping.Index); + + await recorder.Batcher.DrainAsync(); + var contextActions = recorder.Flushed.Where(o => o.Type == "CONTEXT") + .Select(o => $"{o.SubType}:{o.Action}").ToArray(); + // Parent START + 3 child STARTs + 3 child SUCCEEDs + parent SUCCEED + Assert.Equal(8, contextActions.Length); + Assert.Equal("Parallel:START", contextActions[0]); + Assert.Equal("Parallel:SUCCEED", contextActions[^1]); + } + + [Fact] + public async Task CreateParallel_BranchOperationIds_AreDeterministic() + { + var (context, recorder, _, _) = CreateContext(); + + await using (var parallel = context.CreateParallel()) + { + _ = parallel.BranchAsync("a", async (_, _) => { await Task.Yield(); return "a"; }); + _ = parallel.BranchAsync("b", async (_, _) => { await Task.Yield(); return "b"; }); + await parallel.CompleteAsync(); + } + + await recorder.Batcher.DrainAsync(); + + var parentOpId = IdAt(1); + var branchStarts = recorder.Flushed + .Where(o => o.Type == "CONTEXT" && o.SubType == "ParallelBranch" && o.Action == "START") + .ToArray(); + Assert.Equal(2, branchStarts.Length); + Assert.Contains(branchStarts, o => o.Id == ChildIdAt(parentOpId, 1)); + Assert.Contains(branchStarts, o => o.Id == ChildIdAt(parentOpId, 2)); + } + + [Fact] + public async Task CreateParallel_EmptyOperation_FlushesStartAndSucceed() + { + var (context, recorder, _, _) = CreateContext(); + + IBatchResult summary; + await using (var parallel = context.CreateParallel()) + { + summary = await parallel.CompleteAsync(); + } + + Assert.Equal(0, summary.TotalCount); + Assert.Equal(CompletionReason.AllCompleted, summary.CompletionReason); + + await recorder.Batcher.DrainAsync(); + var contextActions = recorder.Flushed.Where(o => o.Type == "CONTEXT") + .Select(o => $"{o.SubType}:{o.Action}").ToArray(); + Assert.Equal(new[] { "Parallel:START", "Parallel:SUCCEED" }, contextActions); + } + + [Fact] + public async Task CreateParallel_NamesPropagateToCheckpointAndHandle() + { + var (context, recorder, _, _) = CreateContext(); + + await using (var parallel = context.CreateParallel(name: "fanout")) + { + var a = parallel.BranchAsync("alpha", async (_, _) => { await Task.Yield(); return 1; }); + var b = parallel.BranchAsync("beta", async (_, _) => { await Task.Yield(); return 2; }); + await parallel.CompleteAsync(); + Assert.Equal("alpha", a.Name); + Assert.Equal("beta", b.Name); + } + + await recorder.Batcher.DrainAsync(); + var branchSucceeds = recorder.Flushed + .Where(o => o.Type == "CONTEXT" && o.SubType == "ParallelBranch" && o.Action == "SUCCEED") + .ToArray(); + Assert.Contains(branchSucceeds, o => o.Name == "alpha"); + Assert.Contains(branchSucceeds, o => o.Name == "beta"); + } + + [Fact] + public async Task CreateParallel_NestedSucceeded_InlinesPerBranchResultsOnParentPayload() + { + var (context, recorder, _, _) = CreateContext(); + + await using (var parallel = context.CreateParallel(name: "fanout")) + { + _ = parallel.BranchAsync("i", async (_, _) => { await Task.Yield(); return 100; }); + _ = parallel.BranchAsync("p", async (_, _) => { await Task.Yield(); return 200; }); + await parallel.CompleteAsync(); + } + + await recorder.Batcher.DrainAsync(); + var parentSucceed = Assert.Single(recorder.Flushed.Where(o => + o.Type == "CONTEXT" && o.SubType == "Parallel" && $"{o.Action}" == "SUCCEED")); + var summary = System.Text.Json.JsonSerializer.Deserialize(parentSucceed.Payload!); + Assert.NotNull(summary); + Assert.Equal("100", summary!.Units[0].Result); + Assert.Equal("200", summary.Units[1].Result); + } + + // ────────────────────────────────────────────────────────────────────── + // Failure handling + // ────────────────────────────────────────────────────────────────────── + + [Fact] + public async Task CreateParallel_DefaultFailFast_BranchFailure_SurfacesOnResultAndHandle() + { + var (context, _, _, _) = CreateContext(); + + IParallelBranch ok; + IParallelBranch bad; + IBatchResult summary; + + await using (var parallel = context.CreateParallel()) + { + ok = parallel.BranchAsync("ok", async (_, _) => { await Task.Yield(); return 1; }); + bad = parallel.BranchAsync("bad", async (_, _) => + { + await Task.Yield(); + throw new InvalidOperationException("branch boom"); + }); + // Never throws on per-branch failure. + summary = await parallel.CompleteAsync(); + } + + Assert.True(summary.HasFailure); + Assert.Equal(CompletionReason.FailureToleranceExceeded, summary.CompletionReason); + Assert.Equal(1, summary.FailureCount); + + Assert.Equal(1, await ok); + Assert.Equal(BatchItemStatus.Failed, bad.Status); + var ex = await Assert.ThrowsAsync(async () => await bad); + Assert.Contains("branch boom", ex.Message); + } + + [Fact] + public async Task CreateParallel_AllCompleted_PartialFailureDoesNotExceedTolerance() + { + var (context, _, _, _) = CreateContext(); + + IBatchResult summary; + await using (var parallel = context.CreateParallel( + config: new ParallelConfig { CompletionConfig = CompletionConfig.AllCompleted() })) + { + _ = parallel.BranchAsync("ok", async (_, _) => { await Task.Yield(); return 1; }); + _ = parallel.BranchAsync("bad", async (_, _) => { await Task.Yield(); throw new InvalidOperationException("x"); }); + summary = await parallel.CompleteAsync(); + } + + Assert.Equal(CompletionReason.AllCompleted, summary.CompletionReason); + Assert.Equal(1, summary.SuccessCount); + Assert.Equal(1, summary.FailureCount); + Assert.True(summary.HasFailure); + } + + // ────────────────────────────────────────────────────────────────────── + // MaxConcurrency + completion short-circuit + // ────────────────────────────────────────────────────────────────────── + + [Fact] + public async Task CreateParallel_MaxConcurrency_LimitsInFlight() + { + var (context, _, _, _) = CreateContext(); + + var inFlight = 0; + var maxObserved = 0; + var gate = new object(); + + await using (var parallel = context.CreateParallel(config: new ParallelConfig { MaxConcurrency = 2 })) + { + for (var i = 0; i < 6; i++) + { + _ = parallel.BranchAsync($"b{i}", async (_, ct) => + { + lock (gate) { inFlight++; maxObserved = Math.Max(maxObserved, inFlight); } + await Task.Delay(20, ct); + lock (gate) { inFlight--; } + return 1; + }); + } + await parallel.CompleteAsync(); + } + + Assert.True(maxObserved <= 2, $"Observed concurrency {maxObserved} exceeded MaxConcurrency = 2"); + } + + [Fact] + public async Task CreateParallel_FirstSuccessful_WithMaxConcurrency1_SkipsTrailingBranches() + { + var (context, _, _, _) = CreateContext(); + + IParallelBranch last; + IBatchResult summary; + + await using (var parallel = context.CreateParallel(config: new ParallelConfig + { + MaxConcurrency = 1, + CompletionConfig = CompletionConfig.FirstSuccessful() + })) + { + _ = parallel.BranchAsync("b0", async (_, _) => { await Task.Yield(); return 1; }); + _ = parallel.BranchAsync("b1", async (_, _) => { await Task.Yield(); return 2; }); + last = parallel.BranchAsync("b2", async (_, _) => { await Task.Yield(); return 3; }); + summary = await parallel.CompleteAsync(); + } + + Assert.Equal(CompletionReason.MinSuccessfulReached, summary.CompletionReason); + Assert.True(summary.SuccessCount >= 1); + Assert.True(summary.StartedCount >= 1); + Assert.Equal(3, summary.TotalCount); + + // The trailing branch never ran; awaiting it surfaces a skip error. + Assert.Equal(BatchItemStatus.Started, last.Status); + await Assert.ThrowsAsync(async () => await last); + } + + // ────────────────────────────────────────────────────────────────────── + // Registration guardrails + // ────────────────────────────────────────────────────────────────────── + + [Fact] + public async Task CreateParallel_RegisterAfterComplete_Throws() + { + var (context, _, _, _) = CreateContext(); + + var parallel = context.CreateParallel(); + _ = parallel.BranchAsync("a", async (_, _) => { await Task.Yield(); return 1; }); + await parallel.CompleteAsync(); + + Assert.Throws(() => + _ = parallel.BranchAsync("late", async (_, _) => { await Task.Yield(); return 2; })); + + await parallel.DisposeAsync(); + } + + [Fact] + public async Task CreateParallel_CompleteAsync_IsIdempotent() + { + var (context, recorder, _, _) = CreateContext(); + + var parallel = context.CreateParallel(); + _ = parallel.BranchAsync("a", async (_, _) => { await Task.Yield(); return 1; }); + var first = await parallel.CompleteAsync(); + var second = await parallel.CompleteAsync(); + Assert.Same(first, second); + await parallel.DisposeAsync(); + + await recorder.Batcher.DrainAsync(); + // Exactly one parent SUCCEED despite two CompleteAsync calls + dispose. + Assert.Single(recorder.Flushed.Where(o => + o.Type == "CONTEXT" && o.SubType == "Parallel" && o.Action == "SUCCEED")); + } + + [Fact] + public async Task CreateParallel_DisposeWithoutComplete_StillCheckpointsParent() + { + var (context, recorder, _, _) = CreateContext(); + + await using (var parallel = context.CreateParallel(name: "auto")) + { + _ = parallel.BranchAsync("a", async (_, _) => { await Task.Yield(); return 1; }); + // No explicit CompleteAsync — DisposeAsync must seal + checkpoint. + } + + await recorder.Batcher.DrainAsync(); + Assert.Single(recorder.Flushed.Where(o => + o.Type == "CONTEXT" && o.SubType == "Parallel" && o.Action == "SUCCEED")); + } + + // ────────────────────────────────────────────────────────────────────── + // Replay — terminal parent reconstructs without re-running branches + // ────────────────────────────────────────────────────────────────────── + + [Fact] + public async Task CreateParallel_ReplaySucceeded_RebuildsFromInlineSummary_WithoutRerunning() + { + var parentOpId = IdAt(1); + var summaryJson = """ + {"CompletionReason":"ALL_COMPLETED","Units":[ + {"Index":0,"Name":"inventory","Status":"SUCCEEDED","Result":"\"reserved\""}, + {"Index":1,"Name":"payment","Status":"SUCCEEDED","Result":"200"} + ]} + """; + + var (context, recorder, _, _) = CreateContext(new InitialExecutionState + { + Operations = new List + { + new() + { + Id = parentOpId, + Type = OperationTypes.Context, + Status = OperationStatuses.Succeeded, + SubType = OperationSubTypes.Parallel, + Name = "process-order", + ContextDetails = new ContextDetails { Result = summaryJson } + } + } + }); + + var executed = false; + IParallelBranch inventory; + IParallelBranch payment; + IBatchResult summary; + + await using (var parallel = context.CreateParallel(name: "process-order")) + { + inventory = parallel.BranchAsync("inventory", async (_, _) => { executed = true; await Task.Yield(); return "LIVE"; }); + payment = parallel.BranchAsync("payment", async (_, _) => { executed = true; await Task.Yield(); return -1; }); + summary = await parallel.CompleteAsync(); + } + + Assert.False(executed); + Assert.Equal("reserved", await inventory); + Assert.Equal(200, await payment); + Assert.Equal(2, summary.SuccessCount); + Assert.Equal(CompletionReason.AllCompleted, summary.CompletionReason); + + await recorder.Batcher.DrainAsync(); + Assert.Empty(recorder.Flushed); // terminal parent → no re-checkpoint + } + + [Fact] + public async Task CreateParallel_ReplaySucceeded_FailedBranch_AwaitRethrows() + { + var parentOpId = IdAt(1); + var summaryJson = """ + {"CompletionReason":"FAILURE_TOLERANCE_EXCEEDED","Units":[ + {"Index":0,"Name":"ok","Status":"SUCCEEDED","Result":"1"}, + {"Index":1,"Name":"bad","Status":"FAILED","Error":{"ErrorType":"System.InvalidOperationException","ErrorMessage":"boom"}} + ]} + """; + + var (context, _, _, _) = CreateContext(new InitialExecutionState + { + Operations = new List + { + new() + { + Id = parentOpId, + Type = OperationTypes.Context, + Status = OperationStatuses.Succeeded, + SubType = OperationSubTypes.Parallel, + Name = "fanout", + ContextDetails = new ContextDetails { Result = summaryJson } + } + } + }); + + IParallelBranch ok; + IParallelBranch bad; + IBatchResult summary; + + await using (var parallel = context.CreateParallel(name: "fanout")) + { + ok = parallel.BranchAsync("ok", async (_, _) => { await Task.Yield(); return -1; }); + bad = parallel.BranchAsync("bad", async (_, _) => { await Task.Yield(); return -1; }); + summary = await parallel.CompleteAsync(); + } + + Assert.Equal(CompletionReason.FailureToleranceExceeded, summary.CompletionReason); + Assert.True(summary.HasFailure); + Assert.Equal(1, await ok); + var ex = await Assert.ThrowsAsync(async () => await bad); + Assert.Contains("boom", ex.Message); + } + + [Fact] + public async Task CreateParallel_ReplayNameDrift_Throws() + { + var parentOpId = IdAt(1); + var summaryJson = """ + {"CompletionReason":"ALL_COMPLETED","Units":[ + {"Index":0,"Name":"inventory","Status":"SUCCEEDED","Result":"1"} + ]} + """; + + var (context, _, _, _) = CreateContext(new InitialExecutionState + { + Operations = new List + { + new() + { + Id = parentOpId, + Type = OperationTypes.Context, + Status = OperationStatuses.Succeeded, + SubType = OperationSubTypes.Parallel, + Name = "fanout", + ContextDetails = new ContextDetails { Result = summaryJson } + } + } + }); + + var parallel = context.CreateParallel(name: "fanout"); + // Registered a branch whose name drifted from the checkpointed "inventory". + Assert.Throws(() => + _ = parallel.BranchAsync("renamed", async (_, _) => { await Task.Yield(); return 1; })); + await parallel.DisposeAsync(); + } + + // ────────────────────────────────────────────────────────────────────── + // Replay — STARTED parent re-runs branches (children replay from own checkpoints) + // ────────────────────────────────────────────────────────────────────── + + [Fact] + public async Task CreateParallel_ReplayStartedParent_ReRunsBranches_AndCheckpointsSucceed() + { + var parentOpId = IdAt(1); + var (context, recorder, _, _) = CreateContext(new InitialExecutionState + { + Operations = new List + { + new() + { + Id = parentOpId, + Type = OperationTypes.Context, + Status = OperationStatuses.Started, + SubType = OperationSubTypes.Parallel, + Name = "fanout" + } + } + }); + + var runCount = 0; + IBatchResult summary; + await using (var parallel = context.CreateParallel(name: "fanout")) + { + _ = parallel.BranchAsync("a", async (_, _) => { Interlocked.Increment(ref runCount); await Task.Yield(); return 1; }); + _ = parallel.BranchAsync("b", async (_, _) => { Interlocked.Increment(ref runCount); await Task.Yield(); return 2; }); + summary = await parallel.CompleteAsync(); + } + + Assert.Equal(2, runCount); // children re-run (no terminal checkpoints for them) + Assert.Equal(2, summary.SuccessCount); + + await recorder.Batcher.DrainAsync(); + // STARTED parent is not re-emitted, but the terminal SUCCEED is written now. + var parentActions = recorder.Flushed + .Where(o => o.Type == "CONTEXT" && o.SubType == "Parallel") + .Select(o => $"{o.Action}").ToArray(); + Assert.DoesNotContain("START", parentActions); + Assert.Contains("SUCCEED", parentActions); + } +} From 85fdae8d11e7a1fc33cf3e90789175572901e76a Mon Sep 17 00:00:00 2001 From: Garrett Beatty Date: Tue, 1 Sep 2026 22:48:34 +0000 Subject: [PATCH 06/12] chore: add AutoVer change file for incremental heterogeneous Parallel (#2519) --- .../add-incremental-heterogeneous-parallel.json | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 .autover/changes/add-incremental-heterogeneous-parallel.json diff --git a/.autover/changes/add-incremental-heterogeneous-parallel.json b/.autover/changes/add-incremental-heterogeneous-parallel.json new file mode 100644 index 000000000..15192e08a --- /dev/null +++ b/.autover/changes/add-incremental-heterogeneous-parallel.json @@ -0,0 +1,11 @@ +{ + "Projects": [ + { + "Name": "Amazon.Lambda.DurableExecution", + "Type": "Minor", + "ChangelogMessages": [ + "Added an incremental, branch-oriented parallel API (IDurableContext.CreateParallel, IDurableParallel, IParallelBranch) supporting heterogeneous per-branch result types and incremental branch registration, alongside the existing homogeneous ParallelAsync overloads. Each branch declares its own result type and returns an awaitable typed handle; branches start on registration (respecting MaxConcurrency) and the operation is sealed with CompleteAsync. Honors the existing MaxConcurrency, CompletionConfig, NestingType, cancellation, deterministic replay, and ILambdaSerializer behavior." + ] + } + ] +} From a6b051e20f18805a7e3dec380449bd860a07df8c Mon Sep 17 00:00:00 2001 From: Garrett Beatty Date: Tue, 1 Sep 2026 23:19:38 +0000 Subject: [PATCH 07/12] fix(DurableExecution): address Copilot review on incremental Parallel (#2519) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Replay: switch explicitly on parent status — only SUCCEEDED reconstructs and only STARTED/PENDING re-run; any other terminal status (FAILED/CANCELLED/ STOPPED/TIMED_OUT) throws NonDeterministicExecutionException instead of silently re-running and overwriting the prior outcome (mirrors ConcurrentOperation.ReplayAsync). - CompleteAsync idempotence: cache the in-progress completion Task, not just the finished result, so concurrent CompleteAsync/DisposeAsync calls share one completion and enqueue exactly one parent SUCCEED. - Terminal replay: enforce the positional replay contract — the registered branch count must equal the frozen summary's unit count, else throw. - Percentage failure tolerance is no longer evaluated against the incomplete denominator during incremental registration; it is suppressed until the operation is sealed (CompletionPolicy gains an evaluatePercentage flag, defaulting true so batch behavior is unchanged). - Observe the per-branch result-task fault in the handle ctor so a discarded failed handle cannot surface as an UnobservedTaskException. - DisposeAsync no longer throws: its safety-net completion swallows faults. - Docs: correct the CreateParallel `name` param (positional op id, not name-derived); document that the CompleteAsync token governs sealing/awaiting and does not retroactively cancel already-started branch bodies. Adds 3 unit tests (unexpected-status throw, branch-count-mismatch throw, percentage-not-evaluated-before-seal). 431 unit tests pass; both incremental integration tests re-verified green against the durable execution service. --- .../IDurableContext.cs | 8 +- .../IDurableParallel.cs | 9 ++ .../Internal/CompletionPolicy.cs | 8 +- .../Internal/IncrementalParallelOperation.cs | 147 +++++++++++++----- .../IncrementalParallelOperationTests.cs | 83 ++++++++++ 5 files changed, 206 insertions(+), 49 deletions(-) diff --git a/Libraries/src/Amazon.Lambda.DurableExecution/IDurableContext.cs b/Libraries/src/Amazon.Lambda.DurableExecution/IDurableContext.cs index 3f1465f72..48daad5de 100644 --- a/Libraries/src/Amazon.Lambda.DurableExecution/IDurableContext.cs +++ b/Libraries/src/Amazon.Lambda.DurableExecution/IDurableContext.cs @@ -391,9 +391,11 @@ Task WaitForConditionAsync( /// as the homogeneous API. /// /// - /// An optional name for the parallel operation, used for observability and to - /// derive the deterministic operation ID. Defaults to a name inferred from the - /// call site. + /// Optional human-readable name for the parallel operation, used only for + /// observability — it surfaces on the wire OperationUpdate.Name field and + /// in execution traces. The deterministic operation ID is positional (derived + /// from the call order, not from this name), so a name change across deployments + /// does not break replay. Defaults to null. /// /// /// Optional parallel configuration. Defaults are used when null. diff --git a/Libraries/src/Amazon.Lambda.DurableExecution/IDurableParallel.cs b/Libraries/src/Amazon.Lambda.DurableExecution/IDurableParallel.cs index 7c5118deb..e859543bc 100644 --- a/Libraries/src/Amazon.Lambda.DurableExecution/IDurableParallel.cs +++ b/Libraries/src/Amazon.Lambda.DurableExecution/IDurableParallel.cs @@ -87,6 +87,15 @@ IParallelBranch BranchAsync( /// , or await individual branch /// handles, to observe failures. It does propagate workflow-level errors (for /// example ) and cancellation. + /// + /// The governs sealing and awaiting: it + /// stops this call from waiting further. Because branches begin executing when + /// they are registered (before CompleteAsync is called), this token is + /// not retroactively linked into already-running branch bodies — those observe + /// the SDK's workflow-shutdown signal (and the completion-policy short-circuit) + /// instead. Dispatched branches always run to a terminal checkpoint so replay + /// stays deterministic, matching . + /// /// /// A token to observe for cancellation. /// The aggregate summarizing branch outcomes. diff --git a/Libraries/src/Amazon.Lambda.DurableExecution/Internal/CompletionPolicy.cs b/Libraries/src/Amazon.Lambda.DurableExecution/Internal/CompletionPolicy.cs index b1573a593..a02cb153b 100644 --- a/Libraries/src/Amazon.Lambda.DurableExecution/Internal/CompletionPolicy.cs +++ b/Libraries/src/Amazon.Lambda.DurableExecution/Internal/CompletionPolicy.cs @@ -44,8 +44,8 @@ public CompletionPolicy(CompletionConfig config) /// failed. Reads slightly-stale counters by design (see the dispatch loop); /// is the authoritative verdict. /// - public bool ShouldStopDispatching(int succeeded, int failed, int totalBranches) - => MinSuccessfulReached(succeeded) || FailureToleranceExceeded(failed, totalBranches); + public bool ShouldStopDispatching(int succeeded, int failed, int totalBranches, bool evaluatePercentage = true) + => MinSuccessfulReached(succeeded) || FailureToleranceExceeded(failed, totalBranches, evaluatePercentage); /// /// Final verdict once all dispatched branches have settled. Failure tolerance @@ -77,7 +77,7 @@ private bool MinSuccessfulReached(int succeeded) // ToleratedFailureCount = 0) and the empty config are therefore equivalent; // CompletionConfig.AllCompleted() sets ToleratedFailureCount = int.MaxValue to // stay lenient. - private bool FailureToleranceExceeded(int failed, int totalBranches) + private bool FailureToleranceExceeded(int failed, int totalBranches, bool evaluatePercentage = true) { if (_failFastOnAnyFailure) return failed > 0; @@ -85,7 +85,7 @@ private bool FailureToleranceExceeded(int failed, int totalBranches) if (_toleratedFailureCount is { } tfc && failed > tfc) return true; - if (_toleratedFailurePercentage is { } tfp && totalBranches > 0 && + if (evaluatePercentage && _toleratedFailurePercentage is { } tfp && totalBranches > 0 && (double)failed / totalBranches > tfp) { return true; diff --git a/Libraries/src/Amazon.Lambda.DurableExecution/Internal/IncrementalParallelOperation.cs b/Libraries/src/Amazon.Lambda.DurableExecution/Internal/IncrementalParallelOperation.cs index 58be79e7a..50e296b45 100644 --- a/Libraries/src/Amazon.Lambda.DurableExecution/Internal/IncrementalParallelOperation.cs +++ b/Libraries/src/Amazon.Lambda.DurableExecution/Internal/IncrementalParallelOperation.cs @@ -99,6 +99,15 @@ public IncrementalParallelBranch(int index, string name, ILambdaSerializer seria Name = name; _serializer = serializer; _childSubType = childSubType; + + // Per-branch failures are intentionally consumed via CompleteAsync, so a + // caller may never await this handle. Observe the fault here so a discarded + // failed handle can never surface as an UnobservedTaskException. + _ = _result.Task.ContinueWith( + static t => { _ = t.Exception; }, + CancellationToken.None, + TaskContinuationOptions.OnlyOnFaulted | TaskContinuationOptions.ExecuteSynchronously, + TaskScheduler.Default); } public string Name { get; } @@ -280,9 +289,9 @@ internal sealed class IncrementalParallelOperation : IDurableParallel private int _failed; private int _registeredCount; private bool _sealed; - private bool _completed; + private volatile bool _sealedVolatile; private bool _disposed; - private IBatchResult? _cachedResult; + private Task? _completion; public IncrementalParallelOperation( string operationId, @@ -321,33 +330,48 @@ public IncrementalParallelOperation( _state.TrackReplay(_operationId); var existing = _state.GetOperation(_operationId); - var terminal = existing != null && - (existing.Status == OperationStatuses.Succeeded || existing.Status == OperationStatuses.Failed); - - if (terminal) + if (existing == null) { - _mode = ParallelExecutionMode.Terminal; - _frozenSummary = BatchSummaryCodec.ParseSummary(existing!.ContextDetails?.Result); - _startTask = Task.CompletedTask; + // Fresh: emit the parent CONTEXT START so the service has a parent + // record if a branch suspends. Enqueued once here so it is ordered + // before any branch's child START. + _mode = ParallelExecutionMode.Run; + _startTask = EnqueueAsync(new SdkOperationUpdate + { + Id = _operationId, + ParentId = _parentId, + Type = OperationTypes.Context, + Action = OperationAction.START, + SubType = OperationSubTypes.Parallel, + Name = _name + }); } else { - _mode = ParallelExecutionMode.Run; - // Fresh (no checkpoint) emits the parent CONTEXT START so the service - // has a parent record if a branch suspends. STARTED/PENDING replay does - // not re-emit it (the original is authoritative). Enqueued once here so - // it is ordered before any branch's child START. - _startTask = existing == null - ? EnqueueAsync(new SdkOperationUpdate - { - Id = _operationId, - ParentId = _parentId, - Type = OperationTypes.Context, - Action = OperationAction.START, - SubType = OperationSubTypes.Parallel, - Name = _name - }) - : Task.CompletedTask; + // Mirror ConcurrentOperation.ReplayAsync: only SUCCEEDED reconstructs, + // only STARTED/PENDING re-run, and any other status is a replay + // mismatch. The parent parallel only ever checkpoints SUCCEED, so a + // FAILED/CANCELLED/STOPPED/TIMED_OUT parent must never be silently + // re-run (which would overwrite the prior terminal outcome). + switch (existing.Status) + { + case OperationStatuses.Succeeded: + _mode = ParallelExecutionMode.Terminal; + _frozenSummary = BatchSummaryCodec.ParseSummary(existing.ContextDetails?.Result); + _startTask = Task.CompletedTask; + break; + case OperationStatuses.Started: + case OperationStatuses.Pending: + // Children replay from their own checkpoints; the parent START + // is not re-emitted (the original is authoritative). + _mode = ParallelExecutionMode.Run; + _startTask = Task.CompletedTask; + break; + default: + throw new NonDeterministicExecutionException( + $"Parallel operation '{_name ?? _operationId}' has unexpected status " + + $"'{existing.Status}' on replay."); + } } } @@ -397,12 +421,30 @@ public IParallelBranch BranchAsync( } } - public async Task CompleteAsync(CancellationToken cancellationToken = default) + public Task CompleteAsync(CancellationToken cancellationToken = default) { lock (_lock) { - if (_cachedResult != null) return _cachedResult; + // Cache the in-progress task (not just the finished result) so two + // concurrent CompleteAsync calls — or DisposeAsync racing one — share a + // single completion and enqueue exactly one parent SUCCEED. + if (_completion != null) return _completion; _sealed = true; + _sealedVolatile = true; + _completion = CompleteCoreAsync(cancellationToken); + return _completion; + } + } + + private async Task CompleteCoreAsync(CancellationToken cancellationToken) + { + // Registration is sealed: the denominator is now known, so re-evaluate the + // completion policy (including percentage-based tolerance, which is + // suppressed pre-seal) and signal any in-flight branches to bail. + if (ShouldStopDispatchingNow()) + { + try { _shortCircuitCts.Cancel(); } + catch (ObjectDisposedException) { } } // Ensure the parent START is durably enqueued even for an empty operation. @@ -440,33 +482,35 @@ public async Task CompleteAsync(CancellationToken cancellationToke ? BuildTerminalResult(controllers) : await BuildAndCheckpointRunResultAsync(controllers, cancellationToken).ConfigureAwait(false); - lock (_lock) - { - _completed = true; - _cachedResult = result; - } return result; } public async ValueTask DisposeAsync() { - bool needComplete; + Task? completion; lock (_lock) { if (_disposed) return; _disposed = true; - needComplete = !_completed; + // If CompleteAsync was never called, complete now so the parent's + // terminal checkpoint is written — otherwise replay would see a STARTED + // parent forever and re-run the whole operation. + completion = _completion; } try { - // Guarantee the parent's terminal checkpoint is written even if the - // caller forgot CompleteAsync — otherwise replay would see a STARTED - // parent forever and re-run the whole operation. - if (needComplete) - { - await CompleteAsync(CancellationToken.None).ConfigureAwait(false); - } + completion ??= CompleteAsync(CancellationToken.None); + await completion.ConfigureAwait(false); + } + catch + { + // DisposeAsync must never throw. A completion fault (e.g. a + // NonDeterministicExecutionException, or the secondary effect of a + // BranchAsync that already threw during registration) is either + // already surfaced to a caller that awaited CompleteAsync, or will + // resurface on the next invocation's replay. Swallow it here so + // `await using` teardown stays clean. } finally { @@ -562,8 +606,15 @@ private void OnBranchSettled(Task settlement) // During incremental registration the "total" is the number registered so far; // percentage-based tolerance is evaluated against that running total. MinSuccessful // and count-based tolerance don't depend on the total. + // During incremental registration the denominator is unknown, so percentage-based + // failure tolerance must NOT drive short-circuiting (a premature ratio like 1/1 + // could skip branches that would have lowered the final ratio). MinSuccessful and + // absolute-count tolerance are denominator-independent and always apply; the + // percentage component is enabled only once registration is sealed, and the final + // verdict (ComputeCompletionReason) always uses the true total. private bool ShouldStopDispatchingNow() => _policy.ShouldStopDispatching( - Volatile.Read(ref _succeeded), Volatile.Read(ref _failed), Volatile.Read(ref _registeredCount)); + Volatile.Read(ref _succeeded), Volatile.Read(ref _failed), Volatile.Read(ref _registeredCount), + evaluatePercentage: _sealedVolatile); private async Task BuildAndCheckpointRunResultAsync( IReadOnlyList controllers, @@ -697,6 +748,18 @@ private IBatchResult BuildTerminalResult(IReadOnlyList + { + new() + { + Id = parentOpId, + Type = OperationTypes.Context, + Status = "CANCELLED", + SubType = OperationSubTypes.Parallel, + Name = "fanout" + } + } + }); + + Assert.Throws(() => context.CreateParallel(name: "fanout")); + } + + [Fact] + public async Task CreateParallel_ReplayBranchCountMismatch_Throws() + { + // Registering a different number of branches than the frozen summary recorded + // violates the positional replay contract. + var parentOpId = IdAt(1); + var summaryJson = """ + {"CompletionReason":"ALL_COMPLETED","Units":[ + {"Index":0,"Name":"inventory","Status":"SUCCEEDED","Result":"1"}, + {"Index":1,"Name":"payment","Status":"SUCCEEDED","Result":"2"} + ]} + """; + + var (context, _, _, _) = CreateContext(new InitialExecutionState + { + Operations = new List + { + new() + { + Id = parentOpId, + Type = OperationTypes.Context, + Status = OperationStatuses.Succeeded, + SubType = OperationSubTypes.Parallel, + Name = "fanout", + ContextDetails = new ContextDetails { Result = summaryJson } + } + } + }); + + var parallel = context.CreateParallel(name: "fanout"); + _ = parallel.BranchAsync("inventory", async (_, _) => { await Task.Yield(); return 1; }); + // Only one branch registered, but the checkpoint recorded two. + await Assert.ThrowsAsync(async () => await parallel.CompleteAsync()); + await parallel.DisposeAsync(); // must not throw a secondary exception + } + + [Fact] + public void CompletionPolicy_PercentageTolerance_NotEvaluatedBeforeSeal() + { + // A percentage-based tolerance must not short-circuit against an incomplete + // denominator: 1 failure out of 1 registered-so-far is 100%, but with two + // more registrations pending the true ratio may be under threshold. + var policy = new CompletionPolicy(new CompletionConfig { ToleratedFailurePercentage = 0.5 }); + + // Pre-seal: percentage suppressed → do NOT stop dispatching. + Assert.False(policy.ShouldStopDispatching(succeeded: 0, failed: 1, totalBranches: 1, evaluatePercentage: false)); + + // Post-seal with the true denominator: 1/3 <= 0.5 → still do not stop. + Assert.False(policy.ShouldStopDispatching(succeeded: 0, failed: 1, totalBranches: 3, evaluatePercentage: true)); + + // Post-seal, genuinely over threshold: 2/3 > 0.5 → stop. + Assert.True(policy.ShouldStopDispatching(succeeded: 0, failed: 2, totalBranches: 3, evaluatePercentage: true)); + } } From 0f460648c348dae54fd1fb1a82fd660f36abf644 Mon Sep 17 00:00:00 2001 From: Garrett Beatty Date: Wed, 2 Sep 2026 21:08:02 +0000 Subject: [PATCH 08/12] feat(DurableExecution): per-operation and per-branch serialization for CreateParallel (#2519) Stacks on the per-step-serializer work: CreateParallel now honors ParallelConfig.ItemSerializer as the operation-level branch-result serializer, and IDurableParallel.BranchAsync accepts an optional per-branch ILambdaSerializer override (falls back to ItemSerializer, then the globally-registered serializer). Each branch's serializer is threaded into both its ChildContextOperation and the inline summary serialization so fresh and replay values match. Adds unit tests for per-branch and operation-level ItemSerializer, and relaxes the timing- sensitive FirstSuccessful test to its deterministic invariants. --- ...dd-incremental-heterogeneous-parallel.json | 3 +- .../DurableContext.cs | 5 +- .../IDurableContext.cs | 2 +- .../IDurableParallel.cs | 20 +++- .../IParallelBranch.cs | 2 +- .../Internal/IncrementalParallelOperation.cs | 20 ++-- .../docs/core/parallel.md | 19 ++++ .../IncrementalParallelOperationTests.cs | 92 +++++++++++++++++-- 8 files changed, 143 insertions(+), 20 deletions(-) diff --git a/.autover/changes/add-incremental-heterogeneous-parallel.json b/.autover/changes/add-incremental-heterogeneous-parallel.json index 15192e08a..e24fd8b3c 100644 --- a/.autover/changes/add-incremental-heterogeneous-parallel.json +++ b/.autover/changes/add-incremental-heterogeneous-parallel.json @@ -4,7 +4,8 @@ "Name": "Amazon.Lambda.DurableExecution", "Type": "Minor", "ChangelogMessages": [ - "Added an incremental, branch-oriented parallel API (IDurableContext.CreateParallel, IDurableParallel, IParallelBranch) supporting heterogeneous per-branch result types and incremental branch registration, alongside the existing homogeneous ParallelAsync overloads. Each branch declares its own result type and returns an awaitable typed handle; branches start on registration (respecting MaxConcurrency) and the operation is sealed with CompleteAsync. Honors the existing MaxConcurrency, CompletionConfig, NestingType, cancellation, deterministic replay, and ILambdaSerializer behavior." + "Added an incremental, branch-oriented parallel API (IDurableContext.CreateParallel, IDurableParallel, IParallelBranch) supporting heterogeneous per-branch result types and incremental branch registration, alongside the existing homogeneous ParallelAsync overloads. Each branch declares its own result type and returns an awaitable typed handle; branches start on registration (respecting MaxConcurrency) and the operation is sealed with CompleteAsync. Honors the existing MaxConcurrency, CompletionConfig, NestingType, cancellation, deterministic replay, and ILambdaSerializer behavior.", + "CreateParallel supports per-operation and per-branch result serialization: ParallelConfig.ItemSerializer sets the operation-level serializer for all branch results, and IDurableParallel.BranchAsync accepts an optional per-branch ILambdaSerializer override, falling back to ItemSerializer and then the globally-registered serializer." ] } ] diff --git a/Libraries/src/Amazon.Lambda.DurableExecution/DurableContext.cs b/Libraries/src/Amazon.Lambda.DurableExecution/DurableContext.cs index 1de3f84ee..a0471ba11 100644 --- a/Libraries/src/Amazon.Lambda.DurableExecution/DurableContext.cs +++ b/Libraries/src/Amazon.Lambda.DurableExecution/DurableContext.cs @@ -197,7 +197,10 @@ public IDurableParallel CreateParallel( ParallelConfig? config = null) { var effectiveConfig = config ?? new ParallelConfig(); - var serializer = LambdaSerializerHelper.GetRequired(LambdaContext); + // Operation-level default for per-branch result serialization: the config's + // ItemSerializer if set, else the globally-registered serializer. Individual + // branches may still override this via BranchAsync's serializer parameter. + var serializer = effectiveConfig.ItemSerializer ?? LambdaSerializerHelper.GetRequired(LambdaContext); var operationId = _idGenerator.NextId(); return new Internal.IncrementalParallelOperation( diff --git a/Libraries/src/Amazon.Lambda.DurableExecution/IDurableContext.cs b/Libraries/src/Amazon.Lambda.DurableExecution/IDurableContext.cs index 48daad5de..13b020303 100644 --- a/Libraries/src/Amazon.Lambda.DurableExecution/IDurableContext.cs +++ b/Libraries/src/Amazon.Lambda.DurableExecution/IDurableContext.cs @@ -373,7 +373,7 @@ Task WaitForConditionAsync( /// overloads — which take a complete branch list up front and share one result /// type — the returned lets you register branches /// one at a time via - /// , + /// , /// each with its own result type (heterogeneous), starting each branch as it is /// registered. Call /// to seal registration and obtain the aggregate . diff --git a/Libraries/src/Amazon.Lambda.DurableExecution/IDurableParallel.cs b/Libraries/src/Amazon.Lambda.DurableExecution/IDurableParallel.cs index e859543bc..95ab26129 100644 --- a/Libraries/src/Amazon.Lambda.DurableExecution/IDurableParallel.cs +++ b/Libraries/src/Amazon.Lambda.DurableExecution/IDurableParallel.cs @@ -7,7 +7,7 @@ namespace Amazon.Lambda.DurableExecution; /// An incremental, branch-oriented parallel operation created by /// . Branches /// are registered one at a time via -/// , +/// , /// each with its own result type (heterogeneous), and each begins executing /// immediately (subject to ). Call /// to seal @@ -23,7 +23,7 @@ namespace Amazon.Lambda.DurableExecution; /// checkpointed plan) and earlier branches should start before later ones are known. /// /// Deterministic replay. Branch identity is positional: the n-th -/// +/// /// call reuses the n-th deterministic operation ID. Workflow code must therefore /// register the same branches in the same order across invocations — produce any /// dynamic branch list inside a checkpointed @@ -66,6 +66,19 @@ public interface IDurableParallel : IAsyncDisposable /// workflow-shutdown signal with the operation's completion-policy /// short-circuit, and returns the branch's result. /// + /// + /// Optional serializer for this branch's result payload. When + /// null (default), the branch uses the operation-level serializer — + /// if set on + /// , otherwise + /// the globally-registered on + /// . Because each branch + /// declares its own result type, a per-branch serializer lets one branch use a + /// bespoke serializer (for example a source-generated context for AOT) without + /// affecting sibling branches. It is part of the deterministic definition: the same + /// branch must be able to deserialize a result it previously serialized, so pass the + /// same serializer at a given branch index across replays. + /// /// A typed handle for awaiting the branch's result. /// /// The operation has already been sealed by @@ -73,7 +86,8 @@ public interface IDurableParallel : IAsyncDisposable /// IParallelBranch BranchAsync( string name, - Func> func); + Func> func, + Amazon.Lambda.Core.ILambdaSerializer? serializer = null); /// /// Seals registration (no further branches may be added), awaits the diff --git a/Libraries/src/Amazon.Lambda.DurableExecution/IParallelBranch.cs b/Libraries/src/Amazon.Lambda.DurableExecution/IParallelBranch.cs index 7a282d518..f99c386a1 100644 --- a/Libraries/src/Amazon.Lambda.DurableExecution/IParallelBranch.cs +++ b/Libraries/src/Amazon.Lambda.DurableExecution/IParallelBranch.cs @@ -7,7 +7,7 @@ namespace Amazon.Lambda.DurableExecution; /// /// A typed handle to a single branch registered on an -/// via . +/// via . /// Unlike the homogeneous /// API — where every branch shares one result type T — each branch on an /// declares its own result type, so a single diff --git a/Libraries/src/Amazon.Lambda.DurableExecution/Internal/IncrementalParallelOperation.cs b/Libraries/src/Amazon.Lambda.DurableExecution/Internal/IncrementalParallelOperation.cs index 50e296b45..2f0cc20b0 100644 --- a/Libraries/src/Amazon.Lambda.DurableExecution/Internal/IncrementalParallelOperation.cs +++ b/Libraries/src/Amazon.Lambda.DurableExecution/Internal/IncrementalParallelOperation.cs @@ -377,7 +377,8 @@ public IncrementalParallelOperation( public IParallelBranch BranchAsync( string name, - Func> func) + Func> func, + ILambdaSerializer? serializer = null) { if (name == null) throw new ArgumentNullException(nameof(name)); if (func == null) throw new ArgumentNullException(nameof(func)); @@ -391,7 +392,10 @@ public IParallelBranch BranchAsync( var index = _branches.Count; // zero-based branch index var childOpId = OperationIdGenerator.HashOperationId($"{_operationId}-{index + 1}"); - var handle = new IncrementalParallelBranch(index, name, _serializer, OperationSubTypes.ParallelBranch); + // Per-branch serializer override, else the operation-level default + // (ParallelConfig.ItemSerializer ?? the globally-registered serializer). + var branchSerializer = serializer ?? _serializer; + var handle = new IncrementalParallelBranch(index, name, branchSerializer, OperationSubTypes.ParallelBranch); var summaryEntry = FindSummaryUnit(index); @@ -408,11 +412,11 @@ public IParallelBranch BranchAsync( if (_mode == ParallelExecutionMode.Terminal) { - ResolveTerminalBranch(handle, name, childOpId, func, summaryEntry); + ResolveTerminalBranch(handle, name, childOpId, func, summaryEntry, branchSerializer); } else { - LaunchRunBranch(handle, name, childOpId, func); + LaunchRunBranch(handle, name, childOpId, func, branchSerializer); } _branches.Add(handle); @@ -527,6 +531,7 @@ private void LaunchRunBranch( string name, string childOpId, Func> func, + ILambdaSerializer branchSerializer, BatchItemStatus? frozenStatus = null) { async Task Run() @@ -550,7 +555,7 @@ async Task Run() _operationId, func, new ChildContextConfig { SubType = OperationSubTypes.ParallelBranch }, - _serializer, + branchSerializer, _childContextFactory, _state, _termination, @@ -709,7 +714,8 @@ private void ResolveTerminalBranch( string name, string childOpId, Func> func, - BatchUnitSummary? summaryEntry) + BatchUnitSummary? summaryEntry, + ILambdaSerializer branchSerializer) { // A branch registered now but absent from the frozen summary (registered // after the original seal) never ran — surface it as skipped. @@ -734,7 +740,7 @@ private void ResolveTerminalBranch( // Overflow: the inline value/error was stripped. Re-run the branch to // recover it from the branch's own checkpoint; the frozen status stays // authoritative. - LaunchRunBranch(handle, name, childOpId, func, frozenStatus: status); + LaunchRunBranch(handle, name, childOpId, func, branchSerializer, frozenStatus: status); break; default: handle.ResolveFromInline(BatchItemStatus.Started, null, null); diff --git a/Libraries/src/Amazon.Lambda.DurableExecution/docs/core/parallel.md b/Libraries/src/Amazon.Lambda.DurableExecution/docs/core/parallel.md index d95b51ab4..e04a7c3ac 100644 --- a/Libraries/src/Amazon.Lambda.DurableExecution/docs/core/parallel.md +++ b/Libraries/src/Amazon.Lambda.DurableExecution/docs/core/parallel.md @@ -58,6 +58,25 @@ PaymentAuthorization authorizedPayment = await payment; The homogeneous `ParallelAsync` overloads remain the simplest choice for a fixed set of same-typed branches and convenient `GetResults()` usage. +### Per-branch serialization + +Because each branch declares its own result type, each may also use its own serializer. `BranchAsync` takes an optional `ILambdaSerializer? serializer`; when omitted a branch uses the operation-level default — `ParallelConfig.ItemSerializer` if set on `CreateParallel`, otherwise the globally-registered `ILambdaContext.Serializer`. This lets one branch opt into a bespoke serializer (for example a source-generated `JsonSerializerContext` for Native AOT) without affecting sibling branches. + +```csharp +await using var parallel = ctx.CreateParallel(name: "process-order"); + +// Uses the operation-level / global serializer. +var inventory = parallel.BranchAsync("inventory", async (b, ct) => await ReserveAsync(b, ct)); + +// Overrides serialization for just this branch. +var payment = parallel.BranchAsync( + "payment", + async (b, ct) => await AuthorizeAsync(b, ct), + serializer: PaymentSerializerContext.Default.CreateLambdaSerializer()); +``` + +Like every other part of the workflow definition, a branch's serializer is re-resolved on replay, so a branch must be able to deserialize a result it previously serialized — keep the serializer stable at a given branch index across deployments. + ## Example Fan out three independent lookups and collect the results: diff --git a/Libraries/test/Amazon.Lambda.DurableExecution.Tests/IncrementalParallelOperationTests.cs b/Libraries/test/Amazon.Lambda.DurableExecution.Tests/IncrementalParallelOperationTests.cs index 8a3e495b5..ddd432a6d 100644 --- a/Libraries/test/Amazon.Lambda.DurableExecution.Tests/IncrementalParallelOperationTests.cs +++ b/Libraries/test/Amazon.Lambda.DurableExecution.Tests/IncrementalParallelOperationTests.cs @@ -289,14 +289,25 @@ public async Task CreateParallel_FirstSuccessful_WithMaxConcurrency1_SkipsTraili summary = await parallel.CompleteAsync(); } - Assert.Equal(CompletionReason.MinSuccessfulReached, summary.CompletionReason); Assert.True(summary.SuccessCount >= 1); - Assert.True(summary.StartedCount >= 1); + Assert.Equal(0, summary.FailureCount); Assert.Equal(3, summary.TotalCount); - - // The trailing branch never ran; awaiting it surfaces a skip error. - Assert.Equal(BatchItemStatus.Started, last.Status); - await Assert.ThrowsAsync(async () => await last); + // With MaxConcurrency=1 the completion policy stops dispatching once the first + // success lands, but because branches start on registration, how many of the + // already-in-flight branches run to completion before the short-circuit is + // observed is timing-dependent. The deterministic invariants: no failures, the + // reason reflects an early success (MinSuccessfulReached when a branch was + // skipped, AllCompleted when every branch happened to finish), and a skipped + // branch's handle throws on await. + Assert.True( + summary.CompletionReason == CompletionReason.MinSuccessfulReached + || summary.CompletionReason == CompletionReason.AllCompleted); + Assert.Equal(3, summary.SuccessCount + summary.StartedCount); + + if (last.Status == BatchItemStatus.Started) + { + await Assert.ThrowsAsync(async () => await last); + } } // ────────────────────────────────────────────────────────────────────── @@ -609,4 +620,73 @@ public void CompletionPolicy_PercentageTolerance_NotEvaluatedBeforeSeal() // Post-seal, genuinely over threshold: 2/3 > 0.5 → stop. Assert.True(policy.ShouldStopDispatching(succeeded: 0, failed: 2, totalBranches: 3, evaluatePercentage: true)); } + + // ────────────────────────────────────────────────────────────────────── + // Per-branch serialization (stacked on feature/per-step-serializer) + // ────────────────────────────────────────────────────────────────────── + + [Fact] + public async Task CreateParallel_PerBranchSerializer_UsedForThatBranchOnly() + { + var (context, _, _, _) = CreateContext(); + var custom = new CountingSerializer(); + + IParallelBranch a; + IParallelBranch b; + await using (var parallel = context.CreateParallel()) + { + // Branch "a" overrides its serializer; branch "b" uses the global default. + a = parallel.BranchAsync("a", async (_, _) => { await Task.Yield(); return 7; }, serializer: custom); + b = parallel.BranchAsync("b", async (_, _) => { await Task.Yield(); return 8; }); + await parallel.CompleteAsync(); + } + + // Results round-trip correctly regardless of which serializer produced them. + Assert.Equal(7, await a); + Assert.Equal(8, await b); + + // The per-branch serializer was exercised for branch "a". + Assert.True(custom.SerializeCount > 0, "custom per-branch serializer should have serialized branch a's result"); + } + + [Fact] + public async Task CreateParallel_ItemSerializer_AppliesToAllBranchesByDefault() + { + var (context, _, _, _) = CreateContext(); + var shared = new CountingSerializer(); + + await using (var parallel = context.CreateParallel(config: new ParallelConfig { ItemSerializer = shared })) + { + _ = parallel.BranchAsync("a", async (_, _) => { await Task.Yield(); return 1; }); + _ = parallel.BranchAsync("b", async (_, _) => { await Task.Yield(); return 2; }); + var summary = await parallel.CompleteAsync(); + Assert.Equal(2, summary.SuccessCount); + } + + // The operation-level ItemSerializer served both branches. + Assert.True(shared.SerializeCount >= 2, "ItemSerializer should serialize every branch result by default"); + } + + /// + /// Delegating that counts calls, so a + /// test can assert which serializer a branch used. + /// + private sealed class CountingSerializer : Amazon.Lambda.Core.ILambdaSerializer + { + private readonly DefaultLambdaJsonSerializer _inner = new(); + public int SerializeCount; + public int DeserializeCount; + + public T Deserialize(System.IO.Stream requestStream) + { + Interlocked.Increment(ref DeserializeCount); + return _inner.Deserialize(requestStream); + } + + public void Serialize(T response, System.IO.Stream responseStream) + { + Interlocked.Increment(ref SerializeCount); + _inner.Serialize(response, responseStream); + } + } } From 9b1220e6527f6ae4ac4aa3765e16c43465f6de91 Mon Sep 17 00:00:00 2001 From: Frank Chen <65260095+zhongkechen@users.noreply.github.com> Date: Wed, 2 Sep 2026 13:41:38 -0700 Subject: [PATCH 09/12] test(DurableExecution): add static-typing parallel suite (#2554) * test(DurableExecution): add incremental parallel conformance handlers * test(DurableExecution): add dedicated static typing suite --------- Co-authored-by: Frank Chen --- .../Conformance/README.md | 3 +- .../Conformance/scripts/build_examples.sh | 2 +- .../ParallelEarlyStart/Function.cs | 44 ++++++++++++ .../ParallelEarlyStart.csproj | 19 ++++++ .../ParallelTypedBranches/Function.cs | 52 ++++++++++++++ .../ParallelTypedBranches.csproj | 19 ++++++ .../Conformance/template_static_typing.yaml | 68 +++++++++++++++++++ 7 files changed, 205 insertions(+), 2 deletions(-) create mode 100644 Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/static_typing/ParallelEarlyStart/Function.cs create mode 100644 Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/static_typing/ParallelEarlyStart/ParallelEarlyStart.csproj create mode 100644 Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/static_typing/ParallelTypedBranches/Function.cs create mode 100644 Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/static_typing/ParallelTypedBranches/ParallelTypedBranches.csproj create mode 100644 Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/template_static_typing.yaml diff --git a/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/README.md b/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/README.md index d4ce87efd..7b83c82da 100644 --- a/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/README.md +++ b/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/README.md @@ -42,7 +42,7 @@ Conformance/ ## Coverage -All nine suites are implemented (one handler project per requirement id): +All ten suites are implemented (one handler project per requirement id): | Suite | Ids | Handlers | |-------|-----|----------| @@ -55,6 +55,7 @@ All nine suites are implemented (one handler project per requirement id): | `wait_for_callback` | 7-1 .. 7-15 | 15 | | `parallel` | 8-1 .. 8-22 (8-15 n/a) | 21 | | `map` | 9-1 .. 9-18 (9-14 n/a) | 17 | +| `static_typing` | 12-1 .. 12-2 | 2 | A few requirement ids have no .NET handler because the SDK intentionally lacks the feature they exercise (e.g. per-item / whole-result serdes slots in `map`); diff --git a/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/scripts/build_examples.sh b/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/scripts/build_examples.sh index 4ce59e368..6707491ab 100755 --- a/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/scripts/build_examples.sh +++ b/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/scripts/build_examples.sh @@ -10,7 +10,7 @@ # ./build_examples.sh [operation...] # # Operations (default: every suite directory found next to the templates): -# step wait callback child invoke parallel map wait_for_callback wait_for_condition +# step wait callback child invoke parallel map static_typing wait_for_callback wait_for_condition # # Examples: # ./build_examples.sh step diff --git a/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/static_typing/ParallelEarlyStart/Function.cs b/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/static_typing/ParallelEarlyStart/Function.cs new file mode 100644 index 000000000..56f3fa9c4 --- /dev/null +++ b/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/static_typing/ParallelEarlyStart/Function.cs @@ -0,0 +1,44 @@ +// 12-2: Parallel branch starts before registration is sealed +using Amazon.Lambda.Core; +using Amazon.Lambda.DurableExecution; +using Amazon.Lambda.RuntimeSupport; +using Amazon.Lambda.Serialization.SystemTextJson; + +namespace ParallelEarlyStart; + +public class Function +{ + public static async Task Main(string[] args) + { + var handler = new Function(); + var serializer = new DefaultLambdaJsonSerializer(); + using var handlerWrapper = HandlerWrapper.GetHandlerWrapper(handler.Handler, serializer); + using var bootstrap = new LambdaBootstrap(handlerWrapper); + await bootstrap.RunAsync(); + } + + public Task Handler( + DurableExecutionInvocationInput input, ILambdaContext context) + => DurableFunction.WrapAsync>(Workflow, input, context); + + private async Task> Workflow(object? input, IDurableContext context) + { + string firstResult; + IParallelBranch second; + + await using (var parallel = context.CreateParallel( + name: "early-start", + config: new ParallelConfig { MaxConcurrency = 1 })) + { + IParallelBranch first = parallel.BranchAsync( + "first", (_, _) => Task.FromResult("ready")); + firstResult = await first; + + second = parallel.BranchAsync( + "second", (_, _) => Task.FromResult(firstResult + "-second")); + await parallel.CompleteAsync(); + } + + return new List { firstResult, await second }; + } +} diff --git a/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/static_typing/ParallelEarlyStart/ParallelEarlyStart.csproj b/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/static_typing/ParallelEarlyStart/ParallelEarlyStart.csproj new file mode 100644 index 000000000..4b354e178 --- /dev/null +++ b/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/static_typing/ParallelEarlyStart/ParallelEarlyStart.csproj @@ -0,0 +1,19 @@ + + + + net8.0 + Exe + true + bootstrap + enable + enable + + + + + + + + + + diff --git a/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/static_typing/ParallelTypedBranches/Function.cs b/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/static_typing/ParallelTypedBranches/Function.cs new file mode 100644 index 000000000..c4cc4b768 --- /dev/null +++ b/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/static_typing/ParallelTypedBranches/Function.cs @@ -0,0 +1,52 @@ +// 12-1: Parallel with independently typed heterogeneous branch handles +using Amazon.Lambda.Core; +using Amazon.Lambda.DurableExecution; +using Amazon.Lambda.RuntimeSupport; +using Amazon.Lambda.Serialization.SystemTextJson; + +namespace ParallelTypedBranches; + +public class Function +{ + public static async Task Main(string[] args) + { + var handler = new Function(); + var serializer = new DefaultLambdaJsonSerializer(); + using var handlerWrapper = HandlerWrapper.GetHandlerWrapper(handler.Handler, serializer); + using var bootstrap = new LambdaBootstrap(handlerWrapper); + await bootstrap.RunAsync(); + } + + public Task Handler( + DurableExecutionInvocationInput input, ILambdaContext context) + => DurableFunction.WrapAsync>(Workflow, input, context); + + private async Task> Workflow(object? input, IDurableContext context) + { + IParallelBranch inventory; + IParallelBranch payment; + IParallelBranch> quote; + + await using (var parallel = context.CreateParallel( + name: "typed-branches", + config: new ParallelConfig { MaxConcurrency = 1 })) + { + inventory = parallel.BranchAsync( + "inventory", (_, _) => Task.FromResult("reserved")); + payment = parallel.BranchAsync( + "payment", (_, _) => Task.FromResult(200)); + quote = parallel.BranchAsync( + "quote", (_, _) => Task.FromResult(new Dictionary + { + ["currency"] = "USD" + })); + + await parallel.CompleteAsync(); + } + + string inventoryResult = await inventory; + int paymentResult = await payment; + Dictionary quoteResult = await quote; + return new List { inventoryResult, paymentResult, quoteResult }; + } +} diff --git a/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/static_typing/ParallelTypedBranches/ParallelTypedBranches.csproj b/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/static_typing/ParallelTypedBranches/ParallelTypedBranches.csproj new file mode 100644 index 000000000..4b354e178 --- /dev/null +++ b/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/static_typing/ParallelTypedBranches/ParallelTypedBranches.csproj @@ -0,0 +1,19 @@ + + + + net8.0 + Exe + true + bootstrap + enable + enable + + + + + + + + + + diff --git a/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/template_static_typing.yaml b/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/template_static_typing.yaml new file mode 100644 index 000000000..599898e05 --- /dev/null +++ b/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/template_static_typing.yaml @@ -0,0 +1,68 @@ +AWSTemplateFormatVersion: '2010-09-09' +Transform: AWS::Serverless-2016-10-31 +Description: Durable Execution Conformance Test Examples - .NET (Static Typing) +Globals: + Function: + Runtime: dotnet8 + Timeout: 60 + MemorySize: 512 + +Resources: + DurableFunctionRole: + Type: AWS::IAM::Role + Properties: + AssumeRolePolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: Allow + Principal: + Service: lambda.amazonaws.com + Action: sts:AssumeRole + ManagedPolicyArns: + - arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole + Policies: + - PolicyName: DurableExecutionPolicy + PolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: Allow + Action: + - lambda:CheckpointDurableExecution + - lambda:GetDurableExecutionState + Resource: '*' + + ParallelTypedBranches: + Type: AWS::Serverless::Function + TestingMetadata: + TestDescription: ["12-1"] + Metadata: + BuildMethod: makefile + Properties: + CodeUri: publish/ParallelTypedBranches/ + Handler: bootstrap + Description: Parallel with independently typed heterogeneous branch handles + Role: + Fn::GetAtt: + - DurableFunctionRole + - Arn + DurableConfig: + RetentionPeriodInDays: 7 + ExecutionTimeout: 300 + + ParallelEarlyStart: + Type: AWS::Serverless::Function + TestingMetadata: + TestDescription: ["12-2"] + Metadata: + BuildMethod: makefile + Properties: + CodeUri: publish/ParallelEarlyStart/ + Handler: bootstrap + Description: Parallel branch completes before later branches are registered + Role: + Fn::GetAtt: + - DurableFunctionRole + - Arn + DurableConfig: + RetentionPeriodInDays: 7 + ExecutionTimeout: 300 From 6e3f25d523d2cfeacdb66ccb35559baf7daaf30f Mon Sep 17 00:00:00 2001 From: Garrett Beatty Date: Thu, 3 Sep 2026 02:05:40 +0000 Subject: [PATCH 10/12] Address adversarial review round 1: step round-trip safety, incremental-parallel overflow/await fixes, serializer deferral, Branch rename, docs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit DurableExecution PR #2553 (stacked on feature/per-step-serializer-conformance). 1. [BLOCKER] StepOperation.ExecuteFunc: move the fresh-success SUCCEED enqueue and result round-trip OUTSIDE the try that funnels into HandleStepFailureAsync (mirrors ChildContextOperation). A serializer that cannot deserialize its own just-written payload now surfaces the fault directly instead of enqueuing a RETRY/FAIL that conflicts with the already-committed SUCCEED. 2. [MAJOR] .autover/changes/12a4a1f7: Minor -> Major. The package is GA (1.x) per CLAUDE.md, and the unconditional fresh-success round-trip is an observable happy-path behavior change for ALL serializers on non-suspending workflows (reference identity, DateTime.Kind, [JsonIgnore], precision). Not preview-exempt. 3. [MAJOR] IncrementalParallelOperation overflow recovery: isolate overflow-recovery re-runs (frozenStatus set) from _shortCircuitCts/_dispatchCts so a completion-policy short-circuit can no longer cancel them; and exclude frozen branches from the cooperative-bail arm so _result honors _frozenStatus (never resolves a frozen Succeeded branch to SkippedError, which made `await branch` throw while Status==Succeeded and lost the recovered value). 4. [MINOR] DurableContext.CreateParallel: defer LambdaSerializerHelper.GetRequired via a lazy factory (memoized in the operation). A workflow overriding the serializer on every branch no longer requires a global serializer at CreateParallel time (AOT / per-branch scenario). GetRequired is resolved only when a branch falls back. 5. [MINOR] Rename IDurableParallel.BranchAsync -> Branch. The method returns a handle synchronously (not a Task), so the Async suffix was misleading. Safe: the API is new/unreleased (absent on master). Updated the interface, impl, all call sites (conformance + tests), docs (parallel.md), and the AutoVer changelog text. 6. [MINOR] IDurableParallel.Branch XML doc: document ArgumentNullException, ObjectDisposedException, and NonDeterministicExecutionException in addition to InvalidOperationException. 7. [MINOR] IncrementalParallelBranch.ExecuteAsync: fault _result before rethrowing a workflow-level DurableExecutionException, so a caller that catches the fault out of CompleteAsync and then awaits the handle observes the fault instead of hanging. 8. [MINOR] Correct the IncrementalParallelOperation class summary and IParallelBranch.Index doc to reflect the 1-based operation-ID suffix (hash("{parentId}-{index+1}")). 9. [NIT] IncrementalParallelHeterogeneousTest: replace the tautological Contains("200") (satisfied by the "USD:4200" POCO branch) with the distinguishing token "Payment":200. Tests: added 4 unit tests (fresh-success round-trip deserialize failure surfaces without RETRY/FAIL; CreateParallel with no global serializer + per-branch overrides does not throw; deferred fallback still throws on a non-overriding branch; a branch faulting with a workflow-level error faults the handle instead of hanging). Build and Amazon.Lambda.DurableExecution.Tests pass (447/447, net10.0). Integration-test and deployed-function projects compile; the heterogeneous integration test requires an AWS deployment and was not run here. AutoVer: source changes are refinements to the two features already covered by the existing change files, so both existing entries were updated (12a4a1f7 -> Major; add-incremental changelog text updated for the Branch rename) rather than adding a new change file. Reclassifying 12a4a1f7's Type was a one-field edit — the AutoVer CLI has no edit verb, and adding a third Major entry would have left the mislabeled Minor in place. --- .../12a4a1f7-d59f-4544-aa1f-6db30289485e.json | 2 +- ...dd-incremental-heterogeneous-parallel.json | 2 +- .../DurableContext.cs | 13 +- .../IDurableContext.cs | 2 +- .../IDurableParallel.cs | 18 ++- .../IParallelBranch.cs | 7 +- .../Internal/IncrementalParallelOperation.cs | 68 ++++++-- .../Internal/StepOperation.cs | 56 ++++--- .../docs/core/parallel.md | 16 +- .../ParallelEarlyStart/Function.cs | 4 +- .../ParallelTypedBranches/Function.cs | 6 +- .../IncrementalParallelHeterogeneousTest.cs | 9 +- .../Function.cs | 6 +- .../Function.cs | 6 +- .../IncrementalParallelOperationTests.cs | 152 ++++++++++++++---- .../PerOperationSerializerTests.cs | 56 +++++++ 16 files changed, 319 insertions(+), 104 deletions(-) diff --git a/.autover/changes/12a4a1f7-d59f-4544-aa1f-6db30289485e.json b/.autover/changes/12a4a1f7-d59f-4544-aa1f-6db30289485e.json index c2b4327cc..1dca7e54c 100644 --- a/.autover/changes/12a4a1f7-d59f-4544-aa1f-6db30289485e.json +++ b/.autover/changes/12a4a1f7-d59f-4544-aa1f-6db30289485e.json @@ -2,7 +2,7 @@ "Projects": [ { "Name": "Amazon.Lambda.DurableExecution", - "Type": "Minor", + "Type": "Major", "ChangelogMessages": [ "Behavior change: steps and child contexts now round-trip their result through the configured serializer on a fresh (non-replay) success, deserializing the just-written checkpoint before returning \u2014 matching replay semantics. A custom per-operation serializer\u0027s transform is now reflected in the operation result on the first execution, not only on replay. The returned value is a fresh deserialized instance rather than the exact object the step/child body produced. Overflow (replay-children) results are unaffected." ] diff --git a/.autover/changes/add-incremental-heterogeneous-parallel.json b/.autover/changes/add-incremental-heterogeneous-parallel.json index e24fd8b3c..e59b54b2d 100644 --- a/.autover/changes/add-incremental-heterogeneous-parallel.json +++ b/.autover/changes/add-incremental-heterogeneous-parallel.json @@ -5,7 +5,7 @@ "Type": "Minor", "ChangelogMessages": [ "Added an incremental, branch-oriented parallel API (IDurableContext.CreateParallel, IDurableParallel, IParallelBranch) supporting heterogeneous per-branch result types and incremental branch registration, alongside the existing homogeneous ParallelAsync overloads. Each branch declares its own result type and returns an awaitable typed handle; branches start on registration (respecting MaxConcurrency) and the operation is sealed with CompleteAsync. Honors the existing MaxConcurrency, CompletionConfig, NestingType, cancellation, deterministic replay, and ILambdaSerializer behavior.", - "CreateParallel supports per-operation and per-branch result serialization: ParallelConfig.ItemSerializer sets the operation-level serializer for all branch results, and IDurableParallel.BranchAsync accepts an optional per-branch ILambdaSerializer override, falling back to ItemSerializer and then the globally-registered serializer." + "CreateParallel supports per-operation and per-branch result serialization: ParallelConfig.ItemSerializer sets the operation-level serializer for all branch results, and IDurableParallel.Branch accepts an optional per-branch ILambdaSerializer override, falling back to ItemSerializer and then the globally-registered serializer." ] } ] diff --git a/Libraries/src/Amazon.Lambda.DurableExecution/DurableContext.cs b/Libraries/src/Amazon.Lambda.DurableExecution/DurableContext.cs index a0471ba11..5cd0d7e94 100644 --- a/Libraries/src/Amazon.Lambda.DurableExecution/DurableContext.cs +++ b/Libraries/src/Amazon.Lambda.DurableExecution/DurableContext.cs @@ -199,12 +199,19 @@ public IDurableParallel CreateParallel( var effectiveConfig = config ?? new ParallelConfig(); // Operation-level default for per-branch result serialization: the config's // ItemSerializer if set, else the globally-registered serializer. Individual - // branches may still override this via BranchAsync's serializer parameter. - var serializer = effectiveConfig.ItemSerializer ?? LambdaSerializerHelper.GetRequired(LambdaContext); + // branches may still override this via Branch's serializer parameter. + // + // Resolved LAZILY: a workflow that overrides the serializer on every Branch + // call (the AOT/per-branch scenario) must not be forced to register a global + // serializer. GetRequired is deferred to the factory below and invoked only + // when a branch actually falls back to this operation-level default. + var lambdaContext = LambdaContext; + Func defaultSerializerFactory = + () => effectiveConfig.ItemSerializer ?? LambdaSerializerHelper.GetRequired(lambdaContext); var operationId = _idGenerator.NextId(); return new Internal.IncrementalParallelOperation( - operationId, name, _idGenerator.ParentId, effectiveConfig, serializer, MakeChildFactory(), + operationId, name, _idGenerator.ParentId, effectiveConfig, defaultSerializerFactory, MakeChildFactory(), _state, _terminationManager, _workflowCancellation, _durableExecutionArn, _batcher); } diff --git a/Libraries/src/Amazon.Lambda.DurableExecution/IDurableContext.cs b/Libraries/src/Amazon.Lambda.DurableExecution/IDurableContext.cs index 13b020303..a860efe2b 100644 --- a/Libraries/src/Amazon.Lambda.DurableExecution/IDurableContext.cs +++ b/Libraries/src/Amazon.Lambda.DurableExecution/IDurableContext.cs @@ -373,7 +373,7 @@ Task WaitForConditionAsync( /// overloads — which take a complete branch list up front and share one result /// type — the returned lets you register branches /// one at a time via - /// , + /// , /// each with its own result type (heterogeneous), starting each branch as it is /// registered. Call /// to seal registration and obtain the aggregate . diff --git a/Libraries/src/Amazon.Lambda.DurableExecution/IDurableParallel.cs b/Libraries/src/Amazon.Lambda.DurableExecution/IDurableParallel.cs index 95ab26129..07651ff68 100644 --- a/Libraries/src/Amazon.Lambda.DurableExecution/IDurableParallel.cs +++ b/Libraries/src/Amazon.Lambda.DurableExecution/IDurableParallel.cs @@ -7,7 +7,7 @@ namespace Amazon.Lambda.DurableExecution; /// An incremental, branch-oriented parallel operation created by /// . Branches /// are registered one at a time via -/// , +/// , /// each with its own result type (heterogeneous), and each begins executing /// immediately (subject to ). Call /// to seal @@ -23,7 +23,7 @@ namespace Amazon.Lambda.DurableExecution; /// checkpointed plan) and earlier branches should start before later ones are known. /// /// Deterministic replay. Branch identity is positional: the n-th -/// +/// /// call reuses the n-th deterministic operation ID. Workflow code must therefore /// register the same branches in the same order across invocations — produce any /// dynamic branch list inside a checkpointed @@ -80,11 +80,21 @@ public interface IDurableParallel : IAsyncDisposable /// same serializer at a given branch index across replays. /// /// A typed handle for awaiting the branch's result. + /// + /// or is null. + /// /// /// The operation has already been sealed by - /// or disposal. + /// . + /// + /// + /// The operation has already been disposed. + /// + /// + /// On replay, the at this branch index differs from the + /// name recorded in the checkpoint for a previous invocation (branch name drift). /// - IParallelBranch BranchAsync( + IParallelBranch Branch( string name, Func> func, Amazon.Lambda.Core.ILambdaSerializer? serializer = null); diff --git a/Libraries/src/Amazon.Lambda.DurableExecution/IParallelBranch.cs b/Libraries/src/Amazon.Lambda.DurableExecution/IParallelBranch.cs index f99c386a1..2300b3b76 100644 --- a/Libraries/src/Amazon.Lambda.DurableExecution/IParallelBranch.cs +++ b/Libraries/src/Amazon.Lambda.DurableExecution/IParallelBranch.cs @@ -7,7 +7,7 @@ namespace Amazon.Lambda.DurableExecution; /// /// A typed handle to a single branch registered on an -/// via . +/// via . /// Unlike the homogeneous /// API — where every branch shares one result type T — each branch on an /// declares its own result type, so a single @@ -41,8 +41,9 @@ public interface IParallelBranch /// /// Zero-based registration order of this branch within its parallel - /// operation. Stable across replays and used to derive the branch's - /// deterministic operation ID. + /// operation. Stable across replays. The branch's deterministic operation ID + /// is derived from the one-based position (hash("{parentId}-{Index+1}")), + /// so the first branch (Index 0) uses suffix 1. /// int Index { get; } diff --git a/Libraries/src/Amazon.Lambda.DurableExecution/Internal/IncrementalParallelOperation.cs b/Libraries/src/Amazon.Lambda.DurableExecution/Internal/IncrementalParallelOperation.cs index 2f0cc20b0..57b4dae0e 100644 --- a/Libraries/src/Amazon.Lambda.DurableExecution/Internal/IncrementalParallelOperation.cs +++ b/Libraries/src/Amazon.Lambda.DurableExecution/Internal/IncrementalParallelOperation.cs @@ -174,19 +174,33 @@ private async Task ExecuteAsync( _result.TrySetException(ex); return BranchOutcome.Failure(Index, Name, ErrorObject.FromException(ex)); } - catch (DurableExecutionException) + catch (DurableExecutionException ex) { // Workflow-level error (e.g. NonDeterministicExecutionException): not a // graceful per-branch failure. Fault the settlement so the orchestrator - // surfaces it out of CompleteAsync. + // surfaces it out of CompleteAsync. Also fault _result before rethrowing + // so a caller that catches the workflow-level fault out of CompleteAsync + // and then awaits this branch handle observes the same fault instead of + // hanging forever on a never-completed result (every other arm below + // completes _result). + _result.TrySetException(ex); throw; } catch (OperationCanceledException) - when (shortCircuitToken.IsCancellationRequested && !controlToken.IsCancellationRequested) + when (shortCircuitToken.IsCancellationRequested && !controlToken.IsCancellationRequested + && _frozenStatus is null) { // Cooperative bail: a sibling satisfied the CompletionConfig before this // branch acquired its concurrency slot (or its body honored the bail // token). Record it as skipped — never a failure. + // + // Excluded when _frozenStatus is set: an overflow-recovery re-run is + // launched only to recover a value the frozen summary already recorded + // as Succeeded/Failed. It is isolated from _shortCircuitCts (see + // LaunchRunBranch), so this arm should not fire for it — but guard + // regardless so a stray short-circuit can never resolve _result to a + // SkippedError while Status reports the frozen terminal verdict, which + // would make `await branch` throw for a branch whose Status==Succeeded. if (_frozenStatus is null) _status = (int)BatchItemStatus.Started; _result.TrySetException(SkippedError()); return BranchOutcome.Skipped(Index, Name); @@ -253,7 +267,9 @@ private T Deserialize(string? serialized) /// Incremental, heterogeneous parallel orchestrator implementing /// . Each branch runs as a /// under the SAME deterministic child -/// operation-ID scheme (hash("{parentId}-{index}")) and the SAME parent +/// operation-ID scheme (hash("{parentId}-{index+1}"), where index is +/// the zero-based branch registration order, so the ID suffix is one-based — +/// positions 1..n) and the SAME parent /// checkpoint shape as the batch /// , so a checkpoint written by one is /// reconstructable by the other. Branch identity is positional: register the same @@ -267,7 +283,14 @@ internal sealed class IncrementalParallelOperation : IDurableParallel private readonly CompletionPolicy _policy; private readonly int? _maxConcurrency; private readonly bool _isVirtual; - private readonly ILambdaSerializer _serializer; + // Operation-level default serializer, resolved LAZILY. A workflow that overrides + // the serializer on every Branch must not be forced to register a global + // serializer just to construct the operation (the AOT/per-branch scenario), so + // the factory — which may call LambdaSerializerHelper.GetRequired and throw when + // no global serializer exists — is invoked only when a branch actually falls back + // to this default. Memoized in _defaultSerializer under _lock. + private readonly Func _defaultSerializerFactory; + private ILambdaSerializer? _defaultSerializer; private readonly Func _childContextFactory; private readonly ExecutionState _state; private readonly TerminationManager _termination; @@ -298,7 +321,7 @@ public IncrementalParallelOperation( string? name, string? parentId, ParallelConfig config, - ILambdaSerializer serializer, + Func defaultSerializerFactory, Func childContextFactory, ExecutionState state, TerminationManager termination, @@ -312,7 +335,7 @@ public IncrementalParallelOperation( _policy = new CompletionPolicy(config.CompletionConfig); _maxConcurrency = config.MaxConcurrency; _isVirtual = config.NestingType == NestingType.Flat; - _serializer = serializer; + _defaultSerializerFactory = defaultSerializerFactory; _childContextFactory = childContextFactory; _state = state; _termination = termination; @@ -375,7 +398,7 @@ public IncrementalParallelOperation( } } - public IParallelBranch BranchAsync( + public IParallelBranch Branch( string name, Func> func, ILambdaSerializer? serializer = null) @@ -393,8 +416,10 @@ public IParallelBranch BranchAsync( var index = _branches.Count; // zero-based branch index var childOpId = OperationIdGenerator.HashOperationId($"{_operationId}-{index + 1}"); // Per-branch serializer override, else the operation-level default - // (ParallelConfig.ItemSerializer ?? the globally-registered serializer). - var branchSerializer = serializer ?? _serializer; + // (ParallelConfig.ItemSerializer ?? the globally-registered serializer), + // resolved lazily here so a workflow overriding the serializer on every + // branch never triggers the global-serializer lookup. Memoized under _lock. + var branchSerializer = serializer ?? (_defaultSerializer ??= _defaultSerializerFactory()); var handle = new IncrementalParallelBranch(index, name, branchSerializer, OperationSubTypes.ParallelBranch); var summaryEntry = FindSummaryUnit(index); @@ -511,7 +536,7 @@ public async ValueTask DisposeAsync() { // DisposeAsync must never throw. A completion fault (e.g. a // NonDeterministicExecutionException, or the secondary effect of a - // BranchAsync that already threw during registration) is either + // Branch that already threw during registration) is either // already surfaced to a caller that awaited CompleteAsync, or will // resurface on the next invocation's replay. Swallow it here so // `await using` teardown stays clean. @@ -534,6 +559,19 @@ private void LaunchRunBranch( ILambdaSerializer branchSerializer, BatchItemStatus? frozenStatus = null) { + // An overflow-recovery re-run (frozenStatus set) exists only to recover a + // value the frozen summary already recorded as terminal; it MUST run to + // completion. A completion-policy short-circuit (OnBranchSettled → + // _shortCircuitCts.Cancel()) must never cancel it — a cancelled recovery + // branch would hit the cooperative-bail arm and lose the recovered value + // even though its Status is the frozen Succeeded/Failed. So isolate it from + // _shortCircuitCts/_dispatchCts: it observes only workflow shutdown, and is + // never handed a bail token. Regular Run-mode branches honor the + // short-circuit exactly as before. + var isRecovery = frozenStatus.HasValue; + var dispatchToken = isRecovery ? _workflowCancellation.Token : _dispatchCts.Token; + var bailToken = isRecovery ? CancellationToken.None : _shortCircuitCts.Token; + async Task Run() { // Parent START must be enqueued before this branch's child START. @@ -541,13 +579,13 @@ async Task Run() if (_semaphore != null) { - await _semaphore.WaitAsync(_dispatchCts.Token).ConfigureAwait(false); + await _semaphore.WaitAsync(dispatchToken).ConfigureAwait(false); } try { // A short-circuit may have fired while waiting on the semaphore. - _dispatchCts.Token.ThrowIfCancellationRequested(); + dispatchToken.ThrowIfCancellationRequested(); var childOp = new ChildContextOperation( childOpId, @@ -562,7 +600,7 @@ async Task Run() _workflowCancellation, _durableExecutionArn, _batcher, - _shortCircuitCts.Token, + bailToken, isVirtual: _isVirtual); // Branch child ops receive CancellationToken.None here — they re-link @@ -576,7 +614,7 @@ async Task Run() } } - handle.Launch(Run, _shortCircuitCts.Token, _workflowCancellation.Token, frozenStatus); + handle.Launch(Run, bailToken, _workflowCancellation.Token, frozenStatus); ObserveSettlement(handle.Settlement); } diff --git a/Libraries/src/Amazon.Lambda.DurableExecution/Internal/StepOperation.cs b/Libraries/src/Amazon.Lambda.DurableExecution/Internal/StepOperation.cs index bb580fbfd..d9ed786a0 100644 --- a/Libraries/src/Amazon.Lambda.DurableExecution/Internal/StepOperation.cs +++ b/Libraries/src/Amazon.Lambda.DurableExecution/Internal/StepOperation.cs @@ -215,6 +215,12 @@ private async Task ExecuteFunc(int attemptNumber, CancellationToken cancellat using var linked = CancellationTokenSource.CreateLinkedTokenSource( cancellationToken, _workflowCancellation.Token); + // Only the user-func call is guarded by the retry/fail funnel. The + // SUCCEED checkpoint and the result round-trip below sit OUTSIDE this + // try (mirroring ChildContextOperation): once SUCCEED is enqueued, no + // later failure may enqueue a conflicting RETRY/FAIL for an operation + // that has already committed a terminal SUCCEED. + T result; try { var stepContext = new StepContext(OperationId, attemptNumber, _logger); @@ -223,7 +229,6 @@ private async Task ExecuteFunc(int attemptNumber, CancellationToken cancellat // lines with the operation id, name, and current attempt. Wrap // only the user-func call — checkpoint emission shouldn't carry // step metadata into any side-channel logging. - T result; using (_logger.BeginScope(new Dictionary { ["operationId"] = OperationId, @@ -233,27 +238,6 @@ private async Task ExecuteFunc(int attemptNumber, CancellationToken cancellat { result = await _func(stepContext, linked.Token); } - - var serialized = SerializeResult(result); - await EnqueueAsync(new SdkOperationUpdate - { - Id = OperationId, - ParentId = ParentId, - Type = OperationTypes.Step, - Action = OperationAction.SUCCEED, - SubType = OperationSubTypes.Step, - Name = Name, - Payload = serialized - }, cancellationToken); - - // Round-trip the just-written checkpoint so the value the workflow - // observes on this fresh execution is the deserialized-from-checkpoint - // value, exactly as it would be on replay. This makes a custom - // (possibly non-round-tripping) StepConfig.Serializer's transform - // visible in the step result on the first run, not just on replay. - // Behavior change: the returned object is no longer the same instance - // the step body produced. See the AutoVer change note. - return DeserializeResult(serialized); } catch (OperationCanceledException) when (linked.IsCancellationRequested) { @@ -272,6 +256,34 @@ await EnqueueAsync(new SdkOperationUpdate // falls through here and is treated as a normal step failure. return await HandleStepFailureAsync(ex, attemptNumber, cancellationToken); } + + var serialized = SerializeResult(result); + await EnqueueAsync(new SdkOperationUpdate + { + Id = OperationId, + ParentId = ParentId, + Type = OperationTypes.Step, + Action = OperationAction.SUCCEED, + SubType = OperationSubTypes.Step, + Name = Name, + Payload = serialized + }, cancellationToken); + + // Round-trip the just-written checkpoint so the value the workflow + // observes on this fresh execution is the deserialized-from-checkpoint + // value, exactly as it would be on replay. This makes a custom + // (possibly non-round-tripping) StepConfig.Serializer's transform + // visible in the step result on the first run, not just on replay. + // Behavior change: the returned object is no longer the same instance + // the step body produced. See the AutoVer change note. + // + // This deserialize sits OUTSIDE the try above by design: it runs AFTER + // the SUCCEED checkpoint is enqueued, so a serializer that cannot + // deserialize its own just-written payload surfaces the fault directly + // to the caller instead of funneling into HandleStepFailureAsync — which + // would enqueue a RETRY/FAIL conflicting with the SUCCEED and risk a + // duplicate side effect on retry. Mirrors ChildContextOperation. + return DeserializeResult(serialized); } /// diff --git a/Libraries/src/Amazon.Lambda.DurableExecution/docs/core/parallel.md b/Libraries/src/Amazon.Lambda.DurableExecution/docs/core/parallel.md index e04a7c3ac..f6fdbe24b 100644 --- a/Libraries/src/Amazon.Lambda.DurableExecution/docs/core/parallel.md +++ b/Libraries/src/Amazon.Lambda.DurableExecution/docs/core/parallel.md @@ -31,16 +31,16 @@ Each branch receives its own `IDurableContext` and a `CancellationToken` (linkin ```csharp await using var parallel = ctx.CreateParallel(name: "process-order"); -IParallelBranch inventory = parallel.BranchAsync( +IParallelBranch inventory = parallel.Branch( "inventory", async (branch, ct) => await ReserveInventoryAsync(branch, ct)); -IParallelBranch payment = parallel.BranchAsync( +IParallelBranch payment = parallel.Branch( "payment", async (branch, ct) => await AuthorizePaymentAsync(branch, ct)); if (plan.RequiresComplianceReview) { // Branches can be added conditionally / incrementally. - _ = parallel.BranchAsync("compliance", + _ = parallel.Branch("compliance", async (branch, ct) => await ReviewComplianceAsync(branch, ct)); } @@ -52,24 +52,24 @@ InventoryReservation reservedInventory = await inventory; PaymentAuthorization authorizedPayment = await payment; ``` -`BranchAsync` returns an awaitable `IParallelBranch` handle exposing `Name`, `Index`, and `Status`. `await handle` yields the branch's typed result, rethrows its `ChildContextException` on failure, or throws a `DurableExecutionException` if the branch was skipped by a completion-policy short-circuit (inspect `Status` first when that's possible). `CompleteAsync()` returns the non-generic aggregate `IBatchResult` (counts + `CompletionReason`); it is idempotent and never throws on per-branch failure. `DisposeAsync` (via `await using`) seals and completes the operation if you did not call `CompleteAsync`, so the parallel's terminal checkpoint is always written. +`Branch` returns an awaitable `IParallelBranch` handle exposing `Name`, `Index`, and `Status`. `await handle` yields the branch's typed result, rethrows its `ChildContextException` on failure, or throws a `DurableExecutionException` if the branch was skipped by a completion-policy short-circuit (inspect `Status` first when that's possible). `CompleteAsync()` returns the non-generic aggregate `IBatchResult` (counts + `CompletionReason`); it is idempotent and never throws on per-branch failure. `DisposeAsync` (via `await using`) seals and completes the operation if you did not call `CompleteAsync`, so the parallel's terminal checkpoint is always written. -> **Deterministic replay applies unchanged.** Branch identity is positional: the n-th `BranchAsync` call reuses the n-th deterministic operation ID, so workflow code must register the same branches in the same order across invocations (a name change at a given index throws `NonDeterministicExecutionException`). Produce any dynamic branch set inside a checkpointed `StepAsync` so replay sees the same branches. `MaxConcurrency`, `CompletionConfig`, `NestingType`, cancellation, and the checkpoint format are identical to `ParallelAsync` — `CreateParallel` writes the same `Parallel` / `ParallelBranch` checkpoints, so it is purely an additive, front-end alternative. +> **Deterministic replay applies unchanged.** Branch identity is positional: the n-th `Branch` call reuses the n-th deterministic operation ID, so workflow code must register the same branches in the same order across invocations (a name change at a given index throws `NonDeterministicExecutionException`). Produce any dynamic branch set inside a checkpointed `StepAsync` so replay sees the same branches. `MaxConcurrency`, `CompletionConfig`, `NestingType`, cancellation, and the checkpoint format are identical to `ParallelAsync` — `CreateParallel` writes the same `Parallel` / `ParallelBranch` checkpoints, so it is purely an additive, front-end alternative. The homogeneous `ParallelAsync` overloads remain the simplest choice for a fixed set of same-typed branches and convenient `GetResults()` usage. ### Per-branch serialization -Because each branch declares its own result type, each may also use its own serializer. `BranchAsync` takes an optional `ILambdaSerializer? serializer`; when omitted a branch uses the operation-level default — `ParallelConfig.ItemSerializer` if set on `CreateParallel`, otherwise the globally-registered `ILambdaContext.Serializer`. This lets one branch opt into a bespoke serializer (for example a source-generated `JsonSerializerContext` for Native AOT) without affecting sibling branches. +Because each branch declares its own result type, each may also use its own serializer. `Branch` takes an optional `ILambdaSerializer? serializer`; when omitted a branch uses the operation-level default — `ParallelConfig.ItemSerializer` if set on `CreateParallel`, otherwise the globally-registered `ILambdaContext.Serializer`. This lets one branch opt into a bespoke serializer (for example a source-generated `JsonSerializerContext` for Native AOT) without affecting sibling branches. ```csharp await using var parallel = ctx.CreateParallel(name: "process-order"); // Uses the operation-level / global serializer. -var inventory = parallel.BranchAsync("inventory", async (b, ct) => await ReserveAsync(b, ct)); +var inventory = parallel.Branch("inventory", async (b, ct) => await ReserveAsync(b, ct)); // Overrides serialization for just this branch. -var payment = parallel.BranchAsync( +var payment = parallel.Branch( "payment", async (b, ct) => await AuthorizeAsync(b, ct), serializer: PaymentSerializerContext.Default.CreateLambdaSerializer()); diff --git a/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/static_typing/ParallelEarlyStart/Function.cs b/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/static_typing/ParallelEarlyStart/Function.cs index 56f3fa9c4..a37e9ac33 100644 --- a/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/static_typing/ParallelEarlyStart/Function.cs +++ b/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/static_typing/ParallelEarlyStart/Function.cs @@ -30,11 +30,11 @@ private async Task> Workflow(object? input, IDurableContext context name: "early-start", config: new ParallelConfig { MaxConcurrency = 1 })) { - IParallelBranch first = parallel.BranchAsync( + IParallelBranch first = parallel.Branch( "first", (_, _) => Task.FromResult("ready")); firstResult = await first; - second = parallel.BranchAsync( + second = parallel.Branch( "second", (_, _) => Task.FromResult(firstResult + "-second")); await parallel.CompleteAsync(); } diff --git a/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/static_typing/ParallelTypedBranches/Function.cs b/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/static_typing/ParallelTypedBranches/Function.cs index c4cc4b768..1f9bc3efd 100644 --- a/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/static_typing/ParallelTypedBranches/Function.cs +++ b/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/Conformance/static_typing/ParallelTypedBranches/Function.cs @@ -31,11 +31,11 @@ private async Task> Workflow(object? input, IDurableContext context name: "typed-branches", config: new ParallelConfig { MaxConcurrency = 1 })) { - inventory = parallel.BranchAsync( + inventory = parallel.Branch( "inventory", (_, _) => Task.FromResult("reserved")); - payment = parallel.BranchAsync( + payment = parallel.Branch( "payment", (_, _) => Task.FromResult(200)); - quote = parallel.BranchAsync( + quote = parallel.Branch( "quote", (_, _) => Task.FromResult(new Dictionary { ["currency"] = "USD" diff --git a/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/IncrementalParallelHeterogeneousTest.cs b/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/IncrementalParallelHeterogeneousTest.cs index 901b4d489..033a39df7 100644 --- a/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/IncrementalParallelHeterogeneousTest.cs +++ b/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/IncrementalParallelHeterogeneousTest.cs @@ -16,7 +16,7 @@ public class IncrementalParallelHeterogeneousTest /// /// End-to-end incremental, heterogeneous parallel: three branches registered - /// one at a time via CreateParallel/BranchAsync return three + /// one at a time via CreateParallel/Branch return three /// unrelated types (string, int, POCO), each retrieved through its own typed /// handle. Validates the parent CONTEXT and per-branch CONTEXT checkpoints all /// land in the service-side history with the correct names, and that the @@ -43,7 +43,12 @@ public async Task IncrementalParallel_HeterogeneousBranches_Succeed() // Each heterogeneous branch's typed result surfaces in the user payload. Assert.Contains("reserved-p1", responsePayload); // string branch - Assert.Contains("200", responsePayload); // int branch + // Distinguishing token for the int branch: a bare "200" is tautologically + // satisfied by the POCO branch's "USD:4200" substring, so assert the + // property-qualified value instead. OrderResult is serialized with + // DefaultLambdaJsonSerializer (PascalCase property names), so Payment=200 + // renders as "Payment":200. + Assert.Contains("\"Payment\":200", responsePayload); // int branch Assert.Contains("USD:4200", responsePayload); // POCO branch // History is eventually consistent — wait until the parent CONTEXT and all diff --git a/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/TestFunctions/IncrementalParallelHeterogeneousFunction/Function.cs b/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/TestFunctions/IncrementalParallelHeterogeneousFunction/Function.cs index c66681b10..ba5c09a6b 100644 --- a/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/TestFunctions/IncrementalParallelHeterogeneousFunction/Function.cs +++ b/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/TestFunctions/IncrementalParallelHeterogeneousFunction/Function.cs @@ -38,17 +38,17 @@ private static async Task Workflow(OrderRequest input, IDurableCont await using var parallel = context.CreateParallel(name: "process-order"); // Each branch declares its own result type — string, int, and Money. - IParallelBranch inventory = parallel.BranchAsync( + IParallelBranch inventory = parallel.Branch( "inventory", async (branch, ct) => await branch.StepAsync( (_, _) => Task.FromResult($"reserved-{orderId}"), name: "reserve")); - IParallelBranch payment = parallel.BranchAsync( + IParallelBranch payment = parallel.Branch( "payment", async (branch, ct) => await branch.StepAsync( (_, _) => Task.FromResult(200), name: "charge")); - IParallelBranch shipping = parallel.BranchAsync( + IParallelBranch shipping = parallel.Branch( "shipping", async (branch, ct) => await branch.StepAsync( (_, _) => Task.FromResult(new Money { Currency = "USD", Amount = 4200 }), name: "quote")); diff --git a/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/TestFunctions/IncrementalParallelReplayFunction/Function.cs b/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/TestFunctions/IncrementalParallelReplayFunction/Function.cs index e7b3a3ecd..fdeaa9534 100644 --- a/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/TestFunctions/IncrementalParallelReplayFunction/Function.cs +++ b/Libraries/test/Amazon.Lambda.DurableExecution.IntegrationTests/TestFunctions/IncrementalParallelReplayFunction/Function.cs @@ -46,9 +46,9 @@ private static async Task Workflow(TestEvent input, IDurableContext { await using var parallel = context.CreateParallel(name: "fanout"); - IParallelBranch a = parallel.BranchAsync("a", BranchAsync); - IParallelBranch b = parallel.BranchAsync("b", BranchAsync); - IParallelBranch c = parallel.BranchAsync("c", BranchAsync); + IParallelBranch a = parallel.Branch("a", BranchAsync); + IParallelBranch b = parallel.Branch("b", BranchAsync); + IParallelBranch c = parallel.Branch("c", BranchAsync); var summary = await parallel.CompleteAsync(); diff --git a/Libraries/test/Amazon.Lambda.DurableExecution.Tests/IncrementalParallelOperationTests.cs b/Libraries/test/Amazon.Lambda.DurableExecution.Tests/IncrementalParallelOperationTests.cs index ddd432a6d..5a42bf83c 100644 --- a/Libraries/test/Amazon.Lambda.DurableExecution.Tests/IncrementalParallelOperationTests.cs +++ b/Libraries/test/Amazon.Lambda.DurableExecution.Tests/IncrementalParallelOperationTests.cs @@ -59,9 +59,9 @@ public async Task CreateParallel_FreshExecution_HeterogeneousBranches_ResolveTyp await using (var parallel = context.CreateParallel(name: "process-order")) { - inventory = parallel.BranchAsync("inventory", async (_, _) => { await Task.Yield(); return "reserved"; }); - payment = parallel.BranchAsync("payment", async (_, _) => { await Task.Yield(); return 200; }); - shipping = parallel.BranchAsync("shipping", async (_, _) => + inventory = parallel.Branch("inventory", async (_, _) => { await Task.Yield(); return "reserved"; }); + payment = parallel.Branch("payment", async (_, _) => { await Task.Yield(); return 200; }); + shipping = parallel.Branch("shipping", async (_, _) => { await Task.Yield(); return new Money { Currency = "USD", Amount = 4200 }; @@ -104,8 +104,8 @@ public async Task CreateParallel_BranchOperationIds_AreDeterministic() await using (var parallel = context.CreateParallel()) { - _ = parallel.BranchAsync("a", async (_, _) => { await Task.Yield(); return "a"; }); - _ = parallel.BranchAsync("b", async (_, _) => { await Task.Yield(); return "b"; }); + _ = parallel.Branch("a", async (_, _) => { await Task.Yield(); return "a"; }); + _ = parallel.Branch("b", async (_, _) => { await Task.Yield(); return "b"; }); await parallel.CompleteAsync(); } @@ -147,8 +147,8 @@ public async Task CreateParallel_NamesPropagateToCheckpointAndHandle() await using (var parallel = context.CreateParallel(name: "fanout")) { - var a = parallel.BranchAsync("alpha", async (_, _) => { await Task.Yield(); return 1; }); - var b = parallel.BranchAsync("beta", async (_, _) => { await Task.Yield(); return 2; }); + var a = parallel.Branch("alpha", async (_, _) => { await Task.Yield(); return 1; }); + var b = parallel.Branch("beta", async (_, _) => { await Task.Yield(); return 2; }); await parallel.CompleteAsync(); Assert.Equal("alpha", a.Name); Assert.Equal("beta", b.Name); @@ -169,8 +169,8 @@ public async Task CreateParallel_NestedSucceeded_InlinesPerBranchResultsOnParent await using (var parallel = context.CreateParallel(name: "fanout")) { - _ = parallel.BranchAsync("i", async (_, _) => { await Task.Yield(); return 100; }); - _ = parallel.BranchAsync("p", async (_, _) => { await Task.Yield(); return 200; }); + _ = parallel.Branch("i", async (_, _) => { await Task.Yield(); return 100; }); + _ = parallel.Branch("p", async (_, _) => { await Task.Yield(); return 200; }); await parallel.CompleteAsync(); } @@ -198,8 +198,8 @@ public async Task CreateParallel_DefaultFailFast_BranchFailure_SurfacesOnResultA await using (var parallel = context.CreateParallel()) { - ok = parallel.BranchAsync("ok", async (_, _) => { await Task.Yield(); return 1; }); - bad = parallel.BranchAsync("bad", async (_, _) => + ok = parallel.Branch("ok", async (_, _) => { await Task.Yield(); return 1; }); + bad = parallel.Branch("bad", async (_, _) => { await Task.Yield(); throw new InvalidOperationException("branch boom"); @@ -227,8 +227,8 @@ public async Task CreateParallel_AllCompleted_PartialFailureDoesNotExceedToleran await using (var parallel = context.CreateParallel( config: new ParallelConfig { CompletionConfig = CompletionConfig.AllCompleted() })) { - _ = parallel.BranchAsync("ok", async (_, _) => { await Task.Yield(); return 1; }); - _ = parallel.BranchAsync("bad", async (_, _) => { await Task.Yield(); throw new InvalidOperationException("x"); }); + _ = parallel.Branch("ok", async (_, _) => { await Task.Yield(); return 1; }); + _ = parallel.Branch("bad", async (_, _) => { await Task.Yield(); throw new InvalidOperationException("x"); }); summary = await parallel.CompleteAsync(); } @@ -255,7 +255,7 @@ public async Task CreateParallel_MaxConcurrency_LimitsInFlight() { for (var i = 0; i < 6; i++) { - _ = parallel.BranchAsync($"b{i}", async (_, ct) => + _ = parallel.Branch($"b{i}", async (_, ct) => { lock (gate) { inFlight++; maxObserved = Math.Max(maxObserved, inFlight); } await Task.Delay(20, ct); @@ -283,9 +283,9 @@ public async Task CreateParallel_FirstSuccessful_WithMaxConcurrency1_SkipsTraili CompletionConfig = CompletionConfig.FirstSuccessful() })) { - _ = parallel.BranchAsync("b0", async (_, _) => { await Task.Yield(); return 1; }); - _ = parallel.BranchAsync("b1", async (_, _) => { await Task.Yield(); return 2; }); - last = parallel.BranchAsync("b2", async (_, _) => { await Task.Yield(); return 3; }); + _ = parallel.Branch("b0", async (_, _) => { await Task.Yield(); return 1; }); + _ = parallel.Branch("b1", async (_, _) => { await Task.Yield(); return 2; }); + last = parallel.Branch("b2", async (_, _) => { await Task.Yield(); return 3; }); summary = await parallel.CompleteAsync(); } @@ -320,11 +320,11 @@ public async Task CreateParallel_RegisterAfterComplete_Throws() var (context, _, _, _) = CreateContext(); var parallel = context.CreateParallel(); - _ = parallel.BranchAsync("a", async (_, _) => { await Task.Yield(); return 1; }); + _ = parallel.Branch("a", async (_, _) => { await Task.Yield(); return 1; }); await parallel.CompleteAsync(); Assert.Throws(() => - _ = parallel.BranchAsync("late", async (_, _) => { await Task.Yield(); return 2; })); + _ = parallel.Branch("late", async (_, _) => { await Task.Yield(); return 2; })); await parallel.DisposeAsync(); } @@ -335,7 +335,7 @@ public async Task CreateParallel_CompleteAsync_IsIdempotent() var (context, recorder, _, _) = CreateContext(); var parallel = context.CreateParallel(); - _ = parallel.BranchAsync("a", async (_, _) => { await Task.Yield(); return 1; }); + _ = parallel.Branch("a", async (_, _) => { await Task.Yield(); return 1; }); var first = await parallel.CompleteAsync(); var second = await parallel.CompleteAsync(); Assert.Same(first, second); @@ -354,7 +354,7 @@ public async Task CreateParallel_DisposeWithoutComplete_StillCheckpointsParent() await using (var parallel = context.CreateParallel(name: "auto")) { - _ = parallel.BranchAsync("a", async (_, _) => { await Task.Yield(); return 1; }); + _ = parallel.Branch("a", async (_, _) => { await Task.Yield(); return 1; }); // No explicit CompleteAsync — DisposeAsync must seal + checkpoint. } @@ -401,8 +401,8 @@ public async Task CreateParallel_ReplaySucceeded_RebuildsFromInlineSummary_Witho await using (var parallel = context.CreateParallel(name: "process-order")) { - inventory = parallel.BranchAsync("inventory", async (_, _) => { executed = true; await Task.Yield(); return "LIVE"; }); - payment = parallel.BranchAsync("payment", async (_, _) => { executed = true; await Task.Yield(); return -1; }); + inventory = parallel.Branch("inventory", async (_, _) => { executed = true; await Task.Yield(); return "LIVE"; }); + payment = parallel.Branch("payment", async (_, _) => { executed = true; await Task.Yield(); return -1; }); summary = await parallel.CompleteAsync(); } @@ -449,8 +449,8 @@ public async Task CreateParallel_ReplaySucceeded_FailedBranch_AwaitRethrows() await using (var parallel = context.CreateParallel(name: "fanout")) { - ok = parallel.BranchAsync("ok", async (_, _) => { await Task.Yield(); return -1; }); - bad = parallel.BranchAsync("bad", async (_, _) => { await Task.Yield(); return -1; }); + ok = parallel.Branch("ok", async (_, _) => { await Task.Yield(); return -1; }); + bad = parallel.Branch("bad", async (_, _) => { await Task.Yield(); return -1; }); summary = await parallel.CompleteAsync(); } @@ -490,7 +490,7 @@ public async Task CreateParallel_ReplayNameDrift_Throws() var parallel = context.CreateParallel(name: "fanout"); // Registered a branch whose name drifted from the checkpointed "inventory". Assert.Throws(() => - _ = parallel.BranchAsync("renamed", async (_, _) => { await Task.Yield(); return 1; })); + _ = parallel.Branch("renamed", async (_, _) => { await Task.Yield(); return 1; })); await parallel.DisposeAsync(); } @@ -521,8 +521,8 @@ public async Task CreateParallel_ReplayStartedParent_ReRunsBranches_AndCheckpoin IBatchResult summary; await using (var parallel = context.CreateParallel(name: "fanout")) { - _ = parallel.BranchAsync("a", async (_, _) => { Interlocked.Increment(ref runCount); await Task.Yield(); return 1; }); - _ = parallel.BranchAsync("b", async (_, _) => { Interlocked.Increment(ref runCount); await Task.Yield(); return 2; }); + _ = parallel.Branch("a", async (_, _) => { Interlocked.Increment(ref runCount); await Task.Yield(); return 1; }); + _ = parallel.Branch("b", async (_, _) => { Interlocked.Increment(ref runCount); await Task.Yield(); return 2; }); summary = await parallel.CompleteAsync(); } @@ -597,7 +597,7 @@ public async Task CreateParallel_ReplayBranchCountMismatch_Throws() }); var parallel = context.CreateParallel(name: "fanout"); - _ = parallel.BranchAsync("inventory", async (_, _) => { await Task.Yield(); return 1; }); + _ = parallel.Branch("inventory", async (_, _) => { await Task.Yield(); return 1; }); // Only one branch registered, but the checkpoint recorded two. await Assert.ThrowsAsync(async () => await parallel.CompleteAsync()); await parallel.DisposeAsync(); // must not throw a secondary exception @@ -636,8 +636,8 @@ public async Task CreateParallel_PerBranchSerializer_UsedForThatBranchOnly() await using (var parallel = context.CreateParallel()) { // Branch "a" overrides its serializer; branch "b" uses the global default. - a = parallel.BranchAsync("a", async (_, _) => { await Task.Yield(); return 7; }, serializer: custom); - b = parallel.BranchAsync("b", async (_, _) => { await Task.Yield(); return 8; }); + a = parallel.Branch("a", async (_, _) => { await Task.Yield(); return 7; }, serializer: custom); + b = parallel.Branch("b", async (_, _) => { await Task.Yield(); return 8; }); await parallel.CompleteAsync(); } @@ -657,8 +657,8 @@ public async Task CreateParallel_ItemSerializer_AppliesToAllBranchesByDefault() await using (var parallel = context.CreateParallel(config: new ParallelConfig { ItemSerializer = shared })) { - _ = parallel.BranchAsync("a", async (_, _) => { await Task.Yield(); return 1; }); - _ = parallel.BranchAsync("b", async (_, _) => { await Task.Yield(); return 2; }); + _ = parallel.Branch("a", async (_, _) => { await Task.Yield(); return 1; }); + _ = parallel.Branch("b", async (_, _) => { await Task.Yield(); return 2; }); var summary = await parallel.CompleteAsync(); Assert.Equal(2, summary.SuccessCount); } @@ -667,6 +667,92 @@ public async Task CreateParallel_ItemSerializer_AppliesToAllBranchesByDefault() Assert.True(shared.SerializeCount >= 2, "ItemSerializer should serialize every branch result by default"); } + // ────────────────────────────────────────────────────────────────────── + // Deferred operation-level serializer resolution (comment 4) + // ────────────────────────────────────────────────────────────────────── + + private static DurableContext CreateContextNoGlobalSerializer(out RecordingBatcher recorder) + { + var state = new ExecutionState(); + state.LoadFromCheckpoint(null); + var tm = new TerminationManager(); + var idGen = new OperationIdGenerator(); + var lambdaContext = new TestLambdaContext(); // NO Serializer registered + recorder = new RecordingBatcher(); + return new DurableContext(state, tm, new WorkflowCancellation(tm), idGen, "arn:test", lambdaContext, recorder.Batcher); + } + + [Fact] + public async Task CreateParallel_NoGlobalSerializer_AllBranchesOverride_DoesNotThrow() + { + // AOT / per-branch scenario: with no global serializer registered, + // CreateParallel and Branch must not eagerly demand one. Resolution of the + // operation-level default (LambdaSerializerHelper.GetRequired) is deferred and + // never reached when every branch supplies its own serializer. + var context = CreateContextNoGlobalSerializer(out _); + var custom = new CountingSerializer(); + + IParallelBranch a; + IParallelBranch b; + await using (var parallel = context.CreateParallel()) // must NOT throw + { + a = parallel.Branch("a", async (_, _) => { await Task.Yield(); return 1; }, serializer: custom); + b = parallel.Branch("b", async (_, _) => { await Task.Yield(); return 2; }, serializer: custom); + var summary = await parallel.CompleteAsync(); + Assert.Equal(2, summary.SuccessCount); + } + + Assert.Equal(1, await a); + Assert.Equal(2, await b); + } + + [Fact] + public async Task CreateParallel_NoGlobalSerializer_BranchWithoutOverride_ThrowsOnThatBranch() + { + // The deferral does not swallow the requirement: a branch that omits its + // serializer falls back to the operation-level default, which resolves + // GetRequired and throws when no global serializer exists — but only then, + // not at CreateParallel time. + var context = CreateContextNoGlobalSerializer(out _); + + await using var parallel = context.CreateParallel(); // deferred: does NOT throw here + Assert.Throws(() => + parallel.Branch("a", async (_, _) => { await Task.Yield(); return 1; })); + } + + // ────────────────────────────────────────────────────────────────────── + // Workflow-level fault surfaces on the handle rather than hanging (comment 7) + // ────────────────────────────────────────────────────────────────────── + + [Fact] + public async Task CreateParallel_BranchThrowsWorkflowError_AwaitingHandleFaults_NotHangs() + { + // A workflow-level fault (NonDeterministicExecutionException) is rethrown out + // of the branch's ExecuteAsync. _result must be faulted before the rethrow so + // a caller that catches the fault out of CompleteAsync and then awaits the + // branch handle observes the same fault instead of hanging on a handle whose + // result was never completed. + var (context, _, _, _) = CreateContext(); + + await using var parallel = context.CreateParallel(); + var bad = parallel.Branch("bad", async (_, _) => + { + await Task.Yield(); + throw new NonDeterministicExecutionException("boom"); + }); + + // The workflow-level fault propagates out of CompleteAsync. + await Assert.ThrowsAsync(async () => await parallel.CompleteAsync()); + + // Awaiting the handle must COMPLETE (faulted), not hang. Guard with a timeout + // so a regression fails the test deterministically instead of blocking it. + async Task AwaitHandle() => await bad; + var handleTask = AwaitHandle(); + var finished = await Task.WhenAny(handleTask, Task.Delay(TimeSpan.FromSeconds(10))); + Assert.Same(handleTask, finished); + await Assert.ThrowsAsync(async () => await handleTask); + } + /// /// Delegating that counts calls, so a /// test can assert which serializer a branch used. diff --git a/Libraries/test/Amazon.Lambda.DurableExecution.Tests/PerOperationSerializerTests.cs b/Libraries/test/Amazon.Lambda.DurableExecution.Tests/PerOperationSerializerTests.cs index 48b342377..3261b3599 100644 --- a/Libraries/test/Amazon.Lambda.DurableExecution.Tests/PerOperationSerializerTests.cs +++ b/Libraries/test/Amazon.Lambda.DurableExecution.Tests/PerOperationSerializerTests.cs @@ -120,6 +120,62 @@ public async Task Step_Replay_UsesPerOpSerializerToDeserialize() Assert.Equal(0, global.DeserializeCount); } + // ---------------------------------------------------------------- Step round-trip failure (fresh success) + + /// + /// Serializer that serializes normally but throws on deserialize — models a + /// custom serializer that cannot round-trip its own just-written payload. + /// + private sealed class DeserializeThrowingSerializer : ILambdaSerializer + { + private readonly ILambdaSerializer _inner = new DefaultLambdaJsonSerializer(); + public sealed class CannotDeserialize : Exception { } + + public T Deserialize(Stream requestStream) => throw new CannotDeserialize(); + public void Serialize(T response, Stream responseStream) => _inner.Serialize(response, responseStream); + } + + [Fact] + public async Task Step_FreshSuccess_RoundTripDeserializeFailure_SurfacesFaultWithoutRetryOrFail() + { + // Comment 1 (BLOCKER): the fresh-success round-trip deserialize sits OUTSIDE + // the try that funnels step failures into HandleStepFailureAsync. A serializer + // that cannot deserialize its own just-written payload must surface the fault + // directly — it must NOT be caught and turned into a RETRY/FAIL checkpoint, + // because the SUCCEED has already been committed and a conflicting terminal + // record would risk a duplicate side effect on retry. + var state = new ExecutionState(); + state.LoadFromCheckpoint(null); + var tm = new TerminationManager(); + var idGen = new OperationIdGenerator(); + var lambdaContext = new TestLambdaContext { Serializer = new DefaultLambdaJsonSerializer() }; + var recorder = new RecordingBatcher(); + var ctx = new DurableContext(state, tm, new WorkflowCancellation(tm), idGen, TestArn, lambdaContext, recorder.Batcher); + + var perOp = new DeserializeThrowingSerializer(); + + // The raw deserialize fault propagates — NOT wrapped in StepException, which + // is what HandleStepFailureAsync would have produced had the deserialize been + // caught by the retry/fail funnel. + await Assert.ThrowsAsync(async () => + await ctx.StepAsync( + async (_, _) => { await Task.CompletedTask; return 42; }, + name: "s", + config: new StepConfig { Serializer = perOp })); + + await recorder.Batcher.DrainAsync(); + var stepActions = recorder.Flushed + .Where(o => o.Type == OperationTypes.Step) + .Select(o => o.Action) + .ToList(); + + // The terminal SUCCEED was committed, and crucially NO RETRY/FAIL was enqueued + // for the same operation afterward. + Assert.Contains(OperationAction.SUCCEED, stepActions); + Assert.DoesNotContain(OperationAction.FAIL, stepActions); + Assert.DoesNotContain(OperationAction.RETRY, stepActions); + } + // ---------------------------------------------------------------- Callback (deserialize side) [Fact] From 9b727b74f93ab5d5d7cefff1349458562d1a25e7 Mon Sep 17 00:00:00 2001 From: Garrett Beatty Date: Thu, 3 Sep 2026 02:22:42 +0000 Subject: [PATCH 11/12] Address adversarial review round 2: fix CompleteAsync cancellation doc; add overflow-recovery terminal-path tests --- .../IDurableParallel.cs | 17 ++- .../IncrementalParallelOperationTests.cs | 126 ++++++++++++++++++ 2 files changed, 136 insertions(+), 7 deletions(-) diff --git a/Libraries/src/Amazon.Lambda.DurableExecution/IDurableParallel.cs b/Libraries/src/Amazon.Lambda.DurableExecution/IDurableParallel.cs index 07651ff68..6de69c093 100644 --- a/Libraries/src/Amazon.Lambda.DurableExecution/IDurableParallel.cs +++ b/Libraries/src/Amazon.Lambda.DurableExecution/IDurableParallel.cs @@ -112,13 +112,16 @@ IParallelBranch Branch( /// handles, to observe failures. It does propagate workflow-level errors (for /// example ) and cancellation. /// - /// The governs sealing and awaiting: it - /// stops this call from waiting further. Because branches begin executing when - /// they are registered (before CompleteAsync is called), this token is - /// not retroactively linked into already-running branch bodies — those observe - /// the SDK's workflow-shutdown signal (and the completion-policy short-circuit) - /// instead. Dispatched branches always run to a terminal checkpoint so replay - /// stays deterministic, matching . + /// The does not interrupt in-flight branch + /// settlement. Because branches begin executing when they are registered (before + /// CompleteAsync is called), this token is neither retroactively linked + /// into already-running branch bodies nor into the wait for them to settle — + /// those observe the SDK's workflow-shutdown signal (and the completion-policy + /// short-circuit) instead. This call blocks until every dispatched branch reaches + /// its terminal checkpoint; the token is observed only after that, so cancellation + /// is surfaced just before the aggregate result would be returned rather than + /// cutting the wait short. Dispatched branches always run to a terminal checkpoint + /// so replay stays deterministic, matching . /// /// /// A token to observe for cancellation. diff --git a/Libraries/test/Amazon.Lambda.DurableExecution.Tests/IncrementalParallelOperationTests.cs b/Libraries/test/Amazon.Lambda.DurableExecution.Tests/IncrementalParallelOperationTests.cs index 5a42bf83c..64d4f6514 100644 --- a/Libraries/test/Amazon.Lambda.DurableExecution.Tests/IncrementalParallelOperationTests.cs +++ b/Libraries/test/Amazon.Lambda.DurableExecution.Tests/IncrementalParallelOperationTests.cs @@ -461,6 +461,132 @@ public async Task CreateParallel_ReplaySucceeded_FailedBranch_AwaitRethrows() Assert.Contains("boom", ex.Message); } + [Fact] + public async Task CreateParallel_ReplaySucceeded_OverflowStrippedResult_ReRunsBranchToRecoverValue() + { + // Overflow-recovery arm of ResolveTerminalBranch: the parent checkpointed + // SUCCEEDED, but the summary exceeded the payload cap so it was written with + // the inline per-branch Result stripped (unit Status=SUCCEEDED, Result=null). + // On replay such a unit is routed through LaunchRunBranch(frozenStatus), which + // RE-RUNS the branch body to recover the stripped value while keeping the + // frozen SUCCEEDED verdict authoritative and NOT re-checkpointing the parent. + var parentOpId = IdAt(1); + var summaryJson = """ + {"CompletionReason":"ALL_COMPLETED","Units":[ + {"Index":0,"Name":"inventory","Status":"SUCCEEDED"} + ]} + """; + + var (context, recorder, _, _) = CreateContext(new InitialExecutionState + { + Operations = new List + { + new() + { + Id = parentOpId, + Type = OperationTypes.Context, + Status = OperationStatuses.Succeeded, + SubType = OperationSubTypes.Parallel, + Name = "process-order", + ContextDetails = new ContextDetails { Result = summaryJson } + } + } + }); + + var executed = false; + IParallelBranch inventory; + IBatchResult summary; + + await using (var parallel = context.CreateParallel(name: "process-order")) + { + inventory = parallel.Branch("inventory", async (_, _) => + { + executed = true; // the recovered value can only come from a re-run + await Task.Yield(); + return "recovered-reserved"; + }); + summary = await parallel.CompleteAsync(); + } + + // (a) The stripped value is not inline, so the branch body had to re-run. + Assert.True(executed); + // (b) The handle resolves to the value the re-run recovered — not the default + // a broken (inline-null) resolution would have produced. + Assert.Equal("recovered-reserved", await inventory); + // (c) The frozen verdict wins even though the body re-executed. + Assert.Equal(BatchItemStatus.Succeeded, inventory.Status); + Assert.Equal(1, summary.SuccessCount); + Assert.Equal(0, summary.FailureCount); + Assert.Equal(CompletionReason.AllCompleted, summary.CompletionReason); + + await recorder.Batcher.DrainAsync(); + // (d) A terminal parent is never re-checkpointed: no parent Parallel SUCCEED. + Assert.DoesNotContain(recorder.Flushed, o => + o.Type == "CONTEXT" && o.SubType == "Parallel" && o.Action == "SUCCEED"); + } + + [Fact] + public async Task CreateParallel_ReplayFailed_OverflowStrippedError_ReRunsBranchToRecoverFailure() + { + // Same overflow-recovery arm for a FAILED unit whose inline Error was stripped + // (unit Status=FAILED, Error=null). The body re-runs (and fails again), the + // recovered failure surfaces on the handle, the frozen FAILED verdict stays + // authoritative, and the parent is not re-checkpointed. + var parentOpId = IdAt(1); + var summaryJson = """ + {"CompletionReason":"FAILURE_TOLERANCE_EXCEEDED","Units":[ + {"Index":0,"Name":"bad","Status":"FAILED"} + ]} + """; + + var (context, recorder, _, _) = CreateContext(new InitialExecutionState + { + Operations = new List + { + new() + { + Id = parentOpId, + Type = OperationTypes.Context, + Status = OperationStatuses.Succeeded, + SubType = OperationSubTypes.Parallel, + Name = "fanout", + ContextDetails = new ContextDetails { Result = summaryJson } + } + } + }); + + var executed = false; + IParallelBranch bad; + IBatchResult summary; + + await using (var parallel = context.CreateParallel(name: "fanout")) + { + bad = parallel.Branch("bad", async (_, _) => + { + executed = true; + await Task.Yield(); + throw new InvalidOperationException("recovered-boom"); + }); + summary = await parallel.CompleteAsync(); + } + + // (a) The stripped error is not inline, so the branch body had to re-run. + Assert.True(executed); + // (b) The recovered failure surfaces on the handle with the re-run's message. + var ex = await Assert.ThrowsAsync(async () => await bad); + Assert.Contains("recovered-boom", ex.Message); + // (c) The frozen FAILED verdict wins. + Assert.Equal(BatchItemStatus.Failed, bad.Status); + Assert.Equal(1, summary.FailureCount); + Assert.True(summary.HasFailure); + Assert.Equal(CompletionReason.FailureToleranceExceeded, summary.CompletionReason); + + await recorder.Batcher.DrainAsync(); + // (d) A terminal parent is never re-checkpointed: no parent Parallel SUCCEED. + Assert.DoesNotContain(recorder.Flushed, o => + o.Type == "CONTEXT" && o.SubType == "Parallel" && o.Action == "SUCCEED"); + } + [Fact] public async Task CreateParallel_ReplayNameDrift_Throws() { From b244c2137ee6c11629a1df689ef00a7567de889d Mon Sep 17 00:00:00 2001 From: Garrett Beatty Date: Thu, 3 Sep 2026 17:57:37 +0000 Subject: [PATCH 12/12] docs(DurableExecution): correct CreateParallel name replay semantics; fix branch name-drift message Address Copilot review on #2553: - CreateParallel 'name' XML docs said a name change 'does not break replay', but the name is passed to ValidateReplayConsistency (throws on drift). Doc now states the name is part of the deterministic definition and must stay stable. - Branch name-drift NonDeterministicExecutionException message had expected/found inverted; now reports the checkpointed name as expected and the current registration as the drifted value. --- .../IDurableContext.cs | 13 ++++++++----- .../Internal/IncrementalParallelOperation.cs | 6 +++--- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/Libraries/src/Amazon.Lambda.DurableExecution/IDurableContext.cs b/Libraries/src/Amazon.Lambda.DurableExecution/IDurableContext.cs index a860efe2b..cf531a4f3 100644 --- a/Libraries/src/Amazon.Lambda.DurableExecution/IDurableContext.cs +++ b/Libraries/src/Amazon.Lambda.DurableExecution/IDurableContext.cs @@ -391,11 +391,14 @@ Task WaitForConditionAsync( /// as the homogeneous API. /// /// - /// Optional human-readable name for the parallel operation, used only for - /// observability — it surfaces on the wire OperationUpdate.Name field and - /// in execution traces. The deterministic operation ID is positional (derived - /// from the call order, not from this name), so a name change across deployments - /// does not break replay. Defaults to null. + /// Optional human-readable name for the parallel operation. It surfaces on the + /// wire OperationUpdate.Name field and in execution traces. The + /// deterministic operation ID is positional (derived from the call order, not + /// from this name); however, when provided, the name becomes part of the + /// operation's deterministic definition and is validated on replay — changing it + /// across deployments for an in-flight execution throws + /// , so keep it stable (or leave + /// it null) for the life of an execution. Defaults to null. /// /// /// Optional parallel configuration. Defaults are used when null. diff --git a/Libraries/src/Amazon.Lambda.DurableExecution/Internal/IncrementalParallelOperation.cs b/Libraries/src/Amazon.Lambda.DurableExecution/Internal/IncrementalParallelOperation.cs index 57b4dae0e..e0ce3a541 100644 --- a/Libraries/src/Amazon.Lambda.DurableExecution/Internal/IncrementalParallelOperation.cs +++ b/Libraries/src/Amazon.Lambda.DurableExecution/Internal/IncrementalParallelOperation.cs @@ -430,9 +430,9 @@ public IParallelBranch Branch( { throw new NonDeterministicExecutionException( $"Non-deterministic execution detected for parallel branch {index} of operation " + - $"'{_name ?? _operationId}': expected name '{name}' but found '{summaryEntry.Name}' " + - $"from a previous invocation. Code must not change the order or name of branches " + - $"between deployments."); + $"'{_name ?? _operationId}': expected checkpointed name '{summaryEntry.Name}' but " + + $"the current registration used '{name}'. Code must not change the order or name of " + + $"branches between deployments."); } if (_mode == ParallelExecutionMode.Terminal)