From 51ac65fbf1b63c9b3e831ed4db4a3d1d7a1ccf4d Mon Sep 17 00:00:00 2001 From: Frank Chen <65260095+zhongkechen@users.noreply.github.com> Date: Tue, 27 Oct 2026 15:42:47 -0700 Subject: [PATCH 1/9] test: cover empty callback failure diagnostics --- .../tests/e2e/empty_callback_error_test.py | 180 +++++++++++++++ .../tests/empty_error_history_test.py | 213 ++++++++++++++++++ 2 files changed, 393 insertions(+) create mode 100644 packages/aws-durable-execution-sdk-python-testing/tests/e2e/empty_callback_error_test.py create mode 100644 packages/aws-durable-execution-sdk-python-testing/tests/empty_error_history_test.py diff --git a/packages/aws-durable-execution-sdk-python-testing/tests/e2e/empty_callback_error_test.py b/packages/aws-durable-execution-sdk-python-testing/tests/e2e/empty_callback_error_test.py new file mode 100644 index 000000000..c41c2fe1d --- /dev/null +++ b/packages/aws-durable-execution-sdk-python-testing/tests/e2e/empty_callback_error_test.py @@ -0,0 +1,180 @@ +"""Real callback failure keeps runtime diagnostics and history consistent.""" + +from datetime import datetime +from pathlib import Path +import time +from typing import Any + +import pytest + +from aws_durable_execution_sdk_python import ( + DurableContext, + durable_execution, + durable_step, +) +from aws_durable_execution_sdk_python.exceptions import ( + CallbackError, + CallbackExternalError, +) +from aws_durable_execution_sdk_python.execution import InvocationStatus +from aws_durable_execution_sdk_python.lambda_service import ( + ErrorObject, + Operation, + OperationStatus, +) +from aws_durable_execution_sdk_python_testing.execution import Execution +from aws_durable_execution_sdk_python_testing.model import ( + GetDurableExecutionHistoryResponse, + events_to_operations, +) +from aws_durable_execution_sdk_python_testing.runner import DurableFunctionTestRunner +from aws_durable_execution_sdk_python_testing.stores.base import ExecutionStore +from aws_durable_execution_sdk_python_testing.stores.filesystem import ( + FileSystemExecutionStore, +) +from aws_durable_execution_sdk_python_testing.stores.memory import ( + InMemoryExecutionStore, +) + + +@pytest.mark.parametrize("filesystem", [False, True]) +@pytest.mark.parametrize("error_kind", ["omitted", "empty", "rich"]) +def test_live_callback_failure_normalizes_before_persistence( + filesystem: bool, + error_kind: str, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + store: ExecutionStore = ( + FileSystemExecutionStore.create(tmp_path / "store") + if filesystem + else InMemoryExecutionStore() + ) + supplied_error = ( + ErrorObject( + message="rejected", + type="RejectedError", + data='{"reason":"test"}', + stack_trace=["original frame"], + ) + if error_kind == "rich" + else ErrorObject.from_dict({}) + ) + expected_error = supplied_error if error_kind == "rich" else None + completed_operations: list[Operation] = [] + original_complete = Execution.complete_callback_failure + + def capture_live_completion( + execution: Execution, + callback_id: str, + error: ErrorObject, + now: datetime | None = None, + ) -> Operation: + # Capture the real API transition before the executor persists it. + # A filesystem round-trip must not hide missing boundary normalization. + operation = original_complete(execution, callback_id, error, now=now) + completed_operations.append(operation) + return operation + + monkeypatch.setattr(Execution, "complete_callback_failure", capture_live_completion) + step_calls: list[str] = [] + caught_errors: list[CallbackError] = [] + + @durable_step + def saved_step(_context: Any) -> str: + step_calls.append("saved") + return "saved" + + def durable_handler(_event: Any, context: DurableContext) -> str: + saved = context.step(saved_step(), name="saved-step") + callback = context.create_callback(name="failed-callback") + try: + callback.result() + except CallbackError as error: + caught_errors.append(error) + return saved + raise AssertionError("Callback failure was not raised") + + handler = durable_execution(durable_handler) + with DurableFunctionTestRunner( + handler=handler, + store=store, + skip_time=False, + execution_timeout=15, + poll_interval=0.01, + ) as runner: + arn = runner.run_async(input="{}") + callback_id = runner.wait_for_callback(arn, name="failed-callback", timeout=5) + deadline = time.monotonic() + 5 + while not store.load(arn).invocation_completions: + assert time.monotonic() < deadline, "Initial invocation did not suspend" + time.sleep(0.01) + if error_kind == "omitted": + runner.send_callback_failure(callback_id) + else: + runner.send_callback_failure(callback_id, supplied_error) + result = runner.wait_for_result(arn, timeout=10) + detailed = runner.get_execution_history(arn, include_execution_data=True) + metadata_only = runner.get_execution_history(arn, include_execution_data=False) + saved_execution = store.load(arn) + + assert result.status is InvocationStatus.SUCCEEDED + assert result.result == '"saved"' + assert step_calls == ["saved"] + assert len(saved_execution.invocation_completions) == 2 + assert len(caught_errors) == 1 + caught = caught_errors[0] + assert isinstance(caught, CallbackExternalError) + assert caught.message == ( + expected_error.message if expected_error else "Callback failed" + ) + assert caught.data == (expected_error.data if expected_error else None) + assert caught.stack_trace == ( + expected_error.stack_trace if expected_error else None + ) + + assert len(completed_operations) == 1 + live = completed_operations[0] + persisted = next( + op for op in saved_execution.operations if op.operation_id == live.operation_id + ) + reloaded = Execution.from_json_dict(saved_execution.to_json_dict()) + roundtripped = next( + op for op in reloaded.operations if op.operation_id == live.operation_id + ) + for operation in (live, persisted, roundtripped): + assert operation.status is OperationStatus.FAILED + assert operation.callback_details is not None + assert operation.callback_details.error == expected_error + + for history, include_data in ((detailed, True), (metadata_only, False)): + wire = history.to_dict() + failed = next( + event for event in wire["Events"] if event["EventType"] == "CallbackFailed" + ) + expected: dict[str, Any] = {"Truncated": not include_data} + if include_data: + expected["Payload"] = expected_error.to_dict() if expected_error else {} + assert failed["CallbackFailedDetails"]["Error"] == expected + decoded = GetDurableExecutionHistoryResponse.from_dict(wire) + decoded_failed = next( + event for event in decoded.events if event.event_type == "CallbackFailed" + ) + assert decoded_failed.callback_failed_details is not None + assert decoded_failed.callback_failed_details.error is not None + assert decoded_failed.callback_failed_details.error.to_dict() == expected + for snapshot in (history, decoded): + reconstructed = next( + op + for op in events_to_operations(snapshot.events) + if op.operation_id == live.operation_id + ) + assert reconstructed.status is OperationStatus.FAILED + assert reconstructed.callback_details is not None + assert reconstructed.callback_details.error == ( + expected_error if include_data else None + ) + assert history.to_dict() == wire + + assert live.callback_details is not None + assert live.callback_details.error == expected_error diff --git a/packages/aws-durable-execution-sdk-python-testing/tests/empty_error_history_test.py b/packages/aws-durable-execution-sdk-python-testing/tests/empty_error_history_test.py new file mode 100644 index 000000000..d40af0036 --- /dev/null +++ b/packages/aws-durable-execution-sdk-python-testing/tests/empty_error_history_test.py @@ -0,0 +1,213 @@ +"""History preserves empty payload presence without inventing SDK diagnostics.""" + +from datetime import UTC, datetime +from typing import Any + +import pytest + +from aws_durable_execution_sdk_python.lambda_service import ( + CallbackDetails, + ErrorObject, + Operation, + OperationStatus, + OperationType, +) +from aws_durable_execution_sdk_python_testing.model import ( + CallbackFailedDetails, + ChainedInvokeFailedDetails, + ContextFailedDetails, + Event, + EventCreationContext, + EventError, + GetDurableExecutionHistoryResponse, + GetDurableExecutionResponse, + StartDurableExecutionInput, + StepFailedDetails, + events_to_operations, +) +from aws_durable_execution_sdk_python_testing.runner import ( + ContextOperation, + DurableFunctionTestResult, + InvokeOperation, +) + + +RICH_ERROR = { + "ErrorType": "RejectedError", + "ErrorMessage": "rejected", + "ErrorData": '{"reason":"test"}', + "StackTrace": ["original frame"], +} +PAYLOADS = [ + {}, + {"ErrorMessage": ""}, + {"ErrorType": ""}, + {"ErrorData": ""}, + {"StackTrace": []}, + RICH_ERROR, +] + + +@pytest.mark.parametrize("payload", PAYLOADS) +@pytest.mark.parametrize("truncated", [False, True]) +def test_event_error_roundtrip_preserves_present_payload( + payload: dict[str, Any], truncated: bool +) -> None: + wire = {"Payload": payload, "Truncated": truncated} + assert EventError.from_dict(wire).to_dict() == wire + + +@pytest.mark.parametrize("payload_field", [{}, {"Payload": None}]) +@pytest.mark.parametrize("truncated", [False, True]) +def test_event_error_omitted_and_null_payload_remain_absent( + payload_field: dict[str, Any], truncated: bool +) -> None: + parsed = EventError.from_dict({**payload_field, "Truncated": truncated}) + assert parsed.payload is None + assert parsed.to_dict() == {"Truncated": truncated} + + +@pytest.mark.parametrize( + "event_type,details_field,details_type,operation_field", + [ + ( + "CallbackFailed", + "callback_failed_details", + CallbackFailedDetails, + "callback_details", + ), + ("StepFailed", "step_failed_details", StepFailedDetails, "step_details"), + ( + "ContextFailed", + "context_failed_details", + ContextFailedDetails, + "context_details", + ), + ( + "ChainedInvokeFailed", + "chained_invoke_failed_details", + ChainedInvokeFailedDetails, + "chained_invoke_details", + ), + ], +) +@pytest.mark.parametrize("payload", [None, *PAYLOADS]) +@pytest.mark.parametrize("decode_wire", [False, True]) +def test_failure_history_reconstruction_normalizes_only_empty_errors( + event_type: str, + details_field: str, + details_type: type[ + CallbackFailedDetails + | StepFailedDetails + | ContextFailedDetails + | ChainedInvokeFailedDetails + ], + operation_field: str, + payload: dict[str, Any] | None, + decode_wire: bool, +) -> None: + timestamp = datetime(2026, 10, 27, tzinfo=UTC) + error = ErrorObject.from_dict(payload) if payload is not None else None + event_details: dict[str, Any] = { + details_field: details_type(error=EventError(payload=error, truncated=False)) + } + history = GetDurableExecutionHistoryResponse( + events=[ + Event( + event_type=event_type, + event_id=1, + event_timestamp=timestamp, + operation_id="operation", + name="failed-operation", + **event_details, + ) + ] + ) + expected_wire = history.to_dict() + if decode_wire: + history = GetDurableExecutionHistoryResponse.from_dict(expected_wire) + assert history.to_dict() == expected_wire + expected_error = error if payload else None + (operation,) = events_to_operations(history.events) + assert operation.status is OperationStatus.FAILED + assert getattr(operation, operation_field).error == expected_error + + execution = GetDurableExecutionResponse( + durable_execution_arn="arn:execution", + durable_execution_name="execution", + function_arn="arn:function", + status="FAILED", + start_timestamp=timestamp, + ) + result = DurableFunctionTestResult.from_execution_history(execution, history) + result_operation = result.get_operation_by_name("failed-operation") + assert isinstance(result_operation, (ContextOperation, InvokeOperation)) + assert result_operation.error == expected_error + assert history.to_dict() == expected_wire + + +@pytest.mark.parametrize("include_data", [False, True]) +@pytest.mark.parametrize("status", [OperationStatus.FAILED, OperationStatus.TIMED_OUT]) +@pytest.mark.parametrize("payload", [None, {}, RICH_ERROR]) +def test_callback_history_projection_preserves_state_and_data_inclusion( + include_data: bool, status: OperationStatus, payload: dict[str, Any] | None +) -> None: + error = ErrorObject.from_dict(payload) if payload is not None else None + operation = Operation( + operation_id="callback", + operation_type=OperationType.CALLBACK, + status=status, + callback_details=CallbackDetails(callback_id="callback-id", error=error), + ) + context = EventCreationContext.create( + operation=operation, + event_id=3, + durable_execution_arn="arn:test", + start_input=StartDurableExecutionInput( + account_id="123456789012", + function_name="test", + function_qualifier="$LATEST", + execution_name="test", + execution_timeout_seconds=30, + execution_retention_period_days=1, + ), + include_execution_data=include_data, + ) + event = Event.create_callback_event(context) + details = ( + event.callback_failed_details + if status is OperationStatus.FAILED + else event.callback_timed_out_details + ) + assert details is not None and details.error is not None + expected: dict[str, Any] = {"Truncated": not include_data} + if include_data and (payload is not None or status is OperationStatus.FAILED): + expected["Payload"] = payload if payload is not None else {} + assert details.error.to_dict() == expected + assert operation.callback_details is not None + assert operation.callback_details.error is error + + +def test_missing_callback_details_do_not_invent_error_payload() -> None: + operation = Operation( + operation_id="callback", + operation_type=OperationType.CALLBACK, + status=OperationStatus.FAILED, + ) + context = EventCreationContext.create( + operation=operation, + event_id=3, + durable_execution_arn="arn:test", + start_input=StartDurableExecutionInput( + account_id="123456789012", + function_name="test", + function_qualifier="$LATEST", + execution_name="test", + execution_timeout_seconds=30, + execution_retention_period_days=1, + ), + include_execution_data=True, + ) + event = Event.create_callback_event_failed(context) + assert event.callback_failed_details is not None + assert event.callback_failed_details.error is None From d69e3989460a1332ab7762acb6022648ed400f5f Mon Sep 17 00:00:00 2001 From: Frank Chen <65260095+zhongkechen@users.noreply.github.com> Date: Tue, 27 Oct 2026 15:43:43 -0700 Subject: [PATCH 2/9] fix: preserve empty callback error history --- .../README.md | 10 +++++++ .../execution.py | 4 ++- .../model.py | 28 +++++++++++++++---- 3 files changed, 36 insertions(+), 6 deletions(-) diff --git a/packages/aws-durable-execution-sdk-python-testing/README.md b/packages/aws-durable-execution-sdk-python-testing/README.md index 58b21e793..346a2b693 100644 --- a/packages/aws-durable-execution-sdk-python-testing/README.md +++ b/packages/aws-durable-execution-sdk-python-testing/README.md @@ -172,6 +172,16 @@ flow, pagination, and a map of the code. ### Error Handling +A callback failure with omitted diagnostics or an empty `ErrorObject` has no +error details in SDK-facing operation state, in both memory and filesystem stores. +Detailed callback-failure history preserves `Error.Payload: {}` with +`Truncated: false`; history requested without execution data omits the payload and +sets `Truncated: true`. Reconstructing operations from detailed history restores +the same absent diagnostics. Present diagnostic fields remain intact, including +explicit empty strings and empty stack-trace lists. The callback still fails and +raises the existing `CallbackExternalError`, including its default message when +diagnostics are absent. Timeout history is unchanged. + The testing framework implements AWS-compliant error responses that match the exact format expected by boto3 and AWS services. For detailed information about error response formats, exception types, and troubleshooting, see: - [Error Response Documentation](docs/error-responses.md) diff --git a/packages/aws-durable-execution-sdk-python-testing/src/aws_durable_execution_sdk_python_testing/execution.py b/packages/aws-durable-execution-sdk-python-testing/src/aws_durable_execution_sdk_python_testing/execution.py index 5190b9b69..04301e697 100644 --- a/packages/aws-durable-execution-sdk-python-testing/src/aws_durable_execution_sdk_python_testing/execution.py +++ b/packages/aws-durable-execution-sdk-python-testing/src/aws_durable_execution_sdk_python_testing/execution.py @@ -709,7 +709,9 @@ def complete_callback_failure( updated_callback_details = None if operation.callback_details: updated_callback_details = replace( - operation.callback_details, error=error + operation.callback_details, + # Match SDK state after serialization: Error={} has no diagnostics. + error=error if error.to_dict() else None, ) self.operations[index] = replace( diff --git a/packages/aws-durable-execution-sdk-python-testing/src/aws_durable_execution_sdk_python_testing/model.py b/packages/aws-durable-execution-sdk-python-testing/src/aws_durable_execution_sdk_python_testing/model.py index 22da52618..86d4d8de8 100644 --- a/packages/aws-durable-execution-sdk-python-testing/src/aws_durable_execution_sdk_python_testing/model.py +++ b/packages/aws-durable-execution-sdk-python-testing/src/aws_durable_execution_sdk_python_testing/model.py @@ -675,7 +675,8 @@ class EventError: @classmethod def from_dict(cls, data: dict) -> EventError: payload = None - if payload_data := data.get("Payload"): + payload_data = data.get("Payload") + if payload_data is not None: payload = ErrorObject.from_dict(payload_data) return cls( @@ -2249,6 +2250,14 @@ def create_callback_event_failed(cls, context: EventCreationContext) -> Event: if callback_details else None ) + if ( + context.include_execution_data + and callback_details is not None + and callback_details.error is None + ): + # Detailed failure history retains empty payload presence. Do not + # mutate the absent diagnostics in the SDK-facing operation state. + event_error = EventError(payload=ErrorObject.from_dict({}), truncated=False) return cls( event_type=EventType.CALLBACK_FAILED.value, event_timestamp=context.end_timestamp, @@ -2606,6 +2615,11 @@ class HistoryEventTypeConfig: } +def _operation_error(error: ErrorObject | None) -> ErrorObject | None: + """Normalize history's empty payload to absent SDK-state diagnostics.""" + return error if error is not None and error.to_dict() else None + + def events_to_operations(events: list[Event]) -> list[Operation]: """Convert a list of history events into operations. @@ -2750,7 +2764,7 @@ def events_to_operations(events: list[Event]) -> list[Operation]: callback_details=CallbackDetails( callback_id=callback_id, result=result, - error=error, + error=_operation_error(error), ), ) @@ -2791,7 +2805,7 @@ def events_to_operations(events: list[Event]) -> list[Operation]: operation, step_details=StepDetails( result=result_val, - error=error_val, + error=_operation_error(error_val), attempt=attempt, next_attempt_timestamp=next_attempt_ts, ), @@ -2824,7 +2838,9 @@ def events_to_operations(events: list[Event]) -> list[Operation]: operation, context_details=ContextDetails( result=None, - error=event.context_failed_details.error.payload, + error=_operation_error( + event.context_failed_details.error.payload + ), ), ) @@ -2849,7 +2865,9 @@ def events_to_operations(events: list[Event]) -> list[Operation]: operation, chained_invoke_details=ChainedInvokeDetails( result=None, - error=event.chained_invoke_failed_details.error.payload, + error=_operation_error( + event.chained_invoke_failed_details.error.payload + ), ), ) From 58567dc8cd1702fb44febbd23c5bccfed3843426 Mon Sep 17 00:00:00 2001 From: Frank Chen <65260095+zhongkechen@users.noreply.github.com> Date: Wed, 28 Oct 2026 12:27:21 -0700 Subject: [PATCH 3/9] fix: preserve empty errors in SDK deserialization Preserve a present empty ErrorObject in callback, step, context, and chained-invoke details by checking presence instead of truthiness. Keep the matching history payload decoder fix and remove the testing library's completion/reconstruction normalization and synthetic payload. Add core decoding and round-trip regressions, and require live callback errors to survive memory and filesystem persistence with the existing default CallbackExternalError message. The focused regressions fail 25 cases on the previous PR revision and pass all 131 with this fix. --- .../README.md | 17 ++-- .../execution.py | 4 +- .../model.py | 25 +----- .../tests/e2e/empty_callback_error_test.py | 18 ++-- .../tests/empty_error_history_test.py | 13 +-- .../lambda_service.py | 8 +- .../tests/lambda_service_test.py | 89 +++++++++++++++++++ 7 files changed, 122 insertions(+), 52 deletions(-) diff --git a/packages/aws-durable-execution-sdk-python-testing/README.md b/packages/aws-durable-execution-sdk-python-testing/README.md index 346a2b693..28fb08ba4 100644 --- a/packages/aws-durable-execution-sdk-python-testing/README.md +++ b/packages/aws-durable-execution-sdk-python-testing/README.md @@ -172,15 +172,16 @@ flow, pagination, and a map of the code. ### Error Handling -A callback failure with omitted diagnostics or an empty `ErrorObject` has no -error details in SDK-facing operation state, in both memory and filesystem stores. -Detailed callback-failure history preserves `Error.Payload: {}` with +A callback failure with omitted diagnostics or an empty `ErrorObject` retains an +empty error object in SDK-facing operation state, in both memory and filesystem +stores. Detailed callback-failure history preserves `Error.Payload: {}` with `Truncated: false`; history requested without execution data omits the payload and -sets `Truncated: true`. Reconstructing operations from detailed history restores -the same absent diagnostics. Present diagnostic fields remain intact, including -explicit empty strings and empty stack-trace lists. The callback still fails and -raises the existing `CallbackExternalError`, including its default message when -diagnostics are absent. Timeout history is unchanged. +sets `Truncated: true`. Reconstructing operations from detailed history preserves +that empty object. Missing or null errors remain `None` when decoded. Present +diagnostic fields remain intact, including explicit empty strings and empty +stack-trace lists. The callback still fails and raises the existing +`CallbackExternalError`, including its default message when diagnostics are absent. +Timeout history is unchanged. The testing framework implements AWS-compliant error responses that match the exact format expected by boto3 and AWS services. For detailed information about error response formats, exception types, and troubleshooting, see: diff --git a/packages/aws-durable-execution-sdk-python-testing/src/aws_durable_execution_sdk_python_testing/execution.py b/packages/aws-durable-execution-sdk-python-testing/src/aws_durable_execution_sdk_python_testing/execution.py index 04301e697..5190b9b69 100644 --- a/packages/aws-durable-execution-sdk-python-testing/src/aws_durable_execution_sdk_python_testing/execution.py +++ b/packages/aws-durable-execution-sdk-python-testing/src/aws_durable_execution_sdk_python_testing/execution.py @@ -709,9 +709,7 @@ def complete_callback_failure( updated_callback_details = None if operation.callback_details: updated_callback_details = replace( - operation.callback_details, - # Match SDK state after serialization: Error={} has no diagnostics. - error=error if error.to_dict() else None, + operation.callback_details, error=error ) self.operations[index] = replace( diff --git a/packages/aws-durable-execution-sdk-python-testing/src/aws_durable_execution_sdk_python_testing/model.py b/packages/aws-durable-execution-sdk-python-testing/src/aws_durable_execution_sdk_python_testing/model.py index 86d4d8de8..052399a1c 100644 --- a/packages/aws-durable-execution-sdk-python-testing/src/aws_durable_execution_sdk_python_testing/model.py +++ b/packages/aws-durable-execution-sdk-python-testing/src/aws_durable_execution_sdk_python_testing/model.py @@ -2250,14 +2250,6 @@ def create_callback_event_failed(cls, context: EventCreationContext) -> Event: if callback_details else None ) - if ( - context.include_execution_data - and callback_details is not None - and callback_details.error is None - ): - # Detailed failure history retains empty payload presence. Do not - # mutate the absent diagnostics in the SDK-facing operation state. - event_error = EventError(payload=ErrorObject.from_dict({}), truncated=False) return cls( event_type=EventType.CALLBACK_FAILED.value, event_timestamp=context.end_timestamp, @@ -2615,11 +2607,6 @@ class HistoryEventTypeConfig: } -def _operation_error(error: ErrorObject | None) -> ErrorObject | None: - """Normalize history's empty payload to absent SDK-state diagnostics.""" - return error if error is not None and error.to_dict() else None - - def events_to_operations(events: list[Event]) -> list[Operation]: """Convert a list of history events into operations. @@ -2764,7 +2751,7 @@ def events_to_operations(events: list[Event]) -> list[Operation]: callback_details=CallbackDetails( callback_id=callback_id, result=result, - error=_operation_error(error), + error=error, ), ) @@ -2805,7 +2792,7 @@ def events_to_operations(events: list[Event]) -> list[Operation]: operation, step_details=StepDetails( result=result_val, - error=_operation_error(error_val), + error=error_val, attempt=attempt, next_attempt_timestamp=next_attempt_ts, ), @@ -2838,9 +2825,7 @@ def events_to_operations(events: list[Event]) -> list[Operation]: operation, context_details=ContextDetails( result=None, - error=_operation_error( - event.context_failed_details.error.payload - ), + error=event.context_failed_details.error.payload, ), ) @@ -2865,9 +2850,7 @@ def events_to_operations(events: list[Event]) -> list[Operation]: operation, chained_invoke_details=ChainedInvokeDetails( result=None, - error=_operation_error( - event.chained_invoke_failed_details.error.payload - ), + error=event.chained_invoke_failed_details.error.payload, ), ) diff --git a/packages/aws-durable-execution-sdk-python-testing/tests/e2e/empty_callback_error_test.py b/packages/aws-durable-execution-sdk-python-testing/tests/e2e/empty_callback_error_test.py index c41c2fe1d..6d1292315 100644 --- a/packages/aws-durable-execution-sdk-python-testing/tests/e2e/empty_callback_error_test.py +++ b/packages/aws-durable-execution-sdk-python-testing/tests/e2e/empty_callback_error_test.py @@ -39,7 +39,7 @@ @pytest.mark.parametrize("filesystem", [False, True]) @pytest.mark.parametrize("error_kind", ["omitted", "empty", "rich"]) -def test_live_callback_failure_normalizes_before_persistence( +def test_live_callback_failure_preserves_error_through_persistence( filesystem: bool, error_kind: str, tmp_path: Path, @@ -60,7 +60,7 @@ def test_live_callback_failure_normalizes_before_persistence( if error_kind == "rich" else ErrorObject.from_dict({}) ) - expected_error = supplied_error if error_kind == "rich" else None + expected_error = supplied_error completed_operations: list[Operation] = [] original_complete = Execution.complete_callback_failure @@ -71,7 +71,7 @@ def capture_live_completion( now: datetime | None = None, ) -> Operation: # Capture the real API transition before the executor persists it. - # A filesystem round-trip must not hide missing boundary normalization. + # The same error object must survive both memory and filesystem storage. operation = original_complete(execution, callback_id, error, now=now) completed_operations.append(operation) return operation @@ -125,13 +125,9 @@ def durable_handler(_event: Any, context: DurableContext) -> str: assert len(caught_errors) == 1 caught = caught_errors[0] assert isinstance(caught, CallbackExternalError) - assert caught.message == ( - expected_error.message if expected_error else "Callback failed" - ) - assert caught.data == (expected_error.data if expected_error else None) - assert caught.stack_trace == ( - expected_error.stack_trace if expected_error else None - ) + assert caught.message == (expected_error.message or "Callback failed") + assert caught.data == expected_error.data + assert caught.stack_trace == expected_error.stack_trace assert len(completed_operations) == 1 live = completed_operations[0] @@ -154,7 +150,7 @@ def durable_handler(_event: Any, context: DurableContext) -> str: ) expected: dict[str, Any] = {"Truncated": not include_data} if include_data: - expected["Payload"] = expected_error.to_dict() if expected_error else {} + expected["Payload"] = expected_error.to_dict() assert failed["CallbackFailedDetails"]["Error"] == expected decoded = GetDurableExecutionHistoryResponse.from_dict(wire) decoded_failed = next( diff --git a/packages/aws-durable-execution-sdk-python-testing/tests/empty_error_history_test.py b/packages/aws-durable-execution-sdk-python-testing/tests/empty_error_history_test.py index d40af0036..80b0850aa 100644 --- a/packages/aws-durable-execution-sdk-python-testing/tests/empty_error_history_test.py +++ b/packages/aws-durable-execution-sdk-python-testing/tests/empty_error_history_test.py @@ -1,4 +1,4 @@ -"""History preserves empty payload presence without inventing SDK diagnostics.""" +"""History and SDK operation state preserve empty error payload presence.""" from datetime import UTC, datetime from typing import Any @@ -93,7 +93,7 @@ def test_event_error_omitted_and_null_payload_remain_absent( ) @pytest.mark.parametrize("payload", [None, *PAYLOADS]) @pytest.mark.parametrize("decode_wire", [False, True]) -def test_failure_history_reconstruction_normalizes_only_empty_errors( +def test_failure_history_reconstruction_preserves_error_presence( event_type: str, details_field: str, details_type: type[ @@ -127,10 +127,13 @@ def test_failure_history_reconstruction_normalizes_only_empty_errors( if decode_wire: history = GetDurableExecutionHistoryResponse.from_dict(expected_wire) assert history.to_dict() == expected_wire - expected_error = error if payload else None + expected_error = error (operation,) = events_to_operations(history.events) assert operation.status is OperationStatus.FAILED assert getattr(operation, operation_field).error == expected_error + if expected_error is not None: + restored = Operation.from_json_dict(operation.to_json_dict()) + assert restored == operation execution = GetDurableExecutionResponse( durable_execution_arn="arn:execution", @@ -181,8 +184,8 @@ def test_callback_history_projection_preserves_state_and_data_inclusion( ) assert details is not None and details.error is not None expected: dict[str, Any] = {"Truncated": not include_data} - if include_data and (payload is not None or status is OperationStatus.FAILED): - expected["Payload"] = payload if payload is not None else {} + if include_data and payload is not None: + expected["Payload"] = payload assert details.error.to_dict() == expected assert operation.callback_details is not None assert operation.callback_details.error is error diff --git a/packages/aws-durable-execution-sdk-python/src/aws_durable_execution_sdk_python/lambda_service.py b/packages/aws-durable-execution-sdk-python/src/aws_durable_execution_sdk_python/lambda_service.py index 233812312..e43c400cf 100644 --- a/packages/aws-durable-execution-sdk-python/src/aws_durable_execution_sdk_python/lambda_service.py +++ b/packages/aws-durable-execution-sdk-python/src/aws_durable_execution_sdk_python/lambda_service.py @@ -209,7 +209,7 @@ def from_dict(cls, data: MutableMapping[str, Any]) -> ContextDetails: return cls( replay_children=data.get("ReplayChildren", False), result=data.get("Result"), - error=ErrorObject.from_dict(error_raw) if error_raw else None, + error=ErrorObject.from_dict(error_raw) if error_raw is not None else None, ) @@ -346,7 +346,7 @@ def from_dict(cls, data: MutableMapping[str, Any]) -> StepDetails: attempt=data.get("Attempt", 0), next_attempt_timestamp=data.get("NextAttemptTimestamp"), result=data.get("Result"), - error=ErrorObject.from_dict(error_raw) if error_raw else None, + error=ErrorObject.from_dict(error_raw) if error_raw is not None else None, ) @@ -371,7 +371,7 @@ def from_dict(cls, data: MutableMapping[str, Any]) -> CallbackDetails: return cls( callback_id=data["CallbackId"], result=data.get("Result"), - error=ErrorObject.from_dict(error_raw) if error_raw else None, + error=ErrorObject.from_dict(error_raw) if error_raw is not None else None, ) @@ -385,7 +385,7 @@ def from_dict(cls, data: MutableMapping[str, Any]) -> ChainedInvokeDetails: error_raw = data.get("Error") return cls( result=data.get("Result"), - error=ErrorObject.from_dict(error_raw) if error_raw else None, + error=ErrorObject.from_dict(error_raw) if error_raw is not None else None, ) diff --git a/packages/aws-durable-execution-sdk-python/tests/lambda_service_test.py b/packages/aws-durable-execution-sdk-python/tests/lambda_service_test.py index 757036d75..975d5302a 100644 --- a/packages/aws-durable-execution-sdk-python/tests/lambda_service_test.py +++ b/packages/aws-durable-execution-sdk-python/tests/lambda_service_test.py @@ -1,8 +1,10 @@ """Tests for the service module.""" +import dataclasses import datetime import json from datetime import UTC +from typing import Any from unittest.mock import Mock, patch import pytest @@ -376,6 +378,93 @@ def test_step_details_minimal(): assert details.error is None +@pytest.mark.parametrize( + "details_type,required_fields", + [ + (CallbackDetails, {"CallbackId": "callback-id"}), + (StepDetails, {}), + (ContextDetails, {}), + (ChainedInvokeDetails, {}), + ], +) +@pytest.mark.parametrize( + "error_field", + [ + {}, + {"Error": None}, + {"Error": {}}, + {"Error": {"ErrorMessage": ""}}, + {"Error": {"ErrorType": ""}}, + {"Error": {"ErrorData": ""}}, + {"Error": {"StackTrace": []}}, + { + "Error": { + "ErrorMessage": "rejected", + "ErrorType": "RejectedError", + "ErrorData": '{"reason":"test"}', + "StackTrace": ["original frame"], + } + }, + ], +) +def test_operation_details_preserve_error_presence( + details_type: type[ + CallbackDetails | StepDetails | ContextDetails | ChainedInvokeDetails + ], + required_fields: dict[str, Any], + error_field: dict[str, Any], +) -> None: + details = details_type.from_dict({**required_fields, **error_field}) + payload = error_field.get("Error") + if payload is None: + assert details.error is None + else: + assert details.error is not None + assert details.error.to_dict() == payload + + +@pytest.mark.parametrize( + "operation_type,details_field,details", + [ + (OperationType.CALLBACK, "callback_details", CallbackDetails("callback-id")), + (OperationType.STEP, "step_details", StepDetails()), + (OperationType.CONTEXT, "context_details", ContextDetails()), + ( + OperationType.CHAINED_INVOKE, + "chained_invoke_details", + ChainedInvokeDetails(), + ), + ], +) +@pytest.mark.parametrize("json_roundtrip", [False, True]) +def test_operation_roundtrip_preserves_empty_error( + operation_type: OperationType, + details_field: str, + details: CallbackDetails | StepDetails | ContextDetails | ChainedInvokeDetails, + json_roundtrip: bool, +) -> None: + error = ErrorObject(message=None, type=None, data=None, stack_trace=None) + operation_details: dict[str, Any] = { + details_field: dataclasses.replace(details, error=error) + } + operation = Operation( + operation_id="failed-operation", + operation_type=operation_type, + status=OperationStatus.FAILED, + **operation_details, + ) + + if json_roundtrip: + restored = Operation.from_json_dict( + json.loads(json.dumps(operation.to_json_dict())) + ) + else: + restored = Operation.from_dict(operation.to_dict()) + + assert restored == operation + assert getattr(restored, details_field).error == error + + def test_wait_details_from_dict(): """Test WaitDetails.from_dict method.""" timestamp = datetime.datetime(2023, 1, 1, 12, 0, 0, tzinfo=datetime.UTC) From 1eb341b1054af2a9a25ad6907ca64d522b909bdd Mon Sep 17 00:00:00 2001 From: Frank Chen <65260095+zhongkechen@users.noreply.github.com> Date: Wed, 28 Oct 2026 15:53:56 -0700 Subject: [PATCH 4/9] fix: restore callback error normalization Revert 58567dc8cd1702fb44febbd23c5bccfed3843426 to restore the original testing-library solution. Normalize empty callback and reconstructed history errors to absent SDK diagnostics while preserving empty payloads in detailed callback history. Restore the original regression expectations and README. Keep the later main-branch merges and unrelated fixes intact. --- .../README.md | 17 ++-- .../execution.py | 4 +- .../model.py | 25 +++++- .../tests/e2e/empty_callback_error_test.py | 18 ++-- .../tests/empty_error_history_test.py | 13 ++- .../lambda_service.py | 8 +- .../tests/lambda_service_test.py | 89 ------------------- 7 files changed, 52 insertions(+), 122 deletions(-) diff --git a/packages/aws-durable-execution-sdk-python-testing/README.md b/packages/aws-durable-execution-sdk-python-testing/README.md index 28fb08ba4..346a2b693 100644 --- a/packages/aws-durable-execution-sdk-python-testing/README.md +++ b/packages/aws-durable-execution-sdk-python-testing/README.md @@ -172,16 +172,15 @@ flow, pagination, and a map of the code. ### Error Handling -A callback failure with omitted diagnostics or an empty `ErrorObject` retains an -empty error object in SDK-facing operation state, in both memory and filesystem -stores. Detailed callback-failure history preserves `Error.Payload: {}` with +A callback failure with omitted diagnostics or an empty `ErrorObject` has no +error details in SDK-facing operation state, in both memory and filesystem stores. +Detailed callback-failure history preserves `Error.Payload: {}` with `Truncated: false`; history requested without execution data omits the payload and -sets `Truncated: true`. Reconstructing operations from detailed history preserves -that empty object. Missing or null errors remain `None` when decoded. Present -diagnostic fields remain intact, including explicit empty strings and empty -stack-trace lists. The callback still fails and raises the existing -`CallbackExternalError`, including its default message when diagnostics are absent. -Timeout history is unchanged. +sets `Truncated: true`. Reconstructing operations from detailed history restores +the same absent diagnostics. Present diagnostic fields remain intact, including +explicit empty strings and empty stack-trace lists. The callback still fails and +raises the existing `CallbackExternalError`, including its default message when +diagnostics are absent. Timeout history is unchanged. The testing framework implements AWS-compliant error responses that match the exact format expected by boto3 and AWS services. For detailed information about error response formats, exception types, and troubleshooting, see: diff --git a/packages/aws-durable-execution-sdk-python-testing/src/aws_durable_execution_sdk_python_testing/execution.py b/packages/aws-durable-execution-sdk-python-testing/src/aws_durable_execution_sdk_python_testing/execution.py index 5190b9b69..04301e697 100644 --- a/packages/aws-durable-execution-sdk-python-testing/src/aws_durable_execution_sdk_python_testing/execution.py +++ b/packages/aws-durable-execution-sdk-python-testing/src/aws_durable_execution_sdk_python_testing/execution.py @@ -709,7 +709,9 @@ def complete_callback_failure( updated_callback_details = None if operation.callback_details: updated_callback_details = replace( - operation.callback_details, error=error + operation.callback_details, + # Match SDK state after serialization: Error={} has no diagnostics. + error=error if error.to_dict() else None, ) self.operations[index] = replace( diff --git a/packages/aws-durable-execution-sdk-python-testing/src/aws_durable_execution_sdk_python_testing/model.py b/packages/aws-durable-execution-sdk-python-testing/src/aws_durable_execution_sdk_python_testing/model.py index 052399a1c..86d4d8de8 100644 --- a/packages/aws-durable-execution-sdk-python-testing/src/aws_durable_execution_sdk_python_testing/model.py +++ b/packages/aws-durable-execution-sdk-python-testing/src/aws_durable_execution_sdk_python_testing/model.py @@ -2250,6 +2250,14 @@ def create_callback_event_failed(cls, context: EventCreationContext) -> Event: if callback_details else None ) + if ( + context.include_execution_data + and callback_details is not None + and callback_details.error is None + ): + # Detailed failure history retains empty payload presence. Do not + # mutate the absent diagnostics in the SDK-facing operation state. + event_error = EventError(payload=ErrorObject.from_dict({}), truncated=False) return cls( event_type=EventType.CALLBACK_FAILED.value, event_timestamp=context.end_timestamp, @@ -2607,6 +2615,11 @@ class HistoryEventTypeConfig: } +def _operation_error(error: ErrorObject | None) -> ErrorObject | None: + """Normalize history's empty payload to absent SDK-state diagnostics.""" + return error if error is not None and error.to_dict() else None + + def events_to_operations(events: list[Event]) -> list[Operation]: """Convert a list of history events into operations. @@ -2751,7 +2764,7 @@ def events_to_operations(events: list[Event]) -> list[Operation]: callback_details=CallbackDetails( callback_id=callback_id, result=result, - error=error, + error=_operation_error(error), ), ) @@ -2792,7 +2805,7 @@ def events_to_operations(events: list[Event]) -> list[Operation]: operation, step_details=StepDetails( result=result_val, - error=error_val, + error=_operation_error(error_val), attempt=attempt, next_attempt_timestamp=next_attempt_ts, ), @@ -2825,7 +2838,9 @@ def events_to_operations(events: list[Event]) -> list[Operation]: operation, context_details=ContextDetails( result=None, - error=event.context_failed_details.error.payload, + error=_operation_error( + event.context_failed_details.error.payload + ), ), ) @@ -2850,7 +2865,9 @@ def events_to_operations(events: list[Event]) -> list[Operation]: operation, chained_invoke_details=ChainedInvokeDetails( result=None, - error=event.chained_invoke_failed_details.error.payload, + error=_operation_error( + event.chained_invoke_failed_details.error.payload + ), ), ) diff --git a/packages/aws-durable-execution-sdk-python-testing/tests/e2e/empty_callback_error_test.py b/packages/aws-durable-execution-sdk-python-testing/tests/e2e/empty_callback_error_test.py index 6d1292315..c41c2fe1d 100644 --- a/packages/aws-durable-execution-sdk-python-testing/tests/e2e/empty_callback_error_test.py +++ b/packages/aws-durable-execution-sdk-python-testing/tests/e2e/empty_callback_error_test.py @@ -39,7 +39,7 @@ @pytest.mark.parametrize("filesystem", [False, True]) @pytest.mark.parametrize("error_kind", ["omitted", "empty", "rich"]) -def test_live_callback_failure_preserves_error_through_persistence( +def test_live_callback_failure_normalizes_before_persistence( filesystem: bool, error_kind: str, tmp_path: Path, @@ -60,7 +60,7 @@ def test_live_callback_failure_preserves_error_through_persistence( if error_kind == "rich" else ErrorObject.from_dict({}) ) - expected_error = supplied_error + expected_error = supplied_error if error_kind == "rich" else None completed_operations: list[Operation] = [] original_complete = Execution.complete_callback_failure @@ -71,7 +71,7 @@ def capture_live_completion( now: datetime | None = None, ) -> Operation: # Capture the real API transition before the executor persists it. - # The same error object must survive both memory and filesystem storage. + # A filesystem round-trip must not hide missing boundary normalization. operation = original_complete(execution, callback_id, error, now=now) completed_operations.append(operation) return operation @@ -125,9 +125,13 @@ def durable_handler(_event: Any, context: DurableContext) -> str: assert len(caught_errors) == 1 caught = caught_errors[0] assert isinstance(caught, CallbackExternalError) - assert caught.message == (expected_error.message or "Callback failed") - assert caught.data == expected_error.data - assert caught.stack_trace == expected_error.stack_trace + assert caught.message == ( + expected_error.message if expected_error else "Callback failed" + ) + assert caught.data == (expected_error.data if expected_error else None) + assert caught.stack_trace == ( + expected_error.stack_trace if expected_error else None + ) assert len(completed_operations) == 1 live = completed_operations[0] @@ -150,7 +154,7 @@ def durable_handler(_event: Any, context: DurableContext) -> str: ) expected: dict[str, Any] = {"Truncated": not include_data} if include_data: - expected["Payload"] = expected_error.to_dict() + expected["Payload"] = expected_error.to_dict() if expected_error else {} assert failed["CallbackFailedDetails"]["Error"] == expected decoded = GetDurableExecutionHistoryResponse.from_dict(wire) decoded_failed = next( diff --git a/packages/aws-durable-execution-sdk-python-testing/tests/empty_error_history_test.py b/packages/aws-durable-execution-sdk-python-testing/tests/empty_error_history_test.py index 80b0850aa..d40af0036 100644 --- a/packages/aws-durable-execution-sdk-python-testing/tests/empty_error_history_test.py +++ b/packages/aws-durable-execution-sdk-python-testing/tests/empty_error_history_test.py @@ -1,4 +1,4 @@ -"""History and SDK operation state preserve empty error payload presence.""" +"""History preserves empty payload presence without inventing SDK diagnostics.""" from datetime import UTC, datetime from typing import Any @@ -93,7 +93,7 @@ def test_event_error_omitted_and_null_payload_remain_absent( ) @pytest.mark.parametrize("payload", [None, *PAYLOADS]) @pytest.mark.parametrize("decode_wire", [False, True]) -def test_failure_history_reconstruction_preserves_error_presence( +def test_failure_history_reconstruction_normalizes_only_empty_errors( event_type: str, details_field: str, details_type: type[ @@ -127,13 +127,10 @@ def test_failure_history_reconstruction_preserves_error_presence( if decode_wire: history = GetDurableExecutionHistoryResponse.from_dict(expected_wire) assert history.to_dict() == expected_wire - expected_error = error + expected_error = error if payload else None (operation,) = events_to_operations(history.events) assert operation.status is OperationStatus.FAILED assert getattr(operation, operation_field).error == expected_error - if expected_error is not None: - restored = Operation.from_json_dict(operation.to_json_dict()) - assert restored == operation execution = GetDurableExecutionResponse( durable_execution_arn="arn:execution", @@ -184,8 +181,8 @@ def test_callback_history_projection_preserves_state_and_data_inclusion( ) assert details is not None and details.error is not None expected: dict[str, Any] = {"Truncated": not include_data} - if include_data and payload is not None: - expected["Payload"] = payload + if include_data and (payload is not None or status is OperationStatus.FAILED): + expected["Payload"] = payload if payload is not None else {} assert details.error.to_dict() == expected assert operation.callback_details is not None assert operation.callback_details.error is error diff --git a/packages/aws-durable-execution-sdk-python/src/aws_durable_execution_sdk_python/lambda_service.py b/packages/aws-durable-execution-sdk-python/src/aws_durable_execution_sdk_python/lambda_service.py index e43c400cf..233812312 100644 --- a/packages/aws-durable-execution-sdk-python/src/aws_durable_execution_sdk_python/lambda_service.py +++ b/packages/aws-durable-execution-sdk-python/src/aws_durable_execution_sdk_python/lambda_service.py @@ -209,7 +209,7 @@ def from_dict(cls, data: MutableMapping[str, Any]) -> ContextDetails: return cls( replay_children=data.get("ReplayChildren", False), result=data.get("Result"), - error=ErrorObject.from_dict(error_raw) if error_raw is not None else None, + error=ErrorObject.from_dict(error_raw) if error_raw else None, ) @@ -346,7 +346,7 @@ def from_dict(cls, data: MutableMapping[str, Any]) -> StepDetails: attempt=data.get("Attempt", 0), next_attempt_timestamp=data.get("NextAttemptTimestamp"), result=data.get("Result"), - error=ErrorObject.from_dict(error_raw) if error_raw is not None else None, + error=ErrorObject.from_dict(error_raw) if error_raw else None, ) @@ -371,7 +371,7 @@ def from_dict(cls, data: MutableMapping[str, Any]) -> CallbackDetails: return cls( callback_id=data["CallbackId"], result=data.get("Result"), - error=ErrorObject.from_dict(error_raw) if error_raw is not None else None, + error=ErrorObject.from_dict(error_raw) if error_raw else None, ) @@ -385,7 +385,7 @@ def from_dict(cls, data: MutableMapping[str, Any]) -> ChainedInvokeDetails: error_raw = data.get("Error") return cls( result=data.get("Result"), - error=ErrorObject.from_dict(error_raw) if error_raw is not None else None, + error=ErrorObject.from_dict(error_raw) if error_raw else None, ) diff --git a/packages/aws-durable-execution-sdk-python/tests/lambda_service_test.py b/packages/aws-durable-execution-sdk-python/tests/lambda_service_test.py index 975d5302a..757036d75 100644 --- a/packages/aws-durable-execution-sdk-python/tests/lambda_service_test.py +++ b/packages/aws-durable-execution-sdk-python/tests/lambda_service_test.py @@ -1,10 +1,8 @@ """Tests for the service module.""" -import dataclasses import datetime import json from datetime import UTC -from typing import Any from unittest.mock import Mock, patch import pytest @@ -378,93 +376,6 @@ def test_step_details_minimal(): assert details.error is None -@pytest.mark.parametrize( - "details_type,required_fields", - [ - (CallbackDetails, {"CallbackId": "callback-id"}), - (StepDetails, {}), - (ContextDetails, {}), - (ChainedInvokeDetails, {}), - ], -) -@pytest.mark.parametrize( - "error_field", - [ - {}, - {"Error": None}, - {"Error": {}}, - {"Error": {"ErrorMessage": ""}}, - {"Error": {"ErrorType": ""}}, - {"Error": {"ErrorData": ""}}, - {"Error": {"StackTrace": []}}, - { - "Error": { - "ErrorMessage": "rejected", - "ErrorType": "RejectedError", - "ErrorData": '{"reason":"test"}', - "StackTrace": ["original frame"], - } - }, - ], -) -def test_operation_details_preserve_error_presence( - details_type: type[ - CallbackDetails | StepDetails | ContextDetails | ChainedInvokeDetails - ], - required_fields: dict[str, Any], - error_field: dict[str, Any], -) -> None: - details = details_type.from_dict({**required_fields, **error_field}) - payload = error_field.get("Error") - if payload is None: - assert details.error is None - else: - assert details.error is not None - assert details.error.to_dict() == payload - - -@pytest.mark.parametrize( - "operation_type,details_field,details", - [ - (OperationType.CALLBACK, "callback_details", CallbackDetails("callback-id")), - (OperationType.STEP, "step_details", StepDetails()), - (OperationType.CONTEXT, "context_details", ContextDetails()), - ( - OperationType.CHAINED_INVOKE, - "chained_invoke_details", - ChainedInvokeDetails(), - ), - ], -) -@pytest.mark.parametrize("json_roundtrip", [False, True]) -def test_operation_roundtrip_preserves_empty_error( - operation_type: OperationType, - details_field: str, - details: CallbackDetails | StepDetails | ContextDetails | ChainedInvokeDetails, - json_roundtrip: bool, -) -> None: - error = ErrorObject(message=None, type=None, data=None, stack_trace=None) - operation_details: dict[str, Any] = { - details_field: dataclasses.replace(details, error=error) - } - operation = Operation( - operation_id="failed-operation", - operation_type=operation_type, - status=OperationStatus.FAILED, - **operation_details, - ) - - if json_roundtrip: - restored = Operation.from_json_dict( - json.loads(json.dumps(operation.to_json_dict())) - ) - else: - restored = Operation.from_dict(operation.to_dict()) - - assert restored == operation - assert getattr(restored, details_field).error == error - - def test_wait_details_from_dict(): """Test WaitDetails.from_dict method.""" timestamp = datetime.datetime(2023, 1, 1, 12, 0, 0, tzinfo=datetime.UTC) From 5defff74e1ddaf7a685a899cbfdff3c1df1952c0 Mon Sep 17 00:00:00 2001 From: Zhongke Chen Date: Thu, 29 Oct 2026 02:26:27 +0000 Subject: [PATCH 5/9] test: address empty error review comments Require exact callback, step, context, and invoke result wrappers. Restore operation persistence checks for populated error payloads, including explicit empty strings and stack-trace lists. Remove redundant history comparisons and duplicate live callback assertions. --- .../tests/e2e/empty_callback_error_test.py | 4 --- .../tests/empty_error_history_test.py | 34 ++++++++++++++----- 2 files changed, 26 insertions(+), 12 deletions(-) diff --git a/packages/aws-durable-execution-sdk-python-testing/tests/e2e/empty_callback_error_test.py b/packages/aws-durable-execution-sdk-python-testing/tests/e2e/empty_callback_error_test.py index c41c2fe1d..a9d8c361e 100644 --- a/packages/aws-durable-execution-sdk-python-testing/tests/e2e/empty_callback_error_test.py +++ b/packages/aws-durable-execution-sdk-python-testing/tests/e2e/empty_callback_error_test.py @@ -174,7 +174,3 @@ def durable_handler(_event: Any, context: DurableContext) -> str: assert reconstructed.callback_details.error == ( expected_error if include_data else None ) - assert history.to_dict() == wire - - assert live.callback_details is not None - assert live.callback_details.error == expected_error diff --git a/packages/aws-durable-execution-sdk-python-testing/tests/empty_error_history_test.py b/packages/aws-durable-execution-sdk-python-testing/tests/empty_error_history_test.py index d40af0036..5db261fd3 100644 --- a/packages/aws-durable-execution-sdk-python-testing/tests/empty_error_history_test.py +++ b/packages/aws-durable-execution-sdk-python-testing/tests/empty_error_history_test.py @@ -1,7 +1,7 @@ """History preserves empty payload presence without inventing SDK diagnostics.""" from datetime import UTC, datetime -from typing import Any +from typing import Any, cast import pytest @@ -26,9 +26,11 @@ events_to_operations, ) from aws_durable_execution_sdk_python_testing.runner import ( + CallbackOperation, ContextOperation, DurableFunctionTestResult, InvokeOperation, + StepOperation, ) @@ -68,26 +70,35 @@ def test_event_error_omitted_and_null_payload_remain_absent( @pytest.mark.parametrize( - "event_type,details_field,details_type,operation_field", + "event_type,details_field,details_type,operation_field,wrapper_type", [ ( "CallbackFailed", "callback_failed_details", CallbackFailedDetails, "callback_details", + CallbackOperation, + ), + ( + "StepFailed", + "step_failed_details", + StepFailedDetails, + "step_details", + StepOperation, ), - ("StepFailed", "step_failed_details", StepFailedDetails, "step_details"), ( "ContextFailed", "context_failed_details", ContextFailedDetails, "context_details", + ContextOperation, ), ( "ChainedInvokeFailed", "chained_invoke_failed_details", ChainedInvokeFailedDetails, "chained_invoke_details", + InvokeOperation, ), ], ) @@ -103,6 +114,7 @@ def test_failure_history_reconstruction_normalizes_only_empty_errors( | ChainedInvokeFailedDetails ], operation_field: str, + wrapper_type: type[ContextOperation | InvokeOperation], payload: dict[str, Any] | None, decode_wire: bool, ) -> None: @@ -123,14 +135,18 @@ def test_failure_history_reconstruction_normalizes_only_empty_errors( ) ] ) - expected_wire = history.to_dict() if decode_wire: + expected_wire = history.to_dict() history = GetDurableExecutionHistoryResponse.from_dict(expected_wire) - assert history.to_dict() == expected_wire + assert history.to_dict() == expected_wire expected_error = error if payload else None (operation,) = events_to_operations(history.events) assert operation.status is OperationStatus.FAILED assert getattr(operation, operation_field).error == expected_error + if expected_error is not None: + roundtripped = Operation.from_json_dict(operation.to_json_dict()) + assert roundtripped.status is OperationStatus.FAILED + assert getattr(roundtripped, operation_field).error == expected_error execution = GetDurableExecutionResponse( durable_execution_arn="arn:execution", @@ -140,10 +156,12 @@ def test_failure_history_reconstruction_normalizes_only_empty_errors( start_timestamp=timestamp, ) result = DurableFunctionTestResult.from_execution_history(execution, history) - result_operation = result.get_operation_by_name("failed-operation") - assert isinstance(result_operation, (ContextOperation, InvokeOperation)) + result_operation = cast( + ContextOperation | InvokeOperation, + result.get_operation_by_name("failed-operation"), + ) + assert type(result_operation) is wrapper_type assert result_operation.error == expected_error - assert history.to_dict() == expected_wire @pytest.mark.parametrize("include_data", [False, True]) From f57129daa5ce83ed00d334996cdfde9cfa1c0c57 Mon Sep 17 00:00:00 2001 From: Zhongke Chen Date: Thu, 29 Oct 2026 03:11:00 +0000 Subject: [PATCH 6/9] fix: preserve empty operation error details Decode present empty errors in callback, step, context, and chained-invoke operation details without collapsing them to None. Retain omitted/null errors and explicit empty diagnostic fields. Add presence and operation round-trip regressions. Before the fix, the new cases report 12 failures and 28 passes; after it, all 247 tests in lambda_service_test.py pass. --- .../lambda_service.py | 8 +- .../tests/lambda_service_test.py | 89 +++++++++++++++++++ 2 files changed, 93 insertions(+), 4 deletions(-) diff --git a/packages/aws-durable-execution-sdk-python/src/aws_durable_execution_sdk_python/lambda_service.py b/packages/aws-durable-execution-sdk-python/src/aws_durable_execution_sdk_python/lambda_service.py index 233812312..e43c400cf 100644 --- a/packages/aws-durable-execution-sdk-python/src/aws_durable_execution_sdk_python/lambda_service.py +++ b/packages/aws-durable-execution-sdk-python/src/aws_durable_execution_sdk_python/lambda_service.py @@ -209,7 +209,7 @@ def from_dict(cls, data: MutableMapping[str, Any]) -> ContextDetails: return cls( replay_children=data.get("ReplayChildren", False), result=data.get("Result"), - error=ErrorObject.from_dict(error_raw) if error_raw else None, + error=ErrorObject.from_dict(error_raw) if error_raw is not None else None, ) @@ -346,7 +346,7 @@ def from_dict(cls, data: MutableMapping[str, Any]) -> StepDetails: attempt=data.get("Attempt", 0), next_attempt_timestamp=data.get("NextAttemptTimestamp"), result=data.get("Result"), - error=ErrorObject.from_dict(error_raw) if error_raw else None, + error=ErrorObject.from_dict(error_raw) if error_raw is not None else None, ) @@ -371,7 +371,7 @@ def from_dict(cls, data: MutableMapping[str, Any]) -> CallbackDetails: return cls( callback_id=data["CallbackId"], result=data.get("Result"), - error=ErrorObject.from_dict(error_raw) if error_raw else None, + error=ErrorObject.from_dict(error_raw) if error_raw is not None else None, ) @@ -385,7 +385,7 @@ def from_dict(cls, data: MutableMapping[str, Any]) -> ChainedInvokeDetails: error_raw = data.get("Error") return cls( result=data.get("Result"), - error=ErrorObject.from_dict(error_raw) if error_raw else None, + error=ErrorObject.from_dict(error_raw) if error_raw is not None else None, ) diff --git a/packages/aws-durable-execution-sdk-python/tests/lambda_service_test.py b/packages/aws-durable-execution-sdk-python/tests/lambda_service_test.py index 757036d75..975d5302a 100644 --- a/packages/aws-durable-execution-sdk-python/tests/lambda_service_test.py +++ b/packages/aws-durable-execution-sdk-python/tests/lambda_service_test.py @@ -1,8 +1,10 @@ """Tests for the service module.""" +import dataclasses import datetime import json from datetime import UTC +from typing import Any from unittest.mock import Mock, patch import pytest @@ -376,6 +378,93 @@ def test_step_details_minimal(): assert details.error is None +@pytest.mark.parametrize( + "details_type,required_fields", + [ + (CallbackDetails, {"CallbackId": "callback-id"}), + (StepDetails, {}), + (ContextDetails, {}), + (ChainedInvokeDetails, {}), + ], +) +@pytest.mark.parametrize( + "error_field", + [ + {}, + {"Error": None}, + {"Error": {}}, + {"Error": {"ErrorMessage": ""}}, + {"Error": {"ErrorType": ""}}, + {"Error": {"ErrorData": ""}}, + {"Error": {"StackTrace": []}}, + { + "Error": { + "ErrorMessage": "rejected", + "ErrorType": "RejectedError", + "ErrorData": '{"reason":"test"}', + "StackTrace": ["original frame"], + } + }, + ], +) +def test_operation_details_preserve_error_presence( + details_type: type[ + CallbackDetails | StepDetails | ContextDetails | ChainedInvokeDetails + ], + required_fields: dict[str, Any], + error_field: dict[str, Any], +) -> None: + details = details_type.from_dict({**required_fields, **error_field}) + payload = error_field.get("Error") + if payload is None: + assert details.error is None + else: + assert details.error is not None + assert details.error.to_dict() == payload + + +@pytest.mark.parametrize( + "operation_type,details_field,details", + [ + (OperationType.CALLBACK, "callback_details", CallbackDetails("callback-id")), + (OperationType.STEP, "step_details", StepDetails()), + (OperationType.CONTEXT, "context_details", ContextDetails()), + ( + OperationType.CHAINED_INVOKE, + "chained_invoke_details", + ChainedInvokeDetails(), + ), + ], +) +@pytest.mark.parametrize("json_roundtrip", [False, True]) +def test_operation_roundtrip_preserves_empty_error( + operation_type: OperationType, + details_field: str, + details: CallbackDetails | StepDetails | ContextDetails | ChainedInvokeDetails, + json_roundtrip: bool, +) -> None: + error = ErrorObject(message=None, type=None, data=None, stack_trace=None) + operation_details: dict[str, Any] = { + details_field: dataclasses.replace(details, error=error) + } + operation = Operation( + operation_id="failed-operation", + operation_type=operation_type, + status=OperationStatus.FAILED, + **operation_details, + ) + + if json_roundtrip: + restored = Operation.from_json_dict( + json.loads(json.dumps(operation.to_json_dict())) + ) + else: + restored = Operation.from_dict(operation.to_dict()) + + assert restored == operation + assert getattr(restored, details_field).error == error + + def test_wait_details_from_dict(): """Test WaitDetails.from_dict method.""" timestamp = datetime.datetime(2023, 1, 1, 12, 0, 0, tzinfo=datetime.UTC) From dfac8b3274f906fcb43be49a9e972e6830235ccd Mon Sep 17 00:00:00 2001 From: Zhongke Chen Date: Thu, 29 Oct 2026 03:13:26 +0000 Subject: [PATCH 7/9] fix: normalize empty error diagnostics Keep checkpointed empty errors intact while omitting diagnostics from plugin snapshots and terminal hooks. Use the existing typed operation error fallback for empty checkpoint errors, preserving supplied empty fields and custom fallback messages. --- .../plugin.py | 20 +++-- .../aws_durable_execution_sdk_python/state.py | 2 +- .../tests/plugin_test.py | 88 +++++++++++++++++++ .../tests/state_test.py | 81 +++++++++++++++++ 4 files changed, 182 insertions(+), 9 deletions(-) diff --git a/packages/aws-durable-execution-sdk-python/src/aws_durable_execution_sdk_python/plugin.py b/packages/aws-durable-execution-sdk-python/src/aws_durable_execution_sdk_python/plugin.py index 6cd688e07..f788ef095 100644 --- a/packages/aws-durable-execution-sdk-python/src/aws_durable_execution_sdk_python/plugin.py +++ b/packages/aws-durable-execution-sdk-python/src/aws_durable_execution_sdk_python/plugin.py @@ -73,14 +73,18 @@ def _extract_result(operation: Operation) -> str | None: def _extract_error(operation: Operation) -> ErrorObject | None: - if operation.step_details and operation.step_details.error: - return operation.step_details.error - if operation.callback_details and operation.callback_details.error: - return operation.callback_details.error - if operation.chained_invoke_details and operation.chained_invoke_details.error: - return operation.chained_invoke_details.error - if operation.context_details and operation.context_details.error: - return operation.context_details.error + for details in ( + operation.step_details, + operation.callback_details, + operation.chained_invoke_details, + operation.context_details, + ): + if ( + details is not None + and details.error is not None + and details.error.to_dict() + ): + return details.error return None diff --git a/packages/aws-durable-execution-sdk-python/src/aws_durable_execution_sdk_python/state.py b/packages/aws-durable-execution-sdk-python/src/aws_durable_execution_sdk_python/state.py index a9883b3ea..ecf4e1b7f 100644 --- a/packages/aws-durable-execution-sdk-python/src/aws_durable_execution_sdk_python/state.py +++ b/packages/aws-durable-execution-sdk-python/src/aws_durable_execution_sdk_python/state.py @@ -335,7 +335,7 @@ def raise_operation_error( ensures async operations whose checkpoint carries a downstream error type (invoke/callback) still surface as the correct per-operation error. """ - if self.error is None: + if self.error is None or not self.error.to_dict(): err_msg: str = ( msg or "Unknown error. No ErrorObject exists on the Checkpoint Operation." diff --git a/packages/aws-durable-execution-sdk-python/tests/plugin_test.py b/packages/aws-durable-execution-sdk-python/tests/plugin_test.py index c6399a7bb..b02acb530 100644 --- a/packages/aws-durable-execution-sdk-python/tests/plugin_test.py +++ b/packages/aws-durable-execution-sdk-python/tests/plugin_test.py @@ -12,6 +12,8 @@ from itertools import product from unittest.mock import MagicMock, patch +import pytest + from aws_durable_execution_sdk_python.identifier import OperationIdentifier from aws_durable_execution_sdk_python.lambda_service import ( CallbackDetails, @@ -2691,6 +2693,92 @@ def test_extract_error_step_details_no_error(self): self.assertEqual(result.message, "boom") +@pytest.mark.parametrize( + "operation_type", + [ + ServiceOperationType.STEP, + ServiceOperationType.CALLBACK, + ServiceOperationType.CHAINED_INVOKE, + ServiceOperationType.CONTEXT, + ], +) +@pytest.mark.parametrize( + "payload", + [ + None, + {}, + {"ErrorMessage": ""}, + {"ErrorType": ""}, + {"ErrorData": ""}, + {"StackTrace": []}, + { + "ErrorMessage": "stored failure", + "ErrorType": "ExternalError", + "ErrorData": "details", + "StackTrace": ["original frame"], + }, + ], +) +def test_plugin_error_projection_preserves_checkpoint_state( + operation_type: ServiceOperationType, payload: dict[str, str | list[str]] | None +) -> None: + error = ErrorObject.from_dict(payload) if payload is not None else None + operation = Operation( + operation_id="op-1", + operation_type=operation_type, + status=OperationStatus.FAILED, + step_details=( + StepDetails(error=error) + if operation_type is ServiceOperationType.STEP + else None + ), + callback_details=( + CallbackDetails(callback_id="callback-1", error=error) + if operation_type is ServiceOperationType.CALLBACK + else None + ), + chained_invoke_details=( + ChainedInvokeDetails(error=error) + if operation_type is ServiceOperationType.CHAINED_INVOKE + else None + ), + context_details=( + ContextDetails(error=error) + if operation_type is ServiceOperationType.CONTEXT + else None + ), + ) + checkpoint = deepcopy(operation.to_dict()) + observed: list[OperationInfo | OperationEndInfo] = [] + + class Observe(DurableInstrumentationPlugin): + def on_invocation_start(self, info: InvocationStartInfo) -> None: + observed.extend([info.operations["op-1"], info.updated_operations["op-1"]]) + + def on_operation_end(self, info: OperationEndInfo) -> None: + observed.append(info) + + executor = PluginExecutor([plugin_factory(Observe())]) + with executor.run(): + executor.on_invocation_start( + execution_arn="arn:exec", + lambda_context=LAMBDA_CTX, + execution_start_time=START_TS, + is_first_invocation=False, + operations_provider=lambda: {"op-1": operation}, + updated_operation_ids=["op-1"], + ) + executor.on_operation_update(operation) + + assert len(observed) == 3 + for info in observed: + assert info.status is OperationStatus.FAILED + assert info.error == (error if payload else None) + assert operation.to_dict() == checkpoint + if error is not None: + assert error.to_dict() == payload + + class TestPluginExecutorIsTerminalStatus(unittest.TestCase): """Tests for PluginExecutor._is_terminal_status static method.""" diff --git a/packages/aws-durable-execution-sdk-python/tests/state_test.py b/packages/aws-durable-execution-sdk-python/tests/state_test.py index 45c41e53c..d9860a3da 100644 --- a/packages/aws-durable-execution-sdk-python/tests/state_test.py +++ b/packages/aws-durable-execution-sdk-python/tests/state_test.py @@ -19,8 +19,11 @@ from aws_durable_execution_sdk_python.exceptions import ( BackgroundThreadError, + ChildContextError, DurableApiErrorCategory, + DurableOperationError, GetExecutionStateError, + InvokeError, NonDeterministicExecutionError, OrphanedChildException, StepError, @@ -417,6 +420,84 @@ def test_checkpointed_result_raise_operation_error_no_error_with_message(): result.raise_operation_error(StepError, "Custom error message") +@pytest.mark.parametrize( + "operation_type,error_cls", + [ + (OperationType.STEP, StepError), + (OperationType.CONTEXT, ChildContextError), + (OperationType.CHAINED_INVOKE, InvokeError), + ], +) +@pytest.mark.parametrize( + "payload", + [ + None, + {}, + {"ErrorMessage": ""}, + {"ErrorType": ""}, + {"ErrorData": ""}, + {"StackTrace": []}, + { + "ErrorMessage": "stored failure", + "ErrorType": "ExternalError", + "ErrorData": "details", + "StackTrace": ["original frame"], + }, + ], +) +@pytest.mark.parametrize("message", [None, "Custom error message"]) +def test_checkpointed_error_uses_only_available_diagnostics( + operation_type: OperationType, + error_cls: type[DurableOperationError], + payload: dict[str, str | list[str]] | None, + message: str | None, +) -> None: + error = ErrorObject.from_dict(payload) if payload is not None else None + operation = Operation( + operation_id="failed-operation", + operation_type=operation_type, + status=OperationStatus.FAILED, + step_details=( + StepDetails(error=error) if operation_type is OperationType.STEP else None + ), + context_details=( + ContextDetails(error=error) + if operation_type is OperationType.CONTEXT + else None + ), + chained_invoke_details=( + ChainedInvokeDetails(error=error) + if operation_type is OperationType.CHAINED_INVOKE + else None + ), + ) + result = CheckpointedResult.create_from_operation(operation) + + with pytest.raises(error_cls) as raised: + result.raise_operation_error(error_cls, message) + + assert type(raised.value) is error_cls + if payload: + assert raised.value.message == payload.get("ErrorMessage") + assert raised.value.error_type == ( + payload.get("ErrorType") + or f"{error_cls.__module__}.{error_cls.__qualname__}" + ) + assert raised.value.data == payload.get("ErrorData") + assert raised.value.stack_trace == payload.get("StackTrace") + assert raised.value.__cause__ is not None + else: + assert raised.value.message == ( + message + or "Unknown error. No ErrorObject exists on the Checkpoint Operation." + ) + assert raised.value.__cause__ is None + assert result.error is error + assert result.status is OperationStatus.FAILED + if error is not None: + assert error.to_dict() == payload + + def test_checkpointed_result_immutable(): """Test that CheckpointedResult is immutable.""" result = CheckpointedResult(status=OperationStatus.SUCCEEDED) From 68a3d7e2b8e3f2ec57920b43ed32f9e1c575028b Mon Sep 17 00:00:00 2001 From: Zhongke Chen Date: Thu, 29 Oct 2026 03:15:29 +0000 Subject: [PATCH 8/9] fix: retain empty detail projection behavior Keep the existing truthiness check for operation detail containers while normalizing empty diagnostic objects. Verify real wire-decoded empty invoke details keep direct, invocation, and terminal plugin information. --- .../plugin.py | 6 +-- .../tests/plugin_test.py | 37 +++++++++++++++++++ 2 files changed, 38 insertions(+), 5 deletions(-) diff --git a/packages/aws-durable-execution-sdk-python/src/aws_durable_execution_sdk_python/plugin.py b/packages/aws-durable-execution-sdk-python/src/aws_durable_execution_sdk_python/plugin.py index f788ef095..d23b6aad9 100644 --- a/packages/aws-durable-execution-sdk-python/src/aws_durable_execution_sdk_python/plugin.py +++ b/packages/aws-durable-execution-sdk-python/src/aws_durable_execution_sdk_python/plugin.py @@ -79,11 +79,7 @@ def _extract_error(operation: Operation) -> ErrorObject | None: operation.chained_invoke_details, operation.context_details, ): - if ( - details is not None - and details.error is not None - and details.error.to_dict() - ): + if details and details.error is not None and details.error.to_dict(): return details.error return None diff --git a/packages/aws-durable-execution-sdk-python/tests/plugin_test.py b/packages/aws-durable-execution-sdk-python/tests/plugin_test.py index b02acb530..ab3996335 100644 --- a/packages/aws-durable-execution-sdk-python/tests/plugin_test.py +++ b/packages/aws-durable-execution-sdk-python/tests/plugin_test.py @@ -2779,6 +2779,43 @@ def on_operation_end(self, info: OperationEndInfo) -> None: assert error.to_dict() == payload +def test_empty_wire_invoke_details_keep_plugin_operation_maps() -> None: + operation = Operation.from_dict( + { + "Id": "op-1", + "Type": "CHAINED_INVOKE", + "Status": "FAILED", + "ChainedInvokeDetails": {}, + } + ) + observed: list[OperationInfo] = [OperationInfo.from_operation(operation)] + + class Observe(DurableInstrumentationPlugin): + def on_invocation_start(self, info: InvocationStartInfo) -> None: + observed.extend([info.operations["op-1"], info.updated_operations["op-1"]]) + + def on_operation_end(self, info: OperationEndInfo) -> None: + observed.append(info) + + executor = PluginExecutor([plugin_factory(Observe())]) + with executor.run(): + executor.on_invocation_start( + execution_arn="arn:exec", + lambda_context=LAMBDA_CTX, + execution_start_time=START_TS, + is_first_invocation=False, + operations_provider=lambda: {"op-1": operation}, + updated_operation_ids=["op-1"], + ) + executor.on_operation_update(operation) + + assert len(observed) == 4 + for info in observed: + assert info.operation_id == "op-1" + assert info.status is OperationStatus.FAILED + assert info.error is None + + class TestPluginExecutorIsTerminalStatus(unittest.TestCase): """Tests for PluginExecutor._is_terminal_status static method.""" From 1709ab7e97fd21679566b2137346161c039502d7 Mon Sep 17 00:00:00 2001 From: Zhongke Chen Date: Thu, 29 Oct 2026 03:17:44 +0000 Subject: [PATCH 9/9] fix: retain empty errors through runner state Keep callback errors and reconstructed history errors intact instead of normalizing them to None and synthesizing detailed history payloads. Retain metadata-only omission and explicit empty diagnostic fields. Require empty errors to survive live, persisted, and reconstructed state. Verify resumed plugin snapshots omit empty diagnostics while the handler keeps its callback fallback message and completed steps stay skipped. The updated regression suite fails 13 cases on the previous PR head. --- .../README.md | 20 +++++---- .../execution.py | 4 +- .../model.py | 25 ++--------- .../tests/e2e/empty_callback_error_test.py | 42 +++++++++++++------ .../tests/empty_error_history_test.py | 10 ++--- 5 files changed, 52 insertions(+), 49 deletions(-) diff --git a/packages/aws-durable-execution-sdk-python-testing/README.md b/packages/aws-durable-execution-sdk-python-testing/README.md index 346a2b693..cae654f2d 100644 --- a/packages/aws-durable-execution-sdk-python-testing/README.md +++ b/packages/aws-durable-execution-sdk-python-testing/README.md @@ -172,15 +172,19 @@ flow, pagination, and a map of the code. ### Error Handling -A callback failure with omitted diagnostics or an empty `ErrorObject` has no -error details in SDK-facing operation state, in both memory and filesystem stores. -Detailed callback-failure history preserves `Error.Payload: {}` with +A callback failure with omitted diagnostics or an empty `ErrorObject` retains an +empty error object in SDK-facing operation state, in both memory and filesystem +stores. Detailed callback-failure history preserves `Error.Payload: {}` with `Truncated: false`; history requested without execution data omits the payload and -sets `Truncated: true`. Reconstructing operations from detailed history restores -the same absent diagnostics. Present diagnostic fields remain intact, including -explicit empty strings and empty stack-trace lists. The callback still fails and -raises the existing `CallbackExternalError`, including its default message when -diagnostics are absent. Timeout history is unchanged. +sets `Truncated: true`. Reconstructing operations from detailed history preserves +that empty object. Missing or null errors remain `None` when decoded. Present +diagnostic fields remain intact, including explicit empty strings and empty +stack-trace lists. The callback still fails and raises the existing +`CallbackExternalError`, including its default message when diagnostics are absent. +Plugin operation information omits wholly empty diagnostics without changing the +stored error. Failed step, child-context, and invoke replay use their existing +default exception message when the stored error has no fields. Timeout history +retains the same data-inclusion rules. The testing framework implements AWS-compliant error responses that match the exact format expected by boto3 and AWS services. For detailed information about error response formats, exception types, and troubleshooting, see: diff --git a/packages/aws-durable-execution-sdk-python-testing/src/aws_durable_execution_sdk_python_testing/execution.py b/packages/aws-durable-execution-sdk-python-testing/src/aws_durable_execution_sdk_python_testing/execution.py index 04301e697..5190b9b69 100644 --- a/packages/aws-durable-execution-sdk-python-testing/src/aws_durable_execution_sdk_python_testing/execution.py +++ b/packages/aws-durable-execution-sdk-python-testing/src/aws_durable_execution_sdk_python_testing/execution.py @@ -709,9 +709,7 @@ def complete_callback_failure( updated_callback_details = None if operation.callback_details: updated_callback_details = replace( - operation.callback_details, - # Match SDK state after serialization: Error={} has no diagnostics. - error=error if error.to_dict() else None, + operation.callback_details, error=error ) self.operations[index] = replace( diff --git a/packages/aws-durable-execution-sdk-python-testing/src/aws_durable_execution_sdk_python_testing/model.py b/packages/aws-durable-execution-sdk-python-testing/src/aws_durable_execution_sdk_python_testing/model.py index 86d4d8de8..052399a1c 100644 --- a/packages/aws-durable-execution-sdk-python-testing/src/aws_durable_execution_sdk_python_testing/model.py +++ b/packages/aws-durable-execution-sdk-python-testing/src/aws_durable_execution_sdk_python_testing/model.py @@ -2250,14 +2250,6 @@ def create_callback_event_failed(cls, context: EventCreationContext) -> Event: if callback_details else None ) - if ( - context.include_execution_data - and callback_details is not None - and callback_details.error is None - ): - # Detailed failure history retains empty payload presence. Do not - # mutate the absent diagnostics in the SDK-facing operation state. - event_error = EventError(payload=ErrorObject.from_dict({}), truncated=False) return cls( event_type=EventType.CALLBACK_FAILED.value, event_timestamp=context.end_timestamp, @@ -2615,11 +2607,6 @@ class HistoryEventTypeConfig: } -def _operation_error(error: ErrorObject | None) -> ErrorObject | None: - """Normalize history's empty payload to absent SDK-state diagnostics.""" - return error if error is not None and error.to_dict() else None - - def events_to_operations(events: list[Event]) -> list[Operation]: """Convert a list of history events into operations. @@ -2764,7 +2751,7 @@ def events_to_operations(events: list[Event]) -> list[Operation]: callback_details=CallbackDetails( callback_id=callback_id, result=result, - error=_operation_error(error), + error=error, ), ) @@ -2805,7 +2792,7 @@ def events_to_operations(events: list[Event]) -> list[Operation]: operation, step_details=StepDetails( result=result_val, - error=_operation_error(error_val), + error=error_val, attempt=attempt, next_attempt_timestamp=next_attempt_ts, ), @@ -2838,9 +2825,7 @@ def events_to_operations(events: list[Event]) -> list[Operation]: operation, context_details=ContextDetails( result=None, - error=_operation_error( - event.context_failed_details.error.payload - ), + error=event.context_failed_details.error.payload, ), ) @@ -2865,9 +2850,7 @@ def events_to_operations(events: list[Event]) -> list[Operation]: operation, chained_invoke_details=ChainedInvokeDetails( result=None, - error=_operation_error( - event.chained_invoke_failed_details.error.payload - ), + error=event.chained_invoke_failed_details.error.payload, ), ) diff --git a/packages/aws-durable-execution-sdk-python-testing/tests/e2e/empty_callback_error_test.py b/packages/aws-durable-execution-sdk-python-testing/tests/e2e/empty_callback_error_test.py index a9d8c361e..52d9add1c 100644 --- a/packages/aws-durable-execution-sdk-python-testing/tests/e2e/empty_callback_error_test.py +++ b/packages/aws-durable-execution-sdk-python-testing/tests/e2e/empty_callback_error_test.py @@ -22,6 +22,11 @@ Operation, OperationStatus, ) +from aws_durable_execution_sdk_python.plugin import ( + DurableInstrumentationPlugin, + InvocationStartInfo, + OperationInfo, +) from aws_durable_execution_sdk_python_testing.execution import Execution from aws_durable_execution_sdk_python_testing.model import ( GetDurableExecutionHistoryResponse, @@ -39,7 +44,7 @@ @pytest.mark.parametrize("filesystem", [False, True]) @pytest.mark.parametrize("error_kind", ["omitted", "empty", "rich"]) -def test_live_callback_failure_normalizes_before_persistence( +def test_live_callback_failure_preserves_error_through_persistence( filesystem: bool, error_kind: str, tmp_path: Path, @@ -60,7 +65,7 @@ def test_live_callback_failure_normalizes_before_persistence( if error_kind == "rich" else ErrorObject.from_dict({}) ) - expected_error = supplied_error if error_kind == "rich" else None + expected_error = supplied_error completed_operations: list[Operation] = [] original_complete = Execution.complete_callback_failure @@ -71,7 +76,7 @@ def capture_live_completion( now: datetime | None = None, ) -> Operation: # Capture the real API transition before the executor persists it. - # A filesystem round-trip must not hide missing boundary normalization. + # The same error fields must survive both memory and filesystem storage. operation = original_complete(execution, callback_id, error, now=now) completed_operations.append(operation) return operation @@ -79,6 +84,19 @@ def capture_live_completion( monkeypatch.setattr(Execution, "complete_callback_failure", capture_live_completion) step_calls: list[str] = [] caught_errors: list[CallbackError] = [] + callback_infos: list[OperationInfo] = [] + + class ObserveCallback(DurableInstrumentationPlugin): + def on_invocation_start(self, info: InvocationStartInfo) -> None: + callback_infos.extend( + operation + for operation in info.operations.values() + if operation.name == "failed-callback" + ) + + class ObserveFactory: + def create_plugin(self, _info: InvocationStartInfo) -> ObserveCallback: + return ObserveCallback() @durable_step def saved_step(_context: Any) -> str: @@ -95,7 +113,7 @@ def durable_handler(_event: Any, context: DurableContext) -> str: return saved raise AssertionError("Callback failure was not raised") - handler = durable_execution(durable_handler) + handler = durable_execution(durable_handler, plugins=[ObserveFactory()]) with DurableFunctionTestRunner( handler=handler, store=store, @@ -125,13 +143,13 @@ def durable_handler(_event: Any, context: DurableContext) -> str: assert len(caught_errors) == 1 caught = caught_errors[0] assert isinstance(caught, CallbackExternalError) - assert caught.message == ( - expected_error.message if expected_error else "Callback failed" - ) - assert caught.data == (expected_error.data if expected_error else None) - assert caught.stack_trace == ( - expected_error.stack_trace if expected_error else None - ) + assert caught.message == (expected_error.message or "Callback failed") + assert caught.data == expected_error.data + assert caught.stack_trace == expected_error.stack_trace + assert callback_infos + for info in callback_infos: + assert info.status is OperationStatus.FAILED + assert info.error == (expected_error if error_kind == "rich" else None) assert len(completed_operations) == 1 live = completed_operations[0] @@ -154,7 +172,7 @@ def durable_handler(_event: Any, context: DurableContext) -> str: ) expected: dict[str, Any] = {"Truncated": not include_data} if include_data: - expected["Payload"] = expected_error.to_dict() if expected_error else {} + expected["Payload"] = expected_error.to_dict() assert failed["CallbackFailedDetails"]["Error"] == expected decoded = GetDurableExecutionHistoryResponse.from_dict(wire) decoded_failed = next( diff --git a/packages/aws-durable-execution-sdk-python-testing/tests/empty_error_history_test.py b/packages/aws-durable-execution-sdk-python-testing/tests/empty_error_history_test.py index 5db261fd3..42dcbee08 100644 --- a/packages/aws-durable-execution-sdk-python-testing/tests/empty_error_history_test.py +++ b/packages/aws-durable-execution-sdk-python-testing/tests/empty_error_history_test.py @@ -1,4 +1,4 @@ -"""History preserves empty payload presence without inventing SDK diagnostics.""" +"""History and SDK operation state preserve empty error payload presence.""" from datetime import UTC, datetime from typing import Any, cast @@ -104,7 +104,7 @@ def test_event_error_omitted_and_null_payload_remain_absent( ) @pytest.mark.parametrize("payload", [None, *PAYLOADS]) @pytest.mark.parametrize("decode_wire", [False, True]) -def test_failure_history_reconstruction_normalizes_only_empty_errors( +def test_failure_history_reconstruction_preserves_error_presence( event_type: str, details_field: str, details_type: type[ @@ -139,7 +139,7 @@ def test_failure_history_reconstruction_normalizes_only_empty_errors( expected_wire = history.to_dict() history = GetDurableExecutionHistoryResponse.from_dict(expected_wire) assert history.to_dict() == expected_wire - expected_error = error if payload else None + expected_error = error (operation,) = events_to_operations(history.events) assert operation.status is OperationStatus.FAILED assert getattr(operation, operation_field).error == expected_error @@ -199,8 +199,8 @@ def test_callback_history_projection_preserves_state_and_data_inclusion( ) assert details is not None and details.error is not None expected: dict[str, Any] = {"Truncated": not include_data} - if include_data and (payload is not None or status is OperationStatus.FAILED): - expected["Payload"] = payload if payload is not None else {} + if include_data and payload is not None: + expected["Payload"] = payload assert details.error.to_dict() == expected assert operation.callback_details is not None assert operation.callback_details.error is error