fix: propagate execution-view invoke context

.github/workflows/otel-conformance-tests.yml

@@ -49,7 +49,7 @@ on:
       conformance_test_ref:
         description: Optional conformance test commit SHA or branch name
         required: false
-        default: "05960beb6c81d7b24b53114f2581b3b6f9fef6d4"
+        default: "c32b40f5548f4776b7d533a76838e02358bc66d3"
         type: string
 
 permissions: {}
@@ -69,15 +69,15 @@ jobs:
       actions: write
       contents: read
       id-token: write
-    uses: aws/aws-durable-execution-conformance-tests/.github/workflows/opentelemetry-orchestrator.yml@d8b8399264628dae57b7a96bf4cbf3e39a99fd8d
+    uses: aws/aws-durable-execution-conformance-tests/.github/workflows/opentelemetry-orchestrator.yml@c32b40f5548f4776b7d533a76838e02358bc66d3
     with:
       case_count: 26
       runs_on: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository && 'ubuntu-latest' || format('codebuild-github-actions-runner-{0}-{1}', github.run_id, github.run_attempt) }}
       language: java
       resource_prefix: j
       sdk_repository: aws/aws-durable-execution-sdk-java
       sdk_ref: ${{ github.event.pull_request.head.sha || github.sha }}
-      conformance_test_ref: ${{ inputs.conformance_test_ref || '05960beb6c81d7b24b53114f2581b3b6f9fef6d4' }}
+      conformance_test_ref: ${{ inputs.conformance_test_ref || 'c32b40f5548f4776b7d533a76838e02358bc66d3' }}
       checkout_sdk: true
       # Build the handlers from this repo's checked-out module instead of the conformance repo's
       # bundled examples/java. Path is relative to the conformance workspace where the SDK is

conformance-tests-otel/README.md

@@ -17,6 +17,14 @@ The [SAM template](template.yaml) maps each case to its handler in both views.
 When investigating a run, read the shared requirements at the
 `conformance_test_ref` selected by the [OTel workflow](../.github/workflows/otel-conformance-tests.yml).
 
+Case 11 pauses at `otel-invoke-replay-barrier` after the child invocation returns.
+The shared driver completes that callback after the second caller invocation
+ends. The third invocation replays the saved invoke result and returns it,
+without starting another child or exporting another terminal invoke span.
+The workflow pins the matching requirements from shared conformance PR #131.
+The shared cloud run checks the service trace carrier and full X-Ray ancestry;
+local replay tests do not establish those service-side properties.
+
 The following notes cover cases 21–26.
 
 | Case | Scenario | Behavior |

conformance-tests-otel/src/main/java/software/amazon/lambda/durable/conformance/otel/Otel11ChainedInvoke.java

@@ -12,10 +12,12 @@ public final class Otel11ChainedInvoke extends OtelConformanceHandler<Map<String
     @Override
     public Map<String, Object> handleRequest(Map<String, Object> event, DurableContext context) {
         requireScenario(event, "chained-invoke");
-        return context.invoke(
+        var result = context.invoke(
                 "otel-invoke",
                 System.getenv("OTEL_INVOKE_TARGET_FUNCTION_NAME"),
                 event,
                 new TypeToken<Map<String, Object>>() {});
+        context.waitForCallback("otel-invoke-replay-barrier", String.class, (callbackId, step) -> {});
+        return result;
     }
 }

docs/advanced/propagation-metadata.md

@@ -38,10 +38,18 @@ invoke path collects after that hook and therefore uses the operation context it
 
 The invocation view can read the sampling flags from an operation span that has already started. The execution view
 defers that span until completion, so its placeholder does not establish a per-span sampling decision. It contributes
-metadata only when the visible provider uses `DurableSampler`, including the application-owned builder setup. It
-abstains for replacement samplers and opaque providers whose effective sampler cannot be verified, including opaque
-agent facades. In those cases the backend's omitted-header fallback applies; no new invoke-parent linkage is promised.
-The invocation view applies the same restriction if asked before an operation span is available.
+metadata when the visible provider uses `DurableSampler`, including the application-owned builder setup. With an
+opaque Java-agent provider, the existing autoconfiguration SPI's post-build callback checks the actual provider's final
+sampler. Only an unwrapped `DurableSampler` is accepted, and only that exact instance can confirm consumption of the
+actual Invocation span's sampling intent through the thread-scoped bridge. A filter around `DurableSampler` cannot
+authorize propagation merely by delegating Invocation sampling. A new configuration clears the previous verification;
+an absent callback or an unrecognized final sampler leaves propagation disabled for an opaque provider.
+The confirmation matches the trace ID, span name and execution ARN, restores an enclosing
+observation when nested, and is cleared when the span-creation scope closes. This lets the producer use the deterministic
+invoke parent ID and actual invocation sampling flags without accessing the agent's provider.
+Replacement samplers and opaque providers without confirmation still abstain. In those cases the backend's omitted-header
+fallback applies; no new invoke-parent linkage is promised. The invocation view still requires a visible durable sampler
+if asked before an operation span is available.
 This does not change the configured sampler, create an early operation span, or query the sampler again.
 
 ## Replay and retry

otel-plugin/README.md

@@ -104,12 +104,17 @@ checkpoints carry that context in the flat optional `ChainedInvokeOptions.XAmznT
 AWS SDK for Java 2.56.0 provides the generated typed setter. Deployed parent/child trace behavior still requires
 cloud validation. See [the implemented path and validation status](../docs/advanced/propagation-metadata.md).
 
-Execution-view operation spans are deferred until completion. Their placeholders can provide sampling metadata only
-with a visible provider using the SDK's `DurableSampler`, as configured by the application-owned builder constructor.
-Replacement samplers and opaque provider/agent facades therefore cause this view to abstain; the backend's existing
-omitted-header fallback applies. Invocation view uses the actual started operation span's flags; before that span is
-available, it applies the same sampling-policy restriction. Neither path changes
-the customer's sampler or creates a span just to obtain a sampling decision.
+Execution-view operation spans are deferred until completion. Their placeholders can provide sampling metadata with
+a visible provider using the SDK's `DurableSampler`, or an agent provider verified after SDK autoconfiguration.
+The post-build callback checks the immutable provider's final sampler, after all customizers have run. Only that exact
+`DurableSampler` instance can confirm consumption of this invocation's sampling intent; a later filtering wrapper is
+rejected even if it delegates Invocation sampling to an inner `DurableSampler`. Confirmation crosses the application/agent class-loader boundary through
+the existing thread-scoped bridge and is cleared after span creation. The propagated parent is the deterministic
+invoke span ID, and sampling flags come from the actual Invocation span.
+Replacement samplers and opaque providers without post-build verification and confirmation still abstain; the backend's existing
+omitted-header fallback applies. Invocation view uses the actual started operation span's flags. Before that span is
+available, it requires a visible durable sampler. Neither path changes the customer's sampler, samples again, or
+creates a span just to obtain propagation metadata.
 
 ## Fallback execution roots
 

otel-plugin/pom.xml

@@ -114,6 +114,7 @@
                     -->
                     <environmentVariables>
                         <_X_AMZN_TRACE_ID/>
+                        <OTEL_INVOKE_TARGET_FUNCTION_NAME>case11-target:1</OTEL_INVOKE_TARGET_FUNCTION_NAME>
                     </environmentVariables>
                 </configuration>
             </plugin>

otel-plugin/src/main/java/software/amazon/lambda/durable/otel/DurableSampler.java

@@ -128,6 +128,10 @@ public SamplingResult shouldSample(
                         key -> delegate.shouldSample(
                                 Context.root(), key.traceId(), name, spanKind, attributes, Collections.emptyList()))
                 : intent.resolved();
+        if (OtelPluginAutoConfigurationState.isConfiguredSampler(this)) {
+            DurableSamplingDecision.confirmConsumption(
+                    traceId, name, attributes.get(SpanAttributes.DURABLE_EXECUTION_ARN));
+        }
         if (name.equals("DurableExecutionRoot")
                 && Boolean.TRUE.equals(attributes.get(SpanAttributes.DURABLE_EXECUTION_SYNTHETIC_ROOT))) {
             // Re-exported anchors carry only SDK identity and the sampled decision. Per-invocation sampler

otel-plugin/src/main/java/software/amazon/lambda/durable/otel/DurableSamplingDecision.java

@@ -91,13 +91,53 @@ static Scope openScope(Intent intent) {
         var key = scopedProperty();
         var previous = System.getProperty(key);
         System.setProperty(key, encode(intent));
-        return () -> {
-            if (previous == null) {
-                System.clearProperty(key);
-            } else {
-                System.setProperty(key, previous);
-            }
-        };
+        return () -> restoreProperty(key, previous);
+    }
+
+    /** Observes whether the actual sampler consumes the intent for this span, including across agent class loaders. */
+    static Observation observe(Intent intent, String traceId, String spanName, String executionArn) {
+        var key = observationProperty();
+        var previous = System.getProperty(key);
+        var expected = spanIdentity(traceId, spanName, executionArn);
+        var scope = openScope(intent);
+        System.setProperty(key, expected);
+        return new Observation(scope, key, previous, expected);
+    }
+
+    /** Called only by the verified final sampler after it resolves an intent for the span being sampled. */
+    static void confirmConsumption(String traceId, String spanName, String executionArn) {
+        var key = observationProperty();
+        var expected = System.getProperty(key);
+        if (expected != null && expected.equals(spanIdentity(traceId, spanName, executionArn))) {
+            System.setProperty(key, "CONSUMED:" + expected);
+        }
+    }
+
+    private static String observationProperty() {
+        // An older agent copy that only observes sampler participation must not satisfy final-sampler verification.
+        return scopedProperty() + ".verifiedSamplerObservation";
+    }
+
+    private static String spanIdentity(String traceId, String spanName, String executionArn) {
+        return traceId + "\n" + spanName + "\n" + executionArn;
+    }
+
+    private static void restoreProperty(String key, String previous) {
+        if (previous == null) System.clearProperty(key);
+        else System.setProperty(key, previous);
+    }
+
+    /** A scoped confirmation for one span; it never infers the sampler from the provider's wrapper type. */
+    record Observation(Scope scope, String key, String previous, String expected) implements AutoCloseable {
+        boolean wasConsumed() {
+            return ("CONSUMED:" + expected).equals(System.getProperty(key));
+        }
+
+        @Override
+        public void close() {
+            restoreProperty(key, previous);
+            scope.close();
+        }
     }
 
     /** Consumes both carriers before callbacks can reuse the parent context or cross-loader bridge. */

otel-plugin/src/main/java/software/amazon/lambda/durable/otel/ExecutionOtelPlugin.java

@@ -136,6 +136,7 @@ public final class ExecutionOtelPlugin implements DurableExecutionPlugin {
     private final DurableSamplingDecision.Intent samplingIntent;
 
     private final Span invocationSpan;
+    private final boolean invocationUsesDurableSampler;
     private final Span executionRootSpan;
 
     /**
@@ -280,6 +281,7 @@ private ExecutionOtelPlugin(OtelPluginEnvironment environment, InvocationInfo in
             this.executionTrace = null;
             this.executionAncestor = null;
             this.invocationSpan = null;
+            this.invocationUsesDurableSampler = false;
             this.executionRootSpan = null;
             this.workflowSpanContext = null;
             return;
@@ -343,7 +345,16 @@ private ExecutionOtelPlugin(OtelPluginEnvironment environment, InvocationInfo in
             spanBuilder.setAttribute(AttributeKey.stringKey("faas.invocation_id"), info.requestId());
         }
 
-        this.invocationSpan = startDurableSpan(spanBuilder);
+        if (OtelPluginSupport.usesDurableSamplingBridge(sdkTracerProvider)) {
+            try (var observation = DurableSamplingDecision.observe(
+                    samplingIntent, canonicalTraceId, "Invocation", durableExecutionArn)) {
+                this.invocationSpan = spanBuilder.startSpan();
+                this.invocationUsesDurableSampler = observation.wasConsumed();
+            }
+        } else {
+            this.invocationSpan = spanBuilder.startSpan();
+            this.invocationUsesDurableSampler = false;
+        }
 
         // Defer the recording Workflow span until terminal completion. The placeholder uses the Invocation span's
         // resolved sampling metadata so operation parents/links match the span that is eventually exported.
@@ -456,9 +467,11 @@ public String providePropagationMetadata(
         if (disabled() || !executionArn.equals(durableExecutionArn)) return null;
         var trace = executionTrace;
         if (trace == null) return null;
-        // These contexts are deferred placeholders, not sampled operation spans. A replacement sampler may decide
-        // differently for Invocation and invoke; an opaque provider does not expose its effective policy.
-        if (sdkTracerProvider == null || !OtelPluginSupport.usesDurableSamplingBridge(sdkTracerProvider)) return null;
+        // Deferred operation contexts may reuse the invocation's decision only when DurableSampler owns it.
+        // An agent hides its provider, so require a bridge confirmation from its verified final sampler instance.
+        // A replacement sampler can decide differently per span and must continue to abstain here.
+        if (!OtelPluginSupport.usesDurableSamplingBridge(sdkTracerProvider)
+                || (sdkTracerProvider == null && !invocationUsesDurableSampler)) return null;
         var context = operationContexts.get(operationId);
         if (context == null) {
             // Before a new operation starts, use the same deterministic ID its initial span will receive.

otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginAutoConfigurationCustomizerProvider.java

@@ -2,14 +2,17 @@
 // SPDX-License-Identifier: Apache-2.0
 package software.amazon.lambda.durable.otel;
 
+import io.opentelemetry.sdk.OpenTelemetrySdk;
 import io.opentelemetry.sdk.autoconfigure.spi.AutoConfigurationCustomizer;
 import io.opentelemetry.sdk.autoconfigure.spi.AutoConfigurationCustomizerProvider;
+import io.opentelemetry.sdk.autoconfigure.spi.internal.AutoConfigureListener;
 
 /**
  * Wraps the Java agent's configured ID generator with scoped durable-execution overrides so durable spans get
  * deterministic IDs.
  */
-public final class OtelPluginAutoConfigurationCustomizerProvider implements AutoConfigurationCustomizerProvider {
+public final class OtelPluginAutoConfigurationCustomizerProvider
+        implements AutoConfigurationCustomizerProvider, AutoConfigureListener {
 
     @Override
     public void customize(AutoConfigurationCustomizer autoConfiguration) {
@@ -23,6 +26,14 @@ public void customize(AutoConfigurationCustomizer autoConfiguration) {
         autoConfiguration.addSamplerCustomizer((sampler, config) -> DurableSampler.wrap(sampler));
     }
 
+    @Override
+    public void afterAutoConfigure(OpenTelemetrySdk sdk) {
+        // This callback runs after every customizer and provider build. Checking the immutable provider here catches
+        // even a later customizer wrapping DurableSampler, without inspecting or modifying another extension's filter.
+        OtelPluginAutoConfigurationState.recordConfiguredSampler(
+                sdk.getSdkTracerProvider().getSampler());
+    }
+
     @Override
     public int order() {
         return Integer.MAX_VALUE;

otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginAutoConfigurationState.java

@@ -2,10 +2,14 @@
 // SPDX-License-Identifier: Apache-2.0
 package software.amazon.lambda.durable.otel;
 
+import io.opentelemetry.sdk.trace.samplers.Sampler;
+
 final class OtelPluginAutoConfigurationState {
 
     private static final String INSTALLED_PROPERTY =
             "software.amazon.lambda.durable.otel.autoConfigurationCustomizerProviderInstalled";
+    // Kept in the agent class loader: only the exact final sampler may confirm an application's propagation request.
+    private static volatile DurableSampler configuredSampler;
 
     private OtelPluginAutoConfigurationState() {}
 
@@ -14,10 +18,20 @@ static boolean isInstalled() {
     }
 
     static void markInstalled() {
+        configuredSampler = null;
         System.setProperty(INSTALLED_PROPERTY, Boolean.TRUE.toString());
     }
 
+    static void recordConfiguredSampler(Sampler sampler) {
+        configuredSampler = sampler instanceof DurableSampler durableSampler ? durableSampler : null;
+    }
+
+    static boolean isConfiguredSampler(DurableSampler sampler) {
+        return sampler == configuredSampler;
+    }
+
     static void resetInstalledForTest() {
+        configuredSampler = null;
         System.clearProperty(INSTALLED_PROPERTY);
     }
 }