Add execution instrumentation for Karate v2

dd-java-agent/instrumentation/karate/karate-2.0/src/main/java/datadog/trace/instrumentation/karate2/KarateExecutionInstrumentation.java

@@ -0,0 +1,87 @@
+package datadog.trace.instrumentation.karate2;
+
+import static datadog.trace.agent.tooling.bytebuddy.matcher.NameMatchers.named;
+import static net.bytebuddy.matcher.ElementMatchers.isBridge;
+import static net.bytebuddy.matcher.ElementMatchers.not;
+import static net.bytebuddy.matcher.ElementMatchers.returns;
+import static net.bytebuddy.matcher.ElementMatchers.takesArgument;
+import static net.bytebuddy.matcher.ElementMatchers.takesArguments;
+import static net.bytebuddy.matcher.ElementMatchers.takesNoArguments;
+
+import com.google.auto.service.AutoService;
+import datadog.trace.agent.tooling.Instrumenter;
+import datadog.trace.agent.tooling.InstrumenterModule;
+import datadog.trace.api.Config;
+import java.util.Collections;
+import java.util.Map;
+
+/**
+ * Drives test-retry execution policies (ATR / EFD / attempt-to-fix) and failure suppression for
+ * Karate v2.
+ *
+ * <ul>
+ *   <li>{@code ScenarioRuntime#call()} is advised to re-run the scenario while the execution policy
+ *       is applicable, overriding the returned {@code ScenarioResult} with the final attempt.
+ *   <li>{@code ScenarioResult#addStepResult(StepResult)} is advised to replace a failing step with
+ *       a skipped one when the policy requests failure suppression.
+ * </ul>
+ *
+ * Compiled for Java 8 (see {@link KarateInstrumentation}); the advice lives in the {@code java21}
+ * source set.
+ */
+@AutoService(InstrumenterModule.class)
+public class KarateExecutionInstrumentation extends InstrumenterModule.CiVisibility
+    implements Instrumenter.ForKnownTypes, Instrumenter.HasMethodAdvice {
+
+  public KarateExecutionInstrumentation() {
+    super("ci-visibility", "karate", "test-retry");
+  }
+
+  @Override
+  public boolean isEnabled() {
+    return super.isEnabled() && Config.get().isCiVisibilityExecutionPoliciesEnabled();
+  }
+
+  @Override
+  public String[] knownMatchingTypes() {
+    return new String[] {"io.karatelabs.core.ScenarioRuntime", "io.karatelabs.core.ScenarioResult"};
+  }
+
+  @Override
+  public String[] helperClassNames() {
+    return new String[] {
+      packageName + ".KarateUtils",
+      packageName + ".TestEventsHandlerHolder",
+      packageName + ".ExecutionContext",
+      packageName + ".KarateTracingListener",
+      packageName + ".KarateScenarioAdvice",
+      packageName + ".KarateScenarioAdvice$RetryAdvice",
+      packageName + ".KarateScenarioAdvice$SuppressErrorAdvice"
+    };
+  }
+
+  @Override
+  public Map<String, String> contextStore() {
+    return Collections.singletonMap(
+        "io.karatelabs.gherkin.Scenario", packageName + ".ExecutionContext");
+  }
+
+  @Override
+  public void methodAdvice(MethodTransformer transformer) {
+    // ScenarioRuntime#call() is the run()-equivalent; match the concrete ScenarioResult-returning
+    // method, not the synthetic Callable#call() bridge.
+    transformer.applyAdvice(
+        named("call")
+            .and(takesNoArguments())
+            .and(returns(named("io.karatelabs.core.ScenarioResult")))
+            .and(not(isBridge())),
+        packageName + ".KarateScenarioAdvice$RetryAdvice");
+
+    // ScenarioResult#addStepResult(StepResult)
+    transformer.applyAdvice(
+        named("addStepResult")
+            .and(takesArguments(1))
+            .and(takesArgument(0, named("io.karatelabs.core.StepResult"))),
+        packageName + ".KarateScenarioAdvice$SuppressErrorAdvice");
+  }
+}

dd-java-agent/instrumentation/karate/karate-2.0/src/main/java/datadog/trace/instrumentation/karate2/KarateInstrumentation.java

@@ -5,6 +5,8 @@
 import com.google.auto.service.AutoService;
 import datadog.trace.agent.tooling.Instrumenter;
 import datadog.trace.agent.tooling.InstrumenterModule;
+import java.util.Collections;
+import java.util.Map;
 
 /**
  * Registers a {@code io.karatelabs.core.RunListener} on every {@code
@@ -32,11 +34,18 @@ public String[] helperClassNames() {
     return new String[] {
       packageName + ".KarateUtils",
       packageName + ".TestEventsHandlerHolder",
+      packageName + ".ExecutionContext",
       packageName + ".KarateTracingListener",
       packageName + ".KarateBuilderAdvice"
     };
   }
 
+  @Override
+  public Map<String, String> contextStore() {
+    return Collections.singletonMap(
+        "io.karatelabs.gherkin.Scenario", packageName + ".ExecutionContext");
+  }
+
   @Override
   public void methodAdvice(MethodTransformer transformer) {
     transformer.applyAdvice(isConstructor(), packageName + ".KarateBuilderAdvice");

dd-java-agent/instrumentation/karate/karate-2.0/src/main/java21/datadog/trace/instrumentation/karate2/ExecutionContext.java

@@ -0,0 +1,56 @@
+package datadog.trace.instrumentation.karate2;
+
+import datadog.trace.api.civisibility.config.TestIdentifier;
+import datadog.trace.api.civisibility.config.TestSourceData;
+import datadog.trace.api.civisibility.execution.TestExecutionPolicy;
+import io.karatelabs.gherkin.Scenario;
+import java.util.Collection;
+
+public class ExecutionContext {
+
+  private final TestExecutionPolicy executionPolicy;
+  private boolean suppressFailures;
+  private Throwable suppressedError;
+
+  public ExecutionContext(TestExecutionPolicy executionPolicy) {
+    this.executionPolicy = executionPolicy;
+  }
+
+  public void setSuppressFailures(boolean suppressFailures) {
+    this.suppressFailures = suppressFailures;
+  }
+
+  public boolean shouldSuppressFailures() {
+    return suppressFailures;
+  }
+
+  public TestExecutionPolicy getExecutionPolicy() {
+    return executionPolicy;
+  }
+
+  /**
+   * Karate v2 {@code StepResult} is immutable (no {@code setFailedReason}/{@code setErrorIgnored}),
+   * so a failure that was suppressed for retry purposes cannot be carried on the replacement step.
+   * We stash it here instead, so the tracing listener can still report the failure to CI
+   * Visibility.
+   */
+  public void setSuppressedError(Throwable suppressedError) {
+    if (this.suppressedError == null) {
+      this.suppressedError = suppressedError;
+    }
+  }
+
+  public Throwable getAndClearSuppressedError() {
+    Throwable suppressedError = this.suppressedError;
+    this.suppressedError = null;
+    return suppressedError;
+  }
+
+  public static ExecutionContext create(Scenario scenario) {
+    TestIdentifier testIdentifier = KarateUtils.toTestIdentifier(scenario);
+    Collection<String> testTags = KarateUtils.getCategories(scenario.getTagsEffective());
+    return new ExecutionContext(
+        TestEventsHandlerHolder.TEST_EVENTS_HANDLER.executionPolicy(
+            testIdentifier, TestSourceData.UNKNOWN, testTags));
+  }
+}

dd-java-agent/instrumentation/karate/karate-2.0/src/main/java21/datadog/trace/instrumentation/karate2/KarateBuilderAdvice.java

@@ -1,16 +1,21 @@
 package datadog.trace.instrumentation.karate2;
 
+import datadog.trace.bootstrap.ContextStore;
+import datadog.trace.bootstrap.InstrumentationContext;
 import io.karatelabs.core.RunEvent;
 import io.karatelabs.core.RunListener;
 import io.karatelabs.core.Runner;
+import io.karatelabs.gherkin.Scenario;
 import net.bytebuddy.asm.Advice;
 
 /** Advice for the {@code io.karatelabs.core.Runner.Builder} constructor. */
 public class KarateBuilderAdvice {
 
   @Advice.OnMethodExit
   public static void onRunnerBuilderConstructorExit(@Advice.This Runner.Builder builder) {
-    builder.listener(new KarateTracingListener());
+    ContextStore<Scenario, ExecutionContext> scenarioContext =
+        InstrumentationContext.get(Scenario.class, ExecutionContext.class);
+    builder.listener(new KarateTracingListener(scenarioContext));
   }
 
   // Karate 2.0.0 and above

dd-java-agent/instrumentation/karate/karate-2.0/src/main/java21/datadog/trace/instrumentation/karate2/KarateScenarioAdvice.java

@@ -0,0 +1,100 @@
+package datadog.trace.instrumentation.karate2;
+
+import datadog.trace.api.civisibility.execution.TestExecutionPolicy;
+import datadog.trace.bootstrap.CallDepthThreadLocalMap;
+import datadog.trace.bootstrap.InstrumentationContext;
+import io.karatelabs.core.RunEvent;
+import io.karatelabs.core.RunListener;
+import io.karatelabs.core.ScenarioResult;
+import io.karatelabs.core.ScenarioRuntime;
+import io.karatelabs.core.StepResult;
+import io.karatelabs.gherkin.Scenario;
+import net.bytebuddy.asm.Advice;
+
+/** Advice classes for {@code io.karatelabs.core.ScenarioRuntime}/{@code ScenarioResult}. */
+public class KarateScenarioAdvice {
+
+  public static class RetryAdvice {
+    @Advice.OnMethodEnter
+    public static void beforeExecute(@Advice.This ScenarioRuntime scenarioRuntime) {
+      ExecutionContext executionContext =
+          InstrumentationContext.get(Scenario.class, ExecutionContext.class)
+              .computeIfAbsent(scenarioRuntime.getScenario(), ExecutionContext::create);
+
+      // Indicate beforehand whether failures should be suppressed. This aligns the ordering with
+      // the rest of the frameworks.
+      TestExecutionPolicy executionPolicy = executionContext.getExecutionPolicy();
+      executionContext.setSuppressFailures(executionPolicy.suppressFailures());
+    }
+
+    @Advice.OnMethodExit
+    public static void afterExecute(
+        @Advice.This ScenarioRuntime scenarioRuntime,
+        @Advice.Return(readOnly = false) ScenarioResult result) {
+      if (CallDepthThreadLocalMap.incrementCallDepth(ScenarioRuntime.class) > 0) {
+        // nested call (a retry invoked below, or a called scenario)
+        return;
+      }
+
+      try {
+        Scenario scenario = scenarioRuntime.getScenario();
+        ExecutionContext context =
+            InstrumentationContext.get(Scenario.class, ExecutionContext.class).get(scenario);
+        if (context == null) {
+          return;
+        }
+
+        ScenarioResult finalResult = result;
+        TestExecutionPolicy executionPolicy = context.getExecutionPolicy();
+        while (executionPolicy.applicable()) {
+          ScenarioRuntime retry =
+              new ScenarioRuntime(scenarioRuntime.getFeatureRuntime(), scenario);
+          finalResult = retry.call();
+        }
+
+        // override the return value so the final attempt is the one recorded.
+        result = finalResult;
+      } finally {
+        CallDepthThreadLocalMap.reset(ScenarioRuntime.class);
+      }
+    }
+
+    // Karate 2.0.0 and above
+    public static void muzzleCheck(RunListener runListener) {
+      runListener.onEvent((RunEvent) null);
+    }
+  }
+
+  public static class SuppressErrorAdvice {
+    @Advice.OnMethodEnter
+    public static void onAddingStepResult(
+        @Advice.Argument(value = 0, readOnly = false) StepResult stepResult,
+        @Advice.FieldValue("scenario") Scenario scenario) {
+
+      if (stepResult.isFailed()) {
+        ExecutionContext executionContext =
+            InstrumentationContext.get(Scenario.class, ExecutionContext.class).get(scenario);
+        if (executionContext == null) {
+          return;
+        }
+
+        // Suppress every failing step of a to-be-retried attempt (not just the first): with
+        // continueOnStepFailure a single attempt can add multiple failing steps, and any leak
+        // would mark the retry attempt's result failed.
+        if (executionContext.shouldSuppressFailures()) {
+          // v2 StepResult is immutable: preserve the error out-of-band, then replace the failing
+          // step with a skipped one so the scenario no longer counts as failed.
+          executionContext.setSuppressedError(stepResult.getError());
+          stepResult = StepResult.skipped(stepResult.getStep(), stepResult.getStartTime());
+        }
+      }
+    }
+
+    // Karate 2.0.0 and above
+    public static void muzzleCheck(RunListener runListener) {
+      runListener.onEvent((RunEvent) null);
+    }
+  }
+
+  private KarateScenarioAdvice() {}
+}