Report failed flaky tests as skip

.gitlab/collect-result/CollectResults.java

@@ -3,12 +3,15 @@
 
 class CollectResults {
   public static void main(String[] args) throws Exception {
+    // Detect flaky jobs
+    var continueOnFailure = Boolean.parseBoolean(System.getenv("CONTINUE_ON_FAILURE"));
+    // Run collector
     var collector =
         new ResultCollector(
             Path.of("results"),
             Path.of("workspace"),
-            List.of(Path.of("workspace"), Path.of("buildSrc")));
-
+            List.of(Path.of("workspace"), Path.of("buildSrc")),
+            continueOnFailure);
     collector.collect();
   }
 }

.gitlab/collect-result/JUnitReport.java

@@ -153,6 +153,29 @@ void tagFinalStatuses() {
     }
   }
 
+  /// Marks every testcase as `test.final_status=skip` for jobs that tolerate failures.
+  ///
+  /// The flaky test jobs (`test_flaky`, `test_flaky_inst`) run with `CONTINUE_ON_FAILURE=true`, so
+  /// their result never gates the pipeline: it runs to completion regardless of pass or fail.
+  /// `test.final_status` records that CI impact rather than the raw outcome, so every test in these
+  /// jobs is `skip` — a failure is a non-blocking failure and a pass is a non-blocking pass. This
+  /// keeps flaky failures from creating false-positive notifications and skewing SLIs, while the
+  /// real per-test outcome stays available in `test.status` (derived from the `<failure>`,
+  /// `<error>`, and `<skipped>` children, which are left in place). Always-green tests that could
+  /// leave the flaky pipeline are then found with `@test.status:pass @test.final_status:skip`.
+  ///
+  /// **Must run before {@link #tagFinalStatuses()}** so the natural pass/fail status is never
+  /// assigned. Testcases already tagged by {@link #tagRetriedAttempts()} or
+  /// {@link #tagSyntheticFailures()} are left untouched (already `skip`).
+  void tagAllAsSkipped() {
+    for (var testcase : testcases()) {
+      if (hasFinalStatusProperty(testcase)) {
+        continue;
+      }
+      addFinalStatusProperty(testcase, "skip", MissingPropertiesPlacement.FIRST_CHILD);
+    }
+  }
+
   void write(Path xmlFile) throws Exception {
     Files.createDirectories(xmlFile.getParent());
     var tmpFile = Files.createTempFile(xmlFile.getParent(), "collect-results-", ".xml");

.gitlab/collect-result/ResultCollector.java

@@ -13,12 +13,15 @@ final class ResultCollector {
   private final Path resultsDir;
   private final Path workspaceDir;
   private final List<Path> searchDirs;
+  private final boolean continueOnFailure;
   private final SourceFileResolver sourceFileResolver;
 
-  ResultCollector(Path resultsDir, Path workspaceDir, List<Path> searchDirs) {
+  ResultCollector(
+      Path resultsDir, Path workspaceDir, List<Path> searchDirs, boolean continueOnFailure) {
     this.resultsDir = resultsDir;
     this.workspaceDir = workspaceDir;
     this.searchDirs = searchDirs;
+    this.continueOnFailure = continueOnFailure;
     this.sourceFileResolver = new SourceFileResolver(workspaceDir);
   }
 
@@ -32,6 +35,10 @@ void collect() throws Exception {
       return;
     }
 
+    if (continueOnFailure) {
+      System.out.println("CONTINUE_ON_FAILURE=true: reporting all tests as skip");
+    }
+
     System.out.println("Saving test results:");
     for (var sourceXml : findXmlFiles(testResultDirs)) {
       collect(sourceXml);
@@ -51,6 +58,11 @@ private void collect(Path sourceXml) throws Exception {
     report.tagRetriedAttempts();
     reportChangedBeforeFinalStatus |= report.normalizeStableTestNames();
     report.tagSyntheticFailures();
+    // Flaky jobs (CONTINUE_ON_FAILURE=true) never gate CI, so record every test as skip before
+    // assigning natural statuses (APMLP-1267).
+    if (continueOnFailure) {
+      report.tagAllAsSkipped();
+    }
     report.tagFinalStatuses();
     report.write(targetXml);
 

AGENTS.md

@@ -68,7 +68,7 @@ docs/                     Developer documentation (see below)
 
 - Title: imperative verb sentence describing user-visible change (e.g. "Fix span sampling rule parsing")
 - Labels: always add `tag: ai generated` and at least one `comp:` or `inst:` label + one `type:` label
-- Use `tag: no release note` for internal/refactoring changes
+- Use `tag: no release notes` for internal/refactoring changes
 - Open as draft first, convert to ready when reviewable
 
 ## Bootstrap constraints (critical)

CONTRIBUTING.md

@@ -118,7 +118,7 @@ Both pull requests and issues should be labelled with at least a component or an
 
 Labels are used to not only categorize but also alter the continuous integration behavior:
 
-* `tag: no release note` to exclude a pull request from the next release changelog. Use it when changes are not relevant to the users like:
+* `tag: no release notes` to exclude a pull request from the next release changelog. Use it when changes are not relevant to the users like:
   * Internal features changes
   * Refactoring pull requests
   * CI and build tools improvements