ci: preserve sanitizer diagnostics and failed suite logs

.github/workflows/_test.yml

@@ -187,6 +187,23 @@ jobs:
           # shard-completeness job still gates the cross-shard union.
           if-no-files-found: warn
 
+      # The job log carries a red suite's SUMMARY; these are the complete logs
+      # behind it, so a failure can be read instead of re-run. The harness
+      # copies the log of every suite that did not end green, plus results.txt,
+      # into test-logs/failed/ (scripts/suite-failure-report.sh) — typically a
+      # few kilobytes for a red suite, not the ~25 MB of a whole run's logs.
+      # Runs only on a failed job. warn, not error: a job that failed OUTSIDE
+      # the suite wave (contract step, build, prod-binary guards) has no such
+      # directory, and a second error would only bury the first.
+      - name: Failing suite logs
+        if: failure()
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: suite-logs-${{ matrix.os }}-${{ matrix.cc }}-${{ strategy.job-index }}
+          path: build/c/test-logs/failed/
+          retention-days: 7
+          if-no-files-found: warn
+
       - name: Compiler cache (save)
         if: always() && (matrix.shard == null || startsWith(matrix.shard, '1/'))
         uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
@@ -282,6 +299,17 @@ jobs:
           scripts/test.sh CC=clang-21 CXX=clang++-21 \
             SANITIZE="-fsanitize=address,undefined -fno-omit-frame-pointer -fno-optimize-sibling-calls"
 
+      # Same step as in test-unix: the complete logs of the suites that did
+      # not end green, only when the job failed.
+      - name: Failing suite logs
+        if: failure()
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: suite-logs-diag
+          path: build/c/test-logs/failed/
+          retention-days: 7
+          if-no-files-found: warn
+
       - name: Compiler cache (save)
         if: always()
         uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
@@ -364,6 +392,17 @@ jobs:
           LLVM_PREFIX="$(brew --prefix llvm@22)"
           scripts/test.sh CC="$LLVM_PREFIX/bin/clang" CXX="$LLVM_PREFIX/bin/clang++"
 
+      # Same step as in test-unix: the complete logs of the suites that did
+      # not end green, only when the job failed.
+      - name: Failing suite logs
+        if: failure()
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: suite-logs-lsan-macos
+          path: build/c/test-logs/failed/
+          retention-days: 7
+          if-no-files-found: warn
+
       - name: Compiler cache (save)
         if: always()
         uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
@@ -531,6 +570,17 @@ jobs:
           # shard-completeness job still gates the cross-shard union.
           if-no-files-found: warn
 
+      # Same step as in test-unix: the complete logs of the suites that did
+      # not end green, only when the job failed.
+      - name: Failing suite logs
+        if: failure()
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: suite-logs-${{ matrix.os }}-${{ matrix.msystem }}-${{ strategy.job-index }}
+          path: build/c/test-logs/failed/
+          retention-days: 7
+          if-no-files-found: warn
+
       # Shard 1 (or the unsharded job) saves the compiler cache; every shard
       # builds the identical runner, so one cache carries the full object set.
       - name: Compiler cache (save)

scripts/README.md

@@ -29,7 +29,10 @@ codes) and rejects unknown flags with exit 2 + `Please consult --help.`
 Internal harnesses — never called directly by a venue (the contract forbids
 it): `smoke-test.sh` (phases; wrappers provide fixture server + sandbox),
 `soak-test.sh` (one soak run; `soak-legs.sh` provides the sequence + guards),
-`run-tests-parallel.sh` (reached through `test.sh`).
+`run-tests-parallel.sh` (reached through `test.sh`), which sources
+`suite-failure-report.sh` for its end-of-run failure summary (failure sites,
+sanitizer report, running test, last lines) and for `test-logs/failed/`, the
+logs of every suite that did not end green.
 
 ## Conventions
 

scripts/run-test-wave.py

@@ -26,6 +26,9 @@
 SUMMARY = re.compile(r"^  (?P<passed>[0-9]+) passed")
 FAILED = re.compile(r"(?:^|, )(?P<failed>[0-9]+) failed")
 SKIPPED = re.compile(r"(?:^|, )(?P<skipped>[0-9]+) skipped")
+# One finished test, as RUN_TEST prints it: "PASS" ending the test's own line,
+# or alone on a line when the test wrote to stderr in between.
+PASS_MARKER = re.compile(r"^(?:  [A-Za-z_].*)?PASS\s*$")
 # Suites whose honest runtime does not fit the default per-suite budget, and so
 # get --slow-timeout instead. This is a statement about SIZE, never about
 # flakiness: every suite here is deterministic and simply long, and a racy suite
@@ -393,6 +396,20 @@ def parse_summary(log_path: pathlib.Path) -> tuple[int, int, int] | None:
     return last_summary
 
 
+def count_pass_markers(log_path: pathlib.Path) -> int:
+    """Tests that finished before a suite died without its completion summary.
+
+    The framework flushes each test's name before running it, so every PASS
+    ahead of the point of death is already in the log.
+    """
+    try:
+        stream = log_path.open(encoding="utf-8", errors="replace")
+    except OSError as exc:
+        raise RuntimeError(f"cannot read suite log {log_path}: {exc}") from exc
+    with stream:
+        return sum(1 for line in stream if PASS_MARKER.match(line) is not None)
+
+
 def record_result(
     active: ActiveSuite,
     returncode: int,
@@ -416,7 +433,15 @@ def record_result(
             f"  FAIL: suite {active.name!r} exited 0 without a completion summary "
             "(ran nothing?)",
         )
-    passed, failed, skipped = summary or (0, 0, 0)
+    # WHY: a suite that aborts, is killed or times out never prints its summary
+    # line, and reporting it as pass=0 said "ran nothing" about a suite that
+    # had finished dozens of tests. Count what the log proves instead. Only
+    # passes: a death is not a counted test failure, and rc already makes the
+    # run red. A suite WITH a summary is never recounted, so green totals are
+    # exactly what they were.
+    if summary is None:
+        summary = (count_pass_markers(active.log_path), 0, 0)
+    passed, failed, skipped = summary
     result = (
         f"{active.name} rc={returncode} pass={passed} fail={failed} "
         f"skip={skipped} secs={elapsed}"

scripts/run-tests-parallel.sh

@@ -24,6 +24,11 @@ RUNNER="${1:?usage: run-tests-parallel.sh <path-to-test-runner> [jobs]}"
 JOBS="${2:-${CBM_TEST_PAR_JOBS:-}}"
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 SCHEDULER="$SCRIPT_DIR/run-test-wave.py"
+# The end-of-run failure summary and the failing-log selection live in one
+# sourced file so tests/test_failure_report_contract.sh can drive the real
+# functions with fabricated logs.
+# shellcheck source=suite-failure-report.sh
+source "$SCRIPT_DIR/suite-failure-report.sh"
 
 if [ -z "$JOBS" ]; then
     if command -v nproc >/dev/null 2>&1; then
@@ -251,6 +256,14 @@ NSHARD=$(wc -l < "$SHARD_EXPECT" | tr -d ' ')
 } > "$LOGDIR/shard-manifest.txt"
 echo "=== parallel test run: $NSHARD of $NSUITES suites (shard ${SHARD_INDEX}/${SHARD_TOTAL}, $(wc -l < "$SER_FILE" | tr -d ' ') serial-tail), $JOBS jobs ==="
 
+# Leave the logs of every non-green suite in $LOGDIR/failed for CI to upload
+# when the job fails. An EXIT trap rather than a call next to the summary
+# below, because the run has three ways out — the scheduler giving up, the
+# union guard, and the normal end — and the first two are exactly the runs
+# whose logs are otherwise unrecoverable. The trap only copies files: it never
+# calls exit, so the status this script ends with is untouched.
+trap 'collect_failed_suite_logs "$LOGDIR" "$RESULTS_FILE" "$SHARD_EXPECT"' EXIT
+
 # Per-suite wall-clock ceilings make a wedged child fail loudly. The
 # `incremental` suite legitimately re-indexes large fixtures (minutes), while
 # `daemon_runtime` measures ~610s solo on arm64 under ASan; those and
@@ -337,10 +350,7 @@ echo "── 8 slowest suites ──"
 sort -t= -k6 -rn "$RESULTS_FILE" | head -8
 grep -v ' rc=0 ' "$RESULTS_FILE" || true
 for f in $(grep -v ' rc=0 ' "$RESULTS_FILE" | awk '{print $1}'); do
-    echo "──── $f: every failure site ────"
-    grep -B2 -A8 "FAIL" "$LOGDIR/$f.log" | head -120
-    echo "──── $f: last 15 lines ────"
-    tail -15 "$LOGDIR/$f.log"
+    suite_failure_summary "$f" "$LOGDIR/$f.log"
 done
 
 echo "────────────────────────────────────────────"