Checkpoint all work and database hot-path regression scope

Author: KSemenenko

.github/workflows/ci.yml

@@ -187,30 +187,30 @@ jobs:
       - run: dotnet build tests/KeyLoad.ComparisonTests --no-restore --configuration Release
       - name: Qualify native image export and import
         id: roundtrip
-        run: dotnet test --project tests/KeyLoad.ComparisonTests --no-build --no-restore --configuration Release --treenode-filter '/*/*/ImageBundleRealTests/*'
+        run: dotnet test --project tests/KeyLoad.ComparisonTests --no-build --no-restore --configuration Release --treenode-filter '/*/*/ImageBundleRealTests/*' --results-directory TestResults/comparison-images/image-roundtrip
       - name: Qualify authenticated current-job identity
-        run: dotnet test --project tests/KeyLoad.ComparisonTests --no-build --no-restore --configuration Release --treenode-filter '/*/*/IsolatedGitHubCurrentJobTests/*'
+        run: dotnet test --project tests/KeyLoad.ComparisonTests --no-build --no-restore --configuration Release --treenode-filter '/*/*/IsolatedGitHubCurrentJobTests/*' --results-directory TestResults/comparison-images/job-identity
       - name: Qualify exact native topology composition models
         run: |
-          dotnet test --project tests/KeyLoad.ComparisonTests --no-build --no-restore --configuration Release --treenode-filter '/*/*/Isolated*Resource*/*'
-          dotnet test --project tests/KeyLoad.ComparisonTests --no-build --no-restore --configuration Release --treenode-filter '/*/*/MongoReadinessResourceTests/*'
-          dotnet test --project tests/KeyLoad.ComparisonTests --no-build --no-restore --configuration Release --treenode-filter '/*/*/MongoReadinessIdentityTests/*'
-          dotnet test --project tests/KeyLoad.ComparisonTests --no-build --no-restore --configuration Release --treenode-filter '/*/*/MongoReadinessTaskFailureTests/*'
-          dotnet test --project tests/KeyLoad.ComparisonTests --no-build --no-restore --configuration Release --treenode-filter '/*/*/IsolatedKurrentVolumeRegressionFailureTests/*'
-          dotnet test --project tests/KeyLoad.ComparisonTests --no-build --no-restore --configuration Release --treenode-filter '/*/*/IsolatedKeyLoadAdmissionTests/*'
-          dotnet test --project tests/KeyLoad.ComparisonTests --no-build --no-restore --configuration Release --treenode-filter '/*/*/ComparisonResourceLogBufferTests/*'
+          dotnet test --project tests/KeyLoad.ComparisonTests --no-build --no-restore --configuration Release --treenode-filter '/*/*/Isolated*Resource*/*' --results-directory TestResults/comparison-images/resource-models
+          dotnet test --project tests/KeyLoad.ComparisonTests --no-build --no-restore --configuration Release --treenode-filter '/*/*/MongoReadinessResourceTests/*' --results-directory TestResults/comparison-images/mongo-resource
+          dotnet test --project tests/KeyLoad.ComparisonTests --no-build --no-restore --configuration Release --treenode-filter '/*/*/MongoReadinessIdentityTests/*' --results-directory TestResults/comparison-images/mongo-identity
+          dotnet test --project tests/KeyLoad.ComparisonTests --no-build --no-restore --configuration Release --treenode-filter '/*/*/MongoReadinessTaskFailureTests/*' --results-directory TestResults/comparison-images/mongo-task-failure
+          dotnet test --project tests/KeyLoad.ComparisonTests --no-build --no-restore --configuration Release --treenode-filter '/*/*/IsolatedKurrentVolumeRegressionFailureTests/*' --results-directory TestResults/comparison-images/kurrent-volume-failure
+          dotnet test --project tests/KeyLoad.ComparisonTests --no-build --no-restore --configuration Release --treenode-filter '/*/*/IsolatedKeyLoadAdmissionTests/*' --results-directory TestResults/comparison-images/keyload-admission
+          dotnet test --project tests/KeyLoad.ComparisonTests --no-build --no-restore --configuration Release --treenode-filter '/*/*/ComparisonResourceLogBufferTests/*' --results-directory TestResults/comparison-images/log-buffer
       - name: Qualify isolated replay admission and native diagnostics
         run: |
-          dotnet test --project tests/KeyLoad.ComparisonTests --no-build --no-restore --configuration Release --treenode-filter '/*/*/IsolatedKeyLoadReplayAdmissionTests/*'
-          dotnet test --project tests/KeyLoad.ComparisonTests --no-build --no-restore --configuration Release --treenode-filter '/*/*/RedisReplicaDiagnosticTests/*'
-          dotnet test --project tests/KeyLoad.ComparisonTests --no-build --no-restore --configuration Release --treenode-filter '/*/*/MongoBootstrapDiagnosticTests/*'
-          dotnet test --project tests/KeyLoad.ComparisonTests --no-build --no-restore --configuration Release --treenode-filter '/*/*/KurrentCleanupDiagnosticTests/*'
-          dotnet test --project tests/KeyLoad.ComparisonTests --no-build --no-restore --configuration Release --treenode-filter '/*/*/ComparisonReplayDiagnosticLogTests/*'
-          dotnet test --project tests/KeyLoad.ComparisonTests --no-build --no-restore --configuration Release --treenode-filter '/*/*/ComparisonReplayDiagnosticRetentionTests/*'
+          dotnet test --project tests/KeyLoad.ComparisonTests --no-build --no-restore --configuration Release --treenode-filter '/*/*/IsolatedKeyLoadReplayAdmissionTests/*' --results-directory TestResults/comparison-images/replay-admission
+          dotnet test --project tests/KeyLoad.ComparisonTests --no-build --no-restore --configuration Release --treenode-filter '/*/*/RedisReplicaDiagnosticTests/*' --results-directory TestResults/comparison-images/redis-diagnostics
+          dotnet test --project tests/KeyLoad.ComparisonTests --no-build --no-restore --configuration Release --treenode-filter '/*/*/MongoBootstrapDiagnosticTests/*' --results-directory TestResults/comparison-images/mongo-diagnostics
+          dotnet test --project tests/KeyLoad.ComparisonTests --no-build --no-restore --configuration Release --treenode-filter '/*/*/KurrentCleanupDiagnosticTests/*' --results-directory TestResults/comparison-images/kurrent-diagnostics
+          dotnet test --project tests/KeyLoad.ComparisonTests --no-build --no-restore --configuration Release --treenode-filter '/*/*/ComparisonReplayDiagnosticLogTests/*' --results-directory TestResults/comparison-images/replay-log
+          dotnet test --project tests/KeyLoad.ComparisonTests --no-build --no-restore --configuration Release --treenode-filter '/*/*/ComparisonReplayDiagnosticRetentionTests/*' --results-directory TestResults/comparison-images/replay-retention
       - name: Qualify time-series workload and aggregation contracts
         run: |
-          dotnet test --project tests/KeyLoad.ComparisonTests --no-build --no-restore --configuration Release --treenode-filter '/*/*/TimeSeriesWorkloadTests/*'
-          dotnet test --project tests/KeyLoad.ComparisonTests --no-build --no-restore --configuration Release --treenode-filter '/*/*/TimeSeriesPackageVersionTests/*'
+          dotnet test --project tests/KeyLoad.ComparisonTests --no-build --no-restore --configuration Release --treenode-filter '/*/*/TimeSeriesWorkloadTests/*' --results-directory TestResults/comparison-images/timeseries-workload
+          dotnet test --project tests/KeyLoad.ComparisonTests --no-build --no-restore --configuration Release --treenode-filter '/*/*/TimeSeriesPackageVersionTests/*' --results-directory TestResults/comparison-images/timeseries-package
       - name: Retain common image bundle
         id: bundle
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a

README.md

@@ -302,10 +302,27 @@ The earlier isolated Linux baseline ([run37087909605](https://github.com/managed
 
 The bounded RAM pool, embedded point cache and local binding passed the development checks in the [R83 source receipt](docs/implementation/cache-memory-source-r83.json). The newer [d45 native source gates](docs/implementation/cache-native-source-r93.json) pass1583/1583 normal and scalar unit cases with identical IDs and no skips,164/164 recovery plus1000 seeded process-cut rows,118 analyzer cases and63/63 SDK/MCP RF3 cases. All17 earlier identity failures and the byte77 oracle now pass. The comparison workflow concluded with24/27 successful preflight jobs and3 failures; the270-cell matrix did not execute. These source gates provide no performance result. The [R100 wire source receipt](docs/implementation/cache-wire-source-r100.json) binds the complete independent review of34 unused internal signed-metadata files and26 corrected test files. Fresh integrated source gates and delivered-SHA native qualification remain pending. Orleans RF3 controls, product coverage, measured benefit, endurance and power-loss gates remain open; server RF3 caches are disabled.
 
+The later ALL186-file `5bbf30f70` checkpoint has [terminal CI evidence](docs/implementation/cache-checkpoint-native-r104.json): real RF3 passes63/63 and analyzer regressions118/118. Its full solution build fails20 Node test-helper diagnostics; normal/scalar unit tests, recovery and comparisons are skipped. The new cache-wire tests therefore remain unqualified. Current helper corrections require a fresh complete source gate and delivered-SHA CI run.
+
+The actual database hot-path investigation has an independently authenticated
+[PointRead baseline](docs/implementation/database-hotpath-baseline-d45-r115.json)
+from that earlier d45 native preflight: median2187ops/s for1 voter,805ops/s for2
+and891ops/s forRF3, at16 closed-loop clients and five10000-operation repetitions.
+Each group runs on one Docker host; this is not multi-host scaling or a complete
+competitor comparison. Server CPU/allocations and phase timings are unavailable.
+The [first preserving repair](docs/Features/ClusterReplication/ReplicaTermMetadata.md)
+targets repeated full-entry decoding during term checks, retaining both public
+authorized quorum barriers. New regressions, implementation and measured benefit
+remain pending. [Current source ownership evidence](docs/implementation/image-http-owner-source-r115.json)
+closes the Node-helper source review; the full R113 development build still fails
+21 other SiteTests diagnostics, and delivered-SHA native tests remain required.
+
 
 The delivered `2f374fc34` [Linux run37093197474](https://github.com/managedcode/KeyLoad/actions/runs/37093197474) now passes the complete source gates,118 analyzer cases,1483 normal and1483 scalar unit cases,164 process-recovery cases and63 Docker RF3 cases without skips. The [original-source receipt](docs/implementation/isolated-source-qualification-37093197474.json) retains exact report/artifact hashes. Comparative image and diagnostic gates pass;27 native preflights finish16 job successes and11 failures, including two explicit unavailable Neo4j topologies among the successes. Complete270 performance and site publication remain unqualified. The separate TimeSeries pure30 cases pass in both modes, while its bounded runner has36 new declared TUnit cases prepared in source and native adapters/all30 cells remain pending.
 
 The later `2ecbeee4d` [run37093992229](https://github.com/managedcode/KeyLoad/actions/runs/37093992229) fails17 normal unit cases and one RF3 queue-receive case; scalar and native comparison jobs are skipped. The [original-failure receipt](docs/implementation/isolated-current-main-baseline-37093992229.json) preserves the exact reports and artifact hashes. Current repairs and a prepared intensive TimeSeries runner remain unqualified until their delivered-source GitHub gates pass.
 
 
 The fixture-only `d45d7f253` [Linux source qualification](docs/implementation/isolated-source-qualification-37098964980.json) passes normal1583/scalar1583, recovery164, analyzer118 and RF363 without skips; all17 prior fixture failures pass in both modes. The [terminal comparison receipt](docs/implementation/isolated-comparison-terminal-37098964980.json) records24successful native preflight jobs and3failures (Mongo2 image import, Mongo3 priority takeover, Kurrent2 cleanup). Two successful jobs explicitly report unavailable Neo4j Community topologies. Full270 was not allocated; aggregation and refreshed site metrics remain unqualified. New ACK ownership, SDK TimeSeries adapter, SQL and transactional lifecycle source require their own delivered-SHA qualification; TimeSeries6/30 remains incomplete.
+
+The later `5bbf30f70` [Linux source run37104211481](docs/implementation/isolated-source-qualification-37104211481.json) fails the full Release build on20 Node-helper diagnostics. Analyzer118 and Docker/Aspire RF363 pass; normal/scalar units, recovery and all native comparison jobs are skipped. No intensive cells execute and no performance metrics are refreshed. Current repairs and the new TimeSeries1/2/3 resource models require a new delivered-source GitHub run; models do not establish native ACKs or copied data.

benchmarks/KeyLoad.Comparisons/Features/BenchmarkComparisons/KurrentCleanupFatalCause.cs

@@ -0,0 +1,45 @@
+using System.Runtime.ExceptionServices;
+
+namespace KeyLoad.Comparisons.Targets;
+
+internal static class KurrentCleanupFatalCause
+{
+    private const int FirstAggregateIndex = 0;
+    private const int PreviousIndexOffset = 1;
+
+    internal static ExceptionDispatchInfo? Find(Exception error)
+    {
+        ArgumentNullException.ThrowIfNull(error);
+        var visited = new HashSet<Exception>(ReferenceEqualityComparer.Instance);
+        var pending = new Stack<Exception>();
+        pending.Push(error);
+        while (pending.TryPop(out var current))
+        {
+            if (!visited.Add(current))
+            {
+                continue;
+            }
+            if (IsFatal(current))
+            {
+                return ExceptionDispatchInfo.Capture(current);
+            }
+            if (current is AggregateException aggregate)
+            {
+                for (var index = aggregate.InnerExceptions.Count - PreviousIndexOffset;
+                    index >= FirstAggregateIndex; index--)
+                {
+                    pending.Push(aggregate.InnerExceptions[index]);
+                }
+            }
+            else if (current.InnerException is { } inner)
+            {
+                pending.Push(inner);
+            }
+        }
+        return null;
+    }
+
+    private static bool IsFatal(Exception error)
+        => error is OutOfMemoryException or StackOverflowException or AccessViolationException
+            or AppDomainUnloadedException or BadImageFormatException or CannotUnloadAppDomainException;
+}

benchmarks/KeyLoad.Comparisons/Features/BenchmarkComparisons/KurrentCleanupState.cs

@@ -9,6 +9,7 @@ internal sealed class KurrentCleanupState(int tracked)
     private readonly Stopwatch clock = Stopwatch.StartNew();
     private int next, submitted, acknowledged, faulted, pending, peak, laterDisposals;
     private ExceptionDispatchInfo? primary;
+    private ExceptionDispatchInfo? firstFatal;
     private KurrentCleanupStage failedStage;
     private KurrentCleanupFailureReason reason;
     private int? grpcStatus;
@@ -123,7 +124,7 @@ internal void ThrowIfFailed()
         ExceptionDispatchInfo? failure;
         lock (gate)
         {
-            failure = primary;
+            failure = firstFatal ?? primary;
         }
         failure?.Throw();
     }
@@ -132,6 +133,10 @@ internal void ThrowIfFailed()
 
     private void Capture(Exception error, KurrentCleanupStage stage)
     {
+        if (firstFatal is null)
+        {
+            firstFatal = KurrentCleanupFatalCause.Find(error);
+        }
         var classified = KurrentCleanupDiagnostics.Classify(error);
         if (primary is not null)
         {

benchmarks/KeyLoad.Comparisons/Features/BenchmarkComparisons/TimeSeries/Intensive/TimeSeriesIntensiveAcknowledgement.cs

@@ -2,6 +2,8 @@ namespace KeyLoad.Comparisons.Features.BenchmarkComparisons.TimeSeries.Intensive
 
 internal readonly record struct TimeSeriesIntensiveAcknowledgement(Guid CommandId, long Sequence)
 {
+    private const long MinimumSequence = 1L;
+    private const string MissingCommand = "An acknowledgement requires an actual nonempty command UUID.";
     internal static TimeSeriesIntensiveAcknowledgement FromReceipt(TimeSeriesIntensiveAppendReceipt receipt)
     {
         ArgumentNullException.ThrowIfNull(receipt);
@@ -13,8 +15,8 @@ internal static void Validate(TimeSeriesIntensiveAcknowledgement acknowledgement
     {
         if (acknowledgement.CommandId == Guid.Empty)
         {
-            throw new ArgumentException("An acknowledgement requires an actual nonempty command UUID.", nameof(acknowledgement));
+            throw new ArgumentException(MissingCommand, nameof(acknowledgement));
         }
-        ArgumentOutOfRangeException.ThrowIfLessThan(acknowledgement.Sequence, 1L, nameof(acknowledgement));
+        ArgumentOutOfRangeException.ThrowIfLessThan(acknowledgement.Sequence, MinimumSequence, nameof(acknowledgement));
     }
 }

benchmarks/KeyLoad.Comparisons/Features/BenchmarkComparisons/TimeSeries/Intensive/TimeSeriesIntensiveCellPhase.cs

@@ -0,0 +1,7 @@
+namespace KeyLoad.Comparisons.Features.BenchmarkComparisons.TimeSeries.Intensive;
+
+internal enum TimeSeriesIntensiveCellPhase
+{
+    Preflight,
+    Intensive
+}

benchmarks/KeyLoad.Comparisons/Features/BenchmarkComparisons/TimeSeries/Intensive/TimeSeriesIntensiveFamilyCell.cs

@@ -0,0 +1,9 @@
+using System.Collections.Immutable;
+
+namespace KeyLoad.Comparisons.Features.BenchmarkComparisons.TimeSeries.Intensive;
+
+internal sealed record TimeSeriesIntensiveFamilyCell(string Id, TimeSeriesIntensiveSelection Selection);
+
+internal sealed record TimeSeriesIntensiveFamilyCells(
+    ImmutableArray<TimeSeriesIntensiveFamilyCell> Preflight,
+    ImmutableArray<TimeSeriesIntensiveFamilyCell> Intensive);