Retain bounded node diagnostics for RF3 failures

Save the last 120 log lines per node on startup or leader-loss failure. Run RF3 verification before the longer crash suite so platform failures surface earlier while keeping every required gate.

Validation: Release integration build and both local RF3 tests passed.

Author: KSemenenko

.github/workflows/ci.yml

@@ -25,8 +25,8 @@ jobs:
       - run: dotnet restore KeyLoad.slnx --locked-mode
       - run: dotnet build KeyLoad.slnx --no-restore --configuration Release
       - run: dotnet test --project tests/KeyLoad.UnitTests --no-build --no-restore --configuration Release
-      - run: dotnet test --project tests/KeyLoad.RecoveryTests --no-build --no-restore --configuration Release
       - run: dotnet test --project tests/KeyLoad.IntegrationTests --no-build --no-restore --configuration Release
+      - run: dotnet test --project tests/KeyLoad.RecoveryTests --no-build --no-restore --configuration Release
       - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         if: always()
         with:

tests/KeyLoad.IntegrationTests/ClusterFixture.cs

@@ -1,4 +1,5 @@
 using Aspire.Hosting;
+using Aspire.Hosting.ApplicationModel;
 using Aspire.Hosting.Testing;
 using KeyLoad.Client;
 using Microsoft.Extensions.DependencyInjection;
@@ -25,13 +26,42 @@ public async ValueTask InitializeAsync()
         await App.StartAsync(timeout.Token);
         using var profile = System.Text.Json.JsonDocument.Parse(File.ReadAllBytes(Path.Combine(Root, "local-profile.json")));
         AdminKey = profile.RootElement.GetProperty("AdminKey").GetString()!;
-        await Task.WhenAll(Enumerable.Range(1, 3).Select(i => App.ResourceNotifications.WaitForResourceHealthyAsync($"node{i}", timeout.Token)));
+        try { await Task.WhenAll(Enumerable.Range(1, 3).Select(i => App.ResourceNotifications.WaitForResourceHealthyAsync($"node{i}", timeout.Token))); }
+        catch { await SaveFailureDiagnosticsAsync(); throw; }
     }
     public KeyLoadClient Client(string node, string? key = null)
     {
         var http = App.CreateHttpClient(node, "http"); http.Timeout = TimeSpan.FromSeconds(30);
         return new(http, key ?? AdminKey);
     }
+    public async Task SaveFailureDiagnosticsAsync()
+    {
+        var repository = new DirectoryInfo(AppContext.BaseDirectory);
+        while (repository.Parent is not null && !File.Exists(Path.Combine(repository.FullName, "KeyLoad.slnx"))) repository = repository.Parent;
+        var output = Path.Combine(repository.FullName, "artifacts", "qualification"); Directory.CreateDirectory(output);
+        var logs = App.Services.GetRequiredService<ResourceLoggerService>();
+        foreach (var number in Enumerable.Range(1, 3))
+        {
+            var name = $"node{number}";
+            try
+            {
+                using var deadline = new CancellationTokenSource(TimeSpan.FromSeconds(5));
+                var lines = new Queue<string>();
+                await foreach (var batch in logs.GetAllAsync(name).WithCancellation(deadline.Token))
+                {
+                    foreach (var line in batch)
+                    {
+                        lines.Enqueue(line.Content);
+                        if (lines.Count > 120) lines.Dequeue();
+                    }
+                }
+                var tail = lines.ToArray();
+                File.WriteAllLines(Path.Combine(output, "rf3-failure-" + name + ".log"), tail);
+                foreach (var line in tail) TestContext.Current.TestOutputHelper?.WriteLine($"{name}: {line}");
+            }
+            catch (OperationCanceledException) { TestContext.Current.TestOutputHelper?.WriteLine($"{name}: log retrieval timed out."); }
+        }
+    }
     public async ValueTask DisposeAsync()
     {
         if (App is not null) { await App.StopAsync(); await App.DisposeAsync(); }

tests/KeyLoad.IntegrationTests/ClusterTests.cs

@@ -17,6 +17,8 @@ public sealed class ClusterTests(ClusterFixture fixture)
     public async Task ReplicatedAtomicBatchSurvivesLeaderProcessKillAndMinorityRejectsWrites()
     {
         using var timeout = new CancellationTokenSource(TimeSpan.FromMinutes(2));
+        try
+        {
         var clients = Enumerable.Range(1, 3).Select(i => fixture.Client($"node{i}")).ToArray();
         var partition = new PartitionRef("integration", "database", "orders", Guid.NewGuid().ToString("N"));
         Success(await clients[0].ConfigureResourceAsync(Guid.NewGuid(), new(partition.TenantId, partition.DatabaseId,
@@ -61,6 +63,8 @@ public async Task ReplicatedAtomicBatchSurvivesLeaderProcessKillAndMinorityRejec
         await EventuallyAsync(async () => (await surviving.StatusAsync(timeout.Token)).IsSuccess, timeout.Token);
         Assert.Null(Success(await surviving.GetAsync(new(partition, "orders", "minority"), timeout.Token)));
         foreach (var index in Enumerable.Range(0, 3)) await fixture.App.ResourceNotifications.WaitForResourceHealthyAsync($"node{index + 1}", timeout.Token);
+        }
+        catch { await fixture.SaveFailureDiagnosticsAsync(); throw; }
     }
     [Fact]
     public async Task UnsignedPeerRequestsAndClientSuppliedPrincipalAreRejected()