Fix native benchmark registry probes and Kurrent startup sequencing

Author: KSemenenko

.github/workflows/benchmarks.yml

@@ -76,6 +76,15 @@ jobs:
           global-json-file: global.json
       - name: Check Docker
         run: docker version
+      - name: Restore .NET packages
+        run: dotnet restore KeyLoad.slnx
+      - name: Build benchmark tests
+        id: benchmark-build
+        run: dotnet build tests/KeyLoad.ComparisonTests --no-restore --configuration Release
+      - name: Build benchmark preparation unit tests
+        run: dotnet build tests/KeyLoad.UnitTests --no-restore --configuration Release
+      - name: Check bounded Docker registry readiness
+        run: dotnet run --project src/KeyLoad.AppHost --no-build --no-restore --configuration Release -- --KeyLoadTests:Suite=unit '--KeyLoadTests:Filter=/*/*/ImageRegistryReadinessTests/*' --KeyLoadTests:ResultsDirectory=TestResults/comparison-images/registry-readiness
       - name: Build KeyLoad server and benchmark Docker images
         id: images
         run: node scripts/Features/BenchmarkComparisons/prepare-images.mjs
@@ -86,11 +95,8 @@ jobs:
         run: |
           test -n "$KEYLOAD_SERVER_IMAGE" && test -n "$KEYLOAD_RUNNER_IMAGE"
           printf 'KeyLoad__ContainerImages__Server=%s\nBenchmarks__ContainerImages__LoadGenerator=%s\nKEYLOAD_IMAGE_RECEIPT=%s\n' "$KEYLOAD_SERVER_IMAGE" "$KEYLOAD_RUNNER_IMAGE" "$RUNNER_TEMP/keyload-images/image-receipt.json" >> "$GITHUB_ENV"
-      - name: Restore .NET packages
-        run: dotnet restore KeyLoad.slnx
-      - name: Build benchmark tests
-        id: benchmark-build
-        run: dotnet build tests/KeyLoad.ComparisonTests --no-restore --configuration Release
+      - name: Check native KurrentDB discovery settings
+        run: dotnet run --project src/KeyLoad.AppHost --no-build --no-restore --configuration Release -- --KeyLoadTests:Suite=comparison '--KeyLoadTests:Filter=/*/*/IsolatedKurrentDiscoverySettingsTests/*' --KeyLoadTests:ResultsDirectory=TestResults/comparison-images/kurrent-discovery
       - name: Check TimescaleDB Docker image and startup tools
         env:
           KEYLOAD_TIMESERIES_IMAGE_FACTS_DIRECTORY: ${{ runner.temp }}/keyload-timeseries-image-facts

benchmarks/KeyLoad.Comparisons/Features/BenchmarkComparisons/KurrentTarget.cs

@@ -57,14 +57,16 @@ public async Task InitializeAsync(BenchmarkDataset dataset, CancellationToken ca
         try
         {
             ownership = new KurrentStreamOwnership(dataset.Options);
-            setupStage = KurrentSetupStage.WriterConstruction;
-            writer = new KurrentDBClient(KurrentNativeSettings.CreateWriter(connectionString));
-            ownedClients.Add(writer);
             setupStage = KurrentSetupStage.MemberVerification;
             var timeout = TimeSpan.FromSeconds(dataset.Options.TimeoutSeconds);
             var proof = await KurrentClusterVerifier.VerifyAsync(connectionString, nodeHttpClients, topology, timeout, cancellationToken);
             nodeClients = proof.NodeClients;
             ownedClients.AddRange(nodeClients);
+            // Native SDK construction eagerly discovers and caches a preferred live member.
+            // Construct the writer after all native views agree on their actual leader.
+            setupStage = KurrentSetupStage.WriterConstruction;
+            writer = new KurrentDBClient(KurrentNativeSettings.CreateWriter(connectionString));
+            ownedClients.Add(writer);
             setupStage = KurrentSetupStage.NoStreamSemantics;
             await VerifyNoStreamConflictAsync(cancellationToken);
 

docs/ADR/ADR-080-benchmark-failure-isolation.md

@@ -1,7 +1,7 @@
 # ADR-080: Isolate benchmark failures during publication
 
 Status: Accepted; source implemented, delivered-source verification pending.
-Date: 2026-10-04. Related: REQ/AC-BC-FAIL-001..005, ADR-056/074/076.
+Date: 2026-10-04. Related: REQ/AC-BC-FAIL-001..007, ADR-056/074/076.
 
 ## Decision
 
@@ -33,14 +33,27 @@ upload; cancellation/timeouts that prevent artifacts remain explicit blockers.
 4. Root owns workflow always-finalization/aggregate condition, shared receipt
    validation, dependency/coverage inventory, source name tests and documentation.
    Diagnostic worker inspects native startup errors without changing KeyLoad engine.
+   Under FAIL-PREP-REGISTRY, the existing image-contracts/manifests/evidence modules
+   own2s real HTTP probes inside the unchanged30s readiness bound and private
+   no-follow121-record/64KiB probe facts. Evidence I/O errors escape transport
+   retry handling. Ten actual HTTP fixture cases belong to UnitTests; the image
+   preparation job must build that runner and invoke its Aspire-owned filter.
+   Run these actual loopback fixture tests before `prepare-images.mjs` owns the
+   same registry port; restore/build the native test runners first. Keep the
+   actual pinned image export/import checks after image construction.
+   Under FAIL-PREP-KURRENT, `KurrentTarget.InitializeAsync` constructs the actual
+   SDK writer only after `KurrentClusterVerifier.VerifyAsync` proves membership.
+   Three SDK/resource-model tests belong to ComparisonTests and run in common
+   preparation; existing native StreamAppend preflights qualify1/2/3-node semantics
+   and copy/cleanup behavior. No provider replacement, URI or ACK/retry change.
 5. Root reviews all diffs, builds solution, runs formatter/governance and focused
    Aspire-owned suites, then checkpoints scoped changes on current main and pushes.
 6. Genuine Linux Benchmarks run qualifies all cells, aggregate, site coverage/browser
    and Pages publication. Local tests are development proof only.
 
 Migration is additive to version4 dispositions; deploy producer/validators/site
 atomically. Rollback reverts this coherent change and restores the conservative
-publication gate, retaining immutable original artifacts. AC-BC-FAIL-001..005 map
+publication gate, retaining immutable original artifacts. AC-BC-FAIL-001..007 map
 to automated and actual-provider evidence in the feature specification. Root alone
 owns integration and shared contract updates; workers never commit or push.
 

docs/Features/BenchmarkComparisons.md

@@ -688,6 +688,8 @@ KeyLoad engine repair and concurrent series-codec work are outside this task.
 | REQ-BC-FAIL-003 authenticated partial results | AC-BC-FAIL-003 failed job accepted only with failed workload, successful result upload and matching failed envelope; missing/malformed/expired/mixed evidence rejected | producer/aggregate/site negative and positive TUnit regressions |
 | REQ-BC-FAIL-004 honest site | AC-BC-FAIL-004 successful competitors retain values; failed cells have no numeric values and expose actual job link | independent numeric oracle, projection validation and real Chrome |
 | REQ-BC-FAIL-005 repair shared preparation | AC-BC-FAIL-005 diagnose exact failed logs, repair benchmark setup/build invocation, retain native isolated topology and Aspire ownership | exact failed-source log, focused regression, delivered-source GitHub rerun |
+| REQ-BC-FAIL-006 bounded registry readiness | AC-BC-FAIL-006 each native HTTP probe has at most2s within the unchanged30s total; only settled non-aborted HTTP200 succeeds; private no-follow diagnostics remain at most121 records/64KiB and evidence-write failures propagate | `ImageRegistryReadinessTests`:10 actual loopback HTTP/error/bounds cases, plus genuine pinned Docker image export/import in GitHub |
+| REQ-BC-FAIL-007 Kurrent writer starts after membership | AC-BC-FAIL-007 verify all native1/2/3-member views before constructing the SDK writer; retain native DNS seeds, TLS verification, leader preference, NoStream semantics, acknowledgements, replica-copy oracle and cleanup | `IsolatedKurrentDiscoverySettingsTests`:3 actual SDK/resource-model cases; genuine Aspire-owned StreamAppend preflights for1/2/3 nodes |
 
 [ADR-080](../ADR/ADR-080-benchmark-failure-isolation.md) owns the boundary change.
 Ordered task graph: FAIL-CONTRACT (root, complete) -> FAIL-SITE (site worker),
@@ -712,6 +714,35 @@ as compact immutable test-only fixtures with original hashes, source and provena
 substitute for current publication input. Final formatting is recorded separately; unrelated concurrent SampleChunk
 formatting is outside this repair. Delivered-source workload/site publication proof remains pending.
 
+The first repair run37158699545/source65e8bf59 passed the clean Linux solution
+build, formatter and pinned Docker image preparation/roundtrip. Actual website
+cell `keyload-n1-document-update` job111309324270/artifact11286559225 failed its
+workload, uploaded successfully and passed unchanged production job/artifact/ZIP/
+envelope/agreement/site validators with its original null report. This is genuine
+failed-cell runtime proof; complete270 aggregation and Pages remain pending.
+
+That run exposed two native preparation defects. Kurrent n3 job111309323415
+started a listening registry but its first HTTP request consumed the entire30s
+probe budget; the underlying transport cause was not logged. FAIL-PREP-REGISTRY
+uses2s attempts inside the same30s total and records bounded safe probe facts.
+Kurrent n2 job111309323419/artifact11286971852 began writer construction while
+native election was still `PreLeader`. Actual pinned SDK1.4.0 constructor/selector
+inspection proves eager discovery can choose a live follower at that stage;
+the exact cached endpoint was not logged. FAIL-PREP-KURRENT moves construction
+after awaited complete membership verification. These are benchmark fixture
+repairs; neither changes a database engine, URI, retry/ACK contract or topology.
+Root records this contract before integration, then runs the focused Aspire
+unit/comparison cases, full build/formatter/governance and an exact-source Linux
+rerun before claiming the two repairs or publication qualified.
+
+Native-stage local development checks: initial full Release build and formatter
+passed; Aspire registry10/10 and actual SDK/resource settings3/3 passed. The new
+mandatory preparation workflow regression passed. The full24-case workflow filter
+had23 passes and exposed its old22-comparison-filter inventory expectation; source
+now enumerates the added23rd filter explicitly. Its clean-source rerun is pending.
+Concurrent uncommitted NativeTextAsync CA1849/IDE0005 prevent repeat shared-checkout
+full checks; those product changes remain outside this repair and are not staged.
+
 ```mermaid
 flowchart LR
   Plan[Complete native cell plan] --> Jobs[Independent Aspire workloads]

docs/implementation/status.json

@@ -3124,12 +3124,30 @@
     "status": "in_progress",
     "feature": "docs/Features/BenchmarkComparisons.md",
     "decision": "docs/ADR/ADR-080-benchmark-failure-isolation.md",
-    "sourceStage": "bounded_transient_github_get_retries_and_authenticated_failed_cell_publication",
+    "sourceStage": "authenticated_failed_cell_publication_with_bounded_registry_probes_and_kurrent_writer_after_membership",
     "baselineRun": 37154664616,
     "originalFailedJob": 111299652762,
     "originalDiagnosticArtifact": 11286445994,
     "provenFailure": "GitHub HTTP503 rejected by rate-only retry policy before database startup",
-    "localDevelopmentVerification": "Local macOS development: full Release solution build passed with zero warnings/errors; Aspire comparison producer/finalizer 2/2 and GitHub evidence 84/84 passed; original PostgreSQL/RabbitMQ report probes 48/48 passed. Aspire workflow regressions 23/23 passed; full formatting remains blocked by unrelated concurrent SampleChunk files. Linux runtime/site/Pages qualification pending.",
+    "failedEnvelopeRuntimeEvidence": {
+      "sourceRevision": "65e8bf59fbac150d7d1d13211abc73f6ad7ea66b",
+      "runId": 37158699545,
+      "jobId": 111309324270,
+      "artifactId": 11286559225,
+      "cellId": "keyload-n1-document-update",
+      "originalArchiveDigest": "sha256:09a4d8e58791a59a230fd3c2137f9108a60571ce892fa073b729a58c2d022439",
+      "verified": "actual failed job/workload, successful upload and original null-report envelope accepted by production archive, agreement and site validators",
+      "completeCohortQualified": false
+    },
+    "nativePreparationRepairEvidence": {
+      "registryFailureJobId": 111309323415,
+      "registryDefect": "One readiness request could consume the entire30s total; native registry was listening but the transport cause was not logged",
+      "kurrentFailureJobId": 111309323419,
+      "kurrentDiagnosticArtifactId": 11286971852,
+      "kurrentDefect": "Writer SDK eagerly discovered before verified membership during PreLeader; pinned selector can cache follower, exact selected endpoint was not logged",
+      "qualification": "Aspire local registry10/10 and native SDK settings3/3 passed; exact-source native Linux rerun pending"
+    },
+    "localDevelopmentVerification": "Local macOS development: initial integrated full Release build and formatter passed with zero warnings/errors. Aspire registry10/10 and SDK settings3/3 passed. Workflow24 cases:23 passed including the new mandatory-preparation regression; the old22-filter inventory assertion was updated for the added23rd native comparison filter and awaits clean-source CI. Repeat full checks are blocked by concurrent uncommitted NativeTextAsync CA1849/IDE0005 outside benchmark scope. Earlier producer/finalizer2/2, GitHub evidence84/84, original report probes48/48 and workflow23/23 passed. Exact-source Linux native/site/Pages qualification pending.",
     "runtimeQualified": false,
     "publicationQualified": false,
     "keyLoadEngineChanges": false

scripts/Features/BenchmarkComparisons/image-contracts.mjs

@@ -19,6 +19,7 @@ export const fileName = Object.freeze({
   registryState: 'registry-state.json',
   nativeCommands: 'native-commands.jsonl',
   registryHeaders: 'registry-headers.jsonl',
+  registryReadiness: 'registry-readiness.jsonl',
   serverDockerfile: 'Dockerfile',
   runnerDockerfile: 'benchmarks/KeyLoad.ComparisonHost/Features/BenchmarkComparisons/Dockerfile',
 });
@@ -198,7 +199,10 @@ export const processLimit = Object.freeze({
   inspectTimeoutMs: 30000,
   cleanupTimeoutMs: 30000,
   readinessTimeoutMs: 30000,
+  readinessProbeTimeoutMs: 2000,
   readinessIntervalMs: 250,
+  maxRegistryReadinessRecords: 121,
+  maxRegistryReadinessBytes: 64 * 1024,
   maxOutputBytes: 256 * 1024,
   maxLogBytes: 128 * 1024,
   maxInspectBytes: 64 * 1024,
@@ -210,6 +214,17 @@ export const processLimit = Object.freeze({
   killGraceMs: 1000,
 });
 
+export const registryProbeErrorCodes = Object.freeze([
+  'ECONNREFUSED', 'ECONNRESET', 'EPIPE', 'ETIMEDOUT', 'EHOSTUNREACH', 'ENETUNREACH',
+  'UND_ERR_CONNECT_TIMEOUT', 'UND_ERR_HEADERS_TIMEOUT', 'UND_ERR_BODY_TIMEOUT', 'UND_ERR_SOCKET',
+]);
+
+export const registryReadinessTokens = Object.freeze({
+  phase: Object.freeze({ request: 'request', bodyCancel: 'body-cancel' }),
+  outcome: Object.freeze({ ready: 'ready', httpStatus: 'http-status', timeout: 'timeout',
+    requestFailed: 'request-failed', bodyCancelFailed: 'body-cancel-failed' }),
+});
+
 export const outputFormat = Object.freeze({
   jsonIndent: 2,
   newline: '\n',

scripts/Features/BenchmarkComparisons/image-evidence.mjs

@@ -2,7 +2,7 @@ import { createHash } from 'node:crypto';
 import { constants as fileFlags } from 'node:fs';
 import { lstat, mkdir, open, readFile, writeFile } from 'node:fs/promises';
 import path from 'node:path';
-import { directoryName, fileName, imageReference, message, outputFormat, processLimit, validation } from './image-contracts.mjs';
+import { directoryName, fileName, imageReference, message, outputFormat, processLimit, registryProbeErrorCodes, registryReadinessTokens, validation } from './image-contracts.mjs';
 
 const privateDirectoryMode = 0o700;
 const privateFileMode = 0o600;
@@ -19,6 +19,9 @@ const diagnosticRedactions = Object.freeze([
   }),
   Object.freeze({ pattern: /\b(?:gh[pousr]_[A-Za-z0-9]{20,}|github_pat_[A-Za-z0-9_]{20,}|AKIA[A-Z0-9]{16})\b/g, replacement: '[REDACTED]' }),
 ]);
+const readinessOutcomes = Object.freeze(Object.values(registryReadinessTokens.outcome));
+const readinessPhases = Object.freeze(Object.values(registryReadinessTokens.phase));
+const utcTimestampPattern = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/;
 
 export async function ensureEvidenceDirectory(context) {
   const expectedPath = path.resolve(context.runnerTemp, directoryName.evidence);
@@ -142,6 +145,23 @@ export async function recordRegistryHeaders(context, imageName, status, headers)
   await appendOwnedLine(context, fileName.registryHeaders, record, processLimit.maxNativeCommandRecords);
 }
 
+export async function recordRegistryReadiness(context, probe) {
+  if (!probe || !Number.isInteger(probe.sequence) || probe.sequence < 1 || probe.sequence > processLimit.maxRegistryReadinessRecords
+    || typeof probe.startedAt !== 'string' || !utcTimestampPattern.test(probe.startedAt) || !Number.isFinite(Date.parse(probe.startedAt))
+    || !Number.isSafeInteger(probe.durationMs) || probe.durationMs < 0
+    || !Number.isInteger(probe.timeoutMs) || probe.timeoutMs <= 0 || probe.timeoutMs > processLimit.readinessProbeTimeoutMs
+    || (probe.status !== null && (!Number.isInteger(probe.status) || probe.status < 100 || probe.status > 599))
+    || typeof probe.aborted !== 'boolean' || !readinessPhases.includes(probe.phase) || !readinessOutcomes.includes(probe.outcome)
+    || (probe.errorCode !== null && !registryProbeErrorCodes.includes(probe.errorCode))) {
+    throw new Error(message.commandOutputLimit);
+  }
+  const record = Object.freeze({ sequence: probe.sequence, startedAt: probe.startedAt, durationMs: probe.durationMs,
+    timeoutMs: probe.timeoutMs, status: probe.status, aborted: probe.aborted, phase: probe.phase,
+    outcome: probe.outcome, errorCode: probe.errorCode });
+  await appendOwnedLine(context, fileName.registryReadiness, record, processLimit.maxRegistryReadinessRecords,
+    processLimit.maxRegistryReadinessBytes);
+}
+
 export async function appendImageOutputs(context, serverReference, loadGeneratorReference) {
   const outputFilePath = context.githubOutput;
   const fileInfo = await lstat(outputFilePath).catch(() => null);
@@ -162,7 +182,7 @@ async function writeOwnedBytes(target, bytes) {
   await writeFile(target, bytes, { flag: existing ? replaceWriteFlag : exclusiveWriteFlag, mode: privateFileMode });
 }
 
-async function appendOwnedLine(context, name, value, maximumRecords) {
+async function appendOwnedLine(context, name, value, maximumRecords, maximumBytes = processLimit.maxCommandEvidenceBytes) {
   const target = await ownedPath(context, name);
   const line = Buffer.from(`${JSON.stringify(value)}${outputFormat.newline}`, outputFormat.utf8);
   if (line.length > processLimit.maxCommandEvidenceRecordBytes) throw new Error(message.commandOutputLimit);
@@ -173,7 +193,7 @@ async function appendOwnedLine(context, name, value, maximumRecords) {
     const info = await handle.stat();
     if (!info.isFile() || info.uid !== process.getuid() || (info.mode & 0o077) !== 0) throw new Error(message.unsafeEvidencePath);
     const recordCount = (await readFile(target, outputFormat.utf8)).split(outputFormat.newline).filter(Boolean).length;
-    if (info.size + line.length > processLimit.maxCommandEvidenceBytes || recordCount >= maximumRecords) {
+    if (info.size + line.length > maximumBytes || recordCount >= maximumRecords) {
       throw new Error(message.commandOutputLimit);
     }
     await handle.writeFile(line);