Skip to content

Commit 43e01af

Browse files
KSemenenkoKSemenenko
committed
Checkpoint functional coverage and native open-loop delivery
Commit the complete shared working-tree scope on main. Three real CLI backup/restore flows passed through Aspire. Formatter, governance and whitespace checks passed at the recorded source cutoff. The current full build, native merge regression, complete Linux suites, global coverage and fault/endurance qualification remain pending.
1 parent e195d24 commit 43e01af

403 files changed

Lines changed: 16842 additions & 1331 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/workflows/benchmarks.yml‎

Lines changed: 30 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,7 @@ jobs:
4646
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
4747
- name: Plan all benchmark scenarios
4848
id: plan
49-
run: node scripts/Features/BenchmarkComparisons/isolated-plan.mjs --output="$RUNNER_TEMP/isolated-plan.json" --scale-output="$RUNNER_TEMP/scaled-plan.json" --vector-output="$RUNNER_TEMP/vector-plan.json" --composite-output="$RUNNER_TEMP/composite-plan.json" --github-output="$GITHUB_OUTPUT"
49+
run: node scripts/Features/BenchmarkComparisons/isolated-plan.mjs --output="$RUNNER_TEMP/isolated-plan.json" --scale-output="$RUNNER_TEMP/scaled-plan.json" --vector-output="$RUNNER_TEMP/vector-plan.json" --composite-output="$RUNNER_TEMP/composite-plan.json" --open-loop-output="$RUNNER_TEMP/open-loop-plan.json" --github-output="$GITHUB_OUTPUT"
5050
- name: Save benchmark plan
5151
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a
5252
with:
@@ -56,6 +56,7 @@ jobs:
5656
${{ runner.temp }}/scaled-plan.json
5757
${{ runner.temp }}/vector-plan.json
5858
${{ runner.temp }}/composite-plan.json
59+
${{ runner.temp }}/open-loop-plan.json
5960
if-no-files-found: error
6061
retention-days: 90
6162
comparison-images:
@@ -207,6 +208,8 @@ jobs:
207208
KEYLOAD_COMPARISON_JOB_NAME: ${{ matrix.jobName }}
208209
KEYLOAD_SCALE_PROFILE: ${{ matrix.scaleProfile }}
209210
KEYLOAD_VECTOR_PROFILE: ${{ matrix.vectorProfile }}
211+
KEYLOAD_OPEN_LOOP_RATE: ${{ matrix.openLoopRate }}
212+
KEYLOAD_OPEN_LOOP_CANCELLATION_PROOF: ${{ matrix.openLoopCancellationProof }}
210213
Benchmarks__Target: ${{ matrix.target }}
211214
Benchmarks__NodeCount: ${{ matrix.nodeCount }}
212215
Benchmarks__Scenario: ${{ matrix.scenario }}
@@ -379,7 +382,8 @@ jobs:
379382
- name: Download source code
380383
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
381384
- name: Verify benchmark plan
382-
run: node scripts/Features/BenchmarkComparisons/isolated-plan.mjs --output="$RUNNER_TEMP/isolated-plan.json" --scale-output="$RUNNER_TEMP/scaled-plan.json" --vector-output="$RUNNER_TEMP/vector-plan.json" --composite-output="$RUNNER_TEMP/composite-plan.json"
385+
id: plan
386+
run: node scripts/Features/BenchmarkComparisons/isolated-plan.mjs --output="$RUNNER_TEMP/isolated-plan.json" --scale-output="$RUNNER_TEMP/scaled-plan.json" --vector-output="$RUNNER_TEMP/vector-plan.json" --composite-output="$RUNNER_TEMP/composite-plan.json" --open-loop-output="$RUNNER_TEMP/open-loop-plan.json"
383387
- name: Download benchmark results
384388
run: node scripts/Features/BenchmarkComparisons/isolated-github-collect.mjs --plan="$RUNNER_TEMP/isolated-plan.json" --scale-plan="$RUNNER_TEMP/scaled-plan.json" --vector-plan="$RUNNER_TEMP/vector-plan.json" --input="$RUNNER_TEMP/isolated-capture"
385389
- name: Check control and complete scale accounting
@@ -409,3 +413,27 @@ jobs:
409413
path: ${{ runner.temp }}/isolated-capture/github/**
410414
if-no-files-found: error
411415
retention-days: 90
416+
- name: Download authenticated open-loop worker results
417+
id: open-loop-intake
418+
if: ${{ always() && !cancelled() && steps.plan.outcome == 'success' }}
419+
run: node scripts/Features/BenchmarkComparisons/open-loop-github-collect.mjs --capture="$RUNNER_TEMP/isolated-capture/github" --plan="$RUNNER_TEMP/open-loop-plan.json" --output="$RUNNER_TEMP/open-loop-intake"
420+
- name: Aggregate open-loop cohort
421+
id: open-loop-aggregate
422+
if: ${{ always() && !cancelled() && steps.open-loop-intake.outcome == 'success' }}
423+
run: node scripts/Features/BenchmarkComparisons/open-loop-cohort-aggregate-cli.mjs --input="$RUNNER_TEMP/open-loop-intake" --output="$RUNNER_TEMP/open-loop-aggregate"
424+
- name: Save original open-loop worker inputs
425+
if: always()
426+
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a
427+
with:
428+
name: comparison-open-loop-intake
429+
path: ${{ runner.temp }}/open-loop-intake/**
430+
if-no-files-found: warn
431+
retention-days: 90
432+
- name: Save open-loop cohort disposition
433+
if: always()
434+
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a
435+
with:
436+
name: comparison-open-loop-cohort
437+
path: ${{ runner.temp }}/open-loop-aggregate/**
438+
if-no-files-found: warn
439+
retention-days: 90

‎.github/workflows/ci.yml‎

Lines changed: 80 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -200,10 +200,87 @@ jobs:
200200
printf 'KEYLOAD_ADMIN_CHROME_PATH=%s\n' "$admin_chrome" >> "$GITHUB_ENV"
201201
- name: Restore .NET packages
202202
run: dotnet restore KeyLoad.slnx
203-
- name: Build three-node database tests
204-
run: dotnet build tests/KeyLoad.IntegrationTests --no-restore --configuration Release
203+
- name: Build the complete native coverage cohort
204+
run: dotnet build KeyLoad.slnx --no-restore --configuration Release
205+
- name: Prepare original source and test-image receipts
206+
shell: pwsh
207+
run: |
208+
$root = $env:GITHUB_WORKSPACE
209+
$evidence = Join-Path $root 'TestResults/functional-coverage/rf3'
210+
& pwsh -NoLogo -NoProfile -File scripts/Features/CodeQuality/functional-coverage.production-source-manifest.ps1 `
211+
-Mode prepare -Root $root -EvidenceRoot $evidence
212+
if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
213+
"KEYLOAD_PRODUCT_COVERAGE_SOURCE_MANIFEST=$evidence/functional-coverage.production-source-manifest.json" | Add-Content $env:GITHUB_ENV
205214
- name: Test three-node database with .NET and MCP clients
206-
run: dotnet run --project src/KeyLoad.AppHost --no-build --no-restore --configuration Release -- --KeyLoadTests:Suite=rf3 --KeyLoadTests:ReportTrx=true
215+
run: dotnet run --project src/KeyLoad.AppHost --no-build --no-restore --configuration Release -- --KeyLoadTests:Suite=rf3 --KeyLoadTests:ResultsDirectory=TestResults/rf3-required --KeyLoadTests:ReportTrx=true
216+
- name: Collect four native functional coverage cohorts
217+
id: functional-coverage
218+
shell: pwsh
219+
run: |
220+
$common = @(
221+
'run', '--project', 'src/KeyLoad.AppHost', '--no-build', '--no-restore', '--configuration', 'Release', '--',
222+
'--KeyLoadTests:ReportTrx=true',
223+
'--KeyLoadTests:CoverageSettings=scripts/Features/CodeQuality/functional-coverage.production.settings.xml',
224+
'--KeyLoadTests:CoverageFormat=coverage')
225+
& dotnet @common '--KeyLoadTests:Suite=unit' '--KeyLoadTests:Filter=/*/*/PartitionQuery*/*' `
226+
'--KeyLoadTests:ResultsDirectory=TestResults/functional-coverage/rf3/unit' `
227+
'--KeyLoadTests:CoverageOutput=TestResults/functional-coverage/rf3/unit/coverage.coverage'
228+
$unitExitCode = $LASTEXITCODE
229+
"unit_exit_code=$unitExitCode" | Add-Content $env:GITHUB_OUTPUT
230+
& dotnet @common '--KeyLoadTests:Suite=unit-scalar' '--KeyLoadTests:Filter=/*/*/PartitionQuery*/*' `
231+
'--KeyLoadTests:ResultsDirectory=TestResults/functional-coverage/rf3/unit-scalar' `
232+
'--KeyLoadTests:CoverageOutput=TestResults/functional-coverage/rf3/unit-scalar/coverage.coverage'
233+
$scalarExitCode = $LASTEXITCODE
234+
"scalar_exit_code=$scalarExitCode" | Add-Content $env:GITHUB_OUTPUT
235+
& dotnet @common '--KeyLoadTests:Suite=recovery' `
236+
'--KeyLoadTests:Filter=/*/*/CommandIdempotencyProcessRecoveryTests/*' `
237+
'--KeyLoadTests:ResultsDirectory=TestResults/functional-coverage/rf3/recovery' `
238+
'--KeyLoadTests:CoverageOutput=TestResults/functional-coverage/rf3/recovery/coverage.coverage'
239+
$recoveryExitCode = $LASTEXITCODE
240+
"recovery_exit_code=$recoveryExitCode" | Add-Content $env:GITHUB_OUTPUT
241+
& dotnet @common '--KeyLoadTests:Suite=rf3' `
242+
'--KeyLoadTests:Filter=/*/*/(PartitionQueryPublicRf3Tests)|(McpDocumentCrudParityTests)/*' `
243+
'--KeyLoadTests:ResultsDirectory=TestResults/functional-coverage/rf3' `
244+
'--KeyLoadTests:CoverageOutput=TestResults/functional-coverage/rf3/coverage.coverage' `
245+
'--KeyLoadTests:NativeCoverage:ServerMode=rf3-original-node-v1' `
246+
"--KeyLoadTests:NativeCoverage:SourceManifest=$env:KEYLOAD_PRODUCT_COVERAGE_SOURCE_MANIFEST"
247+
$rf3ExitCode = $LASTEXITCODE
248+
"rf3_exit_code=$rf3ExitCode" | Add-Content $env:GITHUB_OUTPUT
249+
if ($unitExitCode -ne 0 -or $scalarExitCode -ne 0 -or $recoveryExitCode -ne 0 -or $rf3ExitCode -ne 0) {
250+
exit 1
251+
}
252+
- name: Assemble product descriptor from original reports
253+
id: product-descriptor
254+
if: ${{ success() }}
255+
shell: pwsh
256+
env:
257+
UNIT_EXIT_CODE: ${{ steps.functional-coverage.outputs.unit_exit_code }}
258+
SCALAR_EXIT_CODE: ${{ steps.functional-coverage.outputs.scalar_exit_code }}
259+
RECOVERY_EXIT_CODE: ${{ steps.functional-coverage.outputs.recovery_exit_code }}
260+
RF3_EXIT_CODE: ${{ steps.functional-coverage.outputs.rf3_exit_code }}
261+
run: |
262+
& pwsh -NoLogo -NoProfile -File scripts/Features/CodeQuality/functional-coverage.native-product-descriptor.ps1 `
263+
-Repository $env:GITHUB_WORKSPACE `
264+
-SourceManifestPath $env:KEYLOAD_PRODUCT_COVERAGE_SOURCE_MANIFEST `
265+
-ResultsRoot (Join-Path $env:GITHUB_WORKSPACE 'TestResults/functional-coverage/rf3') `
266+
-UnitExitCode ([int]$env:UNIT_EXIT_CODE) -ScalarExitCode ([int]$env:SCALAR_EXIT_CODE) `
267+
-RecoveryExitCode ([int]$env:RECOVERY_EXIT_CODE) -Rf3ExitCode ([int]$env:RF3_EXIT_CODE)
268+
if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
269+
- name: Validate and merge native product coverage
270+
if: ${{ success() }}
271+
shell: pwsh
272+
run: |
273+
$root = $env:GITHUB_WORKSPACE
274+
$evidence = Join-Path $root 'TestResults/functional-coverage/rf3'
275+
$descriptorPath = Join-Path $evidence 'functional-coverage.native-product-descriptor.v1.json'
276+
$boundsPath = Join-Path $evidence 'functional-coverage.native-options.v1.json'
277+
$descriptor = Get-Content -LiteralPath $descriptorPath -Raw | ConvertFrom-Json
278+
$nativeOptions = Get-Content -LiteralPath $boundsPath -Raw
279+
& pwsh -NoLogo -NoProfile -File scripts/Features/CodeQuality/functional-coverage.native-merge.ps1 `
280+
-Mode Product -Repository $root -EvidenceRoot $evidence -DescriptorPath $descriptorPath `
281+
-ToolPackageRoot $descriptor.tool.packageRoot -ToolVersion $descriptor.tool.version `
282+
-NativeOptionsJson $nativeOptions
283+
if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
207284
- name: Clean up Docker registry
208285
if: ${{ always() && (steps.images.outcome == 'success' || steps.images.outcome == 'failure') }}
209286
run: node scripts/Features/BenchmarkComparisons/cleanup-images.mjs

‎Directory.Build.targets‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,4 +14,24 @@
1414
Text="KeyLoad requires XML documentation generation so IDE0005 runs during builds." />
1515
<MakeDir Directories="$(KeyLoadCodeQualityDirectory)" />
1616
</Target>
17+
<Import Project="$(MSBuildThisFileDirectory)tests/KeyLoad.UnitTests/Features/CodeQuality/Build/FunctionalCompilationIdentity.targets"
18+
Condition="'$(MSBuildProjectName)' == 'KeyLoad.Abstractions' or
19+
'$(MSBuildProjectName)' == 'KeyLoad.Analyzers' or
20+
'$(MSBuildProjectName)' == 'KeyLoad.AppHost' or
21+
'$(MSBuildProjectName)' == 'KeyLoad.Artifacts' or
22+
'$(MSBuildProjectName)' == 'KeyLoad.Cli' or
23+
'$(MSBuildProjectName)' == 'KeyLoad.Client' or
24+
'$(MSBuildProjectName)' == 'KeyLoad.Core' or
25+
'$(MSBuildProjectName)' == 'KeyLoad.Diagnostics' or
26+
'$(MSBuildProjectName)' == 'KeyLoad.Orleans' or
27+
'$(MSBuildProjectName)' == 'KeyLoad.Query' or
28+
'$(MSBuildProjectName)' == 'KeyLoad.Replication' or
29+
'$(MSBuildProjectName)' == 'KeyLoad.Security' or
30+
'$(MSBuildProjectName)' == 'KeyLoad.Server' or
31+
'$(MSBuildProjectName)' == 'KeyLoad.ServiceDefaults' or
32+
'$(MSBuildProjectName)' == 'KeyLoad.Storage.IO' or
33+
'$(MSBuildProjectName)' == 'KeyLoad.Storage.ZoneTree' or
34+
'$(MSBuildProjectName)' == 'KeyLoad.UnitTests' or
35+
'$(MSBuildProjectName)' == 'KeyLoad.RecoveryTests' or
36+
'$(MSBuildProjectName)' == 'KeyLoad.IntegrationTests'" />
1737
</Project>

‎Dockerfile‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
FROM mcr.microsoft.com/dotnet/sdk:10.0.401@sha256:e70cdb7f80b0348f5cb85f19a8f670fca061f033d57eed12fa003d58b0e06317 AS build
22
WORKDIR /source
33
COPY global.json Directory.Build.props Directory.Build.targets Directory.Packages.props NuGet.Config .editorconfig LICENSE ./
4+
COPY KeyLoad.slnx ./KeyLoad.slnx
5+
COPY tests/KeyLoad.UnitTests/Features/CodeQuality/Build/FunctionalCompilationIdentity.targets ./tests/KeyLoad.UnitTests/Features/CodeQuality/Build/FunctionalCompilationIdentity.targets
46
COPY src/ ./src/
57
RUN dotnet restore src/KeyLoad.Server/KeyLoad.Server.csproj --disable-parallel
68
ARG KEYLOAD_RELEASE_VERSION

‎benchmarks/KeyLoad.BenchmarkScenarios/Features/BenchmarkComparisons/Benchmarks/EmbeddedBenchmarks.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -137,7 +137,7 @@ internal void Initialize(PartitionRef partition)
137137

138138
store = new(new(directory), options.Storage, options.PointCache);
139139
database = new(store, new AuthorizationPolicy(), options.Database, options.DueWork, options.EventSource,
140-
options.Messaging, options.GraphExecution, options.ChangeFeedExecution, options.TimeSeriesExecution);
140+
options.Messaging, options.GraphExecution, options.ChangeFeedExecution, options.BlobExecution, options.NativeClaimsExecution, options.TimeSeriesExecution);
141141
database.Bootstrap(new(PrincipalId, PrincipalScope,
142142
[new(Wildcard, Wildcard, Capability.All)], [Wildcard])
143143
{ ClusterAdministrator = true },

‎benchmarks/KeyLoad.BenchmarkScenarios/Features/BenchmarkComparisons/Benchmarks/Serialization/SampleChunkBenchmarkState.cs‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -10,11 +10,16 @@ internal sealed class SampleChunkBenchmarkState
1010
private readonly byte[][] nativeValues;
1111
private readonly byte[] chunk;
1212
private readonly IOptions<DatabaseLimits> limitsOptions;
13+
private readonly IOptions<TimeSeriesExecutionOptions> executionOptions;
1314

14-
internal SampleChunkBenchmarkState(SampleRecord[] records, IOptions<DatabaseLimits> limitsOptions)
15+
internal SampleChunkBenchmarkState(SampleRecord[] records, IOptions<DatabaseLimits> limitsOptions,
16+
IOptions<TimeSeriesExecutionOptions> executionOptions)
1517
{
18+
ArgumentNullException.ThrowIfNull(executionOptions);
19+
executionOptions.Value.Validate();
1620
this.records = records;
1721
this.limitsOptions = limitsOptions;
22+
this.executionOptions = executionOptions;
1823
nativeValues = NativeEncode();
1924
chunk = ChunkEncode();
2025
RequireExact(records, NativeDecode());
@@ -42,7 +47,7 @@ internal byte[][] NativeEncode()
4247
return values;
4348
}
4449

45-
internal byte[] ChunkEncode() => SampleChunkCodec.Encode(records, Budget());
50+
internal byte[] ChunkEncode() => SampleChunkCodec.Encode(records, Budget(), executionOptions);
4651

4752
internal SampleRecord[] NativeDecode()
4853
{
@@ -63,7 +68,7 @@ internal SampleRecord[] ChunkDecode()
6368
{
6469
var budget = Budget();
6570
budget.ChargeBytes(chunk.Length);
66-
return SampleChunkCodec.Decode(chunk, budget);
71+
return SampleChunkCodec.Decode(chunk, budget, executionOptions);
6772
}
6873

6974
private ReadExecutionBudget Budget() => new(limitsOptions);

‎benchmarks/KeyLoad.BenchmarkScenarios/Features/BenchmarkComparisons/Benchmarks/Serialization/SampleChunkSerializationBenchmarks.cs‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,8 +37,9 @@ public void Setup()
3737
{
3838
throw new InvalidOperationException(SetupFailureMessage);
3939
}
40+
var runtime = EmbeddedBenchmarkRuntimeRegistration.Read();
4041
var candidate = new SampleChunkBenchmarkState(SampleChunkBenchmarkCorpus.Create(RecordCount, Corpus),
41-
EmbeddedBenchmarkRuntimeRegistration.Read().Database);
42+
runtime.Database, runtime.TimeSeriesExecution);
4243
SampleChunkBenchmarkManifest.Write(RecordCount, Corpus, candidate, BenchmarkArtifactRegistration.ReadSampleChunk());
4344
state = candidate;
4445
}

‎benchmarks/KeyLoad.BenchmarkScenarios/Features/BenchmarkComparisons/Configuration/EmbeddedBenchmarkRuntimeOptions.cs‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,9 @@ namespace KeyLoad.BenchmarkScenarios.Features.BenchmarkComparisons;
1010
internal sealed record EmbeddedBenchmarkRuntimeOptions(IOptions<DatabaseLimits> Database,
1111
IOptions<DueWorkExecutionOptions> DueWork, IOptions<EventSourceExecutionOptions> EventSource,
1212
IOptions<MessagingExecutionOptions> Messaging, IOptions<GraphExecutionOptions> GraphExecution,
13-
IOptions<ChangeFeedExecutionOptions> ChangeFeedExecution, IOptions<TimeSeriesExecutionOptions> TimeSeriesExecution,
13+
IOptions<ChangeFeedExecutionOptions> ChangeFeedExecution, IOptions<BlobExecutionOptions> BlobExecution,
14+
IOptions<NativeClaimsExecutionOptions> NativeClaimsExecution,
15+
IOptions<TimeSeriesExecutionOptions> TimeSeriesExecution,
1416
IOptions<ZoneTreeStorageExecutionOptions> Storage,
1517
IOptions<ZoneTreePointCacheExecutionOptions> PointCache);
1618

@@ -20,6 +22,7 @@ internal static class EmbeddedBenchmarkRuntimeRegistration
2022
{
2123
internal static EmbeddedBenchmarkRuntimeOptions Read()
2224
{
25+
_ = SerializationExecutionRegistration.Process.Value;
2326
var configuration = new ConfigurationBuilder().AddEnvironmentVariables().Build();
2427
using var configurationLifetime = configuration as IDisposable;
2528
return new(
@@ -35,6 +38,10 @@ internal static EmbeddedBenchmarkRuntimeOptions Read()
3538
settings => settings.IsValid(), GraphExecutionOptions.ValidationMessage),
3639
BenchmarkScenarioOptionsRegistration.Read<ChangeFeedExecutionOptions>(configuration, ChangeFeedExecutionOptions.SectionName,
3740
settings => settings.IsValid(), ChangeFeedExecutionOptions.ValidationMessage),
41+
BenchmarkScenarioOptionsRegistration.Read<BlobExecutionOptions>(configuration, BlobExecutionOptions.SectionName,
42+
settings => settings.IsValid(), BlobExecutionOptions.ValidationMessage),
43+
BenchmarkScenarioOptionsRegistration.Read<NativeClaimsExecutionOptions>(configuration, NativeClaimsExecutionOptions.SectionName,
44+
settings => settings.IsValid(), NativeClaimsExecutionOptions.ValidationMessage),
3845
BenchmarkScenarioOptionsRegistration.Read<TimeSeriesExecutionOptions>(configuration, TimeSeriesExecutionOptions.SectionName,
3946
settings => settings.IsValid(), TimeSeriesExecutionOptions.ValidationMessage),
4047
BenchmarkScenarioOptionsRegistration.Read<ZoneTreeStorageExecutionOptions>(configuration, ZoneTreeStorageExecutionOptions.SectionName,

0 commit comments

Comments
 (0)