Skip to content

Commit 35aa86c

Browse files
KSemenenkoKSemenenko
committed
Checkpoint remaining concurrent runtime and test changes
Include the complete additional working-tree scope on main at the owner request. This is a source checkpoint; final solution and full functional qualification remain incomplete. The last root-verified canonical snapshot failed with 59 CS0103 errors; no passing build or full-suite acceptance is claimed for these concurrent edits.
1 parent beb0248 commit 35aa86c

35 files changed

Lines changed: 403 additions & 153 deletions

File tree

‎Directory.Packages.props‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
<PackageVersion Include="ManagedCode.MCPGateway" Version="0.4.17" />
1919
<PackageVersion Include="ManagedCode.MarkdownLd.Kb" Version="0.2.10" />
2020
<PackageVersion Include="Microsoft.Extensions.AI" Version="10.10.0" />
21+
<PackageVersion Include="Microsoft.Extensions.Configuration.EnvironmentVariables" Version="10.0.12" />
2122
<PackageVersion Include="Microsoft.Extensions.Options" Version="10.0.12" />
2223
<PackageVersion Include="Microsoft.Extensions.Options.ConfigurationExtensions" Version="10.0.12" />
2324
<PackageVersion Include="ModelContextProtocol.AspNetCore" Version="2.2.0" />

‎benchmarks/KeyLoad.BenchmarkScenarios/Features/BenchmarkComparisons/Benchmarks/EmbeddedBenchmarks.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -125,7 +125,7 @@ internal void Initialize(PartitionRef partition)
125125
{
126126
store = new(new(directory), options.Storage, options.PointCache);
127127
database = new(store, new AuthorizationPolicy(), options.Database, options.DueWork, options.EventSource,
128-
options.Messaging);
128+
options.Messaging, options.GraphExecution);
129129
database.Bootstrap(new(PrincipalId, PrincipalScope,
130130
[new(Wildcard, Wildcard, Capability.All)], [Wildcard])
131131
{ ClusterAdministrator = true },

‎benchmarks/KeyLoad.BenchmarkScenarios/Features/BenchmarkComparisons/Configuration/EmbeddedBenchmarkRuntimeOptions.cs‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,8 @@ namespace KeyLoad.BenchmarkScenarios.Features.BenchmarkComparisons;
99
/// <summary>Holds the validated native snapshots borrowed by an embedded benchmark owner.</summary>
1010
internal sealed record EmbeddedBenchmarkRuntimeOptions(IOptions<DatabaseLimits> Database,
1111
IOptions<DueWorkExecutionOptions> DueWork, IOptions<EventSourceExecutionOptions> EventSource,
12-
IOptions<MessagingExecutionOptions> Messaging, IOptions<ZoneTreeStorageExecutionOptions> Storage,
12+
IOptions<MessagingExecutionOptions> Messaging, IOptions<GraphExecutionOptions> GraphExecution,
13+
IOptions<ZoneTreeStorageExecutionOptions> Storage,
1314
IOptions<ZoneTreePointCacheExecutionOptions> PointCache);
1415

1516
/// <summary>Binds the actual generated runner's environment to native fixture policies.</summary>
@@ -28,6 +29,8 @@ internal static EmbeddedBenchmarkRuntimeOptions Read()
2829
settings => settings.IsValid(), EventSourceExecutionOptions.ValidationMessage),
2930
BenchmarkScenarioOptionsRegistration.Read<MessagingExecutionOptions>(configuration, MessagingExecutionOptions.SectionName,
3031
settings => settings.IsValid(), MessagingExecutionOptions.ValidationMessage),
32+
BenchmarkScenarioOptionsRegistration.Read<GraphExecutionOptions>(configuration, GraphExecutionOptions.SectionName,
33+
settings => settings.IsValid(), GraphExecutionOptions.ValidationMessage),
3134
BenchmarkScenarioOptionsRegistration.Read<ZoneTreeStorageExecutionOptions>(configuration, ZoneTreeStorageExecutionOptions.SectionName,
3235
settings => settings.IsValid(), ZoneTreeStorageExecutionOptions.ValidationMessage),
3336
BenchmarkScenarioOptionsRegistration.Read<ZoneTreePointCacheExecutionOptions>(configuration, ZoneTreePointCacheExecutionOptions.SectionName,

‎benchmarks/KeyLoad.BenchmarkScenarios/Features/BenchmarkComparisons/Fixtures/ScaledStorage/ScaledRawStorageFixture.cs‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,27 +1,28 @@
11
using System.Diagnostics;
2+
using Microsoft.Extensions.Options;
23
using System.Runtime.ExceptionServices;
34

45
namespace KeyLoad.BenchmarkScenarios.Features.BenchmarkComparisons;
56

67
internal sealed class ScaledRawStorageFixture : IDisposable
78
{
8-
private const int PreparationLimitMinutes = 20;
9-
internal static readonly TimeSpan PreparationLimit = TimeSpan.FromMinutes(PreparationLimitMinutes);
109
private readonly CancellationTokenSource _lifetime;
1110
private ScaledRawStorageFixtureCore? _core;
1211
private bool _closing;
1312
private bool _disposed;
1413
private const string InactiveMessage = "The scaled native fixture is not initialized.";
1514
private const string DeadlineMessage = "The scaled fixture preparation deadline expired.";
1615

17-
internal ScaledRawStorageFixture(int recordCount, int payloadBytes,
16+
internal ScaledRawStorageFixture(int recordCount, int payloadBytes, IOptions<ScaledStorageExecutionOptions> executionOptions,
1817
CancellationToken cancellationToken = default)
1918
{
19+
var settings = executionOptions.Value;
20+
settings.Validate();
2021
ScaledRawStorageSettings.ValidateInput(recordCount, payloadBytes);
2122
cancellationToken.ThrowIfCancellationRequested();
2223
var deadlineStart = Stopwatch.GetTimestamp();
23-
var processMemoryCeiling = ScaledRawStorageSettings.ValidateFixtureCapacity(recordCount, payloadBytes);
24-
var remaining = PreparationLimit - Stopwatch.GetElapsedTime(deadlineStart);
24+
var processMemoryCeiling = ScaledRawStorageSettings.ValidateFixtureCapacity(recordCount, payloadBytes, settings);
25+
var remaining = settings.PreparationTimeout - Stopwatch.GetElapsedTime(deadlineStart);
2526
if (remaining <= TimeSpan.Zero)
2627
{
2728
throw new TimeoutException(DeadlineMessage);
@@ -52,7 +53,7 @@ internal ScaledRawStorageFixture(int recordCount, int payloadBytes,
5253
try
5354
{
5455
_core = new ScaledRawStorageFixtureCore(recordCount, payloadBytes,
55-
deadlineStart, processMemoryCeiling, _lifetime.Token);
56+
deadlineStart, processMemoryCeiling, _lifetime.Token, executionOptions);
5657
_core.Initialize();
5758
}
5859
catch (Exception primary) when (RawStorageFixtureFailures.IsNonFatal(primary))

‎benchmarks/KeyLoad.BenchmarkScenarios/Features/BenchmarkComparisons/Fixtures/ScaledStorage/ScaledRawStorageFixtureCore.cs‎

Lines changed: 13 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,13 @@
11
using System.Diagnostics;
2+
using Microsoft.Extensions.Options;
23

34
namespace KeyLoad.BenchmarkScenarios.Features.BenchmarkComparisons;
45

56
internal sealed class ScaledRawStorageFixtureCore
67
{
78
private const string InitializedMessage = "The scaled fixture core initialization is one-shot.";
9+
private readonly ScaledStorageExecutionOptions settings;
10+
private readonly IOptions<ScaledStorageExecutionOptions> executionOptions;
811
private readonly CancellationToken _token;
912
private readonly long _deadlineStart;
1013
private readonly long _processMemoryCeilingBytes;
@@ -30,8 +33,10 @@ internal sealed class ScaledRawStorageFixtureCore
3033
private bool _disposed;
3134

3235
internal ScaledRawStorageFixtureCore(int recordCount, int payloadBytes,
33-
long deadlineStart, long processMemoryCeilingBytes, CancellationToken token)
36+
long deadlineStart, long processMemoryCeilingBytes, CancellationToken token, IOptions<ScaledStorageExecutionOptions> executionOptions)
3437
{
38+
settings = executionOptions.Value;
39+
this.executionOptions = executionOptions;
3540
_token = token;
3641
_deadlineStart = deadlineStart;
3742
_processMemoryCeilingBytes = processMemoryCeilingBytes;
@@ -63,12 +68,12 @@ internal void Initialize()
6368
}
6469

6570
_initializeStarted = true;
66-
ScaledRawStoragePreparationGuard.Check(_deadlineStart, _token);
67-
var engine = new ScaledRawStorageZoneTreeEngine(_corpus, _arena, _readScratch, _deadlineStart, _token);
71+
ScaledRawStoragePreparationGuard.Check(_deadlineStart, _token, settings.PreparationTimeout);
72+
var engine = new ScaledRawStorageZoneTreeEngine(_corpus, _arena, _readScratch, _deadlineStart, _token, executionOptions);
6873
_engine = engine;
6974
engine.Initialize();
7075
_seedElapsedTicks = ScaledRawStorageSeedRunner.Run(_corpus, engine, _seedScratch,
71-
RecordCount, _deadlineStart, ref _seedAttempts, ref _successfulSeedWrites, _token);
76+
RecordCount, _deadlineStart, ref _seedAttempts, ref _successfulSeedWrites, _token, settings.PreparationTimeout);
7277
VerifyAllCore(enforceBudget: true);
7378
_initialized = true;
7479
}
@@ -112,7 +117,7 @@ internal ScaledRawStorageSnapshot Capture()
112117
using var process = Process.GetCurrentProcess();
113118
var peak = ScaledRawStorageProcessMemory.ReadPeakBytes(process);
114119
var available = GC.GetGCMemoryInfo().TotalAvailableMemoryBytes;
115-
ScaledRawStorageMemoryGuard.ValidateObserved(peak, available, _processMemoryCeilingBytes);
120+
ScaledRawStorageMemoryGuard.ValidateObserved(peak, available, _processMemoryCeilingBytes, settings.RequiredHeadroomBytes);
116121
return ScaledRawStorageSnapshotFactory.Create(this, native, peak, process.WorkingSet64, available);
117122
}
118123

@@ -175,16 +180,16 @@ private void VerifyAllCore(bool enforceBudget)
175180
{
176181
var result = ScaledRawStorageVerification.Run(_corpus, _engine!, _expectedScratch,
177182
RecordCount, ref _nativeReadCalls, enforceBudget ? _deadlineStart : 0L,
178-
enforceBudget, enforceBudget ? _token : CancellationToken.None);
183+
enforceBudget, enforceBudget ? _token : CancellationToken.None, settings.PreparationTimeout);
179184
_ = _engine!.Capture();
180185
if (enforceBudget)
181186
{
182-
ScaledRawStoragePreparationGuard.Check(_deadlineStart, _token);
187+
ScaledRawStoragePreparationGuard.Check(_deadlineStart, _token, settings.PreparationTimeout);
183188
}
184189

185190
using var process = Process.GetCurrentProcess();
186191
ScaledRawStorageMemoryGuard.ValidateObserved(ScaledRawStorageProcessMemory.ReadPeakBytes(process),
187-
GC.GetGCMemoryInfo().TotalAvailableMemoryBytes, _processMemoryCeilingBytes);
192+
GC.GetGCMemoryInfo().TotalAvailableMemoryBytes, _processMemoryCeilingBytes, settings.RequiredHeadroomBytes);
188193
_verifiedRecords = result.VerifiedRecords;
189194
_verificationPasses++;
190195
_fullValueDigest = result.Digest;

‎benchmarks/KeyLoad.BenchmarkScenarios/Features/BenchmarkComparisons/Fixtures/ScaledStorage/ScaledRawStorageIdentityReader.cs‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -41,10 +41,10 @@ internal static class ScaledRawStoragePreparationGuard
4141
{
4242
private const string DeadlineMessage = "The scaled fixture preparation deadline expired.";
4343

44-
internal static void Check(long deadlineStart, CancellationToken token)
44+
internal static void Check(long deadlineStart, CancellationToken token, TimeSpan preparationTimeout)
4545
{
4646
token.ThrowIfCancellationRequested();
47-
if (Stopwatch.GetElapsedTime(deadlineStart) >= ScaledRawStorageFixture.PreparationLimit)
47+
if (Stopwatch.GetElapsedTime(deadlineStart) >= preparationTimeout)
4848
{
4949
throw new TimeoutException(DeadlineMessage);
5050
}
@@ -57,14 +57,14 @@ internal static class ScaledRawStorageSeedRunner
5757

5858
internal static long Run(ScaledRawStorageCorpus corpus, ScaledRawStorageZoneTreeEngine engine,
5959
byte[] scratch, int recordCount, long deadlineStart, ref long attempts,
60-
ref long successfulWrites, CancellationToken token)
60+
ref long successfulWrites, CancellationToken token, TimeSpan preparationTimeout)
6161
{
6262
var started = Stopwatch.GetTimestamp();
6363
for (var index = 0; index < recordCount; index++)
6464
{
6565
if (index % CancellationCheckStride == 0)
6666
{
67-
ScaledRawStoragePreparationGuard.Check(deadlineStart, token);
67+
ScaledRawStoragePreparationGuard.Check(deadlineStart, token, preparationTimeout);
6868
}
6969

7070
corpus.WriteValue(index, scratch);
@@ -73,7 +73,7 @@ internal static long Run(ScaledRawStorageCorpus corpus, ScaledRawStorageZoneTree
7373
successfulWrites++;
7474
}
7575

76-
ScaledRawStoragePreparationGuard.Check(deadlineStart, token);
76+
ScaledRawStoragePreparationGuard.Check(deadlineStart, token, preparationTimeout);
7777
return Stopwatch.GetTimestamp() - started;
7878
}
7979
}

‎benchmarks/KeyLoad.BenchmarkScenarios/Features/BenchmarkComparisons/Fixtures/ScaledStorage/ScaledRawStorageMemoryGuard.cs‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2,19 +2,17 @@ namespace KeyLoad.BenchmarkScenarios.Features.BenchmarkComparisons;
22

33
internal static class ScaledRawStorageMemoryGuard
44
{
5-
internal const long MaximumProcessBytes = 12L * 1024 * 1024 * 1024;
6-
internal const long RequiredHeadroomBytes = 2L * 1024 * 1024 * 1024;
75
private const string MemoryMessage = "Observed process memory exceeded the frozen ceiling or headroom.";
86
private const string UnavailablePeakMessage = "The process did not provide a positive lifetime peak memory observation.";
97

10-
internal static void ValidateObserved(long peak, long available, long ceiling)
8+
internal static void ValidateObserved(long peak, long available, long ceiling, long requiredHeadroomBytes)
119
{
1210
if (peak <= 0)
1311
{
1412
throw new InvalidDataException(UnavailablePeakMessage);
1513
}
1614

17-
if (peak > ceiling || checked(peak + RequiredHeadroomBytes) > available)
15+
if (peak > ceiling || checked(peak + requiredHeadroomBytes) > available)
1816
{
1917
throw new InvalidOperationException(MemoryMessage);
2018
}

‎benchmarks/KeyLoad.BenchmarkScenarios/Features/BenchmarkComparisons/Fixtures/ScaledStorage/ScaledRawStorageSettings.cs‎

Lines changed: 7 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,6 @@ internal static class ScaledRawStorageSettings
66
private const int OneMillion = 1_000_000;
77
private const int SmallPayloadBytes = 32;
88
private const int LargePayloadBytes = 1024;
9-
private const long FourteenGiB = 15_032_385_536L;
109
private const int KeyBytes = 16;
1110
private const int ReservedMissKeyCount = 1;
1211
private const int OrderBytes = sizeof(int);
@@ -28,36 +27,36 @@ internal static void ValidateInput(int recordCount, int payloadBytes)
2827
}
2928
}
3029

31-
internal static long ValidateFixtureCapacity(int recordCount, int payloadBytes)
30+
internal static long ValidateFixtureCapacity(int recordCount, int payloadBytes, ScaledStorageExecutionOptions settings)
3231
{
3332
ValidateInput(recordCount, payloadBytes);
3433
using var process = System.Diagnostics.Process.GetCurrentProcess();
3534
var projectedCeiling = checked(ScaledRawStorageProcessMemory.ReadPeakBytes(process)
36-
+ CapacityBound(recordCount, payloadBytes));
35+
+ CapacityBound(recordCount, payloadBytes, settings.RequiredHeadroomBytes));
3736
var available = GC.GetGCMemoryInfo().TotalAvailableMemoryBytes;
3837
var qualificationCount = recordCount is HundredThousand or OneMillion;
39-
if (qualificationCount && available < FourteenGiB)
38+
if (qualificationCount && available < settings.MinimumQualificationCapacityBytes)
4039
{
4140
throw new InvalidOperationException(InsufficientMemoryMessage);
4241
}
4342

44-
var ceiling = qualificationCount ? ScaledRawStorageMemoryGuard.MaximumProcessBytes : projectedCeiling;
45-
if (projectedCeiling > ScaledRawStorageMemoryGuard.MaximumProcessBytes || ceiling > available)
43+
var ceiling = qualificationCount ? settings.MaximumProcessBytes : projectedCeiling;
44+
if (projectedCeiling > settings.MaximumProcessBytes || ceiling > available)
4645
{
4746
throw new InvalidOperationException(InsufficientMemoryMessage);
4847
}
4948

5049
return ceiling;
5150
}
5251

53-
internal static long CapacityBound(int recordCount, int payloadBytes)
52+
internal static long CapacityBound(int recordCount, int payloadBytes, long requiredHeadroomBytes)
5453
{
5554
var keysAndOrder = checked(((long)recordCount + ReservedMissKeyCount) * KeyBytes
5655
+ ((long)recordCount * OrderBytes));
5756
var values = checked((long)recordCount * payloadBytes);
5857
var scratch = checked((long)payloadBytes * ScratchCopies);
5958
return checked(keysAndOrder + values + scratch
60-
+ ScaledRawStorageMemoryGuard.RequiredHeadroomBytes);
59+
+ requiredHeadroomBytes);
6160
}
6261

6362
}

‎benchmarks/KeyLoad.BenchmarkScenarios/Features/BenchmarkComparisons/Fixtures/ScaledStorage/ScaledRawStorageVerification.cs‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -13,14 +13,14 @@ internal static class ScaledRawStorageVerification
1313

1414
internal static ScaledRawStorageVerificationResult Run(ScaledRawStorageCorpus corpus,
1515
ScaledRawStorageZoneTreeEngine engine, byte[] expectedScratch, int recordCount, ref long readCalls,
16-
long deadlineStart, bool enforceBudget, CancellationToken token)
16+
long deadlineStart, bool enforceBudget, CancellationToken token, TimeSpan preparationTimeout)
1717
{
1818
var started = Stopwatch.GetTimestamp();
1919
using var digest = IncrementalHash.CreateHash(HashAlgorithmName.SHA256);
2020
var verified = 0L;
2121
for (var index = 0; index < recordCount; index++)
2222
{
23-
CheckBoundary(index, deadlineStart, enforceBudget, token);
23+
CheckBoundary(index, deadlineStart, enforceBudget, token, preparationTimeout);
2424
readCalls++;
2525
if (!engine.TryRead(index, out var actual))
2626
{
@@ -37,43 +37,43 @@ internal static ScaledRawStorageVerificationResult Run(ScaledRawStorageCorpus co
3737
verified++;
3838
}
3939

40-
CheckBoundary(recordCount, deadlineStart, enforceBudget, token);
40+
CheckBoundary(recordCount, deadlineStart, enforceBudget, token, preparationTimeout);
4141
readCalls++;
4242
if (engine.TryRead(recordCount, out _))
4343
{
4444
throw new InvalidOperationException(PresentMessage);
4545
}
4646

47-
CheckCompletion(deadlineStart, enforceBudget, token);
47+
CheckCompletion(deadlineStart, enforceBudget, token, preparationTimeout);
4848
return new ScaledRawStorageVerificationResult(
4949
verified,
5050
Convert.ToHexStringLower(digest.GetHashAndReset()),
5151
Stopwatch.GetTimestamp() - started);
5252
}
5353

54-
private static void CheckBoundary(int operation, long deadlineStart, bool enforceBudget, CancellationToken token)
54+
private static void CheckBoundary(int operation, long deadlineStart, bool enforceBudget, CancellationToken token, TimeSpan preparationTimeout)
5555
{
5656
if (!enforceBudget || operation % CancellationCheckStride != 0)
5757
{
5858
return;
5959
}
6060

6161
token.ThrowIfCancellationRequested();
62-
if (Stopwatch.GetElapsedTime(deadlineStart) >= ScaledRawStorageFixture.PreparationLimit)
62+
if (Stopwatch.GetElapsedTime(deadlineStart) >= preparationTimeout)
6363
{
6464
throw new TimeoutException(DeadlineMessage);
6565
}
6666
}
6767

68-
private static void CheckCompletion(long deadlineStart, bool enforceBudget, CancellationToken token)
68+
private static void CheckCompletion(long deadlineStart, bool enforceBudget, CancellationToken token, TimeSpan preparationTimeout)
6969
{
7070
if (!enforceBudget)
7171
{
7272
return;
7373
}
7474

7575
token.ThrowIfCancellationRequested();
76-
if (Stopwatch.GetElapsedTime(deadlineStart) >= ScaledRawStorageFixture.PreparationLimit)
76+
if (Stopwatch.GetElapsedTime(deadlineStart) >= preparationTimeout)
7777
{
7878
throw new TimeoutException(DeadlineMessage);
7979
}

0 commit comments

Comments
 (0)