Skip to content

Commit 35ea896

Browse files
KSemenenkoKSemenenko
committed
Implement bounded lossless sample chunk codec and source-bound controls
1 parent 65e8bf5 commit 35ea896

57 files changed

Lines changed: 5433 additions & 42 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎README.md‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,8 @@ The default server is a three-node replicated cluster (RF3), built on Orleans wi
4545

4646
The server uses ZoneTree.FullTextSearch for bounded derived text candidates while preserving exact authorized ranking. Its [development receipt](docs/implementation/native-full-text-development-2026-10-03.json) records 26 native unit and 10 process-recovery checks through Aspire. Full Linux/RF3 qualification and incremental projection replay remain pending; these checks establish no speed improvement.
4747

48+
A bounded lossless time-series chunk codec has [development evidence](docs/implementation/sample-chunk-codec-development-2026-10-03.json) from full Aspire unit/scalar suites and matched native-codec controls. Canonical chunk storage, rewrite/recovery and RF3 qualification remain in progress under KL-078.
49+
4850
## Get started
4951

5052
Install the .NET SDK selected in [global.json](global.json) and Docker, then build from the repository root:

‎benchmarks/KeyLoad.BenchmarkScenarios/Features/BenchmarkComparisons/SampleChunkBenchmarkCorpus.cs‎

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
using System.Buffers.Binary;
2+
using System.Globalization;
3+
using System.Security.Cryptography;
4+
5+
namespace KeyLoad.BenchmarkScenarios.Features.BenchmarkComparisons;
6+
7+
internal static class SampleChunkBenchmarkCorpus
8+
{
9+
internal const string Regular = "regular";
10+
internal const string Late = "late-equal";
11+
internal const string Random = "deterministic-random";
12+
internal const int Seed = 78_031;
13+
private const string Series = "metrics-Київ🌍";
14+
private static readonly DateTimeOffset Start = new(2024, 1, 1, 0, 0, 0, TimeSpan.Zero);
15+
16+
internal static SampleRecord[] Create(int count, string corpus)
17+
{
18+
if (count is not (1 or 32 or 256) || corpus is not (Regular or Late or Random))
19+
{
20+
throw new ArgumentException("The sample chunk control corpus is unsupported.");
21+
}
22+
var records = new SampleRecord[count];
23+
for (var index = 0; index < count; index++)
24+
{
25+
var ticks = TimestampTicks(count, index, corpus);
26+
var timestamp = Start.AddTicks(ticks).ToOffset(Offset(index));
27+
var value = corpus == Random
28+
? ((RandomPart(index, 1) >> 11) * (1d / (1UL << 53)) - 0.5) * 10_000 : 100d + index / 10d;
29+
var tags = "{\"region\":\"Київ🌍\",\"sensor\":\"sensor-"
30+
+ (corpus == Regular ? 0 : index % 8).ToString(CultureInfo.InvariantCulture) + "\"}";
31+
var id = "sample-" + index.ToString(CultureInfo.InvariantCulture);
32+
records[index] = new(Series, new(id, timestamp, value), index + 1, tags);
33+
}
34+
Array.Sort(records, static (left, right) =>
35+
{
36+
var time = left.Sample.Timestamp.UtcTicks.CompareTo(right.Sample.Timestamp.UtcTicks);
37+
return time != 0 ? time : left.Sequence.CompareTo(right.Sequence);
38+
});
39+
return records;
40+
}
41+
42+
private static long TimestampTicks(int count, int index, string corpus)
43+
=> corpus switch
44+
{
45+
Regular => index * TimeSpan.TicksPerMinute,
46+
Late => ((index * 17 % count) / 2) * TimeSpan.TicksPerMinute,
47+
Random => (long)(RandomPart(index, 0) % TimeSpan.TicksPerDay),
48+
_ => throw new ArgumentException("The sample chunk control corpus is unsupported.")
49+
};
50+
51+
private static TimeSpan Offset(int index)
52+
=> TimeSpan.FromMinutes((index % 3) switch { 0 => -330, 1 => 0, _ => 120 });
53+
54+
private static ulong RandomPart(int index, int lane)
55+
{
56+
Span<byte> input = stackalloc byte[sizeof(int) * 3];
57+
BinaryPrimitives.WriteInt32LittleEndian(input, Seed);
58+
BinaryPrimitives.WriteInt32LittleEndian(input[sizeof(int)..], index);
59+
BinaryPrimitives.WriteInt32LittleEndian(input[(sizeof(int) * 2)..], lane);
60+
Span<byte> digest = stackalloc byte[SHA256.HashSizeInBytes];
61+
_ = SHA256.HashData(input, digest);
62+
return BinaryPrimitives.ReadUInt64LittleEndian(digest);
63+
}
64+
}

‎benchmarks/KeyLoad.BenchmarkScenarios/Features/BenchmarkComparisons/SampleChunkBenchmarkManifest.cs‎

Lines changed: 95 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,95 @@
1+
using System.Buffers.Binary;
2+
using System.Runtime.InteropServices;
3+
using System.Security.Cryptography;
4+
using System.Text.Json;
5+
6+
namespace KeyLoad.BenchmarkScenarios.Features.BenchmarkComparisons;
7+
8+
internal static class SampleChunkBenchmarkManifest
9+
{
10+
internal const string DirectoryEnvironment = "KEYLOAD_CHUNK_BENCHMARK_MANIFEST_DIRECTORY";
11+
internal const string SourceHeadEnvironment = "KEYLOAD_CHUNK_SOURCE_HEAD";
12+
internal const string SourceInventoryEnvironment = "KEYLOAD_CHUNK_SOURCE_INVENTORY_SHA256";
13+
private const string InvalidSourceMessage = "The sample chunk measurement source identity is missing or malformed.";
14+
15+
internal static void Write(int records, string corpus, SampleChunkBenchmarkState state)
16+
{
17+
var directory = Environment.GetEnvironmentVariable(DirectoryEnvironment);
18+
if (string.IsNullOrWhiteSpace(directory))
19+
{
20+
return;
21+
}
22+
var head = RequireHex(SourceHeadEnvironment, 40);
23+
var sourceInventory = RequireHex(SourceInventoryEnvironment, 64);
24+
var output = Describe(records, corpus, state, head, sourceInventory);
25+
var bytes = JsonSerializer.SerializeToUtf8Bytes(output, JsonDefaults.Options);
26+
Directory.CreateDirectory(directory);
27+
var path = Path.Combine(directory, records.ToString(System.Globalization.CultureInfo.InvariantCulture) + "-" + corpus + ".json");
28+
if (File.Exists(path))
29+
{
30+
if (!File.ReadAllBytes(path).AsSpan().SequenceEqual(bytes))
31+
{
32+
throw new InvalidOperationException("The sample chunk corpus manifest already belongs to a different source or corpus.");
33+
}
34+
return;
35+
}
36+
using var stream = new FileStream(path, FileMode.CreateNew, FileAccess.Write, FileShare.None);
37+
stream.Write(bytes);
38+
}
39+
40+
private static object Describe(int records, string corpus, SampleChunkBenchmarkState state,
41+
string head, string sourceInventory)
42+
=> new
43+
{
44+
schemaVersion = 1,
45+
scope = "development_codec_microbenchmark_control_only",
46+
fixture = nameof(SampleChunkSerializationBenchmarks),
47+
sourceHead = head,
48+
sourceInventorySha256 = sourceInventory,
49+
corpus,
50+
seed = SampleChunkBenchmarkCorpus.Seed,
51+
corpusGenerator = "SHA256_seed_sample_lane_Int32LE",
52+
actualRecordCount = records,
53+
nativeValueBytes = state.NativeBytes,
54+
chunkValueBytes = state.ChunkBytes,
55+
nativeValueBytesPerSample = (double)state.NativeBytes / state.RecordCount,
56+
chunkValueBytesPerSample = (double)state.ChunkBytes / state.RecordCount,
57+
orderedNativeValuesSha256 = HashNative(state.NativeValues),
58+
chunkValueSha256 = Convert.ToHexStringLower(SHA256.HashData(state.ChunkValue)),
59+
correctness = "every_field_original_offset_and_IEEE_bits_verified_before_timing",
60+
accounting = "complete_native_value_envelopes_excluding_keys_WAL_replication_indexes_and_storage_overhead",
61+
machineName = Environment.MachineName,
62+
processorCount = Environment.ProcessorCount,
63+
architecture = RuntimeInformation.ProcessArchitecture.ToString(),
64+
operatingSystem = RuntimeInformation.OSDescription,
65+
runtime = RuntimeInformation.FrameworkDescription,
66+
databaseScaleEvidence = false,
67+
githubQualified = false,
68+
canonicalChunkStorage = false,
69+
rewriteCostQualified = false,
70+
correctionRecoveryQualified = false
71+
};
72+
73+
private static string HashNative(IReadOnlyList<byte[]> values)
74+
{
75+
using var hash = IncrementalHash.CreateHash(HashAlgorithmName.SHA256);
76+
Span<byte> length = stackalloc byte[sizeof(int)];
77+
foreach (var value in values)
78+
{
79+
BinaryPrimitives.WriteInt32LittleEndian(length, value.Length);
80+
hash.AppendData(length);
81+
hash.AppendData(value);
82+
}
83+
return Convert.ToHexStringLower(hash.GetHashAndReset());
84+
}
85+
86+
private static string RequireHex(string environment, int length)
87+
{
88+
var value = Environment.GetEnvironmentVariable(environment);
89+
if (value is null || value.Length != length || value.Any(character => !char.IsAsciiHexDigit(character)))
90+
{
91+
throw new InvalidOperationException(InvalidSourceMessage);
92+
}
93+
return value;
94+
}
95+
}

‎benchmarks/KeyLoad.BenchmarkScenarios/Features/BenchmarkComparisons/SampleChunkBenchmarkState.cs‎

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
using KeyLoad.Core;
2+
using KeyLoad.Core.Features.TimeSeries;
3+
4+
namespace KeyLoad.BenchmarkScenarios.Features.BenchmarkComparisons;
5+
6+
internal sealed class SampleChunkBenchmarkState
7+
{
8+
private readonly SampleRecord[] records;
9+
private readonly byte[][] nativeValues;
10+
private readonly byte[] chunk;
11+
12+
internal SampleChunkBenchmarkState(SampleRecord[] records)
13+
{
14+
this.records = records;
15+
nativeValues = NativeEncode();
16+
chunk = ChunkEncode();
17+
RequireExact(records, NativeDecode());
18+
RequireExact(records, ChunkDecode());
19+
}
20+
21+
internal long NativeBytes => nativeValues.Sum(value => (long)value.Length);
22+
internal int ChunkBytes => chunk.Length;
23+
internal int RecordCount => records.Length;
24+
internal IReadOnlyList<byte[]> NativeValues => nativeValues;
25+
internal ReadOnlySpan<byte> ChunkValue => chunk;
26+
27+
internal byte[][] NativeEncode()
28+
{
29+
var budget = Budget();
30+
var values = new byte[records.Length][];
31+
for (var index = 0; index < records.Length; index++)
32+
{
33+
budget.Check();
34+
values[index] = NativeSerialization.Serialize(records[index]);
35+
budget.Check();
36+
}
37+
return values;
38+
}
39+
40+
internal byte[] ChunkEncode() => SampleChunkCodec.Encode(records, Budget());
41+
42+
internal SampleRecord[] NativeDecode()
43+
{
44+
var budget = Budget();
45+
var decoded = new SampleRecord[records.Length];
46+
for (var index = 0; index < records.Length; index++)
47+
{
48+
budget.ChargeBytes(nativeValues[index].Length);
49+
decoded[index] = NativeSerialization.Deserialize<SampleRecord>(nativeValues[index]);
50+
budget.Check();
51+
}
52+
return decoded;
53+
}
54+
55+
internal SampleRecord[] ChunkDecode()
56+
{
57+
var budget = Budget();
58+
budget.ChargeBytes(chunk.Length);
59+
return SampleChunkCodec.Decode(chunk, budget);
60+
}
61+
62+
private static ReadExecutionBudget Budget() => new(new DatabaseLimits());
63+
64+
private static void RequireExact(SampleRecord[] expected, SampleRecord[] actual)
65+
{
66+
if (actual.Length != expected.Length)
67+
{
68+
throw new InvalidOperationException("The sample chunk benchmark corpus failed exact roundtrip verification.");
69+
}
70+
for (var index = 0; index < expected.Length; index++)
71+
{
72+
var left = expected[index];
73+
var right = actual[index];
74+
if (left.SeriesId != right.SeriesId || left.Sequence != right.Sequence || left.TagsJson != right.TagsJson
75+
|| left.Sample.EventId != right.Sample.EventId
76+
|| left.Sample.Timestamp.Ticks != right.Sample.Timestamp.Ticks
77+
|| left.Sample.Timestamp.Offset != right.Sample.Timestamp.Offset
78+
|| BitConverter.DoubleToInt64Bits(left.Sample.Value) != BitConverter.DoubleToInt64Bits(right.Sample.Value))
79+
{
80+
throw new InvalidOperationException("The sample chunk benchmark corpus failed exact roundtrip verification.");
81+
}
82+
}
83+
}
84+
}

‎benchmarks/KeyLoad.BenchmarkScenarios/Features/BenchmarkComparisons/SampleChunkSerializationBenchmarks.cs‎

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
using BenchmarkDotNet.Attributes;
2+
using BenchmarkDotNet.Configs;
3+
using BenchmarkDotNet.Jobs;
4+
5+
namespace KeyLoad.BenchmarkScenarios.Features.BenchmarkComparisons;
6+
7+
/// <summary>Measures bounded lossless chunk values against the actual native per-record codec; these are small development controls.</summary>
8+
[MemoryDiagnoser]
9+
[SimpleJob(RuntimeMoniker.Net10_0, launchCount: 1, warmupCount: 3, iterationCount: 8, id: "SampleChunkCodecDevelopment")]
10+
[GroupBenchmarksBy(BenchmarkLogicalGroupRule.ByCategory)]
11+
[CategoriesColumn]
12+
public class SampleChunkSerializationBenchmarks
13+
{
14+
private SampleChunkBenchmarkState? state;
15+
16+
/// <summary>Gets or sets the actual number of independently identified samples in each encoded batch.</summary>
17+
[Params(1, 32, 256)]
18+
public int RecordCount { get; set; } = 32;
19+
20+
/// <summary>Gets or sets the deterministic timestamp/value/tag distribution, not a database dataset size.</summary>
21+
[Params(SampleChunkBenchmarkCorpus.Regular, SampleChunkBenchmarkCorpus.Late, SampleChunkBenchmarkCorpus.Random)]
22+
public string Corpus { get; set; } = SampleChunkBenchmarkCorpus.Regular;
23+
24+
/// <summary>Prepares both encodings and verifies every decoded field outside measurement.</summary>
25+
[GlobalSetup]
26+
public void Setup()
27+
{
28+
if (state is not null)
29+
{
30+
throw new InvalidOperationException("The sample chunk benchmark is already initialized.");
31+
}
32+
var candidate = new SampleChunkBenchmarkState(SampleChunkBenchmarkCorpus.Create(RecordCount, Corpus));
33+
SampleChunkBenchmarkManifest.Write(RecordCount, Corpus, candidate);
34+
state = candidate;
35+
}
36+
37+
/// <summary>Encodes one current native SampleRecord value per actual sample.</summary>
38+
/// <returns>The complete independent native value envelopes, excluding storage keys and journals.</returns>
39+
[Benchmark(Baseline = true)]
40+
[BenchmarkCategory("Encode")]
41+
public byte[][] NativeRecordsEncode() => RequireState().NativeEncode();
42+
43+
/// <summary>Encodes the same samples through the bounded checksummed generated chunk envelope.</summary>
44+
/// <returns>The complete native chunk value envelope.</returns>
45+
[Benchmark]
46+
[BenchmarkCategory("Encode")]
47+
public byte[] ChunkEncode() => RequireState().ChunkEncode();
48+
49+
/// <summary>Decodes the current independent native value envelopes under one operation budget.</summary>
50+
/// <returns>The owned decoded sample records.</returns>
51+
[Benchmark(Baseline = true)]
52+
[BenchmarkCategory("Decode")]
53+
public SampleRecord[] NativeRecordsDecode() => RequireState().NativeDecode();
54+
55+
/// <summary>Decodes the same samples from the bounded lossless chunk after charging its actual bytes.</summary>
56+
/// <returns>The owned decoded sample records.</returns>
57+
[Benchmark]
58+
[BenchmarkCategory("Decode")]
59+
public SampleRecord[] ChunkDecode() => RequireState().ChunkDecode();
60+
61+
/// <summary>Releases the corpus after a measurement process; repeated cleanup is harmless.</summary>
62+
[GlobalCleanup]
63+
public void Cleanup() => state = null;
64+
65+
private SampleChunkBenchmarkState RequireState()
66+
=> state ?? throw new InvalidOperationException("The sample chunk benchmark is not initialized.");
67+
}

‎docs/ADR/ADR-079-lossless-series-chunk-codecs.md‎

Lines changed: 39 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
# ADR-079 — Bounded lossless time-series chunk codec qualification
22

3-
Status: Accepted for the codec qualification stage; source and runtime evidence pending.
3+
Status: Accepted; codec source and local development verification delivered.
4+
Exact-source Linux and subsequent canonical layout qualification remain pending.
45
Related: KL-078, REQ-SERIES-006/017–020 and AC-CHUNK-001–006 in
56
[TimeSeries](../Features/TimeSeries.md), architecture §27.6, ADR-035, ADR-073 and
67
ADR-077. The owner authorized completing the implementation plan; root owns this
@@ -150,6 +151,31 @@ exhaustion remains BudgetExceeded. Failure cannot mutate canonical source data.
150151
Add only the exact benchmark friend visibility needed. Each result records
151152
source, machine, corpus and actual serialized bytes; these small controls are
152153
not database scale/comparison evidence or a public acceleration claim.
154+
New UnitTests `Features/BenchmarkComparisons/SampleChunkBenchmark*` own
155+
the fixture metadata and real external generated-consumer execution oracle;
156+
the existing bounded child/capture owner is reused without changing it.
157+
TASK-CHUNK-PROVENANCE, cluster_wave Luna/high, owns only new
158+
`scripts/Features/BenchmarkComparisons/sample-chunk-development*.mjs`.
159+
A bounded development CLI captures the complete tracked/untracked nonignored
160+
source inventory and actual Release benchmark dependency DLLs, including Core,
161+
before execution. It sets the frozen manifest environment, awaits the real
162+
BenchmarkDotNet child and validates all36 original cases and9 corpus manifests.
163+
It refuses changed source/binaries, missing/duplicate cases, failed execution,
164+
mismatched source or corpus and substituted artifacts. A new owned output
165+
directory holds original reports, bounded logs, inventories and a derived
166+
receipt explicitly marked local development only. Default settings match the
167+
fixture's1 launch,3 warmups,8 measurements and100ms iteration time, with
168+
explicit DontRemove outlier mode retaining every actual measurement; optional
169+
Dry mode remains an execution oracle with no performance qualification.
170+
No existing public/GitHub evidence module, workflow or website is modified.
171+
TASK-CHUNK-PROVENANCE-ORACLE, lifecycle_wave Luna/high, owns only new UnitTests
172+
BenchmarkComparisons `SampleChunkBenchmarkRejection*` helpers. After the real
173+
Dry consumer completes, validate its receipt and reject independently corrupted
174+
copies of its original reports/manifests: missing or duplicated cases, wrong
175+
launch/settings/parameters, nonpositive measurements, source-hash mismatch and
176+
forbidden qualification flags. Leave all originals byte-exact. Execute the
177+
actual checked-in validators in a bounded awaited child inside TUnit/Aspire;
178+
supplied parser fixtures never become measured or authenticated evidence.
153179
6. TASK-CHUNK-JOIN, root: inspect every diff and acceptance mapping, enforce
154180
numeric complexity, strict full build/formatter/governance, run actual TUnit
155181
through Aspire, retain original reports and commit/push the completed stage.
@@ -161,3 +187,15 @@ Coding workers stop and escalate on contract ambiguity, bound/compatibility
161187
conflict, shared-file changes or any proposed canonical storage integration.
162188
Unit/store suites prove codec behavior only. Process-kill, power-loss, endurance,
163189
database performance and complete KL-078 acceptance are not inferred from them.
190+
191+
## Development evidence
192+
193+
The [2026-10-03 receipt](../implementation/sample-chunk-codec-development-2026-10-03.json)
194+
retains the complete frozen source/runtime inventories, full Release and formatter
195+
results, Aspire normal/scalar2852/2852 reports, two real36-case Dry consumers and
196+
12 copied-report rejection cases per consumer. The ordinary36-case BenchmarkDotNet
197+
control retains all8 measurements,3 warmups,9 actual corpora and original
198+
encode/decode cost, allocations and native-envelope bytes for1/32/256 samples.
199+
Both the initial failed validator/settings integration and its originals remain
200+
recorded. Canonical storage, rewrite/correction recovery, RF3 and delivered-source
201+
Linux qualification are still open.

0 commit comments

Comments
 (0)