diff --git a/ts/packages/benchmarks/README.AUTOGEN.md b/ts/packages/benchmarks/README.AUTOGEN.md index 825ed00a37..084c5705b5 100644 --- a/ts/packages/benchmarks/README.AUTOGEN.md +++ b/ts/packages/benchmarks/README.AUTOGEN.md @@ -3,7 +3,7 @@ - + # @typeagent/benchmarks — AI-generated documentation @@ -52,10 +52,10 @@ _None._ - [./src/core/prices.ts](./src/core/prices.ts) - [./src/core/rateLimiter.ts](./src/core/rateLimiter.ts) - [./src/core/tokenEstimate.ts](./src/core/tokenEstimate.ts) -- _…and 44 more under `./src/`._ +- _…and 45 more under `./src/`._ --- -_Auto-generated against commit `95c2a1d9ba80426f522f7ece727da39f8a577d9e` on `2026-08-19T17:31:21.317Z` by `docs-generate.yml`. Links validated at that commit; the working tree may have drifted by up to 24h. Re-run `pnpm --filter @typeagent/benchmarks docs:verify-links` to spot-check._ +_Auto-generated against commit `cdc601cd99a2f6dae1d83e6501d158c8cfcf1421` on `2026-08-21T00:46:31.017Z` by `docs-generate.yml`. Links validated at that commit; the working tree may have drifted by up to 24h. Re-run `pnpm --filter @typeagent/benchmarks docs:verify-links` to spot-check._ diff --git a/ts/packages/benchmarks/src/translationBench/synthesizer/generationSupport.ts b/ts/packages/benchmarks/src/translationBench/synthesizer/generationSupport.ts index 3ab3422564..880fba5495 100644 --- a/ts/packages/benchmarks/src/translationBench/synthesizer/generationSupport.ts +++ b/ts/packages/benchmarks/src/translationBench/synthesizer/generationSupport.ts @@ -6,6 +6,11 @@ import fs from "node:fs"; import { z } from "zod"; import type { TranslationBenchBenchmarkSchema } from "./benchmark.js"; +import { + appendSyncedJsonlRecords, + initializeSyncedJsonlFile, + readRecoverableJsonlLines, +} from "./jsonlCheckpoint.js"; import { parseJsonText, parseVersionedWithZod, @@ -193,7 +198,7 @@ export function translationBenchResumeKey( ]); } -function validateRowShard( +function validateTranslationBenchCheckpointRowShard( row: TranslationBenchCheckpointRow, header: TranslationBenchCheckpointHeader, ): void { @@ -208,11 +213,14 @@ function validateRowShard( } } -function settingsEqual(left: unknown, right: unknown): boolean { +function translationBenchCheckpointSettingsEqual( + left: unknown, + right: unknown, +): boolean { return canonicalJson(left) === canonicalJson(right); } -function assertCompatibleHeaders( +function assertTranslationBenchCheckpointHeadersCompatible( actual: TranslationBenchCheckpointHeader, expected: TranslationBenchCheckpointHeader, ): void { @@ -221,7 +229,12 @@ function assertCompatibleHeaders( "Translation bench checkpoint run fingerprint is incompatible", ); } - if (!settingsEqual(actual.settings, expected.settings)) { + if ( + !translationBenchCheckpointSettingsEqual( + actual.settings, + expected.settings, + ) + ) { throw new Error( "Translation bench checkpoint settings are incompatible", ); @@ -245,19 +258,12 @@ export function createTranslationBenchRunFingerprint( export function readTranslationBenchCheckpoint( filePath: string, ): TranslationBenchCheckpoint { - const text = fs.readFileSync(filePath, "utf8"); - const lines = text.endsWith("\n") - ? text.slice(0, -1).split("\n") - : text.split("\n"); - if (lines.length === 0 || (lines.length === 1 && lines[0] === "")) { - throw new Error(`Translation bench checkpoint '${filePath}' is empty`); - } - if (lines.some((line) => line.trim().length === 0)) { + const lines = readRecoverableJsonlLines(filePath); + if (lines.length === 0 || lines.some((line) => line.trim().length === 0)) { throw new Error( - `Translation bench checkpoint '${filePath}' contains a blank line`, + `Translation bench checkpoint '${filePath}' is empty or contains a blank line`, ); } - const header = parseTranslationBenchCheckpointHeader( parseJsonText(lines[0]!, `checkpoint '${filePath}' line 1`), ); @@ -270,7 +276,7 @@ export function readTranslationBenchCheckpoint( `checkpoint '${filePath}' line ${index + 1}`, ), ); - validateRowShard(row, header); + validateTranslationBenchCheckpointRowShard(row, header); const key = translationBenchResumeKey(row); if (resumeKeys.has(key)) { throw new Error(`Duplicate translation bench resume key '${key}'`); @@ -281,6 +287,10 @@ export function readTranslationBenchCheckpoint( return { header, rows, resumeKeys }; } +/** + * Appends checkpoint rows for one owning writer. Concurrent writers are not + * supported; the caller must serialize all access to the checkpoint path. + */ export function appendTranslationBenchCheckpointRows( filePath: string, checkpointHeader: TranslationBenchCheckpointHeader, @@ -290,7 +300,7 @@ export function appendTranslationBenchCheckpointRows( const batchKeys = new Set(); const normalizedRows = rows.map((row) => { const parsed = parseTranslationBenchCheckpointRow(row); - validateRowShard(parsed, header); + validateTranslationBenchCheckpointRowShard(parsed, header); const key = translationBenchResumeKey(parsed); if (batchKeys.has(key)) { throw new Error(`Duplicate translation bench resume key '${key}'`); @@ -302,34 +312,23 @@ export function appendTranslationBenchCheckpointRows( let current: TranslationBenchCheckpoint; if (fs.existsSync(filePath)) { current = readTranslationBenchCheckpoint(filePath); - assertCompatibleHeaders(current.header, header); + assertTranslationBenchCheckpointHeadersCompatible( + current.header, + header, + ); } else { - try { - fs.writeFileSync(filePath, `${canonicalJson(header)}\n`, { - flag: "wx", - }); - current = { - header, - rows: [], - resumeKeys: new Set(), - }; - } catch (error) { - const code = (error as NodeJS.ErrnoException).code; - if (code !== "EEXIST") throw error; - current = readTranslationBenchCheckpoint(filePath); - assertCompatibleHeaders(current.header, header); - } + initializeSyncedJsonlFile(filePath, canonicalJson(header)); + current = { header, rows: [], resumeKeys: new Set() }; } - for (const key of batchKeys) { if (current.resumeKeys.has(key)) { throw new Error(`Duplicate translation bench resume key '${key}'`); } } if (normalizedRows.length > 0) { - fs.appendFileSync( + appendSyncedJsonlRecords( filePath, - normalizedRows.map((row) => `${canonicalJson(row)}\n`).join(""), + normalizedRows.map((row) => canonicalJson(row)), ); } return { diff --git a/ts/packages/benchmarks/src/translationBench/synthesizer/jsonlCheckpoint.ts b/ts/packages/benchmarks/src/translationBench/synthesizer/jsonlCheckpoint.ts new file mode 100644 index 0000000000..51f1b0bf6f --- /dev/null +++ b/ts/packages/benchmarks/src/translationBench/synthesizer/jsonlCheckpoint.ts @@ -0,0 +1,96 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import fs from "node:fs"; +import path from "node:path"; + +import { splitTranslationBenchCheckpointLines } from "../runner/scale.js"; + +function fsyncDirectory(filePath: string): void { + if (process.platform === "win32") return; + const directory = fs.openSync(path.dirname(filePath), "r"); + try { + fs.fsyncSync(directory); + } finally { + fs.closeSync(directory); + } +} + +function writeAll(handle: number, buffer: Buffer, position: number): void { + let offset = 0; + while (offset < buffer.length) { + const written = fs.writeSync( + handle, + buffer, + offset, + buffer.length - offset, + position + offset, + ); + if (written === 0) throw new Error("Unable to complete JSONL write"); + offset += written; + } +} + +/** Initializes a JSONL file owned by a single writer. */ +export function initializeSyncedJsonlFile( + filePath: string, + firstRecord: string, +): void { + const temporaryPath = `${filePath}.tmp`; + let handle: number | undefined; + try { + handle = fs.openSync(temporaryPath, "w"); + fs.writeFileSync(handle, `${firstRecord}\n`, "utf8"); + fs.fsyncSync(handle); + } finally { + if (handle !== undefined) fs.closeSync(handle); + } + fs.renameSync(temporaryPath, filePath); + fsyncDirectory(filePath); +} + +export function readRecoverableJsonlLines(filePath: string): string[] { + return splitTranslationBenchCheckpointLines( + fs.readFileSync(filePath, "utf8"), + ); +} + +/** + * Repairs a torn final line and appends records for one owning writer. + * Concurrent calls for the same path are not supported. + */ +export function appendSyncedJsonlRecords( + filePath: string, + records: readonly string[], +): void { + if (records.length === 0) return; + const handle = fs.openSync(filePath, "r+"); + try { + const content = fs.readFileSync(handle); + let appendOffset = content.length; + let separator = ""; + if (appendOffset > 0 && content.at(-1) !== 0x0a) { + const lastNewline = content.lastIndexOf(0x0a); + if (lastNewline < 0) { + throw new Error( + `JSONL file '${filePath}' has no complete line`, + ); + } + const tail = content.subarray(lastNewline + 1).toString("utf8"); + try { + JSON.parse(tail); + separator = "\n"; + } catch { + appendOffset = lastNewline + 1; + fs.ftruncateSync(handle, appendOffset); + } + } + const payload = Buffer.from( + separator + records.map((record) => `${record}\n`).join(""), + ); + writeAll(handle, payload, appendOffset); + fs.fsyncSync(handle); + } finally { + fs.closeSync(handle); + } +} diff --git a/ts/packages/benchmarks/test/translationBench.checkpointPrimitives.spec.ts b/ts/packages/benchmarks/test/translationBench.checkpointPrimitives.spec.ts index 833cf7ed76..955d2a93ac 100644 --- a/ts/packages/benchmarks/test/translationBench.checkpointPrimitives.spec.ts +++ b/ts/packages/benchmarks/test/translationBench.checkpointPrimitives.spec.ts @@ -1,27 +1,49 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. - -import { describe, expect, it } from "@jest/globals"; - +import { afterAll, describe, expect, it } from "@jest/globals"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; import { + appendTranslationBenchCheckpointRows, createTranslationBenchRunFingerprint, - getTranslationBenchShardIndex, - splitTranslationBenchCheckpointLines, -} from "../src/translationBench/runner/scale.js"; + readTranslationBenchCheckpoint, + type TranslationBenchCheckpointHeader, + type TranslationBenchCheckpointRow, +} from "../src/translationBench/synthesizer/generationSupport.js"; -describe("translation bench checkpoint primitives", () => { - it("uses canonical fingerprints and stable shards", () => { - expect(createTranslationBenchRunFingerprint({ b: 2, a: 1 })).toBe( - createTranslationBenchRunFingerprint({ a: 1, b: 2 }), - ); - expect(getTranslationBenchShardIndex("case-1", 8)).toBe( - getTranslationBenchShardIndex("case-1", 8), - ); - }); +const directory = fs.mkdtempSync(path.join(os.tmpdir(), "translation-bench-")); +afterAll(() => fs.rmSync(directory, { recursive: true, force: true })); +const header: TranslationBenchCheckpointHeader = { + kind: "translation-bench-checkpoint", + version: 1, + runFingerprint: createTranslationBenchRunFingerprint({ run: 1 }), + settings: { model: "test" }, + shardIndex: 0, + shardCount: 1, +}; +const row = (caseId: string): TranslationBenchCheckpointRow => ({ + kind: "translation-bench-row", + version: 1, + phase: "generate", + model: "test", + scenario: "default", + caseId, + value: caseId, +}); - it("drops only an incomplete trailing JSONL row", () => { +describe("translation bench checkpoints", () => { + it("recovers a torn final row before appending", () => { + const checkpointPath = path.join(directory, "checkpoint.jsonl"); + appendTranslationBenchCheckpointRows(checkpointPath, header, [ + row("1"), + ]); + fs.appendFileSync(checkpointPath, '{"kind":"translation-bench-row"'); + appendTranslationBenchCheckpointRows(checkpointPath, header, [ + row("2"), + ]); expect( - splitTranslationBenchCheckpointLines('{"header":1}\n{"row":'), - ).toEqual(['{"header":1}']); + readTranslationBenchCheckpoint(checkpointPath).rows, + ).toEqual([row("1"), row("2")]); }); });