diff --git a/scripts/typeperf/README.md b/scripts/typeperf/README.md index c76f316352d5..4e7ee59f2242 100644 --- a/scripts/typeperf/README.md +++ b/scripts/typeperf/README.md @@ -12,8 +12,10 @@ yarn check:typeperf --update # re-record budget.json after an intended change It fails when: - a stress fixture's **instantiation count** rises more than 10% over [budget.json](./budget.json). + It type-checks with TypeScript 7 (`@typescript/native`, the compiler the repo builds with). Instantiations are deterministic for a given TypeScript version, so they make a stable - signal; check times are printed for context only. + signal; check times are printed for context only. Counts shift when the compiler is + upgraded, so re-record the budget with `--update` alongside a TypeScript bump. - a fixture has type errors. The `patheq` fixture's errors mean the path types (`PathKeys`, `PathArgs`, `ShortenPath`, `PathArgsAndSearch`, `KeysToArgs`) disagree with the frozen reference implementation in @@ -41,7 +43,7 @@ When a count drops more than 10% below its budget, the check says so; re-record | setUpdaters | `ctrl.set()` updaters spreading `prev` on a 30-member Union | | patheq | path types against [patheq/orig.ts](./patheq/orig.ts) on fixed-seed fuzzed paths | -To compare TypeScript 6 and 7 or check time, run a compiler on a fixture directly: +To compare with TypeScript 6 or dig into one fixture, run a compiler on it directly: ```bash npx tsc6 -p scripts/typeperf/scenarios/paths/tsconfig.json --extendedDiagnostics diff --git a/scripts/typeperf/budget.json b/scripts/typeperf/budget.json index 0b716924ef9b..5723578597d2 100644 --- a/scripts/typeperf/budget.json +++ b/scripts/typeperf/budget.json @@ -1,16 +1,16 @@ { "instantiations": { - "resources": 121763, - "bigEntity": 12783, - "union": 16142, - "paths": 345562, - "nested": 2895, - "vue": 47288, - "schemas": 99251, - "typical": 13038, - "setValues": 26029, - "setUpdaters": 66033, - "patheq": 935715 + "resources": 123569, + "bigEntity": 12790, + "union": 16394, + "paths": 345603, + "nested": 3001, + "vue": 47903, + "schemas": 109095, + "typical": 13109, + "setValues": 26027, + "setUpdaters": 66031, + "patheq": 934395 }, - "typescript": "6.0.3" + "typescript": "7.0.2" } diff --git a/scripts/typeperf/check.mjs b/scripts/typeperf/check.mjs index 773e4419568d..8c8c1b22443c 100644 --- a/scripts/typeperf/check.mjs +++ b/scripts/typeperf/check.mjs @@ -1,16 +1,17 @@ // Type-check performance budget for @data-client's public types. // -// Generates the fixtures (gen.mjs), type-checks each with the `typescript` package, -// and fails when a fixture has errors or its instantiation count exceeds budget.json -// by more than TOLERANCE. Unlike check time, instantiations are deterministic for a -// given TypeScript version, so they are what the budget tracks. +// Generates the fixtures (gen.mjs), type-checks each with the repo's `tsc` +// (TypeScript 7, @typescript/native), and fails when a fixture has errors or its +// instantiation count exceeds budget.json by more than TOLERANCE. +// Unlike check time, instantiations are deterministic for a given TypeScript +// version, so they are what the budget tracks. // // usage: yarn check:typeperf [--update] [scenario...] // --update rewrite budget.json with the current counts (after an intended change) +import { spawnSync } from 'node:child_process'; import fs from 'node:fs'; import path from 'node:path'; import { fileURLToPath } from 'node:url'; -import ts from 'typescript'; import { generate } from './gen.mjs'; @@ -18,64 +19,59 @@ const dir = path.dirname(fileURLToPath(import.meta.url)); const budgetFile = path.join(dir, 'budget.json'); const TOLERANCE = 0.1; +// The `tsc` bin yarn puts on PATH, so the check follows the compiler the repo builds with +const tsc = (...args) => + spawnSync('tsc', args, { + encoding: 'utf8', + shell: process.platform === 'win32', + }); +const tsVersion = tsc('--version').stdout.match(/[\d.]+/)?.[0]; + const args = process.argv.slice(2); const update = args.includes('--update'); const only = args.filter(a => !a.startsWith('--')); const budget = JSON.parse(fs.readFileSync(budgetFile, 'utf8')); -// Share parsed lib and @data-client .d.ts files between fixtures; each program still -// gets its own checker, so instantiation counts are unaffected. -const sourceFiles = new Map(); -const host = options => { - const h = ts.createCompilerHost(options); - const getSourceFile = h.getSourceFile; - h.getSourceFile = (file, ...rest) => { - if (file.includes('/scenarios/')) return getSourceFile(file, ...rest); - if (!sourceFiles.has(file)) - sourceFiles.set(file, getSourceFile(file, ...rest)); - return sourceFiles.get(file); - }; - return h; -}; - const failures = []; const lowered = []; let needsUpdate = false; // A full --update re-records from scratch, dropping removed fixtures if (update && !only.length) budget.instantiations = {}; -console.log(`TypeScript ${ts.version}\n`); +console.log(`TypeScript ${tsVersion}\n`); console.log('scenario\tinstantiations\tbudget\tchange\ttime'); for (const s of generate(only)) { - const cfg = ts.getParsedCommandLineOfConfigFile( - path.join(dir, 'scenarios', s, 'tsconfig.json'), - {}, - { ...ts.sys, onUnRecoverableConfigFileDiagnostic: () => {} }, - ); - const start = performance.now(); - const program = ts.createProgram( - cfg.fileNames, - cfg.options, - host(cfg.options), + const { status, stdout, stderr } = tsc( + '-p', + // relative, so a Windows shell doesn't split a checkout path with spaces + path.relative( + process.cwd(), + path.join(dir, 'scenarios', s, 'tsconfig.json'), + ), + '--extendedDiagnostics', + '--pretty', + 'false', ); - const diagnostics = ts.getPreEmitDiagnostics(program); - const seconds = (performance.now() - start) / 1000; - const count = program.getInstantiationCount(); - if (diagnostics.length) { - failures.push(`${s}: ${diagnostics.length} type errors`); - console.error( - ts.formatDiagnostics(diagnostics.slice(0, 10), { - getCanonicalFileName: f => f, - getCurrentDirectory: () => process.cwd(), - getNewLine: () => '\n', - }), + const stat = name => + stdout.match(new RegExp(`^${name}:\\s+(\\S+)`, 'm'))?.[1]; + const count = Number(stat('Instantiations')); + if (Number.isNaN(count)) { + console.error(stdout, stderr); + failures.push(`${s}: tsc did not report instantiations`); + continue; + } + if (status !== 0) { + const errors = stdout.match(/^.* error TS\d+:.*$/gm) ?? []; + failures.push( + `${s}: tsc exited with ${status}, ${errors.length} type errors`, ); + console.error(errors.slice(0, 10).join('\n') || stderr); } const max = budget.instantiations[s]; const change = max ? (count - max) / max : 0; const pct = max ? `${change >= 0 ? '+' : ''}${(change * 100).toFixed(1)}%` : 'new'; - console.log(`${s}\t${count}\t${max ?? '-'}\t${pct}\t${seconds.toFixed(1)}s`); + console.log(`${s}\t${count}\t${max ?? '-'}\t${pct}\t${stat('Total time')}`); if (update) { budget.instantiations[s] = count; } else if (!max) { @@ -92,10 +88,10 @@ for (const s of generate(only)) { } if (update) { - budget.typescript = ts.version; + budget.typescript = tsVersion; fs.writeFileSync(budgetFile, JSON.stringify(budget, null, 2) + '\n'); console.log(`\nUpdated ${path.relative(process.cwd(), budgetFile)}`); -} else if (budget.typescript !== ts.version) { +} else if (budget.typescript !== tsVersion) { console.log( `\nbudget.json was recorded with TypeScript ${budget.typescript}; re-record it with --update if the compiler upgrade moved the counts.`, );