ci: Measure the type-check budget with TypeScript 7

scripts/typeperf/README.md

@@ -12,8 +12,10 @@ yarn check:typeperf --update    # re-record budget.json after an intended change
 It fails when:
 
 - a stress fixture's **instantiation count** rises more than 10% over [budget.json](./budget.json).
+  It type-checks with TypeScript 7 (`@typescript/native`, the compiler the repo builds with).
   Instantiations are deterministic for a given TypeScript version, so they make a stable
-  signal; check times are printed for context only.
+  signal; check times are printed for context only. Counts shift when the compiler is
+  upgraded, so re-record the budget with `--update` alongside a TypeScript bump.
 - a fixture has type errors.
   The `patheq` fixture's errors mean the path types (`PathKeys`, `PathArgs`, `ShortenPath`,
   `PathArgsAndSearch`, `KeysToArgs`) disagree with the frozen reference implementation in
@@ -41,7 +43,7 @@ When a count drops more than 10% below its budget, the check says so; re-record
 | setUpdaters | `ctrl.set()` updaters spreading `prev` on a 30-member Union |
 | patheq | path types against [patheq/orig.ts](./patheq/orig.ts) on fixed-seed fuzzed paths |
 
-To compare TypeScript 6 and 7 or check time, run a compiler on a fixture directly:
+To compare with TypeScript 6 or dig into one fixture, run a compiler on it directly:
 
 ```bash
 npx tsc6 -p scripts/typeperf/scenarios/paths/tsconfig.json --extendedDiagnostics

scripts/typeperf/budget.json

@@ -1,16 +1,16 @@
 {
   "instantiations": {
-    "resources": 121763,
-    "bigEntity": 12783,
-    "union": 16142,
-    "paths": 345562,
-    "nested": 2895,
-    "vue": 47288,
-    "schemas": 99251,
-    "typical": 13038,
-    "setValues": 26029,
-    "setUpdaters": 66033,
-    "patheq": 935715
+    "resources": 123569,
+    "bigEntity": 12790,
+    "union": 16394,
+    "paths": 345603,
+    "nested": 3001,
+    "vue": 47903,
+    "schemas": 109095,
+    "typical": 13109,
+    "setValues": 26027,
+    "setUpdaters": 66031,
+    "patheq": 934395
   },
-  "typescript": "6.0.3"
+  "typescript": "7.0.2"
 }

scripts/typeperf/check.mjs

@@ -1,81 +1,77 @@
 // Type-check performance budget for @data-client's public types.
 //
-// Generates the fixtures (gen.mjs), type-checks each with the `typescript` package,
-// and fails when a fixture has errors or its instantiation count exceeds budget.json
-// by more than TOLERANCE. Unlike check time, instantiations are deterministic for a
-// given TypeScript version, so they are what the budget tracks.
+// Generates the fixtures (gen.mjs), type-checks each with the repo's `tsc`
+// (TypeScript 7, @typescript/native), and fails when a fixture has errors or its
+// instantiation count exceeds budget.json by more than TOLERANCE.
+// Unlike check time, instantiations are deterministic for a given TypeScript
+// version, so they are what the budget tracks.
 //
 // usage: yarn check:typeperf [--update] [scenario...]
 //   --update  rewrite budget.json with the current counts (after an intended change)
+import { spawnSync } from 'node:child_process';
 import fs from 'node:fs';
 import path from 'node:path';
 import { fileURLToPath } from 'node:url';
-import ts from 'typescript';
 
 import { generate } from './gen.mjs';
 
 const dir = path.dirname(fileURLToPath(import.meta.url));
 const budgetFile = path.join(dir, 'budget.json');
 const TOLERANCE = 0.1;
 
+// The `tsc` bin yarn puts on PATH, so the check follows the compiler the repo builds with
+const tsc = (...args) =>
+  spawnSync('tsc', args, {
+    encoding: 'utf8',
+    shell: process.platform === 'win32',
+  });
+const tsVersion = tsc('--version').stdout.match(/[\d.]+/)?.[0];
+
 const args = process.argv.slice(2);
 const update = args.includes('--update');
 const only = args.filter(a => !a.startsWith('--'));
 const budget = JSON.parse(fs.readFileSync(budgetFile, 'utf8'));
 
-// Share parsed lib and @data-client .d.ts files between fixtures; each program still
-// gets its own checker, so instantiation counts are unaffected.
-const sourceFiles = new Map();
-const host = options => {
-  const h = ts.createCompilerHost(options);
-  const getSourceFile = h.getSourceFile;
-  h.getSourceFile = (file, ...rest) => {
-    if (file.includes('/scenarios/')) return getSourceFile(file, ...rest);
-    if (!sourceFiles.has(file))
-      sourceFiles.set(file, getSourceFile(file, ...rest));
-    return sourceFiles.get(file);
-  };
-  return h;
-};
-
 const failures = [];
 const lowered = [];
 let needsUpdate = false;
 // A full --update re-records from scratch, dropping removed fixtures
 if (update && !only.length) budget.instantiations = {};
-console.log(`TypeScript ${ts.version}\n`);
+console.log(`TypeScript ${tsVersion}\n`);
 console.log('scenario\tinstantiations\tbudget\tchange\ttime');
 for (const s of generate(only)) {
-  const cfg = ts.getParsedCommandLineOfConfigFile(
-    path.join(dir, 'scenarios', s, 'tsconfig.json'),
-    {},
-    { ...ts.sys, onUnRecoverableConfigFileDiagnostic: () => {} },
-  );
-  const start = performance.now();
-  const program = ts.createProgram(
-    cfg.fileNames,
-    cfg.options,
-    host(cfg.options),
+  const { status, stdout, stderr } = tsc(
+    '-p',
+    // relative, so a Windows shell doesn't split a checkout path with spaces
+    path.relative(
+      process.cwd(),
+      path.join(dir, 'scenarios', s, 'tsconfig.json'),
+    ),
+    '--extendedDiagnostics',
+    '--pretty',
+    'false',
   );
-  const diagnostics = ts.getPreEmitDiagnostics(program);
-  const seconds = (performance.now() - start) / 1000;
-  const count = program.getInstantiationCount();
-  if (diagnostics.length) {
-    failures.push(`${s}: ${diagnostics.length} type errors`);
-    console.error(
-      ts.formatDiagnostics(diagnostics.slice(0, 10), {
-        getCanonicalFileName: f => f,
-        getCurrentDirectory: () => process.cwd(),
-        getNewLine: () => '\n',
-      }),
+  const stat = name =>
+    stdout.match(new RegExp(`^${name}:\\s+(\\S+)`, 'm'))?.[1];
+  const count = Number(stat('Instantiations'));
+  if (Number.isNaN(count)) {
+    console.error(stdout, stderr);
+    failures.push(`${s}: tsc did not report instantiations`);
+    continue;
+  }
+  if (status !== 0) {
+    const errors = stdout.match(/^.* error TS\d+:.*$/gm) ?? [];
+    failures.push(
+      `${s}: tsc exited with ${status}, ${errors.length} type errors`,
     );
+    console.error(errors.slice(0, 10).join('\n') || stderr);
   }
 
   const max = budget.instantiations[s];
   const change = max ? (count - max) / max : 0;
   const pct =
     max ? `${change >= 0 ? '+' : ''}${(change * 100).toFixed(1)}%` : 'new';
-  console.log(`${s}\t${count}\t${max ?? '-'}\t${pct}\t${seconds.toFixed(1)}s`);
+  console.log(`${s}\t${count}\t${max ?? '-'}\t${pct}\t${stat('Total time')}`);
   if (update) {
     budget.instantiations[s] = count;
   } else if (!max) {
@@ -92,10 +88,10 @@ for (const s of generate(only)) {
 }
 
 if (update) {
-  budget.typescript = ts.version;
+  budget.typescript = tsVersion;
   fs.writeFileSync(budgetFile, JSON.stringify(budget, null, 2) + '\n');
   console.log(`\nUpdated ${path.relative(process.cwd(), budgetFile)}`);
-} else if (budget.typescript !== ts.version) {
+} else if (budget.typescript !== tsVersion) {
   console.log(
     `\nbudget.json was recorded with TypeScript ${budget.typescript}; re-record it with --update if the compiler upgrade moved the counts.`,
   );