Repair invalid compile caches and clean failed artifacts

apps/server/src/compile-artifacts.ts

@@ -1,17 +1,46 @@
-import { open } from 'node:fs/promises';
+import { open, type FileHandle } from 'node:fs/promises';
+
+const PDF_TAIL_BYTES = 1024;
+const PDF_WHITESPACE = new Set([0, 9, 10, 12, 13, 32]);
 
 export async function hasPdfSignature(path: string): Promise<boolean> {
   const file = await open(path, 'r');
   try {
     const signature = Buffer.alloc(5);
-    let offset = 0;
-    while (offset < signature.length) {
-      const { bytesRead } = await file.read(signature, offset, signature.length - offset, offset);
-      if (!bytesRead) return false;
-      offset += bytesRead;
-    }
-    return signature.toString('utf8') === '%PDF-';
+    return await readAt(file, signature, 0) && signature.toString('utf8') === '%PDF-';
+  } finally {
+    await file.close();
+  }
+}
+
+// A bounded completeness check for compiler-generated artifacts. This rejects
+// missing headers/terminal EOF markers, without claiming semantic PDF validation.
+export async function hasPdfEnvelope(path: string): Promise<boolean> {
+  const file = await open(path, 'r');
+  try {
+    const signature = Buffer.alloc(5);
+    if (!await readAt(file, signature, 0) || signature.toString('utf8') !== '%PDF-') return false;
+    const { size } = await file.stat();
+    const tailPosition = Math.max(signature.length, size - PDF_TAIL_BYTES);
+    if (size - tailPosition < 5) return false;
+    const tail = Buffer.alloc(size - tailPosition);
+    if (!await readAt(file, tail, tailPosition)) return false;
+    let end = tail.length;
+    while (end > 0 && PDF_WHITESPACE.has(tail[end - 1]!)) end--;
+    const marker = end - 5;
+    return marker >= 0 && tail.subarray(marker, end).toString('latin1') === '%%EOF' &&
+      (marker === 0 || tail[marker - 1] === 10 || tail[marker - 1] === 13);
   } finally {
     await file.close();
   }
 }
+
+async function readAt(file: FileHandle, buffer: Buffer, position: number): Promise<boolean> {
+  let offset = 0;
+  while (offset < buffer.length) {
+    const { bytesRead } = await file.read(buffer, offset, buffer.length - offset, position + offset);
+    if (!bytesRead) return false;
+    offset += bytesRead;
+  }
+  return true;
+}

apps/server/src/compiler.ts

@@ -9,7 +9,7 @@ import { HttpError } from './errors.js';
 import type { ProjectStore } from './project-store.js';
 import type { ActorIdentity, ProjectRecord } from './types.js';
 import { stageCompileSources, type CompileSource } from './compile-staging.js';
-import { hasPdfSignature } from './compile-artifacts.js';
+import { hasPdfEnvelope } from './compile-artifacts.js';
 
 export interface CompileJob {
   id: string;
@@ -127,7 +127,7 @@ export class Compiler {
 
       const cachedPdf = join(this.cacheDir, `${sourceHash}.pdf`);
       const cachedLog = join(this.cacheDir, `${sourceHash}.log`);
-      if (await fileExists(cachedPdf)) {
+      if (await fileExists(cachedPdf) && await validCachedPdf(cachedPdf)) {
         const cachedSynctex = join(this.cacheDir, `${sourceHash}.synctex.gz`);
         const synctexPath = await fileExists(cachedSynctex) ? cachedSynctex : null;
         const now = new Date().toISOString();
@@ -263,6 +263,7 @@ export class Compiler {
 
     const workDir = join(this.jobsDir, job.id);
     const outputDir = join(workDir, '.overroot-output');
+    const temporaryArtifacts: string[] = [];
     try {
       await stageCompileSources(workDir, sources);
 
@@ -313,13 +314,16 @@ export class Compiler {
       const synctexPath = (await fileExists(generatedSynctex)) ? join(this.cacheDir, `${job.source_hash}.synctex.gz`) : null;
       const pdfTemp = `${pdfPath}.${job.id}.tmp`;
       const logTemp = `${logPath}.${job.id}.tmp`;
+      // Register before writing: even a failed copy can leave a partial file.
+      temporaryArtifacts.push(pdfTemp, logTemp);
       await copyFile(generatedPdf, pdfTemp);
-      if (!(await hasPdfSignature(pdfTemp))) throw new Error('Compiler produced an invalid PDF');
+      if (!(await hasPdfEnvelope(pdfTemp))) throw new Error('Compiler produced an invalid PDF');
       await writeFile(logTemp, log, { mode: 0o600 });
       await rename(pdfTemp, pdfPath);
       await rename(logTemp, logPath);
       if (synctexPath) {
         const synctexTemp = `${synctexPath}.${job.id}.tmp`;
+        temporaryArtifacts.push(synctexTemp);
         await copyFile(generatedSynctex, synctexTemp);
         await rename(synctexTemp, synctexPath);
       }
@@ -349,7 +353,7 @@ export class Compiler {
       const failed = this.get(job.id, project.id)!;
       this.events.emit(project.id, { type: 'compile.failed', job: publicCompileJob(failed) });
     } finally {
-      await rm(workDir, { recursive: true, force: true });
+      await cleanupCompileFiles(workDir, temporaryArtifacts);
     }
   }
 
@@ -495,6 +499,28 @@ async function fileExists(path: string): Promise<boolean> {
   }
 }
 
+async function validCachedPdf(path: string): Promise<boolean> {
+  try {
+    return await hasPdfEnvelope(path);
+  } catch (error) {
+    // A cache artifact can disappear between stat and open. Recompile the
+    // already frozen snapshot instead of reporting a successful broken cache hit.
+    if ((error as NodeJS.ErrnoException).code === 'ENOENT') return false;
+    throw error;
+  }
+}
+
+async function cleanupCompileFiles(workDir: string, temporaryArtifacts: readonly string[]): Promise<void> {
+  const cleanup = await Promise.allSettled([
+    rm(workDir, { recursive: true, force: true }),
+    ...temporaryArtifacts.map(path => rm(path, { force: true }))
+  ]);
+  // Drain every cleanup operation before releasing the job's admission slot.
+  // Final cache artifacts and other jobs' temporary files are never removed.
+  const failure = cleanup.find(result => result.status === 'rejected');
+  if (failure?.status === 'rejected') throw failure.reason;
+}
+
 async function readOptional(path: string): Promise<string> {
   try {
     return await readFile(path, 'utf8');

apps/server/tests/compiler-artifacts.test.ts

@@ -2,11 +2,12 @@ import { mkdtemp, rm, writeFile } from 'node:fs/promises';
 import { tmpdir } from 'node:os';
 import { join } from 'node:path';
 import { afterEach, expect, it, vi } from 'vitest';
-import { hasPdfSignature } from '../src/compile-artifacts.js';
+import { hasPdfEnvelope, hasPdfSignature } from '../src/compile-artifacts.js';
 
 const control = vi.hoisted(() => ({
-  read: null as (() => Promise<never>) | null,
+  read: null as ((position: number) => Promise<void>) | null,
   reads: [] as Array<{ offset: number; length: number; position: number }>,
+  bytesRead: 0,
   closes: 0
 }));
 vi.mock('node:fs/promises', async (importOriginal) => {
@@ -16,10 +17,13 @@ vi.mock('node:fs/promises', async (importOriginal) => {
     return {
       read: async (buffer: Buffer, offset: number, length: number, position: number) => {
         control.reads.push({ offset, length, position });
-        if (control.read) return control.read();
+        await control.read?.(position);
         // Exercise legal short reads too.
-        return handle.read(buffer, offset, Math.min(length, 2), position);
+        const result = await handle.read(buffer, offset, Math.min(length, 2), position);
+        control.bytesRead += result.bytesRead;
+        return result;
       },
+      stat: () => handle.stat(),
       close: async () => { control.closes++; await handle.close(); }
     };
   } };
@@ -29,6 +33,7 @@ let dir: string;
 afterEach(async () => {
   control.read = null;
   control.reads.length = 0;
+  control.bytesRead = 0;
   control.closes = 0;
   if (dir) await rm(dir, { recursive: true, force: true });
 });
@@ -62,3 +67,40 @@ it('closes the PDF handle if reading fails', async () => {
   await expect(hasPdfSignature(path)).rejects.toThrow('Controlled read failure');
   expect(control.closes).toBe(1);
 });
+
+it.each(['%PDF-1.7\ntruncated', '%PDF-1.7\n%%EO', '%PDF-1.7\n%%EOF\ntrailing', 'not PDF\n%%EOF\n'])(
+  'rejects an incomplete compiler artifact %j', async content => {
+    dir = await mkdtemp(join(tmpdir(), 'overroot-pdf-'));
+    const path = join(dir, 'incomplete.pdf');
+    await writeFile(path, content);
+    expect(await hasPdfEnvelope(path)).toBe(false);
+    expect(control.closes).toBe(1);
+  }
+);
+
+it.each(['\n', '\r\n', '\n \t\f\r\n', '\n\0'])('accepts a terminal compiler PDF marker followed by %j', async whitespace => {
+  dir = await mkdtemp(join(tmpdir(), 'overroot-pdf-'));
+  const path = join(dir, 'envelope.pdf');
+  await writeFile(path, `%PDF-1.7\ntrailer\n<< /Size 1 >>\nstartxref\n9\n%%EOF${whitespace}`);
+  expect(await hasPdfEnvelope(path)).toBe(true);
+  expect(control.closes).toBe(1);
+});
+
+it('checks only five header bytes and at most 1024 tail bytes even with short reads', async () => {
+  dir = await mkdtemp(join(tmpdir(), 'overroot-pdf-'));
+  const path = join(dir, 'large.pdf');
+  await writeFile(path, Buffer.concat([Buffer.from('%PDF-1.7\n'), Buffer.alloc(1024 * 1024), Buffer.from('\nstartxref\n9\n%%EOF\n')]));
+  expect(await hasPdfEnvelope(path)).toBe(true);
+  expect(control.bytesRead).toBe(5 + 1024);
+  expect(control.closes).toBe(1);
+});
+
+it('closes the PDF handle if reading the footer fails after a valid header', async () => {
+  dir = await mkdtemp(join(tmpdir(), 'overroot-pdf-'));
+  const path = join(dir, 'footer-failure.pdf');
+  await writeFile(path, '%PDF-1.7\nstartxref\n9\n%%EOF\n');
+  control.read = async position => { if (position >= 5) throw new Error('Controlled footer read failure'); };
+  await expect(hasPdfEnvelope(path)).rejects.toThrow('Controlled footer read failure');
+  expect(control.bytesRead).toBe(5);
+  expect(control.closes).toBe(1);
+});

apps/server/tests/compiler-cache.test.ts

@@ -0,0 +1,106 @@
+import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { afterEach, beforeEach, expect, it, vi } from 'vitest';
+import { buildApp } from '../src/app.js';
+import { config } from '../src/config.js';
+import type { AppContext } from '../src/context.js';
+
+const cacheIo = vi.hoisted(() => ({
+  beforeOpen: null as ((path: string) => Promise<void>) | null,
+  writes: new Map<string, Buffer>()
+}));
+vi.mock('node:fs/promises', async (importOriginal) => {
+  const fs = await importOriginal<typeof import('node:fs/promises')>();
+  return { ...fs, open: async (...args: Parameters<typeof fs.open>) => {
+    await cacheIo.beforeOpen?.(String(args[0]));
+    return fs.open(...args);
+  }, writeFile: async (...args: Parameters<typeof fs.writeFile>) => {
+    await fs.writeFile(...args);
+    if (typeof args[0] === 'string' && (typeof args[1] === 'string' || args[1] instanceof Uint8Array)) {
+      cacheIo.writes.set(args[0], Buffer.from(args[1]));
+    }
+  } };
+});
+
+const previous = { fake: config.fakeCompiler, jobs: config.compileMaxJobs };
+let app: Awaited<ReturnType<typeof buildApp>>['app'];
+let context: AppContext;
+let dir: string;
+
+beforeEach(() => {
+  config.fakeCompiler = true;
+  config.compileMaxJobs = 1;
+});
+afterEach(async () => {
+  cacheIo.beforeOpen = null;
+  cacheIo.writes.clear();
+  if (app) await app.close();
+  vi.restoreAllMocks();
+  config.fakeCompiler = previous.fake;
+  config.compileMaxJobs = previous.jobs;
+  if (dir) await rm(dir, { recursive: true, force: true });
+});
+
+async function setup() {
+  dir = await mkdtemp(join(tmpdir(), 'overroot-cache-'));
+  ({ app, context } = await buildApp({ databasePath: join(dir, 'test.sqlite3'), logger: false }));
+  const user = await context.auth.createUser('cache-test', 'cache-test-password', true);
+  const actor = { type: 'user' as const, id: user.id, name: user.username };
+  const project = context.projects.createImportedProject(user.id, 'Cache recovery', [
+    { path: 'main.tex', content: Buffer.from('Frozen revision') }
+  ]);
+  const request = () => context.compiler.request(project, actor);
+  const first = await request();
+  await vi.waitFor(() => expect(context.compiler.get(first.id, project.id)?.status).toBe('succeeded'));
+  // A completed job's final staging cleanup still owns its admission slot.
+  await vi.waitFor(async () => expect((await request()).status).toBe('succeeded'));
+  const original = context.compiler.get(first.id, project.id)!;
+  const pdf = await readFile(original.pdf_path!);
+  const events = vi.spyOn(context.events, 'emit');
+  return { request, project, actor, original, pdf, events };
+}
+
+it.each(['', '%PD', 'broken cache artifact', '%PDF-1.7\ntruncated'])('recompiles a corrupt cached PDF %j and reuses the repaired output', async corruption => {
+  const h = await setup();
+  await writeFile(h.original.pdf_path!, corruption);
+  const repaired = await h.request();
+  expect(repaired.status).toBe('queued');
+  expect(h.events.mock.calls.some(([, event]) => event.type === 'compile.ready' && event.cached === true)).toBe(false);
+  await vi.waitFor(() => expect(context.compiler.get(repaired.id, h.project.id)?.status).toBe('succeeded'));
+  expect(await readFile(h.original.pdf_path!)).toEqual(h.pdf);
+  const cached = await vi.waitFor(() => h.request());
+  expect(cached.status).toBe('succeeded');
+  expect(h.events.mock.calls.some(([, event]) => event.type === 'compile.ready' && event.cached === false)).toBe(true);
+  expect(h.events.mock.calls.at(-1)?.[1]).toMatchObject({ type: 'compile.ready', cached: true });
+});
+
+it('recompiles the frozen snapshot when a cached PDF disappears between stat and validation', async () => {
+  const h = await setup();
+  let removed = false;
+  cacheIo.beforeOpen = async path => {
+    if (path !== h.original.pdf_path || removed) return;
+    removed = true;
+    await rm(path);
+    const main = context.projects.getFile(h.project.id, 'main.tex')!;
+    context.projects.saveFile(main.id, Buffer.from('A later revision'), h.actor);
+  };
+  const repaired = await h.request();
+  expect(removed).toBe(true);
+  expect(repaired.status).toBe('queued');
+  expect(repaired.source_hash).toBe(h.original.source_hash);
+  await vi.waitFor(() => expect(context.compiler.get(repaired.id, h.project.id)?.status).toBe('succeeded'));
+  expect(await readFile(h.original.pdf_path!)).toEqual(h.pdf);
+  expect(cacheIo.writes.get(join(dir, 'jobs', repaired.id, 'main.tex'))?.toString()).toBe('Frozen revision');
+  expect(Buffer.from(context.projects.getFile(h.project.id, 'main.tex')!.content).toString()).toBe('A later revision');
+});
+
+it('reports a current cache validation I/O failure and releases admission for a retry', async () => {
+  const h = await setup();
+  cacheIo.beforeOpen = async path => {
+    if (path === h.original.pdf_path) throw Object.assign(new Error('Controlled cache read failure'), { code: 'EACCES' });
+  };
+  await expect(h.request()).rejects.toThrow('Controlled cache read failure');
+  cacheIo.beforeOpen = null;
+  expect((await h.request()).status).toBe('succeeded');
+});