fix(studio-server): undo and restore keep the media they delete or replace

packages/studio-server/src/history/blobStore.ts

@@ -16,6 +16,8 @@ export interface BlobStore {
   writeTo(hash: string, absPath: string, beforeReplace?: () => void): Promise<void>;
   bytes(): number;
   size(hash: string): number;
+  keep(path: string, absPath: string, remove: boolean): Promise<void>;
+  release(held: Iterable<readonly [hash: string, path: string]>): Promise<void>;
   prune(keep: ReadonlySet<string>): Promise<void>;
   drop(hashes: Iterable<string>): Promise<void>;
 }
@@ -58,14 +60,45 @@ async function cloneOrCopy(from: string, to: string, beforeReplace?: () => void)
   }
 }
 
+const listDir = (path: string) =>
+  readdir(path, { withFileTypes: true }).catch((error: NodeJS.ErrnoException) => {
+    if (error.code === "ENOENT") return [];
+    throw error;
+  });
+
 export async function openBlobStore(dir: string): Promise<BlobStore> {
   const sizes = new Map<string, number>();
+  const asideDir = join(dirname(dir), "aside");
+  const asideSizes = new Map<string, number>();
+  // The project paths still counting on each set-aside copy; it goes when the last one has its file back.
+  const holds = new Map<string, Set<string>>();
   await mkdir(dir, { recursive: true });
-  for (const shard of await readdir(dir))
-    for (const hash of await readdir(join(dir, shard)).catch(() => []))
-      sizes.set(hash, (await stat(join(dir, shard, hash))).size);
-  const pathOf = (hash: string) => blobPath(dir, hash);
+  for (const [root, found] of [
+    [dir, sizes],
+    [asideDir, asideSizes],
+  ] as const)
+    for (const shard of await listDir(root)) {
+      if (!shard.isDirectory()) {
+        // A crash mid-copy leaves its unpublished temp file beside the shards.
+        if (shard.name.startsWith("incoming-")) await rm(join(root, shard.name), { force: true });
+        continue;
+      }
+      for (const hash of await listDir(join(root, shard.name)))
+        found.set(hash.name, (await stat(join(root, shard.name, hash.name))).size);
+    }
+  for (const hash of asideSizes.keys()) holds.set(hash, new Set(["*"]));
+  const pathOf = (hash: string) => blobPath(sizes.has(hash) ? dir : asideDir, hash);
   let total = [...sizes.values()].reduce((sum, size) => sum + size, 0);
+  const forget = async (hash: string) => {
+    await rm(blobPath(dir, hash), { force: true });
+    total -= sizes.get(hash) ?? 0;
+    sizes.delete(hash);
+  };
+  const forgetAside = async (hash: string) => {
+    await rm(blobPath(asideDir, hash), { force: true });
+    asideSizes.delete(hash);
+    holds.delete(hash);
+  };
 
   return {
     async put(absPath) {
@@ -82,36 +115,58 @@ export async function openBlobStore(dir: string): Promise<BlobStore> {
         await rm(temp, { force: true });
         return hash;
       }
-      await mkdir(dirname(pathOf(hash)), { recursive: true });
+      await mkdir(dirname(blobPath(dir, hash)), { recursive: true });
       const size = (await stat(temp)).size;
-      await rename(temp, pathOf(hash));
+      await rename(temp, blobPath(dir, hash));
       sizes.set(hash, size);
       total += size;
       return hash;
     },
     reference: (absPath) => hashFile(absPath),
-    has: (hash) => sizes.has(hash),
-    read: async (hash) => readBlob(dir, hash),
+    has: (hash) => sizes.has(hash) || asideSizes.has(hash),
+    read: (hash) => readFile(pathOf(hash)),
     writeTo: async (hash, absPath, beforeReplace) =>
       cloneOrCopy(pathOf(hash), absPath, beforeReplace),
+    async release(held) {
+      for (const [hash, path] of held) {
+        holds.get(hash)?.delete(path);
+        if (holds.get(hash)?.size === 0) await forgetAside(hash);
+      }
+    },
+    async keep(path, absPath, remove) {
+      const hash = await hashFile(absPath);
+      if (sizes.has(hash)) return;
+      if (asideSizes.has(hash)) {
+        holds.get(hash)!.add(path);
+        return;
+      }
+      const to = blobPath(asideDir, hash);
+      await mkdir(dirname(to), { recursive: true });
+      const moved =
+        remove &&
+        (await rename(absPath, to).then(
+          () => true,
+          () => false,
+        ));
+      if (!moved) {
+        const temp = join(asideDir, `incoming-${randomUUID()}`);
+        await cloneOrStream(absPath, temp).catch(async (error) => {
+          await rm(temp, { force: true });
+          throw error;
+        });
+        await rename(temp, to);
+      }
+      asideSizes.set(hash, (await stat(to)).size);
+      holds.set(hash, new Set([path]));
+    },
     bytes: () => total,
     size: (hash) => sizes.get(hash) ?? 0,
     async drop(hashes) {
-      for (const hash of hashes) {
-        const size = sizes.get(hash);
-        if (size === undefined) continue;
-        await rm(pathOf(hash), { force: true });
-        sizes.delete(hash);
-        total -= size;
-      }
+      for (const hash of hashes) if (sizes.has(hash)) await forget(hash);
     },
     async prune(keep) {
-      for (const [hash, size] of sizes) {
-        if (keep.has(hash)) continue;
-        await rm(pathOf(hash), { force: true });
-        sizes.delete(hash);
-        total -= size;
-      }
+      for (const hash of [...sizes.keys()]) if (!keep.has(hash)) await forget(hash);
+      for (const hash of [...asideSizes.keys()]) if (!keep.has(hash)) await forgetAside(hash);
     },
   };
 }

packages/studio-server/src/history/projectHistory.ts

@@ -1122,6 +1122,9 @@ class Engine {
     this.windows.push(group);
     try {
       for (const [path, hash] of changes) await this.writeProjectFile(path, hash);
+      await this.blobs.release(
+        changes.flatMap(([path, hash]) => (hash ? [[hash, path] as const] : [])),
+      );
       await this.sweep();
     } finally {
       this.windows = this.windows.filter((open) => open !== group);
@@ -1138,6 +1141,9 @@ class Engine {
       const version = hash === null ? DELETED_VERSION : hashVersion(hash);
       recordFileWriteReceipt(absPath, { path, version, writeToken });
     }
+    const standing = lstatSync(absPath, { throwIfNoEntry: false });
+    if (standing?.isFile() && isUncopied(path, standing.size))
+      await this.blobs.keep(path, absPath, hash === null);
     if (hash === null) {
       this.assertWritable();
       rmSync(absPath, { force: true });

packages/studio-server/src/history/undoUncopied.test.ts

@@ -0,0 +1,231 @@
+import {
+  chmodSync,
+  existsSync,
+  mkdirSync,
+  mkdtempSync,
+  readdirSync,
+  readFileSync,
+  rmSync,
+  writeFileSync,
+} from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { afterEach, describe, expect, it } from "vitest";
+import { openBlobStore } from "./blobStore";
+import { openProjectHistory, type ProjectHistory } from "./projectHistory";
+import type { HistoryWho } from "./historyLog";
+
+const you: HistoryWho = { kind: "person", name: "You" };
+const MiB = 1024 ** 2;
+const cleanup: Array<() => unknown> = [];
+afterEach(async () => {
+  for (const step of cleanup.splice(0).reverse()) await step();
+});
+
+function tempDir(prefix: string): string {
+  const dir = mkdtempSync(join(tmpdir(), prefix));
+  cleanup.push(() => rmSync(dir, { recursive: true, force: true }));
+  return dir;
+}
+
+async function open(projectDir: string, historyRoot: string) {
+  const history = await openProjectHistory({ projectDir, historyRoot, quietMs: 30 });
+  cleanup.push(() => history.close());
+  return history;
+}
+
+async function change(history: ProjectHistory, label: string, writes: () => void) {
+  const window = await history.beginWindow(you, label);
+  writes();
+  const entry = await window.close();
+  if (!entry) throw new Error("the window recorded nothing");
+  return entry;
+}
+
+async function project() {
+  const projectDir = tempDir("hf-undo-media-project-");
+  const historyRoot = tempDir("hf-undo-media-root-");
+  writeFileSync(join(projectDir, "index.html"), "A");
+  const history = await open(projectDir, historyRoot);
+  const media = join(projectDir, "join.mp4");
+  return { projectDir, historyRoot, history, media };
+}
+
+describe("undoing a change that made or replaced media", () => {
+  it("keeps the made file's bytes, so redo brings the same file back", async () => {
+    const { history, media } = await project();
+    const bytes = Buffer.alloc(2 * MiB, 7);
+    const made = await change(history, "Place join", () => writeFileSync(media, bytes));
+
+    const undone = await history.undo(made.id, { who: you });
+    expect(existsSync(media)).toBe(false);
+
+    const redone = await history.undo(undone.ok ? undone.entry!.id : "", { who: you });
+    expect(redone).toMatchObject({ ok: true });
+    expect(readFileSync(media).equals(bytes)).toBe(true);
+  });
+
+  it("still brings the file back after the history is closed and opened again", async () => {
+    const { history, media, projectDir, historyRoot } = await project();
+    const bytes = Buffer.alloc(2 * MiB, 9);
+    const made = await change(history, "Place join", () => writeFileSync(media, bytes));
+    const undone = await history.undo(made.id, { who: you });
+    await history.close();
+
+    const reopened = await open(projectDir, historyRoot);
+    const redone = await reopened.undo(undone.ok ? undone.entry!.id : "", { who: you });
+    expect(redone).toMatchObject({ ok: true });
+    expect(readFileSync(media).equals(bytes)).toBe(true);
+  });
+
+  it("keeps the bytes a restore overwrites, so undoing the restore returns them", async () => {
+    const { history, media, projectDir } = await project();
+    const first = Buffer.alloc(2 * MiB, 1);
+    const second = Buffer.alloc(2 * MiB, 2);
+    const made = await change(history, "Place join", () => writeFileSync(media, first));
+    await history.undo(made.id, { who: you });
+    await change(history, "New join", () => writeFileSync(media, second));
+    mkdirSync(join(projectDir, "x"), { recursive: true });
+
+    const restored = await history.restore(made.id, you);
+    expect(readFileSync(media).equals(first)).toBe(true);
+
+    const back = await history.undo(restored!.id, { who: you });
+    expect(back).toMatchObject({ ok: true });
+    expect(readFileSync(media).equals(second)).toBe(true);
+  });
+
+  it("does not spend the set-aside bytes when an old version is only checked out", async () => {
+    const { history, media } = await project();
+    const bytes = Buffer.alloc(2 * MiB, 3);
+    const made = await change(history, "Place join", () => writeFileSync(media, bytes));
+    const undone = await history.undo(made.id, { who: you });
+
+    await history.checkout(made.id, "after", tempDir("hf-undo-media-out-"));
+
+    const redone = await history.undo(undone.ok ? undone.entry!.id : "", { who: you });
+    expect(redone).toMatchObject({ ok: true });
+    expect(readFileSync(media).equals(bytes)).toBe(true);
+  });
+
+  it("brings back two files with the same bytes made in one change", async () => {
+    const { history, projectDir } = await project();
+    const bytes = Buffer.alloc(2 * MiB, 4);
+    const made = await change(history, "Place joins", () => {
+      writeFileSync(join(projectDir, "a.mp4"), bytes);
+      writeFileSync(join(projectDir, "b.mp4"), bytes);
+    });
+    const undone = await history.undo(made.id, { who: you });
+
+    const redone = await history.undo(undone.ok ? undone.entry!.id : "", { who: you });
+    expect(redone).toMatchObject({ ok: true });
+    for (const name of ["a.mp4", "b.mp4"])
+      expect(readFileSync(join(projectDir, name)).equals(bytes)).toBe(true);
+  });
+
+  it("does not fold the history away when the media it keeps is larger than the budget", async () => {
+    const projectDir = tempDir("hf-undo-media-project-");
+    const historyRoot = tempDir("hf-undo-media-root-");
+    writeFileSync(join(projectDir, "index.html"), "A");
+    const history = await openProjectHistory({ projectDir, historyRoot, budgetBytes: MiB });
+    cleanup.push(() => history.close());
+    const media = join(projectDir, "join.mp4");
+    const bytes = Buffer.alloc(2 * MiB, 5);
+    const made = await change(history, "Place join", () => writeFileSync(media, bytes));
+
+    const undone = await history.undo(made.id, { who: you });
+    const redone = await history.undo(undone.ok ? undone.entry!.id : "", { who: you });
+
+    expect(redone).toMatchObject({ ok: true });
+    expect(readFileSync(media).equals(bytes)).toBe(true);
+    expect(history.list().length).toBeGreaterThanOrEqual(3);
+  });
+});
+
+describe("the same bytes at two paths in separate entries", () => {
+  it("brings both files back through undo and redo in any order", async () => {
+    const { history, projectDir } = await project();
+    const bytes = Buffer.alloc(2 * MiB, 8);
+    const first = await change(history, "Place a", () =>
+      writeFileSync(join(projectDir, "a.mp4"), bytes),
+    );
+    const second = await change(history, "Place b", () =>
+      writeFileSync(join(projectDir, "b.mp4"), bytes),
+    );
+    const undoneSecond = await history.undo(second.id, { who: you });
+    const undoneFirst = await history.undo(first.id, { who: you });
+
+    const redoneFirst = await history.undo(undoneFirst.ok ? undoneFirst.entry!.id : "", {
+      who: you,
+    });
+    const redoneSecond = await history.undo(undoneSecond.ok ? undoneSecond.entry!.id : "", {
+      who: you,
+    });
+
+    expect([redoneFirst, redoneSecond]).toMatchObject([{ ok: true }, { ok: true }]);
+    for (const name of ["a.mp4", "b.mp4"])
+      expect(readFileSync(join(projectDir, name)).equals(bytes)).toBe(true);
+  });
+});
+
+describe("the set-aside store beside the blobs", () => {
+  async function stores() {
+    const home = tempDir("hf-aside-home-");
+    const source = join(home, "join.mp4");
+    writeFileSync(source, Buffer.alloc(2 * MiB, 6));
+    const blobs = await openBlobStore(join(home, "blobs"));
+    const hash = await blobs.reference(source);
+    return { home, source, blobs, hash };
+  }
+
+  it("keeps a copied blob of the same bytes when the set-aside copy is released", async () => {
+    const { home, source, blobs, hash } = await stores();
+    await blobs.keep("join.mp4", source, false);
+    const copy = join(home, "backup.txt");
+    writeFileSync(copy, readFileSync(source));
+
+    expect(await blobs.put(copy)).toBe(hash);
+    await blobs.release([[hash, "join.mp4"]]);
+
+    expect(blobs.has(hash)).toBe(true);
+    expect((await blobs.read(hash)).length).toBe(2 * MiB);
+  });
+
+  it("counts for no bytes and no size until it is released", async () => {
+    const { source, blobs, hash } = await stores();
+    await blobs.keep("join.mp4", source, true);
+
+    expect([blobs.has(hash), blobs.bytes(), blobs.size(hash)]).toEqual([true, 0, 0]);
+    expect(existsSync(source)).toBe(false);
+  });
+
+  it("leaves only the finished copy in the set-aside folder", async () => {
+    const { home, source, blobs, hash } = await stores();
+    await blobs.keep("join.mp4", source, false);
+
+    const aside = join(home, "aside");
+    expect(readdirSync(aside)).toEqual([hash.slice(0, 2)]);
+    expect(readdirSync(join(aside, hash.slice(0, 2)))).toEqual([hash]);
+  });
+
+  it("opens past a temp file a crash left mid-copy, and clears it", async () => {
+    const { home, source, blobs, hash } = await stores();
+    await blobs.keep("join.mp4", source, false);
+    const leftovers = [join(home, "blobs", "incoming-1234"), join(home, "aside", "incoming-5678")];
+    for (const leftover of leftovers) writeFileSync(leftover, "half a copy");
+
+    const reopened = await openBlobStore(join(home, "blobs"));
+
+    expect(reopened.has(hash)).toBe(true);
+    expect(leftovers.some((leftover) => existsSync(leftover))).toBe(false);
+  });
+
+  it("still fails the open when the folder cannot be read", async () => {
+    const { home } = await stores();
+    const blobsDir = join(home, "blobs");
+    chmodSync(blobsDir, 0o111);
+    cleanup.push(() => chmodSync(blobsDir, 0o755));
+
+    await expect(openBlobStore(blobsDir)).rejects.toMatchObject({ code: "EACCES" });
+  });
+});

scripts/package-size-budgets.json

@@ -52,7 +52,7 @@
     "unpacked": 24360208
   },
   "@hyperframes/studio-server": {
-    "packed": 143700,
-    "unpacked": 559199
+    "packed": 144500,
+    "unpacked": 568500
   }
 }