diff --git a/packages/studio-server/src/history/blobStore.ts b/packages/studio-server/src/history/blobStore.ts index 52dc05ab4ea..d7b0a784fb0 100644 --- a/packages/studio-server/src/history/blobStore.ts +++ b/packages/studio-server/src/history/blobStore.ts @@ -16,6 +16,8 @@ export interface BlobStore { writeTo(hash: string, absPath: string, beforeReplace?: () => void): Promise; bytes(): number; size(hash: string): number; + keep(path: string, absPath: string, remove: boolean): Promise; + release(held: Iterable): Promise; prune(keep: ReadonlySet): Promise; drop(hashes: Iterable): Promise; } @@ -58,14 +60,45 @@ async function cloneOrCopy(from: string, to: string, beforeReplace?: () => void) } } +const listDir = (path: string) => + readdir(path, { withFileTypes: true }).catch((error: NodeJS.ErrnoException) => { + if (error.code === "ENOENT") return []; + throw error; + }); + export async function openBlobStore(dir: string): Promise { const sizes = new Map(); + const asideDir = join(dirname(dir), "aside"); + const asideSizes = new Map(); + // The project paths still counting on each set-aside copy; it goes when the last one has its file back. + const holds = new Map>(); await mkdir(dir, { recursive: true }); - for (const shard of await readdir(dir)) - for (const hash of await readdir(join(dir, shard)).catch(() => [])) - sizes.set(hash, (await stat(join(dir, shard, hash))).size); - const pathOf = (hash: string) => blobPath(dir, hash); + for (const [root, found] of [ + [dir, sizes], + [asideDir, asideSizes], + ] as const) + for (const shard of await listDir(root)) { + if (!shard.isDirectory()) { + // A crash mid-copy leaves its unpublished temp file beside the shards. + if (shard.name.startsWith("incoming-")) await rm(join(root, shard.name), { force: true }); + continue; + } + for (const hash of await listDir(join(root, shard.name))) + found.set(hash.name, (await stat(join(root, shard.name, hash.name))).size); + } + for (const hash of asideSizes.keys()) holds.set(hash, new Set(["*"])); + const pathOf = (hash: string) => blobPath(sizes.has(hash) ? dir : asideDir, hash); let total = [...sizes.values()].reduce((sum, size) => sum + size, 0); + const forget = async (hash: string) => { + await rm(blobPath(dir, hash), { force: true }); + total -= sizes.get(hash) ?? 0; + sizes.delete(hash); + }; + const forgetAside = async (hash: string) => { + await rm(blobPath(asideDir, hash), { force: true }); + asideSizes.delete(hash); + holds.delete(hash); + }; return { async put(absPath) { @@ -82,36 +115,58 @@ export async function openBlobStore(dir: string): Promise { await rm(temp, { force: true }); return hash; } - await mkdir(dirname(pathOf(hash)), { recursive: true }); + await mkdir(dirname(blobPath(dir, hash)), { recursive: true }); const size = (await stat(temp)).size; - await rename(temp, pathOf(hash)); + await rename(temp, blobPath(dir, hash)); sizes.set(hash, size); total += size; return hash; }, reference: (absPath) => hashFile(absPath), - has: (hash) => sizes.has(hash), - read: async (hash) => readBlob(dir, hash), + has: (hash) => sizes.has(hash) || asideSizes.has(hash), + read: (hash) => readFile(pathOf(hash)), writeTo: async (hash, absPath, beforeReplace) => cloneOrCopy(pathOf(hash), absPath, beforeReplace), + async release(held) { + for (const [hash, path] of held) { + holds.get(hash)?.delete(path); + if (holds.get(hash)?.size === 0) await forgetAside(hash); + } + }, + async keep(path, absPath, remove) { + const hash = await hashFile(absPath); + if (sizes.has(hash)) return; + if (asideSizes.has(hash)) { + holds.get(hash)!.add(path); + return; + } + const to = blobPath(asideDir, hash); + await mkdir(dirname(to), { recursive: true }); + const moved = + remove && + (await rename(absPath, to).then( + () => true, + () => false, + )); + if (!moved) { + const temp = join(asideDir, `incoming-${randomUUID()}`); + await cloneOrStream(absPath, temp).catch(async (error) => { + await rm(temp, { force: true }); + throw error; + }); + await rename(temp, to); + } + asideSizes.set(hash, (await stat(to)).size); + holds.set(hash, new Set([path])); + }, bytes: () => total, size: (hash) => sizes.get(hash) ?? 0, async drop(hashes) { - for (const hash of hashes) { - const size = sizes.get(hash); - if (size === undefined) continue; - await rm(pathOf(hash), { force: true }); - sizes.delete(hash); - total -= size; - } + for (const hash of hashes) if (sizes.has(hash)) await forget(hash); }, async prune(keep) { - for (const [hash, size] of sizes) { - if (keep.has(hash)) continue; - await rm(pathOf(hash), { force: true }); - sizes.delete(hash); - total -= size; - } + for (const hash of [...sizes.keys()]) if (!keep.has(hash)) await forget(hash); + for (const hash of [...asideSizes.keys()]) if (!keep.has(hash)) await forgetAside(hash); }, }; } diff --git a/packages/studio-server/src/history/projectHistory.ts b/packages/studio-server/src/history/projectHistory.ts index ec389369ef2..8acaa9e75c6 100644 --- a/packages/studio-server/src/history/projectHistory.ts +++ b/packages/studio-server/src/history/projectHistory.ts @@ -1122,6 +1122,9 @@ class Engine { this.windows.push(group); try { for (const [path, hash] of changes) await this.writeProjectFile(path, hash); + await this.blobs.release( + changes.flatMap(([path, hash]) => (hash ? [[hash, path] as const] : [])), + ); await this.sweep(); } finally { this.windows = this.windows.filter((open) => open !== group); @@ -1138,6 +1141,9 @@ class Engine { const version = hash === null ? DELETED_VERSION : hashVersion(hash); recordFileWriteReceipt(absPath, { path, version, writeToken }); } + const standing = lstatSync(absPath, { throwIfNoEntry: false }); + if (standing?.isFile() && isUncopied(path, standing.size)) + await this.blobs.keep(path, absPath, hash === null); if (hash === null) { this.assertWritable(); rmSync(absPath, { force: true }); diff --git a/packages/studio-server/src/history/undoUncopied.test.ts b/packages/studio-server/src/history/undoUncopied.test.ts new file mode 100644 index 00000000000..bab3e765236 --- /dev/null +++ b/packages/studio-server/src/history/undoUncopied.test.ts @@ -0,0 +1,231 @@ +import { + chmodSync, + existsSync, + mkdirSync, + mkdtempSync, + readdirSync, + readFileSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { openBlobStore } from "./blobStore"; +import { openProjectHistory, type ProjectHistory } from "./projectHistory"; +import type { HistoryWho } from "./historyLog"; + +const you: HistoryWho = { kind: "person", name: "You" }; +const MiB = 1024 ** 2; +const cleanup: Array<() => unknown> = []; +afterEach(async () => { + for (const step of cleanup.splice(0).reverse()) await step(); +}); + +function tempDir(prefix: string): string { + const dir = mkdtempSync(join(tmpdir(), prefix)); + cleanup.push(() => rmSync(dir, { recursive: true, force: true })); + return dir; +} + +async function open(projectDir: string, historyRoot: string) { + const history = await openProjectHistory({ projectDir, historyRoot, quietMs: 30 }); + cleanup.push(() => history.close()); + return history; +} + +async function change(history: ProjectHistory, label: string, writes: () => void) { + const window = await history.beginWindow(you, label); + writes(); + const entry = await window.close(); + if (!entry) throw new Error("the window recorded nothing"); + return entry; +} + +async function project() { + const projectDir = tempDir("hf-undo-media-project-"); + const historyRoot = tempDir("hf-undo-media-root-"); + writeFileSync(join(projectDir, "index.html"), "A"); + const history = await open(projectDir, historyRoot); + const media = join(projectDir, "join.mp4"); + return { projectDir, historyRoot, history, media }; +} + +describe("undoing a change that made or replaced media", () => { + it("keeps the made file's bytes, so redo brings the same file back", async () => { + const { history, media } = await project(); + const bytes = Buffer.alloc(2 * MiB, 7); + const made = await change(history, "Place join", () => writeFileSync(media, bytes)); + + const undone = await history.undo(made.id, { who: you }); + expect(existsSync(media)).toBe(false); + + const redone = await history.undo(undone.ok ? undone.entry!.id : "", { who: you }); + expect(redone).toMatchObject({ ok: true }); + expect(readFileSync(media).equals(bytes)).toBe(true); + }); + + it("still brings the file back after the history is closed and opened again", async () => { + const { history, media, projectDir, historyRoot } = await project(); + const bytes = Buffer.alloc(2 * MiB, 9); + const made = await change(history, "Place join", () => writeFileSync(media, bytes)); + const undone = await history.undo(made.id, { who: you }); + await history.close(); + + const reopened = await open(projectDir, historyRoot); + const redone = await reopened.undo(undone.ok ? undone.entry!.id : "", { who: you }); + expect(redone).toMatchObject({ ok: true }); + expect(readFileSync(media).equals(bytes)).toBe(true); + }); + + it("keeps the bytes a restore overwrites, so undoing the restore returns them", async () => { + const { history, media, projectDir } = await project(); + const first = Buffer.alloc(2 * MiB, 1); + const second = Buffer.alloc(2 * MiB, 2); + const made = await change(history, "Place join", () => writeFileSync(media, first)); + await history.undo(made.id, { who: you }); + await change(history, "New join", () => writeFileSync(media, second)); + mkdirSync(join(projectDir, "x"), { recursive: true }); + + const restored = await history.restore(made.id, you); + expect(readFileSync(media).equals(first)).toBe(true); + + const back = await history.undo(restored!.id, { who: you }); + expect(back).toMatchObject({ ok: true }); + expect(readFileSync(media).equals(second)).toBe(true); + }); + + it("does not spend the set-aside bytes when an old version is only checked out", async () => { + const { history, media } = await project(); + const bytes = Buffer.alloc(2 * MiB, 3); + const made = await change(history, "Place join", () => writeFileSync(media, bytes)); + const undone = await history.undo(made.id, { who: you }); + + await history.checkout(made.id, "after", tempDir("hf-undo-media-out-")); + + const redone = await history.undo(undone.ok ? undone.entry!.id : "", { who: you }); + expect(redone).toMatchObject({ ok: true }); + expect(readFileSync(media).equals(bytes)).toBe(true); + }); + + it("brings back two files with the same bytes made in one change", async () => { + const { history, projectDir } = await project(); + const bytes = Buffer.alloc(2 * MiB, 4); + const made = await change(history, "Place joins", () => { + writeFileSync(join(projectDir, "a.mp4"), bytes); + writeFileSync(join(projectDir, "b.mp4"), bytes); + }); + const undone = await history.undo(made.id, { who: you }); + + const redone = await history.undo(undone.ok ? undone.entry!.id : "", { who: you }); + expect(redone).toMatchObject({ ok: true }); + for (const name of ["a.mp4", "b.mp4"]) + expect(readFileSync(join(projectDir, name)).equals(bytes)).toBe(true); + }); + + it("does not fold the history away when the media it keeps is larger than the budget", async () => { + const projectDir = tempDir("hf-undo-media-project-"); + const historyRoot = tempDir("hf-undo-media-root-"); + writeFileSync(join(projectDir, "index.html"), "A"); + const history = await openProjectHistory({ projectDir, historyRoot, budgetBytes: MiB }); + cleanup.push(() => history.close()); + const media = join(projectDir, "join.mp4"); + const bytes = Buffer.alloc(2 * MiB, 5); + const made = await change(history, "Place join", () => writeFileSync(media, bytes)); + + const undone = await history.undo(made.id, { who: you }); + const redone = await history.undo(undone.ok ? undone.entry!.id : "", { who: you }); + + expect(redone).toMatchObject({ ok: true }); + expect(readFileSync(media).equals(bytes)).toBe(true); + expect(history.list().length).toBeGreaterThanOrEqual(3); + }); +}); + +describe("the same bytes at two paths in separate entries", () => { + it("brings both files back through undo and redo in any order", async () => { + const { history, projectDir } = await project(); + const bytes = Buffer.alloc(2 * MiB, 8); + const first = await change(history, "Place a", () => + writeFileSync(join(projectDir, "a.mp4"), bytes), + ); + const second = await change(history, "Place b", () => + writeFileSync(join(projectDir, "b.mp4"), bytes), + ); + const undoneSecond = await history.undo(second.id, { who: you }); + const undoneFirst = await history.undo(first.id, { who: you }); + + const redoneFirst = await history.undo(undoneFirst.ok ? undoneFirst.entry!.id : "", { + who: you, + }); + const redoneSecond = await history.undo(undoneSecond.ok ? undoneSecond.entry!.id : "", { + who: you, + }); + + expect([redoneFirst, redoneSecond]).toMatchObject([{ ok: true }, { ok: true }]); + for (const name of ["a.mp4", "b.mp4"]) + expect(readFileSync(join(projectDir, name)).equals(bytes)).toBe(true); + }); +}); + +describe("the set-aside store beside the blobs", () => { + async function stores() { + const home = tempDir("hf-aside-home-"); + const source = join(home, "join.mp4"); + writeFileSync(source, Buffer.alloc(2 * MiB, 6)); + const blobs = await openBlobStore(join(home, "blobs")); + const hash = await blobs.reference(source); + return { home, source, blobs, hash }; + } + + it("keeps a copied blob of the same bytes when the set-aside copy is released", async () => { + const { home, source, blobs, hash } = await stores(); + await blobs.keep("join.mp4", source, false); + const copy = join(home, "backup.txt"); + writeFileSync(copy, readFileSync(source)); + + expect(await blobs.put(copy)).toBe(hash); + await blobs.release([[hash, "join.mp4"]]); + + expect(blobs.has(hash)).toBe(true); + expect((await blobs.read(hash)).length).toBe(2 * MiB); + }); + + it("counts for no bytes and no size until it is released", async () => { + const { source, blobs, hash } = await stores(); + await blobs.keep("join.mp4", source, true); + + expect([blobs.has(hash), blobs.bytes(), blobs.size(hash)]).toEqual([true, 0, 0]); + expect(existsSync(source)).toBe(false); + }); + + it("leaves only the finished copy in the set-aside folder", async () => { + const { home, source, blobs, hash } = await stores(); + await blobs.keep("join.mp4", source, false); + + const aside = join(home, "aside"); + expect(readdirSync(aside)).toEqual([hash.slice(0, 2)]); + expect(readdirSync(join(aside, hash.slice(0, 2)))).toEqual([hash]); + }); + + it("opens past a temp file a crash left mid-copy, and clears it", async () => { + const { home, source, blobs, hash } = await stores(); + await blobs.keep("join.mp4", source, false); + const leftovers = [join(home, "blobs", "incoming-1234"), join(home, "aside", "incoming-5678")]; + for (const leftover of leftovers) writeFileSync(leftover, "half a copy"); + + const reopened = await openBlobStore(join(home, "blobs")); + + expect(reopened.has(hash)).toBe(true); + expect(leftovers.some((leftover) => existsSync(leftover))).toBe(false); + }); + + it("still fails the open when the folder cannot be read", async () => { + const { home } = await stores(); + const blobsDir = join(home, "blobs"); + chmodSync(blobsDir, 0o111); + cleanup.push(() => chmodSync(blobsDir, 0o755)); + + await expect(openBlobStore(blobsDir)).rejects.toMatchObject({ code: "EACCES" }); + }); +}); diff --git a/scripts/package-size-budgets.json b/scripts/package-size-budgets.json index 1e5ae92bcd9..49fe0f79aae 100644 --- a/scripts/package-size-budgets.json +++ b/scripts/package-size-budgets.json @@ -52,7 +52,7 @@ "unpacked": 24360208 }, "@hyperframes/studio-server": { - "packed": 143700, - "unpacked": 559199 + "packed": 144500, + "unpacked": 568500 } }