diff --git a/src/database.js b/src/database.js index 28de58b..ce563ab 100644 --- a/src/database.js +++ b/src/database.js @@ -60,81 +60,90 @@ export class RedXaiDatabase { } setById(id, value) { - const node = this.requireById(id); - node.value = valueFrom(value); - this.commitMutation(); - return node; + return this.atomicMutation(() => { + const node = this.requireById(id); + node.value = valueFrom(value); + return node; + }); } renameById(id, name) { if (!/^[A-Za-z_][A-Za-z0-9_.-]*$/.test(name)) throw new TypeError('Invalid RedXai variable name'); - const node = this.requireById(id); - node.name = name; - this.commitMutation(); - return node; + return this.atomicMutation(() => { + const node = this.requireById(id); + node.name = name; + return node; + }); } add(databaseIndex, name, value, id = null) { - const database = this.document.databases[databaseIndex]; - if (!database) throw new RangeError(`Database index ${databaseIndex} does not exist`); - const actualId = id ?? this.nextId(); - const node = createAssignment(name, value, actualId); - database.entries.push(node); - this.commitMutation(); - return node; + return this.atomicMutation(() => { + const database = this.document.databases[databaseIndex]; + if (!database) throw new RangeError(`Database index ${databaseIndex} does not exist`); + const actualId = id ?? this.nextId(); + const node = createAssignment(name, value, actualId); + database.entries.push(node); + return node; + }); } addToArray(arrayId, name, value, id = null) { - const owner = this.requireById(arrayId); - if (owner.value.valueType !== 'array') throw new TypeError(`ID ${arrayId} is not an array`); - const actualId = id ?? this.nextId(); - const node = createAssignment(name, value, actualId); - owner.value.items.push(node); - this.commitMutation(); - return node; + return this.atomicMutation(() => { + const owner = this.requireById(arrayId); + if (owner.value.valueType !== 'array') throw new TypeError(`ID ${arrayId} is not an array`); + const actualId = id ?? this.nextId(); + const node = createAssignment(name, value, actualId); + owner.value.items.push(node); + return node; + }); } deleteById(id) { - const record = this.idIndex.get(id); - if (!record) return false; - record.container.splice(record.index, 1); - this.commitMutation(); - return true; + if (!this.idIndex.has(id)) return false; + return this.atomicMutation(() => { + const record = this.idIndex.get(id); + if (!record) return false; + record.container.splice(record.index, 1); + return true; + }); } moveById(id, targetArrayId, index = null) { - const record = this.idIndex.get(id); - if (!record) throw new RangeError(`RedXai ID ${id} was not found`); - if (id === targetArrayId) throw new TypeError('A value cannot be moved inside itself'); - const target = this.requireById(targetArrayId); - if (target.value.valueType !== 'array') throw new TypeError(`Target ID ${targetArrayId} is not an array`); - if (containsId(record.node, targetArrayId)) throw new TypeError('A value cannot be moved into one of its descendants'); - - const [node] = record.container.splice(record.index, 1); - const insertion = index == null ? target.value.items.length : Math.max(0, Math.min(index, target.value.items.length)); - target.value.items.splice(insertion, 0, node); - this.commitMutation(); - return node; + return this.atomicMutation(() => { + const record = this.idIndex.get(id); + if (!record) throw new RangeError(`RedXai ID ${id} was not found`); + if (id === targetArrayId) throw new TypeError('A value cannot be moved inside itself'); + const target = this.requireById(targetArrayId); + if (target.value.valueType !== 'array') throw new TypeError(`Target ID ${targetArrayId} is not an array`); + if (containsId(record.node, targetArrayId)) throw new TypeError('A value cannot be moved into one of its descendants'); + + const [node] = record.container.splice(record.index, 1); + const insertion = index == null ? target.value.items.length : Math.max(0, Math.min(index, target.value.items.length)); + target.value.items.splice(insertion, 0, node); + return node; + }); } copyById(id, options = {}) { - const record = this.idIndex.get(id); - if (!record) throw new RangeError(`RedXai ID ${id} was not found`); - const clone = structuredClone(record.node); - const idMap = new Map(); - let next = this.nextId(); - remapIds(clone, (oldId, isRoot) => { - const replacement = isRoot && options.newId != null ? options.newId : next++; - if (oldId != null) idMap.set(oldId, replacement); - return replacement; - }, true); - if (options.newName) clone.name = options.newName; - - const target = options.targetArrayId == null ? record.container : this.requireArray(options.targetArrayId).value.items; - const insertion = options.index == null ? target.length : Math.max(0, Math.min(options.index, target.length)); - target.splice(insertion, 0, clone); - this.commitMutation(); - return { node: clone, idMap }; + return this.atomicMutation(() => { + const record = this.idIndex.get(id); + if (!record) throw new RangeError(`RedXai ID ${id} was not found`); + const clone = structuredClone(record.node); + const idMap = new Map(); + let next = this.nextId(); + remapIds(clone, (oldId, isRoot) => { + const replacement = isRoot && options.newId != null ? options.newId : next++; + if (oldId != null) idMap.set(oldId, replacement); + return replacement; + }, true); + remapInternalReferences(clone, idMap); + if (options.newName) clone.name = options.newName; + + const target = options.targetArrayId == null ? record.container : this.requireArray(options.targetArrayId).value.items; + const insertion = options.index == null ? target.length : Math.max(0, Math.min(options.index, target.length)); + target.splice(insertion, 0, clone); + return { node: clone, idMap }; + }); } requireArray(id) { @@ -166,6 +175,20 @@ export class RedXaiDatabase { return structuredClone(this.document); } + atomicMutation(callback) { + const before = structuredClone(this.document); + try { + const result = callback(); + if (this.options.validate) assertValid(this.document); + this.rebuildIndex(); + return result; + } catch (error) { + this.document = before; + this.rebuildIndex(); + throw error; + } + } + commitMutation() { if (this.options.validate) assertValid(this.document); this.rebuildIndex(); @@ -188,3 +211,30 @@ function remapIds(node, allocator, isRoot = false) { } } } + +function remapInternalReferences(node, idMap) { + walkNodeValues(node, (value) => { + if (value.valueType !== 'reference' || value.target?.type !== 'id') return; + const replacement = idMap.get(value.target.value); + if (replacement != null) value.target.value = replacement; + }); +} + +function walkNodeValues(node, visitor) { + if (node.kind !== 'assignment') return; + visitor(node.value); + if (node.value.valueType !== 'array' && node.value.valueType !== 'collection') return; + node.value.items.forEach((item) => { + if (item.kind === 'assignment') walkNodeValues(item, visitor); + else if (item.kind === 'value') walkValue(item, visitor); + }); +} + +function walkValue(value, visitor) { + visitor(value); + if (value.valueType !== 'array' && value.valueType !== 'collection') return; + value.items.forEach((item) => { + if (item.kind === 'assignment') walkNodeValues(item, visitor); + else if (item.kind === 'value') walkValue(item, visitor); + }); +} diff --git a/src/store.js b/src/store.js index 8802f5b..1ffd95d 100644 --- a/src/store.js +++ b/src/store.js @@ -1,3 +1,4 @@ +import { randomUUID } from 'node:crypto'; import { open, readFile, rename, copyFile, mkdir, stat, unlink } from 'node:fs/promises'; import { dirname } from 'node:path'; import { parse } from './parser.js'; @@ -46,46 +47,98 @@ export class RedXaiFileStore { async save(filePath, document) { assertExtension(filePath); - const warnings = assertValid(document); - const source = serialize(document, { indent: this.options.indent }); + const prepared = this.prepareDocument(document); + await mkdir(dirname(filePath), { recursive: true }); + const release = await this.acquireLock(filePath); + try { + return await this.writeLocked(filePath, prepared); + } finally { + await release(); + } + } + + async transaction(filePath, callback) { + assertExtension(filePath); await mkdir(dirname(filePath), { recursive: true }); const release = await this.acquireLock(filePath); - const temporaryPath = `${filePath}.tmp-${process.pid}-${Date.now()}`; try { - if (this.options.backup && await exists(filePath)) await copyFile(filePath, `${filePath}.bak`); + // Loading while holding the same lock prevents the classic read-modify-write + // race where two writers both commit from an identical stale snapshot. + const loaded = await this.load(filePath); + const result = await loaded.database.transaction(callback); + const prepared = this.prepareDocument(loaded.database.document); + await this.writeLocked(filePath, prepared); + return result; + } finally { + await release(); + } + } + + prepareDocument(document) { + const warnings = assertValid(document); + const source = serialize(document, { indent: this.options.indent }); + return { source, warnings }; + } + + async writeLocked(filePath, prepared) { + const temporaryPath = `${filePath}.tmp-${process.pid}-${randomUUID()}`; + const backupPath = `${filePath}.bak`; + const backupTemporaryPath = `${backupPath}.tmp-${process.pid}-${randomUUID()}`; + try { + if (this.options.backup && await exists(filePath)) { + await copyFile(filePath, backupTemporaryPath); + await syncFile(backupTemporaryPath); + await rename(backupTemporaryPath, backupPath); + } + const handle = await open(temporaryPath, 'wx', 0o600); try { - await handle.writeFile(source, 'utf8'); + await handle.writeFile(prepared.source, 'utf8'); await handle.sync(); } finally { await handle.close(); } await rename(temporaryPath, filePath); await syncDirectory(dirname(filePath)); - return { filePath, bytes: Buffer.byteLength(source), warnings }; + return { + filePath, + bytes: Buffer.byteLength(prepared.source), + warnings: prepared.warnings, + }; } catch (error) { await unlink(temporaryPath).catch(() => {}); + await unlink(backupTemporaryPath).catch(() => {}); throw new RedXaiStorageError(`Failed to save ${filePath}`, { cause: error }); - } finally { - await release(); } } - async transaction(filePath, callback) { - const loaded = await this.load(filePath); - const result = await loaded.database.transaction(callback); - await this.save(filePath, loaded.database.document); - return result; - } - async acquireLock(filePath) { const lockPath = `${filePath}.lock`; const started = Date.now(); while (true) { try { const handle = await open(lockPath, 'wx', 0o600); - await handle.writeFile(JSON.stringify({ pid: process.pid, createdAt: new Date().toISOString() })); + await handle.writeFile(JSON.stringify({ + pid: process.pid, + createdAt: new Date().toISOString(), + token: randomUUID(), + })); + await handle.sync(); + + // Refresh the lock mtime while a valid writer is alive. A long transaction + // must not be mistaken for a crashed process merely because staleLockMs elapsed. + const heartbeatMs = Math.max(100, Math.floor(this.options.staleLockMs / 3)); + const heartbeat = setInterval(() => { + const now = new Date(); + handle.utimes(now, now).catch(() => {}); + }, heartbeatMs); + heartbeat.unref?.(); + + let released = false; return async () => { + if (released) return; + released = true; + clearInterval(heartbeat); await handle.close().catch(() => {}); await unlink(lockPath).catch(() => {}); }; @@ -129,12 +182,21 @@ async function isStale(lockPath, staleMs) { } } +async function syncFile(filePath) { + const handle = await open(filePath, 'r'); + try { + await handle.sync(); + } finally { + await handle.close(); + } +} + async function syncDirectory(directory) { try { const handle = await open(directory, 'r'); try { await handle.sync(); } finally { await handle.close(); } } catch { - // Some platforms do not support fsync on directories. The file itself is already synced. + // Some platforms do not support fsync on directories. The files themselves are synced. } } diff --git a/test/mutation-hardening.test.js b/test/mutation-hardening.test.js new file mode 100644 index 0000000..db5e0bc --- /dev/null +++ b/test/mutation-hardening.test.js @@ -0,0 +1,113 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { parse, RedXaiDatabase, serialize } from '../src/index.js'; + +const referencedSource = `{RedXaiStore[ID=1]=[ +{Root[ID=2]}=;[ + {Child[ID=3]}="value", + {Link[ID=4]}=@3, + {Nested[ID=5]}=;[{Deep[ID=6]}=TRUE]: +]:, +{Count[ID=7]}=1 +]};`; + +function fresh() { + return new RedXaiDatabase(parse(referencedSource)); +} + +function validationIssue(code, messagePart) { + return (error) => Array.isArray(error?.issues) + && error.issues.some((entry) => entry.code === code && entry.message.includes(messagePart)); +} + +test('failed duplicate-ID add restores the exact document and indexes', () => { + const db = fresh(); + const before = serialize(db.snapshot()); + + assert.throws(() => db.add(0, 'Duplicate', 99, 3), validationIssue('RXH113', 'Duplicate ID 3')); + + assert.equal(serialize(db.snapshot()), before); + assert.equal(db.getById(3).name, 'Child'); + assert.equal(db.getByName('duplicate').length, 0); + assert.equal(db.nextId(), 8); +}); + +test('deleting a referenced value fails atomically and restores it', () => { + const db = fresh(); + const before = serialize(db.snapshot()); + + assert.throws(() => db.deleteById(3), validationIssue('RXH150', 'Reference target ID 3')); + + assert.equal(serialize(db.snapshot()), before); + assert.equal(db.requireById(3).value.value, 'value'); + assert.equal(db.requireById(4).value.target.value, 3); +}); + +test('failed copy with a duplicate explicit root ID leaves no partial subtree', () => { + const db = fresh(); + const before = serialize(db.snapshot()); + + assert.throws( + () => db.copyById(2, { newId: 7, newName: 'BadCopy' }), + validationIssue('RXH113', 'Duplicate ID 7'), + ); + + assert.equal(serialize(db.snapshot()), before); + assert.equal(db.getByName('badcopy').length, 0); + assert.equal(db.nextId(), 8); +}); + +test('copy remaps references that point inside the copied subtree', () => { + const db = fresh(); + const result = db.copyById(2, { newName: 'RootCopy' }); + + const copiedChildId = result.idMap.get(3); + const copiedLinkId = result.idMap.get(4); + assert.ok(Number.isInteger(copiedChildId)); + assert.ok(Number.isInteger(copiedLinkId)); + assert.notEqual(copiedChildId, 3); + assert.equal(db.requireById(copiedLinkId).value.target.value, copiedChildId); + assert.equal(db.requireById(4).value.target.value, 3); +}); + +test('moving a value into its descendant is rejected without altering order', () => { + const db = fresh(); + const before = serialize(db.snapshot()); + + assert.throws(() => db.moveById(2, 5), /descendants/); + + assert.equal(serialize(db.snapshot()), before); + assert.equal(db.document.databases[0].entries[0].id, 2); +}); + +test('async transaction rejection never reaches the live database', async () => { + const db = fresh(); + const before = serialize(db.snapshot()); + + await assert.rejects( + db.transaction(async (draft) => { + draft.setById(7, 999); + await Promise.resolve(); + throw new Error('abort async transaction'); + }), + /abort async transaction/, + ); + + assert.equal(serialize(db.snapshot()), before); + assert.equal(db.requireById(7).value.value, 1); +}); + +test('failed mutation after a successful mutation restores the latest valid state', () => { + const db = fresh(); + db.setById(7, 42); + const committed = serialize(db.snapshot()); + + assert.throws( + () => db.addToArray(2, 'DuplicateChild', false, 3), + validationIssue('RXH113', 'Duplicate ID 3'), + ); + + assert.equal(serialize(db.snapshot()), committed); + assert.equal(db.requireById(7).value.value, 42); + assert.equal(db.getByName('duplicatechild').length, 0); +}); diff --git a/test/store-concurrency.test.js b/test/store-concurrency.test.js new file mode 100644 index 0000000..9449ea2 --- /dev/null +++ b/test/store-concurrency.test.js @@ -0,0 +1,99 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { mkdtemp, readFile, writeFile, stat, utimes } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { RedXaiFileStore, createDocument, assignment } from '../src/index.js'; + +async function fixture() { + const directory = await mkdtemp(join(tmpdir(), 'redxaihm-hardening-')); + return { directory, file: join(directory, 'Counter.RedXai') }; +} + +const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); + +test('concurrent transactions serialize the entire read-modify-write cycle', async () => { + const { file } = await fixture(); + const firstStore = new RedXaiFileStore({ lockTimeoutMs: 5_000 }); + const secondStore = new RedXaiFileStore({ lockTimeoutMs: 5_000 }); + await firstStore.save(file, createDocument({ entries: [assignment('Count', 0, 2)] })); + + const first = firstStore.transaction(file, async (db) => { + const current = db.requireById(2).value.value; + await sleep(150); + db.setById(2, current + 1); + }); + await sleep(20); + const second = secondStore.transaction(file, async (db) => { + const current = db.requireById(2).value.value; + await sleep(20); + db.setById(2, current + 1); + }); + + await Promise.all([first, second]); + const loaded = await firstStore.load(file); + assert.equal(loaded.database.requireById(2).value.value, 2); +}); + +test('lock heartbeat prevents a long live transaction from being stolen as stale', async () => { + const { file } = await fixture(); + const options = { staleLockMs: 150, lockTimeoutMs: 3_000 }; + const firstStore = new RedXaiFileStore(options); + const secondStore = new RedXaiFileStore(options); + await firstStore.save(file, createDocument({ entries: [assignment('Count', 0, 2)] })); + + const first = firstStore.transaction(file, async (db) => { + await sleep(450); + db.setById(2, 1); + }); + await sleep(250); + const second = secondStore.transaction(file, (db) => { + db.setById(2, db.requireById(2).value.value + 1); + }); + + await Promise.all([first, second]); + const loaded = await firstStore.load(file); + assert.equal(loaded.database.requireById(2).value.value, 2); +}); + +test('an abandoned stale lock is removed and does not block a valid save', async () => { + const { file } = await fixture(); + const store = new RedXaiFileStore({ staleLockMs: 50, lockTimeoutMs: 1_000 }); + const lockPath = `${file}.lock`; + await writeFile(lockPath, '{"pid":999999,"createdAt":"old"}', { mode: 0o600 }); + const old = new Date(Date.now() - 10_000); + await utimes(lockPath, old, old); + + await store.save(file, createDocument({ entries: [assignment('Ready', true, 2)] })); + const loaded = await store.load(file); + assert.equal(loaded.database.requireById(2).value.value, true); +}); + +test('primary and backup corruption are both rejected', async () => { + const { file } = await fixture(); + const store = new RedXaiFileStore(); + const document = createDocument({ entries: [assignment('Count', 1, 2)] }); + await store.save(file, document); + document.databases[0].entries[0].value.value = 2; + await store.save(file, document); + await writeFile(file, 'corrupt primary', 'utf8'); + await writeFile(`${file}.bak`, 'corrupt backup', 'utf8'); + + await assert.rejects(() => store.load(file), /Unable to load/); +}); + +test('database, backup, and lock files are never created world-readable', async () => { + if (process.platform === 'win32') return; + const { file } = await fixture(); + const store = new RedXaiFileStore(); + const document = createDocument({ entries: [assignment('SecretMetadata', 'not-a-secret', 2)] }); + await store.save(file, document); + document.databases[0].entries[0].value.value = 'changed'; + await store.save(file, document); + + const databaseMode = (await stat(file)).mode & 0o777; + const backupMode = (await stat(`${file}.bak`)).mode & 0o777; + assert.equal(databaseMode, 0o600); + assert.equal(backupMode, 0o600); + assert.match(await readFile(`${file}.bak`, 'utf8'), /not-a-secret/); +});