diff --git a/CHANGELOG.md b/CHANGELOG.md index 1efdd09..70b41f5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,8 @@ Every entry says what changed as a caller sees it, why, and what to watch for ### Added +- Synthetic installed-Drive decision history checks: replacement on acceptance, reversal, event + time and document/account/policy boundaries, with a network/browser/keyring guard. - `security preflight` checks scanner pins, command availability, canonical repository paths and aliases, GitHub analysis access and an optional comparison baseline without scanning source. JSON diagnostics distinguish scanner readiness from the separately approved audit stages. diff --git a/performance/documents/README.md b/performance/documents/README.md index ead4ca8..9692d71 100644 --- a/performance/documents/README.md +++ b/performance/documents/README.md @@ -2,6 +2,9 @@ Measurements of how documents become text before they are indexed and searched. +- [drive-decisions](drive-decisions/README.md): installed-package decision replacement, confirmation, + reversal, event time and local account/policy boundaries. + - [drive-ai](drive-ai/README.md): executable extraction checks with a loopback model responder, source evidence, explicit saves, error handling and an in-flight source change. diff --git a/performance/documents/drive-decisions/README.md b/performance/documents/drive-decisions/README.md new file mode 100644 index 0000000..b51d804 --- /dev/null +++ b/performance/documents/drive-decisions/README.md @@ -0,0 +1,40 @@ +# Drive decision history + +Drive could record a decision but could not record its replacement or that it stopped holding. +This check asks whether the new commands preserve history, respect document/account boundaries +and behave the same through an installed executable as through the shared store. + +## Repeat + +Install the exact published package into temporary state, then give the runner its package directory: + +```sh +T=$(mktemp -d -p /var/tmp) && HOME=$T MESSAGING_STORE=$T/store WIRECAT_STATE_DIR=$T/state NPM_CONFIG_USERCONFIG=/dev/null npm install --prefix "$T/installed" --ignore-scripts --no-audit --no-fund @wirecat/drive-cli@0.13.0 +T=$(mktemp -d -p /var/tmp) && HOME=$T MESSAGING_STORE=$T/store WIRECAT_STATE_DIR=$T/state node performance/documents/drive-decisions/run.mjs /absolute/path/to/installed/node_modules/@wirecat/drive-cli +``` + +Use the first command's installation path in the second command. The runner creates a separate +synthetic config, database and snapshots under `/var/tmp`. Core's public store API seeds the fixture; +all checks invoke the installed Drive bin in a child process. The child preloader denies fetch, +HTTP requests, subprocesses and keyring construction. No Google account is connected. + +## Results + +The [released result](results-released.json) records 24 executable invocations of Drive 0.13.0, +with exact Core 0.294.0 and Base 0.35.0 dependency pins. [npm metadata](npm-integrity.json) +records the published version and integrity. Source changes shipped in +[Drive #87](https://github.com/WireCatLabs/drive-cli/pull/87). + +| Case | Result | +| --- | --- | +| Agent proposal before review | Earlier decision remains accepted; proposed decision cannot be reversed | +| Confirmation | Replacement becomes accepted and earlier decision becomes superseded | +| Owner replacement | Accepted immediately, retaining the earlier reference and normalized event time | +| Different document/account | Reversal and replacement refused; no extra fact or status change | +| Dry-run/read-only | Accepted decision remains unchanged | +| Offline reversal | Reversed status persists in a subsequent process | +| Ordinary fact/unknown argument | Refused; decision history remains queryable | + +The runner checks behavior with synthetic local records. It does not measure live Google access, +remote writes, AI accuracy or model transfers. MCP filtering and public command effects are covered +by Drive's source tests; this runner measures CLI execution only. diff --git a/performance/documents/drive-decisions/guard.mjs b/performance/documents/drive-decisions/guard.mjs new file mode 100644 index 0000000..1138cb6 --- /dev/null +++ b/performance/documents/drive-decisions/guard.mjs @@ -0,0 +1,31 @@ +import { createRequire, syncBuiltinESMExports } from "node:module" + +const require = createRequire(`${process.env.DRIVE_TEST_PACKAGE}/package.json`) +const deny = () => { + throw new Error("Unexpected network/browser/keyring access") +} +globalThis.fetch = deny +for (const name of ["node:http", "node:https"]) { + const module = require(name) + module.request = deny + module.get = deny +} +for (const name of ["exec", "execFile", "spawn", "fork", "execSync", "execFileSync", "spawnSync"]) + require("node:child_process")[name] = deny +require("node:net").Socket.prototype.connect = deny +require("node:tls").connect = deny +syncBuiltinESMExports() +const baseRequire = createRequire(require.resolve("@wirecat/cli-base")) +const keyring = baseRequire.resolve("@napi-rs/keyring") +require.cache[keyring] = { + id: keyring, + filename: keyring, + loaded: true, + exports: { + Entry: class { + constructor() { + deny() + } + }, + }, +} diff --git a/performance/documents/drive-decisions/npm-integrity.json b/performance/documents/drive-decisions/npm-integrity.json new file mode 100644 index 0000000..3a94173 --- /dev/null +++ b/performance/documents/drive-decisions/npm-integrity.json @@ -0,0 +1,4 @@ +{ + "version": "0.13.0", + "dist.integrity": "sha512-xgtVsAOHwoQSYUB+/bSzeffHNnTEo/0SF2FPDh/oR88qwd88QsU0Zbo1sE2exG5rQvNuMh9HoEW0DJDwCfp8sg==" +} diff --git a/performance/documents/drive-decisions/results-released.json b/performance/documents/drive-decisions/results-released.json new file mode 100644 index 0000000..02807a7 --- /dev/null +++ b/performance/documents/drive-decisions/results-released.json @@ -0,0 +1,31 @@ +{ + "version": "0.13.0", + "dependencies": { + "@bomb.sh/tab": "0.0.22", + "@googleapis/docs": "14.1.0", + "@googleapis/drive": "26.0.2", + "@googleapis/driveactivity": "11.0.1", + "@modelcontextprotocol/server": "2.1.0", + "@wirecat/cli-base": "0.35.0", + "@wirecat/cli-core": "0.294.0", + "commander": "15.0.0", + "google-auth-library": "11.2.0", + "mammoth": "1.13.0", + "proper-lockfile": "4.1.2", + "unpdf": "1.8.1", + "valibot": "1.5.0" + }, + "node": "v24.19.0", + "commands": 24, + "checks": [ + "proposal leaves earlier decision accepted", + "confirm supersedes earlier decision", + "owner replacement and event time survive executable round trip", + "wrong document and account cannot reverse or replace", + "dry-run and readonly preserve decision; offline reversal persists", + "nondecisions and unknown arguments refused; history retained" + ], + "guards": "child network/browser/keyring denied", + "package_manifest_sha256": "020f7568bcd2f0f3167de0949a85fbc36bfa79228c1a41d4eecf66b03f6f99b1", + "limitations": "synthetic local decisions only; no live Google writes, AI calls or model accuracy measurement" +} diff --git a/performance/documents/drive-decisions/run.mjs b/performance/documents/drive-decisions/run.mjs new file mode 100644 index 0000000..7aa669f --- /dev/null +++ b/performance/documents/drive-decisions/run.mjs @@ -0,0 +1,206 @@ +import assert from "node:assert/strict" +import { spawnSync } from "node:child_process" +import { createHash } from "node:crypto" +import { mkdirSync, mkdtempSync, readFileSync, writeFileSync } from "node:fs" +import { createRequire } from "node:module" +import { join, resolve } from "node:path" +import { fileURLToPath, pathToFileURL } from "node:url" + +const [input] = process.argv.slice(2) +assert.ok(input && process.argv.length === 3, "supply the installed Drive package directory") +const pkg = resolve(input) +const metadata = JSON.parse(readFileSync(join(pkg, "package.json"), "utf8")) +assert.equal(metadata.name, "@wirecat/drive-cli") +assert.equal(metadata.version, "0.13.0") +assert.equal(metadata.dependencies["@wirecat/cli-core"], "0.294.0") +assert.equal(metadata.dependencies["@wirecat/cli-base"], "0.35.0") +const require = createRequire(join(pkg, "package.json")) +const { openStore } = await import(pathToFileURL(require.resolve("@wirecat/cli-core/store"))) +const root = mkdtempSync("/var/tmp/drive-decisions-") +const env = { + HOME: root, + PATH: process.env.PATH, + NPM_CONFIG_USERCONFIG: "/dev/null", + NO_COLOR: "1", + DRIVE_CONFIG_DIR: join(root, "config"), + DRIVE_STATE_DIR: join(root, "state"), + DRIVE_CACHE_DIR: join(root, "cache"), + DRIVE_STORE: join(root, "drive.db"), + MESSAGING_STORE: join(root, "drive.db"), + WIRECAT_STATE_DIR: join(root, "wirecat"), + XDG_CONFIG_HOME: join(root, "xdg-config"), + XDG_DATA_HOME: join(root, "xdg-data"), + XDG_STATE_HOME: join(root, "xdg-state"), + XDG_CACHE_HOME: join(root, "xdg-cache"), + DRIVE_TEST_PACKAGE: pkg, +} +mkdirSync(env.DRIVE_CONFIG_DIR) +writeFileSync( + join(env.DRIVE_CONFIG_DIR, "config.json"), + JSON.stringify({ + profiles: { + synthetic: { principal: "synthetic-principal", roots: ["root"], storage: "keyring" }, + other: { principal: "other-principal", roots: ["root"], storage: "keyring" }, + }, + }), + { mode: 0o600 }, +) +const store = await openStore({ path: env.DRIVE_STORE, env }) +const folder = await store.notes.addFolder({ name: "google-drive:synthetic-principal" }) +const documents = {} +for (const id of ["doc", "other-doc"]) { + const { note } = await store.notes.saveFileNote({ + folderId: folder.id, + path: `${id}.md`, + title: id, + text: "A synthetic budget decision", + contentHash: "synthetic-hash", + }) + documents[id] = { + metadata: { + id, + name: id, + parents: ["root"], + mime: "application/vnd.google-apps.document", + url: `https://example.invalid/${id}`, + trashed: false, + canDownload: true, + }, + hash: "synthetic-hash", + reference: note.ref, + tabs: 0, + } +} +const initial = await store.facts.add({ + type: "decision", + statement: "Use the April budget", + by: "owner", + subject: documents.doc.reference, + evidence: [documents.doc.reference], +}) +const proposal = await store.facts.add({ + type: "decision", + statement: "Use the May budget", + by: { bot: "synthetic-agent", model: "synthetic-model" }, + supersedes: initial.ref, + subject: documents.doc.reference, + evidence: [documents.doc.reference], +}) +await store.close() +const state = { version: 1, principal: "synthetic-principal", scope: "", folders: {}, documents, skipped: [] } +const statePath = join(env.DRIVE_STATE_DIR, "profiles/synthetic") +mkdirSync(statePath, { recursive: true }) +writeFileSync(join(statePath, "state.json"), JSON.stringify(state)) +mkdirSync(join(statePath, "documents")) +for (const [id, doc] of Object.entries(documents)) + writeFileSync( + join(statePath, `documents/${id}.json`), + JSON.stringify({ + metadata: doc.metadata, + text: "A synthetic budget decision", + markdown: "A synthetic budget decision", + tabs: [], + }), + ) +const otherPath = join(env.DRIVE_STATE_DIR, "profiles/other") +mkdirSync(join(otherPath, "documents"), { recursive: true }) +writeFileSync(join(otherPath, "state.json"), JSON.stringify({ ...state, principal: "other-principal" })) +for (const [id, doc] of Object.entries(documents)) + writeFileSync( + join(otherPath, `documents/${id}.json`), + JSON.stringify({ + metadata: doc.metadata, + text: "A synthetic budget decision", + markdown: "A synthetic budget decision", + tabs: [], + }), + ) +const guard = fileURLToPath(new URL("./guard.mjs", import.meta.url)) +let commands = 0 +const checks = [] +const call = (words, { profile = "synthetic", exit = 0, error } = {}) => { + const result = spawnSync( + process.execPath, + ["--import", guard, join(pkg, "dist/bin/drive.js"), profile, "--json", ...words], + { env, encoding: "utf8", timeout: 30000 }, + ) + assert.equal(result.error, undefined) + assert.equal(result.status, exit, `CLI exit differs for ${words.slice(0, 2).join(" ")}: ${result.stderr}`) + const answer = JSON.parse(result.stdout.trim() || result.stderr.trim()) + if (error) assert.equal(answer.error.code, error) + commands++ + return answer.item ?? answer +} +const show = (ref) => call(["facts", "show", "doc", ref]) +assert.equal(show(initial.ref).status, "accepted") +assert.equal(show(proposal.ref).status, "proposed") +call(["facts", "reverse", "doc", proposal.ref], { exit: 2, error: "validation_error" }) +assert.equal(show(initial.ref).status, "accepted") +checks.push("proposal leaves earlier decision accepted") +assert.equal(call(["facts", "confirm", "doc", proposal.ref]).status, "accepted") +assert.equal(show(initial.ref).status, "superseded") +checks.push("confirm supersedes earlier decision") +const replacement = call([ + "facts", + "add", + "doc", + "Pause the budget", + "--type", + "decision", + "--supersedes", + proposal.ref, + "--occurred-time", + "2026-05-01T12:00:00Z", +]) +assert.equal(replacement.status, "accepted") +assert.equal(replacement.supersedes, proposal.ref) +assert.equal(replacement.occurred_at, "2026-05-01T12:00:00.000Z") +assert.equal(show(proposal.ref).status, "superseded") +checks.push("owner replacement and event time survive executable round trip") +for (const [doc, profile] of [ + ["other-doc", "synthetic"], + ["doc", "other"], +]) { + call(["facts", "reverse", doc, replacement.ref], { profile, exit: 6, error: "not_found" }) + call(["facts", "add", doc, "Wrong source", "--type", "decision", "--supersedes", replacement.ref], { + profile, + exit: 6, + error: "not_found", + }) +} +assert.equal(show(replacement.ref).status, "accepted") +assert.deepEqual(call(["facts", "list", "other-doc"]).items, []) +checks.push("wrong document and account cannot reverse or replace") +assert.deepEqual(call(["--dry-run", "facts", "reverse", "doc", replacement.ref]).effects, { + remote: "none", + local: "write", +}) +call(["--read-only", "facts", "reverse", "doc", replacement.ref], { exit: 5, error: "permission_error" }) +assert.equal(show(replacement.ref).status, "accepted") +assert.equal(call(["--offline", "facts", "reverse", "doc", replacement.ref]).status, "reversed") +assert.equal(show(replacement.ref).status, "reversed") +checks.push("dry-run and readonly preserve decision; offline reversal persists") +const ordinary = call(["facts", "add", "doc", "Budget discussed"]) +call(["facts", "reverse", "doc", ordinary.ref], { exit: 2, error: "validation_error" }) +assert.equal(show(ordinary.ref).status, "accepted") +assert.equal(call(["facts", "list", "doc", "kind:decision"]).items.length, 3) +call(["facts", "reverse", "doc", replacement.ref, "--unknown"], { exit: 2, error: "validation_error" }) +checks.push("nondecisions and unknown arguments refused; history retained") +process.stdout.write( + `${JSON.stringify( + { + version: metadata.version, + dependencies: metadata.dependencies, + node: process.version, + commands, + checks, + guards: "child network/browser/keyring denied", + package_manifest_sha256: createHash("sha256") + .update(readFileSync(join(pkg, "package.json"))) + .digest("hex"), + limitations: "synthetic local decisions only; no live Google writes, AI calls or model accuracy measurement", + }, + null, + 2, + )}\n`, +)