From eee3d99e5a819c1e4c6e1020a1abd78b13d53245 Mon Sep 17 00:00:00 2001 From: leemour Date: Sat, 10 Oct 2026 19:15:42 +0200 Subject: [PATCH] test(search): verify discovery in published CLI binaries --- CHANGELOG.md | 7 ++ performance/search/README.md | 5 +- performance/search/released/README.md | 48 ++++++++++ performance/search/released/offline.mjs | 18 ++++ performance/search/released/verify.mjs | 113 ++++++++++++++++++++++++ 5 files changed, 189 insertions(+), 2 deletions(-) create mode 100644 performance/search/released/README.md create mode 100644 performance/search/released/offline.mjs create mode 100644 performance/search/released/verify.mjs diff --git a/CHANGELOG.md b/CHANGELOG.md index 8020d63..ce1623a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,13 @@ Notable changes to `@wirecat/cli-testing`, one section per version, newest first Every entry says what changed as a caller sees it, why, and what to watch for — the rules are [`docs/dev/CONVENTIONS.md`](docs/dev/CONVENTIONS.md#the-changelog). +## Unreleased + +### Added + +- On-demand offline search verification for exact published Telegram and MAX packages, with + isolated synthetic stores, network guards and shape-only reports. + ## 0.1.0 — 10.10.2026 ### Fixed diff --git a/performance/search/README.md b/performance/search/README.md index f9e58fb..9ba1166 100644 --- a/performance/search/README.md +++ b/performance/search/README.md @@ -9,7 +9,8 @@ then [MODELS.md](MODELS.md) for the plain-language model explanation, then This area is an offline research harness, outside the published `src/` suites. It uses synthetic messages and an explicitly selected cli-messaging **build**, including the experimental internal service. It never imports that project's TypeScript source, opens an existing message store or -contacts a messenger. The public CLI/MCP search defaults are unchanged. +contacts a messenger. The public CLI/MCP search defaults are unchanged. Explicit discovery is now released; the +[released-package contract check](released/README.md) verifies both published CLI binaries offline. The [tool review](TOOL-RESEARCH.md) examines QMD, Meilisearch and Typesense. The [lightweight matrix](matrix/README.md) records 23 tested approaches, fresh held-out results and @@ -37,7 +38,7 @@ preserved and missing-fact partial hits explicitly retained. ## Run against the prototype The selected cli-messaging checkout must contain the combined-search prototype from -[draft PR 803](https://github.com/leemour/cli-messaging/pull/803), installed dependencies and a fresh +[merged PR 803](https://github.com/WireCatLabs/cli-messaging/pull/803), installed dependencies and a fresh `pnpm build`. Build it in its own checkout, then run these commands from the cli-testing root: ```sh diff --git a/performance/search/released/README.md b/performance/search/released/README.md new file mode 100644 index 0000000..9a38e0d --- /dev/null +++ b/performance/search/released/README.md @@ -0,0 +1,48 @@ +# Released search contract + +Use this on-demand check to verify explicit discovery in an exact published Telegram or MAX +package. It executes the installed CLI binary for both strict and discovery searches. It is not +part of the published `src/` suite or the ranking benchmark. + +## Prepare a runtime + +Package installation is the network step. Choose exact versions deliberately; this example +records the releases used for the initial verification. Installation scripts are disabled. + +```sh +mkdir -p /tmp/released-search-runtime +npm install --prefix /tmp/released-search-runtime --ignore-scripts --no-audit --no-fund --userconfig /dev/null @wirecat/tg-cli@0.44.0 @wirecat/max-cli@0.43.0 +``` + +Keep the generated package lock with the run evidence: it records resolved artifacts and +integrity hashes. An existing runtime with those exact packages also works. + +## Run offline + +From the cli-testing root: + +```sh +node --import ./performance/search/released/offline.mjs performance/search/released/verify.mjs /tmp/released-search-runtime tg 0.44.0 /tmp/released-search-tg.json +node --import ./performance/search/released/offline.mjs performance/search/released/verify.mjs /tmp/released-search-runtime max 0.43.0 /tmp/released-search-max.json +``` + +The published store API seeds three synthetic messages in a new temporary database. Published +app metadata selects the tool's configuration; searches are judged through the actual binary, +not an internal search service. The parent asks about a fictional export; its eligible reply +contains the time without the question's keywords. Another author's reply must be excluded. + +The verifier checks that discovery retrieves the eligible reply with its parent locator, +respects the author filter, and leaves default strict search empty. It asserts the requested +package version and records its shared messaging dependency. Each binary has a 15-second timeout. +An error or failed assertion exits unsuccessfully; reports are written only after success. + +Configuration, XDG paths and the store are isolated; HOME is not inherited. The verifier drops inherited environment +variables except PATH. The preloader denies Node network, DNS and keyring socket access in the +fixture process and both CLI processes. It is a regression guard, not an operating-system sandbox +against malicious native code. Nothing logs into an account, sends a message or reads an owner's +archive. Temporary synthetic stores are retained for inspection. + +Reports contain versions, verifier and guard hashes, and assertion outcomes. This tiny fixture proves a released-package +contract, not general ranking quality, archive scaling, live provider behavior or vulnerability +absence. Use the [evaluation guide](../combined-search-evaluation.md) and +[research index](../README.md) for those separate questions. diff --git a/performance/search/released/offline.mjs b/performance/search/released/offline.mjs new file mode 100644 index 0000000..d542f88 --- /dev/null +++ b/performance/search/released/offline.mjs @@ -0,0 +1,18 @@ +import dgram from "node:dgram" +import dns from "node:dns" +import { syncBuiltinESMExports } from "node:module" +import net from "node:net" + +const denied = () => { + throw new Error("Released search verification cannot access network or keyring sockets") +} +net.Socket.prototype.connect = denied +net.connect = denied +net.createConnection = denied +dgram.createSocket = denied +dns.lookup = denied +dns.resolve = denied +dns.promises.lookup = denied +dns.promises.resolve = denied +globalThis.fetch = denied +syncBuiltinESMExports() diff --git a/performance/search/released/verify.mjs b/performance/search/released/verify.mjs new file mode 100644 index 0000000..8bccde5 --- /dev/null +++ b/performance/search/released/verify.mjs @@ -0,0 +1,113 @@ +import assert from "node:assert/strict" +import { execFileSync } from "node:child_process" +import { createHash } from "node:crypto" +import { mkdtempSync, readFileSync, writeFileSync } from "node:fs" +import { createRequire } from "node:module" +import { tmpdir } from "node:os" +import { join, resolve } from "node:path" +import { fileURLToPath, pathToFileURL } from "node:url" + +const [runtime, tool, version, output] = process.argv.slice(2) +assert.ok( + runtime && output && /^(tg|max)$/.test(tool) && /^\d+\.\d+\.\d+$/.test(version), + "Usage: node verify.mjs RUNTIME tg|max EXACT_VERSION REPORT.json", +) +const sandbox = mkdtempSync(join(tmpdir(), "published-search-discovery-")) +const inheritedPath = process.env.PATH +for (const name of Object.keys(process.env)) delete process.env[name] +process.env.PATH = inheritedPath +process.env.TMPDIR = sandbox +const prefix = tool.toUpperCase() +for (const kind of ["CONFIG", "STATE", "CACHE"]) process.env[`${prefix}_${kind}_DIR`] = join(sandbox, kind) +for (const kind of ["CONFIG", "DATA", "CACHE"]) process.env[`XDG_${kind}_HOME`] = join(sandbox, `xdg-${kind}`) +process.env.MESSAGING_STORE = join(sandbox, "synthetic.db") +process.env.NODE_ENV = "test" +process.env.CI = "true" +const directory = resolve(runtime, "node_modules/@wirecat", `${tool}-cli`) +const manifest = JSON.parse(readFileSync(join(directory, "package.json"), "utf8")) +assert.equal(manifest.version, version) +const require = createRequire(join(directory, "package.json")) +const load = (name) => import(pathToFileURL(require.resolve(name)).href) +const { rememberAccount } = await load("@wirecat/cli-messaging/cli") +const { openStore } = await load("@wirecat/cli-messaging/store") +const app = await import(pathToFileURL(join(directory, "dist/app.js")).href) +const description = app[tool === "tg" ? "TG" : "MAX_APP"] +const account = { provider: tool === "tg" ? "telegram" : "max", account: "500" } +rememberAccount(description, "default", account.account, process.env) +const store = await openStore({ path: process.env.MESSAGING_STORE }) +await store.saveChats(account, [ + { id: "990", title: "Synthetic discovery", kind: "group", unreadCount: 0, lastMessageAt: null, participantsCount: 3 }, +]) +await store.saveMessages( + account, + "990", + [ + { id: "9901", text: "Helix export: what time should the daily export run?", senderId: "700" }, + { id: "9902", text: "Every day at 06:45 UTC.", senderId: "700", replyToId: "9901" }, + { id: "9903", text: "Every day at 02:00 UTC.", senderId: "701", replyToId: "9901" }, + ].map((m) => ({ + ...m, + chatId: "990", + senderName: "Synthetic", + timestamp: "2026-10-08T12:00:00.000Z", + editedAt: null, + outgoing: false, + attachments: [], + replyTo: null, + forwardedFrom: null, + reactions: null, + })), + { via: "synthetic" }, +) +await store.fillSearchIndex({}) +await store.fillStems({}) +await store.close() +const query = "What time does Helix export run? chat:990 from:700 date:2026-10-08" +const invoke = (extra) => + JSON.parse( + execFileSync( + process.execPath, + [ + "--import", + fileURLToPath(new URL("./offline.mjs", import.meta.url)), + join(directory, `dist/bin/${tool}.js`), + "search", + "messages", + query, + ...extra, + "--timezone", + "UTC", + "--json", + "--offline", + "--no-record", + ], + { env: process.env, encoding: "utf8", timeout: 15000 }, + ), + ) +const discovery = invoke(["--discover"]) +const strict = invoke([]) +assert.equal(discovery.query.discovery.method, "lexical-partial") +assert.ok(discovery.items.some((m) => m.id === "9902" && m.discovery.parent === `msg:${account.provider}/500/990/9901`)) +assert.ok(!discovery.items.some((m) => m.id === "9903")) +assert.deepEqual(strict.items, []) +const evidence = { + tool, + version, + messaging: manifest.dependencies["@wirecat/cli-messaging"], + discoveryItems: discovery.items.length, + strictItems: strict.items.length, + replyRetrieved: true, + otherSenderExcluded: true, + parentVerified: true, + publishedBinaryVerified: true, + syntheticOnly: true, + node: process.version, + verifierSha256: createHash("sha256") + .update(readFileSync(fileURLToPath(import.meta.url))) + .digest("hex"), + offlineGuardSha256: createHash("sha256") + .update(readFileSync(new URL("./offline.mjs", import.meta.url))) + .digest("hex"), +} +writeFileSync(resolve(output), `${JSON.stringify(evidence, null, 2)}\n`) +process.stdout.write(`${JSON.stringify(evidence)}\n`)