From 5d7fb1410d206ec94aaafca58813c1a302b9516d Mon Sep 17 00:00:00 2001 From: leemour Date: Thu, 15 Oct 2026 23:35:29 +0200 Subject: [PATCH] =?UTF-8?q?perf(mentions):=20find=20a=20letter=20by=20its?= =?UTF-8?q?=20tracking=20link=20=E2=80=94=20url:=20and=20link:=20on=20a=20?= =?UTF-8?q?copy=20of=20the=20owner's=20store?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Totals only. url: 86 ms median, link: 504 ms median, every sampled letter found. Co-Authored-By: Claude Opus 5.5 (1M context) --- performance/search/mentions/README.md | 29 +++++++ .../search/mentions/mail-link-search.mjs | 86 +++++++++++++++++++ 2 files changed, 115 insertions(+) create mode 100644 performance/search/mentions/mail-link-search.mjs diff --git a/performance/search/mentions/README.md b/performance/search/mentions/README.md index 0868894..ef005d2 100644 --- a/performance/search/mentions/README.md +++ b/performance/search/mentions/README.md @@ -133,3 +133,32 @@ What it showed: tracking links are most of the mail link weight and almost never for one finds nothing useful. Keeping one per letter and site, valued as the site, removes 60% of the table and its indexes and keeps every letter findable by `domain:`. Not measured: how many tracking-like links were real pages a person would search for (a YouTube `watch?v=` link is collapsed too); the effect on `messages sync` time. + +## Fifth run: finding a letter by its tracking link (2026-10-15) + +Why: after the fourth run mail keeps a tracking link only as its site, so `url:""` found nothing for it. +The owner wants a letter found by the whole link, by a part of a link and by site, without storing every link again. +The question: is reading the stored letter text fast enough for that. It decided the SQL in cli-core's +`mention-links.ts`: `url:` narrows to letters with a row of the link's site, then looks for the link in their body; +`link:` (a part of a link) matches stored links and the body of letters that hold a tracking link. + +Setup: same machine, Node 24.19.0; a `sqlite3 .backup` copy of the owner's store with the `url` rule version 2 +rebuilt on it (51 s); cli-core branch `feat/mail-link-search` built with `pnpm build`; +`node mail-link-search.mjs /dist 30 --no-rebuild`. It takes 30 random letters that hold a +tracking link, reads the first such link from each in memory, and runs three mail searches over both mail accounts +of the copy, 50 results a page: `url:` with the link as written, `link:` with its path, and `domain:` with its site. +Totals only; the copy was deleted after. + +| | letters searched | median | slowest | the letter found | +|---|---|---|---|---| +| `url:` the whole link | 23 | 86 ms | 385 ms | 23 of 23 | +| `link:` its path | 22 | 504 ms | 861 ms | 8 of 8; 14 more had over 50 results, not checked | +| `domain:` its site | 23 | 27 ms | 71 ms | | + +The copy has 257,020 letters; 67,429 of them (460 MB of body text) hold a tracking link, and `link:` reads all of +those bodies. Before the body was read only inside a subquery of letters with the site's row, `url:` took 0.9 s +(the `OR` read every letter's text); `lower()` on the body made `link:` take 1.06 s, `LIKE` 0.5 s. + +What it showed: `url:` costs one indexed lookup plus a few bodies, so it is as fast as before for a person. +`link:` reads every body that holds a tracking link and stays under a second on this mailbox. Not measured: how +often `link:` finds a part outside a link in such a letter (it reads the whole body); stores with more mail. diff --git a/performance/search/mentions/mail-link-search.mjs b/performance/search/mentions/mail-link-search.mjs new file mode 100644 index 0000000..b30fe12 --- /dev/null +++ b/performance/search/mentions/mail-link-search.mjs @@ -0,0 +1,86 @@ +// Times `url:` (a whole tracking link) and `link:` (its path) in mail search on a copy of a store, after the url +// rebuild. Prints totals only: never a link, a subject or a text. +// node mail-link-search.mjs [sample] [--no-rebuild] +import { DatabaseSync } from "node:sqlite" + +const [, , dist, path, size = "30", skip] = process.argv +const { openStore } = await import(`${dist}/store/store.js`) +const { findMentions, trackedOrigin } = await import(`${dist}/domain/mentions.js`) + +const store = await openStore({ path }) +const t = Date.now() +if (skip !== "--no-rebuild") await store.rebuildMentions({ kinds: ["url"] }) +const rebuildSeconds = (Date.now() - t) / 1000 + +const raw = new DatabaseSync(path, { readOnly: true }) +const accounts = raw + .prepare("SELECT provider, external_id AS account FROM accounts WHERE provider = 'email'") + .all() + .map(({ provider, account }) => ({ provider, account })) +const letters = raw + .prepare( + `SELECT e.id, e.external_id, coalesce(e.subject, '') || char(10, 10) || coalesce(e.body_text, '') AS text + FROM emails e WHERE e.deleted_at IS NULL AND e.id IN (SELECT source_id FROM mentions + WHERE source_type = 'email' AND kind = 'url' AND value NOT LIKE '%://%/%') + ORDER BY random() LIMIT ?`, + ) + .all(Number(size)) +raw.close() + +const run = async (field, value) => { + const started = process.hrtime.bigint() + const page = await store.matchQuery({ + root: { kind: "predicate", field, operator: "phrase", value, span: { start: 0, end: 1 } }, + accounts, + limit: 50, + corpus: "mail", + }) + return { + ms: Number(process.hrtime.bigint() - started) / 1e6, + ids: page.items.map(({ id }) => id), + more: page.hasMore, + } +} + +const stats = { url: [], link: [], domain: [] } +const found = { url: 0, link: 0 } +const full = { url: 0, link: 0 } +for (const letter of letters) { + const link = findMentions(String(letter.text)).find( + (one) => one.kind === "url" && trackedOrigin(one.value) !== undefined, + ) + if (!link) continue + const written = String(letter.text).slice(link.start, link.end) + const url = await run("url", written) + stats.url.push(url.ms) + if (url.ids.includes(letter.external_id) || url.more) found.url++ + if (url.more) full.url++ + const part = new URL(link.value).pathname + if (part.length > 1) { + const linked = await run("link", part) + stats.link.push(linked.ms) + if (linked.ids.includes(letter.external_id) || linked.more) found.link++ + if (linked.more) full.link++ + } + stats.domain.push((await run("domain", link.domain)).ms) +} +await store.close() + +const summary = (list) => { + const sorted = [...list].sort((a, b) => a - b) + return { + n: sorted.length, + medianMs: Math.round(sorted[sorted.length >> 1] ?? 0), + maxMs: Math.round(sorted.at(-1) ?? 0), + } +} +process.stdout.write( + `${JSON.stringify({ + rebuildSeconds, + mailAccounts: accounts.length, + sampled: letters.length, + url: { ...summary(stats.url), foundTheLetterOrPageFull: found.url, pageFull: full.url }, + link: { ...summary(stats.link), foundTheLetterOrPageFull: found.link, pageFull: full.link }, + domain: summary(stats.domain), + })}\n`, +)