diff --git a/package-lock.json b/package-lock.json index c35e300..3b277cb 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8570,7 +8570,7 @@ }, "packages/plugin": { "name": "@harperfast/prerender", - "version": "0.102.0", + "version": "0.102.1", "license": "Apache-2.0", "dependencies": { "fast-xml-parser": "^5.0.9", diff --git a/packages/plugin/package.json b/packages/plugin/package.json index 500a41f..5decf2e 100644 --- a/packages/plugin/package.json +++ b/packages/plugin/package.json @@ -1,6 +1,6 @@ { "name": "@harperfast/prerender", - "version": "0.102.0", + "version": "0.102.1", "type": "module", "description": "Configurable Harper plugin for prerendering pages for bots and crawlers", "license": "Apache-2.0", diff --git a/packages/plugin/src/util/renderSchedule.js b/packages/plugin/src/util/renderSchedule.js index 52720c6..8a380ab 100644 --- a/packages/plugin/src/util/renderSchedule.js +++ b/packages/plugin/src/util/renderSchedule.js @@ -500,10 +500,17 @@ export async function* walkScheduleRowsDescending({ above, chunkSize = 1000, onU * iterator per event). `omitCurrent`: the keeper loads the current rows itself, by the walk above. * * What a node's subscription receives on a residency-pinned table, measured by bench/queue-keeper - * (#215, #217): a `put` with the value for every write to a row this node owns, local or replicated; - * a `delete` with no value for each write THIS node makes to a row it does not own (nothing is stored - * locally, so the delivery re-read finds nothing); nothing for rows it neither owns nor wrote. After a - * replication base copy that carries any row of the table, the whole table is re-sent as puts. + * (#215, #217) on Harper 5.2: a `put` with the value for every write to a row this node owns, local + * or replicated; a `delete` with no value for each write THIS node makes to a row it does not own + * (nothing is stored locally, so the delivery re-read finds nothing); nothing for rows it neither owns + * nor wrote. After a replication base copy that carries any row of the table, the whole table is + * re-sent as puts. + * + * Harper 5.3 (harper#2767) delivers a write only while the row's local entry still carries that + * write's version, so the not-owned `delete`s above are no longer sent at all, and a write already + * superseded by a later one is skipped (the later write's event carries the row). Neither changes + * the keeper: it ignores rows this node does not own (`classifyScheduleRow`), and every entry it + * publishes is checked against its durable row before a claim grants it. */ export const subscribeScheduleChanges = (listener) => scheduleTable().subscribe({ omitCurrent: true, listener }); diff --git a/packages/plugin/src/util/urlWalk.js b/packages/plugin/src/util/urlWalk.js index b5f8746..4a830a3 100644 --- a/packages/plugin/src/util/urlWalk.js +++ b/packages/plugin/src/util/urlWalk.js @@ -27,11 +27,14 @@ * tables (the default), `cacheKey` for `PrerenderedPage`. The unreadable-row rules are the same for * any string key — a row whose key attribute did not decode is skipped and counted, never a cursor. * `endBound` (exclusive) keeps the verification probes inside the caller's range so a row from - * the next keyspace region can neither resume nor fail a prefix-scoped walk. The bounded probe - * shape (two conditions on the key) falls back to an unbounded probe filtered in code when the - * store rejects it. Residual (documented, not fixable from this layer): a range whose LAST key is - * itself unreadable can still end a walk early when every probe path aborts on it; escalate such - * rows to the database layer. + * the next keyspace region can neither resume nor fail a prefix-scoped walk. The bounded probe is + * ONE `gtlt` condition on the key, not a `greater_than` + `less_than` pair: Harper 5.3's planner + * orders separate conditions by a storage-level size estimate, and when `< endBound` estimates + * narrower it leads, so finding the one row past the cursor can walk every row below the bound. One + * exclusive range is a single bounded read on 5.2 and 5.3 alike. It falls back to an unbounded + * probe filtered in code when the store rejects it. Residual (documented, not fixable from this + * layer): a range whose LAST key is itself unreadable can still end a walk early when every probe + * path aborts on it; escalate such rows to the database layer. * * `searchOptions` is passed as the second argument of every search the walk makes, so a caller on a * residency-pinned table can keep every read local (`{ replicateFrom: false }`). @@ -85,12 +88,14 @@ export async function* walkUrlRange( * treat that as unknown, never as empty. */ const probeSearch = async (descending) => { - const range = [{ attribute: key, comparator: 'greater_than', value: cursor ?? '' }]; - const bounded = endBound ? [...range, { attribute: key, comparator: 'less_than', value: endBound }] : range; + const after = cursor ?? ''; + const range = [{ attribute: key, comparator: 'greater_than', value: after }]; + // One exclusive range (`> after` and `< endBound`), never two conditions the planner may reorder. + const bounded = endBound ? [{ attribute: key, comparator: 'gtlt', value: [after, endBound] }] : range; try { return await search(bounded, descending, 1); } catch { - // The two-condition shape is refused here. Ascending degrades cleanly: probe unbounded + // The bounded shape is refused here. Ascending degrades cleanly: probe unbounded // and apply the bound in code — a READABLE first row at or past endBound is proof the // range itself is clear. Descending cannot degrade (the table's top row says nothing // about this range), so it reports unknown. diff --git a/packages/plugin/test/discoveredPurge.test.js b/packages/plugin/test/discoveredPurge.test.js index ec74f64..3595c87 100644 --- a/packages/plugin/test/discoveredPurge.test.js +++ b/packages/plugin/test/discoveredPurge.test.js @@ -198,8 +198,15 @@ test('walkPrefix: keyset chunks, ge-then-gt, and the walk ends at the first URL search({ conditions, limit }) { calls.push(conditions[0]); const { comparator, value } = conditions[0]; + // The walk's verification probe is one `gtlt` range: `> value[0]` and `< value[1]`. + const keep = + comparator === 'gtlt' + ? (url) => url > value[0] && url < value[1] + : comparator === 'greater_than_equal' + ? (url) => url >= value + : (url) => url > value; return all - .filter((url) => (comparator === 'greater_than_equal' ? url >= value : url > value)) + .filter(keep) .slice(0, limit) .map((url) => ({ url, sitemapUrl: null })); }, @@ -489,9 +496,10 @@ test('a stop issued from another worker reaches the running pass — through sta static async put() {} static async delete() {} static search({ conditions = [], limit = 50, sort } = {}) { - const gt = conditions.find((c) => c.comparator === 'greater_than')?.value ?? ''; + const range = conditions.find((c) => c.comparator === 'gtlt')?.value; + const gt = range?.[0] ?? conditions.find((c) => c.comparator === 'greater_than')?.value ?? ''; const ge = conditions.find((c) => c.comparator === 'greater_than_or_equal')?.value; - const lt = conditions.find((c) => c.comparator === 'less_than')?.value; + const lt = range?.[1] ?? conditions.find((c) => c.comparator === 'less_than')?.value; let keys = URLS.filter((u) => (ge !== undefined ? u >= ge : u > gt) && (lt === undefined || u < lt)); if (sort?.descending) keys = [...keys].reverse(); const page = keys.slice(0, limit); diff --git a/packages/plugin/test/urlWalk.test.js b/packages/plugin/test/urlWalk.test.js index 83d10ba..16a3693 100644 --- a/packages/plugin/test/urlWalk.test.js +++ b/packages/plugin/test/urlWalk.test.js @@ -12,6 +12,15 @@ import { recoverKeylessKeys, walkUrlRange } from '../src/util/urlWalk.js'; const U = (n) => `https://example.com/product/prd-${String(n).padStart(4, '0')}/`; +/** The walk's key conditions as [lower, upper?]: a `gtlt` range is `> value[0]` and `< value[1]`. */ +const bounds = (conditions) => + conditions[0].comparator === 'gtlt' + ? [ + { comparator: 'greater_than', value: conditions[0].value[0] }, + { comparator: 'less_than', value: conditions[0].value[1] }, + ] + : [conditions[0], conditions[1]]; + /** A table whose search answers from a scripted, ordered row list. `aborts` is a set of urls the * PROJECTED read path silently stops in front of (the select-abort personality); rows with * `url: undefined` model the tolerant personality. */ @@ -19,7 +28,7 @@ const fakeTable = (rows, { abortsBefore = new Set() } = {}) => ({ calls: [], search({ conditions, sort, select, limit }) { this.calls.push({ conditions, select, limit, descending: !!sort?.descending }); - const [range, upper] = [conditions[0], conditions[1]]; + const [range, upper] = bounds(conditions); let view = rows.filter((r) => { const key = r.key; if (range.comparator === 'greater_than' ? key <= range.value : key < range.value) return false; @@ -103,10 +112,9 @@ test('descending end-check catches the fully-blind store: probe empty, top shows const table = { ...fakeTable(rows, { abortsBefore: new Set([U(2)]) }), search({ conditions, sort, limit }) { - const upper = conditions[1]; + const [lower, upper] = bounds(conditions); let view = rows.filter((r) => { - if (conditions[0].comparator === 'greater_than' ? r.key <= conditions[0].value : r.key < conditions[0].value) - return false; + if (lower.comparator === 'greater_than' ? r.key <= lower.value : r.key < lower.value) return false; if (upper && r.key >= upper.value) return false; return true; }); @@ -128,6 +136,26 @@ test('descending end-check catches the fully-blind store: probe empty, top shows ); }); +test('the bounded probe is ONE gtlt range, never a pair of conditions the planner may reorder', async () => { + // Harper 5.3 orders separate conditions by a storage-level estimate; a `less_than endBound` that + // estimates narrower would lead and walk every row below the bound to find one row past the cursor. + const inside = row('https://example.com/catalog.jsp?CN=a'); + const outsidePoison = row('https://example.com/product/prd-0001/', false); + const table = fakeTable([inside, outsidePoison], { abortsBefore: new Set([outsidePoison.key]) }); + const endBound = 'https://example.com/catalog.jsq'; + const urls = await collect( + walkUrlRange(table, { startAt: 'https://example.com/catalog.jsp', select: ['url'], chunkSize: 10, endBound }) + ); + assert.deepEqual(urls, [inside.key]); + const probes = table.calls.filter((call) => !call.select); + assert.ok(probes.length > 0, 'the short chunk was verified by a probe'); + for (const probe of probes) { + assert.equal(probe.conditions.length, 1); + assert.equal(probe.conditions[0].comparator, 'gtlt'); + assert.equal(probe.conditions[0].value[1], endBound); + } +}); + test('endBound keeps the probes inside the range: a next-region poison row cannot false-alarm the walk', async () => { // The unreadable row lives in the NEXT keyspace region. Bounded probes exclude it, so this // range finishes cleanly; unbounded probes would see it and throw a false partial-coverage @@ -203,7 +231,7 @@ test('ambiguous url-less probe row with endBound: the descending probe decides, }); test('bounded probe shape unsupported: ascending falls back to an unbounded probe filtered in code', async () => { - // The store refuses two-condition searches AND its projected reads stop at the region + // The store refuses the bounded `gtlt` probe AND its projected reads stop at the region // boundary, so the short chunk needs verifying. The range is genuinely finished (the only row // past the cursor is beyond endBound and readable) — the fallback must prove that instead of // erroring or lying. @@ -213,7 +241,7 @@ test('bounded probe shape unsupported: ascending falls back to an unbounded prob const base = fakeTable(rows, { abortsBefore: new Set([outside.key]) }); const table = { search(q) { - if (q.conditions.length > 1) throw new Error('two-condition search unsupported'); + if (q.conditions.some((c) => c.comparator === 'gtlt')) throw new Error('bounded range search unsupported'); return base.search.call(base, q); }, };