fix(plugin): bound the url-walk probe with one gtlt range, ready for Harper 5.3; v0.102.1

packages/plugin/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@harperfast/prerender",
-	"version": "0.102.0",
+	"version": "0.102.1",
 	"type": "module",
 	"description": "Configurable Harper plugin for prerendering pages for bots and crawlers",
 	"license": "Apache-2.0",

packages/plugin/src/util/renderSchedule.js

@@ -500,10 +500,17 @@ export async function* walkScheduleRowsDescending({ above, chunkSize = 1000, onU
  * iterator per event). `omitCurrent`: the keeper loads the current rows itself, by the walk above.
  *
  * What a node's subscription receives on a residency-pinned table, measured by bench/queue-keeper
- * (#215, #217): a `put` with the value for every write to a row this node owns, local or replicated;
- * a `delete` with no value for each write THIS node makes to a row it does not own (nothing is stored
- * locally, so the delivery re-read finds nothing); nothing for rows it neither owns nor wrote. After a
- * replication base copy that carries any row of the table, the whole table is re-sent as puts.
+ * (#215, #217) on Harper 5.2: a `put` with the value for every write to a row this node owns, local
+ * or replicated; a `delete` with no value for each write THIS node makes to a row it does not own
+ * (nothing is stored locally, so the delivery re-read finds nothing); nothing for rows it neither owns
+ * nor wrote. After a replication base copy that carries any row of the table, the whole table is
+ * re-sent as puts.
+ *
+ * Harper 5.3 (harper#2767) delivers a write only while the row's local entry still carries that
+ * write's version, so the not-owned `delete`s above are no longer sent at all, and a write already
+ * superseded by a later one is skipped (the later write's event carries the row). Neither changes
+ * the keeper: it ignores rows this node does not own (`classifyScheduleRow`), and every entry it
+ * publishes is checked against its durable row before a claim grants it.
  */
 export const subscribeScheduleChanges = (listener) => scheduleTable().subscribe({ omitCurrent: true, listener });
 

packages/plugin/src/util/urlWalk.js

@@ -27,11 +27,14 @@
  * tables (the default), `cacheKey` for `PrerenderedPage`. The unreadable-row rules are the same for
  * any string key — a row whose key attribute did not decode is skipped and counted, never a cursor.
  * `endBound` (exclusive) keeps the verification probes inside the caller's range so a row from
- * the next keyspace region can neither resume nor fail a prefix-scoped walk. The bounded probe
- * shape (two conditions on the key) falls back to an unbounded probe filtered in code when the
- * store rejects it. Residual (documented, not fixable from this layer): a range whose LAST key is
- * itself unreadable can still end a walk early when every probe path aborts on it; escalate such
- * rows to the database layer.
+ * the next keyspace region can neither resume nor fail a prefix-scoped walk. The bounded probe is
+ * ONE `gtlt` condition on the key, not a `greater_than` + `less_than` pair: Harper 5.3's planner
+ * orders separate conditions by a storage-level size estimate, and when `< endBound` estimates
+ * narrower it leads, so finding the one row past the cursor can walk every row below the bound. One
+ * exclusive range is a single bounded read on 5.2 and 5.3 alike. It falls back to an unbounded
+ * probe filtered in code when the store rejects it. Residual (documented, not fixable from this
+ * layer): a range whose LAST key is itself unreadable can still end a walk early when every probe
+ * path aborts on it; escalate such rows to the database layer.
  *
  * `searchOptions` is passed as the second argument of every search the walk makes, so a caller on a
  * residency-pinned table can keep every read local (`{ replicateFrom: false }`).
@@ -85,12 +88,14 @@ export async function* walkUrlRange(
 	 * treat that as unknown, never as empty.
 	 */
 	const probeSearch = async (descending) => {
-		const range = [{ attribute: key, comparator: 'greater_than', value: cursor ?? '' }];
-		const bounded = endBound ? [...range, { attribute: key, comparator: 'less_than', value: endBound }] : range;
+		const after = cursor ?? '';
+		const range = [{ attribute: key, comparator: 'greater_than', value: after }];
+		// One exclusive range (`> after` and `< endBound`), never two conditions the planner may reorder.
+		const bounded = endBound ? [{ attribute: key, comparator: 'gtlt', value: [after, endBound] }] : range;
 		try {
 			return await search(bounded, descending, 1);
 		} catch {
-			// The two-condition shape is refused here. Ascending degrades cleanly: probe unbounded
+			// The bounded shape is refused here. Ascending degrades cleanly: probe unbounded
 			// and apply the bound in code — a READABLE first row at or past endBound is proof the
 			// range itself is clear. Descending cannot degrade (the table's top row says nothing
 			// about this range), so it reports unknown.

packages/plugin/test/discoveredPurge.test.js

@@ -198,8 +198,15 @@ test('walkPrefix: keyset chunks, ge-then-gt, and the walk ends at the first URL
 		search({ conditions, limit }) {
 			calls.push(conditions[0]);
 			const { comparator, value } = conditions[0];
+			// The walk's verification probe is one `gtlt` range: `> value[0]` and `< value[1]`.
+			const keep =
+				comparator === 'gtlt'
+					? (url) => url > value[0] && url < value[1]
+					: comparator === 'greater_than_equal'
+						? (url) => url >= value
+						: (url) => url > value;
 			return all
-				.filter((url) => (comparator === 'greater_than_equal' ? url >= value : url > value))
+				.filter(keep)
 				.slice(0, limit)
 				.map((url) => ({ url, sitemapUrl: null }));
 		},
@@ -489,9 +496,10 @@ test('a stop issued from another worker reaches the running pass — through sta
 		static async put() {}
 		static async delete() {}
 		static search({ conditions = [], limit = 50, sort } = {}) {
-			const gt = conditions.find((c) => c.comparator === 'greater_than')?.value ?? '';
+			const range = conditions.find((c) => c.comparator === 'gtlt')?.value;
+			const gt = range?.[0] ?? conditions.find((c) => c.comparator === 'greater_than')?.value ?? '';
 			const ge = conditions.find((c) => c.comparator === 'greater_than_or_equal')?.value;
-			const lt = conditions.find((c) => c.comparator === 'less_than')?.value;
+			const lt = range?.[1] ?? conditions.find((c) => c.comparator === 'less_than')?.value;
 			let keys = URLS.filter((u) => (ge !== undefined ? u >= ge : u > gt) && (lt === undefined || u < lt));
 			if (sort?.descending) keys = [...keys].reverse();
 			const page = keys.slice(0, limit);

packages/plugin/test/urlWalk.test.js

@@ -12,14 +12,23 @@ import { recoverKeylessKeys, walkUrlRange } from '../src/util/urlWalk.js';
 
 const U = (n) => `https://example.com/product/prd-${String(n).padStart(4, '0')}/`;
 
+/** The walk's key conditions as [lower, upper?]: a `gtlt` range is `> value[0]` and `< value[1]`. */
+const bounds = (conditions) =>
+	conditions[0].comparator === 'gtlt'
+		? [
+				{ comparator: 'greater_than', value: conditions[0].value[0] },
+				{ comparator: 'less_than', value: conditions[0].value[1] },
+			]
+		: [conditions[0], conditions[1]];
+
 /** A table whose search answers from a scripted, ordered row list. `aborts` is a set of urls the
  *  PROJECTED read path silently stops in front of (the select-abort personality); rows with
  *  `url: undefined` model the tolerant personality. */
 const fakeTable = (rows, { abortsBefore = new Set() } = {}) => ({
 	calls: [],
 	search({ conditions, sort, select, limit }) {
 		this.calls.push({ conditions, select, limit, descending: !!sort?.descending });
-		const [range, upper] = [conditions[0], conditions[1]];
+		const [range, upper] = bounds(conditions);
 		let view = rows.filter((r) => {
 			const key = r.key;
 			if (range.comparator === 'greater_than' ? key <= range.value : key < range.value) return false;
@@ -103,10 +112,9 @@ test('descending end-check catches the fully-blind store: probe empty, top shows
 	const table = {
 		...fakeTable(rows, { abortsBefore: new Set([U(2)]) }),
 		search({ conditions, sort, limit }) {
-			const upper = conditions[1];
+			const [lower, upper] = bounds(conditions);
 			let view = rows.filter((r) => {
-				if (conditions[0].comparator === 'greater_than' ? r.key <= conditions[0].value : r.key < conditions[0].value)
-					return false;
+				if (lower.comparator === 'greater_than' ? r.key <= lower.value : r.key < lower.value) return false;
 				if (upper && r.key >= upper.value) return false;
 				return true;
 			});
@@ -128,6 +136,26 @@ test('descending end-check catches the fully-blind store: probe empty, top shows
 	);
 });
 
+test('the bounded probe is ONE gtlt range, never a pair of conditions the planner may reorder', async () => {
+	// Harper 5.3 orders separate conditions by a storage-level estimate; a `less_than endBound` that
+	// estimates narrower would lead and walk every row below the bound to find one row past the cursor.
+	const inside = row('https://example.com/catalog.jsp?CN=a');
+	const outsidePoison = row('https://example.com/product/prd-0001/', false);
+	const table = fakeTable([inside, outsidePoison], { abortsBefore: new Set([outsidePoison.key]) });
+	const endBound = 'https://example.com/catalog.jsq';
+	const urls = await collect(
+		walkUrlRange(table, { startAt: 'https://example.com/catalog.jsp', select: ['url'], chunkSize: 10, endBound })
+	);
+	assert.deepEqual(urls, [inside.key]);
+	const probes = table.calls.filter((call) => !call.select);
+	assert.ok(probes.length > 0, 'the short chunk was verified by a probe');
+	for (const probe of probes) {
+		assert.equal(probe.conditions.length, 1);
+		assert.equal(probe.conditions[0].comparator, 'gtlt');
+		assert.equal(probe.conditions[0].value[1], endBound);
+	}
+});
+
 test('endBound keeps the probes inside the range: a next-region poison row cannot false-alarm the walk', async () => {
 	// The unreadable row lives in the NEXT keyspace region. Bounded probes exclude it, so this
 	// range finishes cleanly; unbounded probes would see it and throw a false partial-coverage
@@ -203,7 +231,7 @@ test('ambiguous url-less probe row with endBound: the descending probe decides,
 });
 
 test('bounded probe shape unsupported: ascending falls back to an unbounded probe filtered in code', async () => {
-	// The store refuses two-condition searches AND its projected reads stop at the region
+	// The store refuses the bounded `gtlt` probe AND its projected reads stop at the region
 	// boundary, so the short chunk needs verifying. The range is genuinely finished (the only row
 	// past the cursor is beyond endBound and readable) — the fallback must prove that instead of
 	// erroring or lying.
@@ -213,7 +241,7 @@ test('bounded probe shape unsupported: ascending falls back to an unbounded prob
 	const base = fakeTable(rows, { abortsBefore: new Set([outside.key]) });
 	const table = {
 		search(q) {
-			if (q.conditions.length > 1) throw new Error('two-condition search unsupported');
+			if (q.conditions.some((c) => c.comparator === 'gtlt')) throw new Error('bounded range search unsupported');
 			return base.search.call(base, q);
 		},
 	};