From 54e311f15cc749fcbe8dda26f94747b51b5ce5c4 Mon Sep 17 00:00:00 2001 From: ayom04 Date: Sat, 22 Aug 2026 04:37:19 +0100 Subject: [PATCH] feat: cursor pagination for transfer and audit history Offset pagination over transfer and audit history was both unstable and expensive. The window is defined by a row count, so a transfer created (or archived) while a client is paging shifts every later page: rows get repeated or skipped. Each request also re-materialised and re-filtered the whole collection to reach the requested slice. Replace it with indexed cursor pagination on both history endpoints: - Add an append-only OrderedIndex behind transfers and the audit log. Each record gets a dense, immutable sequence number, so ordering is a total order even when many records share a millisecond, and a page seek is O(1) instead of O(offset). The audit log also gains a secondary index by resourceId. - Order by an immutable creation position rather than updatedAt, so claiming, cancelling or archiving a transfer never moves it within a page. - Cursors are HMAC-signed and bound to the sort order, the normalised filter set, the collection, and the calling API token. A cursor replayed under a different token is rejected with 403; a different filter set or sort order with 400. The actor fingerprint is keyed, so a captured cursor cannot be brute-forced back to a token. - Bound every history query: limits above the maximum are rejected rather than silently clamped, offsets beyond the scan budget are refused, and a scan examines at most PAGINATION_MAX_SCAN records. A budget-truncated page is still gap-free and resumable through its nextCursor. Offset pagination keeps working unchanged - same total/count/limit/offset fields, same default ordering - and offset responses now carry a nextCursor so clients can migrate mid-walk. Co-Authored-By: Claude Opus 5 --- .env.example | 12 + CHANGELOG.md | 45 ++ README.md | 100 +++- src/config/index.js | 10 + src/controllers/auditController.js | 35 +- src/controllers/transferController.js | 23 +- src/services/auditService.js | 88 +++- src/services/transferService.js | 128 ++++- src/store/index.js | 11 + src/utils/cursor.js | 202 ++++++++ src/utils/historyPage.js | 114 +++++ src/utils/orderedIndex.js | 250 ++++++++++ src/utils/pagination.js | 118 +++++ test/paginationApi.test.js | 682 ++++++++++++++++++++++++++ test/paginationCursor.test.js | 371 ++++++++++++++ 15 files changed, 2139 insertions(+), 50 deletions(-) create mode 100644 src/utils/cursor.js create mode 100644 src/utils/historyPage.js create mode 100644 src/utils/orderedIndex.js create mode 100644 test/paginationApi.test.js create mode 100644 test/paginationCursor.test.js diff --git a/.env.example b/.env.example index d244c57..dcba576 100644 --- a/.env.example +++ b/.env.example @@ -39,6 +39,18 @@ DB_POOL_CONNECTION_TIMEOUT_MS=2000 CACHE_DEFAULT_POLICY=no-store CACHE_RATES_MAX_AGE_SECONDS=10 +# History pagination (GET /api/transfers, GET /api/audit) +PAGINATION_DEFAULT_LIMIT=50 +# Requests above this limit are rejected with 400, not clamped. +PAGINATION_MAX_LIMIT=200 +# Max records a single history query may examine before returning a +# resumable, truncated page. +PAGINATION_MAX_SCAN=10000 +# HMAC key used to sign pagination cursors. Left unset a random key is +# generated per process; set it explicitly when running more than one instance, +# or cursors minted by one will be rejected by another. +# PAGINATION_CURSOR_SECRET= + # API Token authentication # JSON map of token → array of scopes. If omitted, hardcoded demo tokens are used. # Valid scopes: transfers:read, transfers:write, users:read, users:write, audit:read diff --git a/CHANGELOG.md b/CHANGELOG.md index dee7ce3..a8e4e0a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,51 @@ When preparing a new release: ### Added +- Cursor pagination for `GET /api/transfers` and `GET /api/audit`. Pass + `?cursor=` (with optional `?order=asc|desc`) to page by an indexed position + instead of a row offset; responses carry a `pageInfo` block with + `hasMore`, `nextCursor`, `endCursor`, `scanned` and `scanTruncated`. + Cursors are HMAC-signed and bound to the API token, filter set and sort + order they were issued for, so they cannot be replayed across actor scopes + or filters. Signing key: `PAGINATION_CURSOR_SECRET`. +- Append-only ordered indexes behind transfer and audit history + (`src/utils/orderedIndex.js`), giving both collections a deterministic total + order and constant-cost page seeks regardless of how deep the page is. The + audit log also gains a secondary index by `resourceId`. +- Per-request work budget for history queries (`PAGINATION_MAX_SCAN`, + default `10000`), so a highly selective filter over a large history costs a + bounded amount of work. A budget-truncated page is gap-free and resumable + via its `nextCursor`. +- `PAGINATION_DEFAULT_LIMIT` and `PAGINATION_MAX_LIMIT` configuration. + +### Changed + +- **Breaking:** `GET /api/transfers` and `GET /api/audit` now reject a `limit` + above `PAGINATION_MAX_LIMIT` (200) with `400 LIMIT_TOO_LARGE` instead of + silently clamping it, and reject malformed `limit`/`offset`/`order` values + instead of falling back to defaults. Silent clamping left callers unable to + tell a truncated page from a complete one. Other collections + (`GET /api/users`) keep the previous lenient behaviour. +- **Breaking:** `?offset=` beyond `PAGINATION_MAX_SCAN` is rejected with + `400 OFFSET_TOO_DEEP`; deep pages must use `?cursor=`. `?cursor=` and + `?offset=` cannot be combined. +- `?offset=` is deprecated on both history endpoints but otherwise unchanged: + the `total`, `count`, `limit` and `offset` fields and the default ordering + are all preserved, and offset responses now also carry a `nextCursor` so + clients can migrate mid-walk. `total` is not returned in cursor mode, + because computing it requires the full-collection pass that cursor + pagination exists to avoid. + +### Fixed + +- Offset pagination over transfer and audit history repeated or skipped rows + when records were written while a client was paging, because the window was + defined by a row count rather than a position. Cursor pagination anchors to + an immutable creation position instead, so concurrent writes cannot shift a + page boundary. + +### Added + - Error tracking integration hook (`src/services/errorTrackingService.js`) that captures every error through a replaceable transport (console by default) and enriches it with request context (id, method, url). diff --git a/README.md b/README.md index ac8013d..4fc42db 100644 --- a/README.md +++ b/README.md @@ -45,6 +45,10 @@ The application is configured using environment variables (typically defined in | `DB_POOL_CONNECTION_TIMEOUT_MS` | Time to wait for a connection before timing out (ms) | `2000` | | `CACHE_DEFAULT_POLICY` | Default cache policy for endpoints (`no-store`, `public`, `private`) | `no-store` | | `CACHE_RATES_MAX_AGE_SECONDS` | Cache duration for rates endpoints (seconds) | `10` | +| `PAGINATION_DEFAULT_LIMIT` | Page size used when a request omits `limit` | `50` | +| `PAGINATION_MAX_LIMIT` | Largest accepted `limit`; bigger requests are rejected | `200` | +| `PAGINATION_MAX_SCAN` | Max records a single history query may examine | `10000` | +| `PAGINATION_CURSOR_SECRET` | HMAC key used to sign pagination cursors | *(random per process)* | | `API_TOKENS` | JSON object mapping API tokens to their allowed scopes (see [Authentication](#authentication)) | *(demo tokens)* | @@ -109,7 +113,7 @@ src/ routes/ Express routers services/ business logic (rates, quotes, transfers, users, error tracking) store/ in-memory store and seed data - utils/ logger, ids, money, ApiError, asyncHandler + utils/ logger, ids, money, ApiError, asyncHandler, pagination/cursors validators/ request validators app.js Express app assembly index.js server bootstrap @@ -127,6 +131,91 @@ consistent envelope: Every response carries an `X-Request-Id` header (echoed from the request when supplied) so logs and errors can be correlated. +### Pagination + +`GET /api/transfers` and `GET /api/audit` page with **opaque cursors**. A cursor +names a position in the collection rather than a row count, so pages stay +correct while records are being written. + +| Parameter | Description | +|-----------|-------------| +| `limit` | Page size. Defaults to `PAGINATION_DEFAULT_LIMIT`; values above `PAGINATION_MAX_LIMIT` are **rejected with 400**, not clamped. | +| `order` | `asc` or `desc`. Transfers default to `asc`, audit entries to `desc`. | +| `cursor` | Opaque cursor from a previous response's `pageInfo`. | +| `offset` | Legacy, deprecated. Cannot be combined with `cursor`. | + +Every response carries a `pageInfo` block: + +```json +{ + "count": 50, + "limit": 50, + "order": "asc", + "pageInfo": { + "hasMore": true, + "nextCursor": "eyJ2Ijox...", + "endCursor": "eyJ2Ijox...", + "scanned": 51, + "scanTruncated": false + }, + "transfers": [] +} +``` + +- `nextCursor` — pass as `?cursor=` to fetch the next page; `null` on the last page. +- `endCursor` — the position this page ended at, present even on the last page. + With `order=asc` a client can park here and poll for records appended later + without re-reading anything. +- `scanned` / `scanTruncated` — how many records the query examined, and whether + it stopped at `PAGINATION_MAX_SCAN` rather than at the end of the collection. + A truncated page is still gap-free: follow `nextCursor` to continue. + +To page a collection, walk it: + +```bash +curl -H "Authorization: Bearer $TOKEN" \ + "http://localhost:3000/api/transfers?limit=50" +# then, with pageInfo.nextCursor from the previous response: +curl -H "Authorization: Bearer $TOKEN" \ + "http://localhost:3000/api/transfers?limit=50&cursor=$CURSOR" +``` + +Cursors are signed and bound to the query that produced them. Reusing one +against a different filter set, a different sort order, or a different API +token is rejected rather than reinterpreted: + +| Condition | Status | `details.code` | +|-----------|--------|----------------| +| Cursor presented with a different API token | `403` | `CURSOR_ACTOR_MISMATCH` | +| Filters changed mid-walk | `400` | `CURSOR_FILTER_MISMATCH` | +| Sort order changed mid-walk | `400` | `CURSOR_ORDER_MISMATCH` | +| Cursor edited, forged or truncated | `400` | `INVALID_CURSOR` | +| Cursor predates a store restart | `400` | `STALE_CURSOR` | +| `limit` above the maximum | `400` | `LIMIT_TOO_LARGE` | +| `offset` beyond `PAGINATION_MAX_SCAN` | `400` | `OFFSET_TOO_DEEP` | +| `cursor` and `offset` sent together | `400` | `CONFLICTING_PAGINATION` | + +Cursors are signed with `PAGINATION_CURSOR_SECRET`. Without it a random key is +generated per process, which is fine for the in-memory store — cursors do not +outlive a restart anyway — but **set it explicitly before running more than one +instance behind a load balancer**, or cursors minted by one instance will be +rejected by another. + +#### Deprecated: offset pagination + +`?offset=` still works and still returns the original `total`/`offset` fields, +so existing clients keep working. It has two problems cursors fix: + +- **Instability.** The offset window is defined by a row count, so a record + inserted (or archived) while a client is paging shifts every later page: rows + get repeated or skipped entirely. +- **Cost.** The server must walk every skipped record on each request, and + `total` costs a full pass over the filtered collection. Offsets beyond + `PAGINATION_MAX_SCAN` are refused for that reason. + +Offset responses include `pageInfo.nextCursor` as well, so a client can start on +offsets and switch to cursors mid-walk. + ### Caching and headers The API implements Cache-Control response headers for security and efficiency: @@ -160,8 +249,9 @@ rounded away. - `POST /api/transfers` — create a transfer. Body: `{ senderName, recipientName, amount, from, to }` - `GET /api/transfers` — list transfers. Supports `?status=`, `?q=` (name - search), `?archived=` (true/false/all), and `?limit=`/`?offset=` pagination. - Archived transfers are excluded from results by default. + search), `?archived=` (true/false/all), and [cursor pagination](#pagination) + via `?cursor=`/`?limit=`/`?order=`. Archived transfers are excluded from + results by default; the default order is `asc` (oldest first). - `GET /api/transfers/stats` — aggregate counts and volume by currency. - `GET /api/transfers/:id` — fetch one transfer. - `POST /api/transfers/:id/claim` — recipient claims the transfer. @@ -230,4 +320,8 @@ curl -H "Authorization: Bearer $TOKEN" "http://localhost:3000/api/users" # View audit log (requires audit:read) curl -H "Authorization: Bearer $TOKEN" "http://localhost:3000/api/audit" + +# Page the audit log for one resource, newest first (requires audit:read) +curl -H "Authorization: Bearer $TOKEN" \ + "http://localhost:3000/api/audit?resourceId=&limit=20" ``` diff --git a/src/config/index.js b/src/config/index.js index 995151e..41f9a71 100644 --- a/src/config/index.js +++ b/src/config/index.js @@ -59,6 +59,16 @@ const config = { ratesMaxAge: parseInt(process.env.CACHE_RATES_MAX_AGE_SECONDS, 10) || 10, }, + pagination: { + // Page size used when a request does not ask for one. + defaultLimit: parseInt(process.env.PAGINATION_DEFAULT_LIMIT, 10) || 50, + // Hard ceiling on a single page. Larger requests are rejected, not clamped. + maxLimit: parseInt(process.env.PAGINATION_MAX_LIMIT, 10) || 200, + // Ceiling on records a single history query may examine. Bounds the cost of + // a highly selective filter (or a deep offset) over a large history. + maxScan: parseInt(process.env.PAGINATION_MAX_SCAN, 10) || 10000, + }, + apiTokens: (() => { try { if (process.env.API_TOKENS) { diff --git a/src/controllers/auditController.js b/src/controllers/auditController.js index cc2a427..799b428 100644 --- a/src/controllers/auditController.js +++ b/src/controllers/auditController.js @@ -1,7 +1,7 @@ 'use strict'; const auditService = require('../services/auditService'); -const { parsePagination } = require('../utils/pagination'); +const { buildHistoryPage } = require('../utils/historyPage'); /** * Audit log controllers. @@ -9,26 +9,29 @@ const { parsePagination } = require('../utils/pagination'); /** * GET /api/audit - * Return all audit log entries, newest first, with limit/offset pagination. - * Supports optional ?resourceId= query param to filter by resource. + * Return audit log entries, newest first by default. + * + * Supports ?resourceId= to filter by resource, ?order=asc|desc, ?limit=, and + * either ?cursor= (stable under concurrent writes) or the legacy ?offset=. */ function listAuditEntries(req, res) { - const { resourceId } = req.query; + const resourceId = req.query.resourceId == null || req.query.resourceId === '' + ? null + : String(req.query.resourceId); - const all = resourceId - ? auditService.getEntriesForResource(resourceId) - : auditService.getEntries(); + const filters = { resourceId }; - const { limit, offset } = parsePagination(req.query); - const entries = all.slice(offset, offset + limit); - - res.json({ - total: all.length, - count: entries.length, - limit, - offset, - entries, + const { items, envelope } = buildHistoryPage({ + req, + collection: 'audit', + filters, + defaultOrder: 'desc', + query: (args) => auditService.queryEntries({ resourceId, ...args }), + countTotal: () => auditService.countEntries(resourceId), + resolvePosition: (seq) => auditService.positionKeyAt(seq, resourceId), }); + + res.json({ ...envelope, entries: items }); } module.exports = { diff --git a/src/controllers/transferController.js b/src/controllers/transferController.js index 5815bf7..611b7df 100644 --- a/src/controllers/transferController.js +++ b/src/controllers/transferController.js @@ -1,7 +1,7 @@ 'use strict'; const transferService = require('../services/transferService'); -const { parsePagination } = require('../utils/pagination'); +const { buildHistoryPage } = require('../utils/historyPage'); /** * Transfer controllers. @@ -21,6 +21,10 @@ function createTransfer(req, res) { * List transfers, optionally filtered by ?status= and/or ?q= (name search). * Archived transfers are excluded by default; pass ?archived=true to see only archived, * or ?archived=all to include both archived and non-archived. + * + * Pagination: pass ?cursor= to page by the creation-order index (stable while + * transfers are being created), or the legacy ?offset=. ?order= selects asc + * (oldest first, the default) or desc. ?limit= is capped by config.pagination.maxLimit. */ function listTransfers(req, res) { const archivedParam = req.query.archived; @@ -33,14 +37,23 @@ function listTransfers(req, res) { archived = false; } - const all = transferService.listTransfers({ + const filters = transferService.normaliseTransferFilters({ status: req.query.status, search: req.query.q, archived, }); - const { limit, offset } = parsePagination(req.query); - const transfers = all.slice(offset, offset + limit); - res.json({ total: all.length, count: transfers.length, limit, offset, transfers }); + + const { items, envelope } = buildHistoryPage({ + req, + collection: 'transfers', + filters, + defaultOrder: 'asc', + query: (args) => transferService.queryTransfers({ ...filters, ...args }), + countTotal: () => transferService.listTransfers(filters).length, + resolvePosition: (seq) => transferService.positionKeyAt(seq), + }); + + res.json({ ...envelope, transfers: items }); } /** diff --git a/src/services/auditService.js b/src/services/auditService.js index 34e7476..4e51403 100644 --- a/src/services/auditService.js +++ b/src/services/auditService.js @@ -1,6 +1,8 @@ 'use strict'; const { newId } = require('../utils/ids'); +const { OrderedIndex } = require('../utils/orderedIndex'); +const config = require('../config'); /** * Audit log service. @@ -23,8 +25,18 @@ const { newId } = require('../utils/ids'); * at — ISO-8601 timestamp of when the entry was recorded */ -/** @type {Array} */ -const auditLog = []; +/** + * Entries live in an append-only ordered index rather than a plain array. + * + * The index adds two things a plain array cannot: a dense sequence number per + * entry, which gives cursor pagination a deterministic tie-breaker when several + * entries share a millisecond, and a secondary index by `resourceId`, so + * filtering by resource no longer scans the whole log. + */ +const auditIndex = new OrderedIndex({ + sortKeyOf: (entry) => entry.at, + groupKeyOf: (entry) => entry.resourceId, +}); /** * Append a new entry to the audit log. @@ -49,7 +61,7 @@ function addEntry({ action, resourceId, payload = {}, requestId } = {}) { at: new Date().toISOString(), }; - auditLog.push(entry); + auditIndex.append(entry); return entry; } @@ -58,7 +70,7 @@ function addEntry({ action, resourceId, payload = {}, requestId } = {}) { * @returns {Array} */ function getEntries() { - return auditLog.slice().reverse(); + return auditIndex.records.map((record) => record.item).reverse(); } /** @@ -67,19 +79,83 @@ function getEntries() { * @returns {Array} */ function getEntriesForResource(resourceId) { - return auditLog.filter((e) => e.resourceId === resourceId).reverse(); + // A nullish id matches no resource. Guarded explicitly because the index + // treats a null group key as "the whole index". + if (resourceId == null || resourceId === '') return []; + return auditIndex.recordsFor(String(resourceId)).map((record) => record.item).reverse(); +} + +/** + * Page through the audit log using the ordered index. + * + * Entries are immutable once written, so the sort position of an entry never + * changes. That is what makes a cursor into this log stable: a page boundary + * recorded now still means the same thing after any number of later appends. + * + * @param {object} [options] + * @param {string} [options.resourceId] - restrict to one resource via the secondary index. + * @param {'asc'|'desc'} [options.order] - defaults to newest first. + * @param {number} [options.limit] + * @param {number|null} [options.afterSeq] - exclusive start position from a cursor. + * @param {number} [options.skip] - legacy offset support. + * @param {number} [options.maxScan] - per-request work budget. + * @returns {{ items: object[], last: object|null, hasMore: boolean, scanned: number, + * scanTruncated: boolean, skipped: number }} + */ +function queryEntries({ + resourceId, + order = 'desc', + limit = config.pagination.defaultLimit, + afterSeq = null, + skip = 0, + maxScan = config.pagination.maxScan, +} = {}) { + return auditIndex.scan({ + group: resourceId == null || resourceId === '' ? null : String(resourceId), + order, + limit, + afterSeq, + skip, + maxScan, + }); +} + +/** + * Timestamp of the entry occupying a given index position within the same + * grouping the query uses, or null when no such position exists. + * @param {number} seq + * @param {string} [resourceId] + * @returns {string|null} + */ +function positionKeyAt(seq, resourceId) { + const group = resourceId == null || resourceId === '' ? null : String(resourceId); + const record = auditIndex.recordAt(seq, group); + return record ? record.key : null; +} + +/** + * Number of entries recorded for a resource, or in the whole log. + * @param {string} [resourceId] + * @returns {number} + */ +function countEntries(resourceId) { + if (resourceId == null || resourceId === '') return auditIndex.size; + return auditIndex.recordsFor(String(resourceId)).length; } /** * Clear all audit entries. Primarily used in tests and when the store is reset. */ function reset() { - auditLog.length = 0; + auditIndex.reset(); } module.exports = { addEntry, + countEntries, getEntries, getEntriesForResource, + positionKeyAt, + queryEntries, reset, }; diff --git a/src/services/transferService.js b/src/services/transferService.js index 11554ea..86ca529 100644 --- a/src/services/transferService.js +++ b/src/services/transferService.js @@ -7,6 +7,7 @@ const { TRANSFER_STATUS, TRANSFER_TRANSITIONS } = require('../config/constants') const quoteService = require('./quoteService'); const stellarService = require('./stellarService'); const auditService = require('./auditService'); +const config = require('../config'); // Keep lifecycle timestamps strictly increasing even when multiple operations // happen within the same millisecond (common in tests and API batches). @@ -33,17 +34,23 @@ function nextTimestamp(previous) { * @returns {Array} */ function listTransfers(filters = {}) { + const match = buildTransferFilter(filters); + return Array.from(store.transfers.values()).filter(match); +} + +/** + * Normalise a transfer filter set into a canonical description. + * + * The canonical form is what a cursor is fingerprinted against, so two requests + * that mean the same thing (`?status=pending` and `?status=pending&archived=false`) + * produce interchangeable cursors, while two that mean different things never do. + * + * @param {object} [filters] + * @returns {{ status: string|null, search: string|null, archived: boolean|'all' }} + * @throws {ApiError} 400 when the status filter is not a known status. + */ +function normaliseTransferFilters(filters = {}) { const { status, search, archived } = filters; - let transfers = Array.from(store.transfers.values()); - - // Filter by archived state: default excludes archived transfers - if (archived === true) { - transfers = transfers.filter((t) => t.archivedAt != null); - } else if (archived !== 'all') { - // archived === false or undefined: exclude archived - transfers = transfers.filter((t) => !t.archivedAt); - } - // archived === 'all' includes both archived and non-archived if (status) { const validStatuses = Object.values(TRANSFER_STATUS); @@ -53,19 +60,96 @@ function listTransfers(filters = {}) { { allowed: validStatuses } ); } - transfers = transfers.filter((t) => t.status === status); } - if (search) { - const needle = String(search).trim().toLowerCase(); - transfers = transfers.filter( - (t) => - t.senderName.toLowerCase().includes(needle) || - t.recipientName.toLowerCase().includes(needle) - ); - } + const needle = search == null ? '' : String(search).trim().toLowerCase(); + + return { + status: status || null, + search: needle === '' ? null : needle, + archived: archived === true ? true : (archived === 'all' ? 'all' : false), + }; +} + +/** + * Build the predicate for a transfer filter set. + * @param {object} [filters] + * @returns {(transfer: object) => boolean} + */ +function buildTransferFilter(filters = {}) { + const { status, search, archived } = normaliseTransferFilters(filters); - return transfers; + return function matches(transfer) { + // Archived state: `true` returns only archived, `'all'` returns both, + // anything else (the default) excludes archived transfers. + if (archived === true) { + if (transfer.archivedAt == null) return false; + } else if (archived !== 'all') { + if (transfer.archivedAt) return false; + } + + if (status && transfer.status !== status) return false; + + if (search) { + const inSender = transfer.senderName.toLowerCase().includes(search); + const inRecipient = transfer.recipientName.toLowerCase().includes(search); + if (!inSender && !inRecipient) return false; + } + + return true; + }; +} + +/** + * Page through transfer history using the creation-order index. + * + * Ordering is by creation position, which is immutable: claiming, cancelling + * or archiving a transfer never moves it. A cursor therefore stays valid across + * arbitrary concurrent writes - new transfers only ever appear at the newest + * edge of the ordering, never in the middle of a page the client already read. + * + * @param {object} [options] + * @param {string} [options.status] + * @param {string} [options.search] + * @param {boolean|'all'} [options.archived] + * @param {'asc'|'desc'} [options.order] - defaults to oldest first. + * @param {number} [options.limit] + * @param {number|null} [options.afterSeq] - exclusive start position from a cursor. + * @param {number} [options.skip] - legacy offset support. + * @param {number} [options.maxScan] - per-request work budget. + * @returns {{ items: object[], last: object|null, hasMore: boolean, scanned: number, + * scanTruncated: boolean, skipped: number }} + */ +function queryTransfers({ + status, + search, + archived, + order = 'asc', + limit = config.pagination.defaultLimit, + afterSeq = null, + skip = 0, + maxScan = config.pagination.maxScan, +} = {}) { + return store.transferIndex.scan({ + match: buildTransferFilter({ status, search, archived }), + order, + limit, + afterSeq, + skip, + maxScan, + }); +} + +/** + * Timestamp of the transfer occupying a given index position, or null when no + * such position exists. Used to detect cursors that survived a store reset and + * now point at an unrelated record. + * @param {number} seq + * @returns {string|null} + */ +function positionKeyAt(seq) { + const record = store.transferIndex.recordAt(seq); + return record ? record.key : null; } /** @@ -136,6 +220,7 @@ function createTransfer(data, requestId) { transfer.updatedAt = nextTimestamp(transfer.createdAt); store.transfers.set(transfer.id, transfer); + store.transferIndex.append(transfer); auditService.addEntry({ action: 'transfer.created', @@ -246,6 +331,9 @@ function unarchiveTransfer(id) { module.exports = { listTransfers, + normaliseTransferFilters, + positionKeyAt, + queryTransfers, getStats, getTransferOrThrow, createTransfer, diff --git a/src/store/index.js b/src/store/index.js index b70fbcd..6cbffdf 100644 --- a/src/store/index.js +++ b/src/store/index.js @@ -1,6 +1,7 @@ 'use strict'; const auditService = require('../services/auditService'); +const { OrderedIndex } = require('../utils/orderedIndex'); /** * Simple in-memory data store. @@ -10,12 +11,22 @@ const auditService = require('../services/auditService'); const store = { users: new Map(), transfers: new Map(), + + /** + * Append-only creation-order index over `transfers`, maintained alongside the + * map by transferService. It gives transfer history a stable total order and + * O(1) seeks, which a Map cannot provide. Transfers are never deleted - the + * lifecycle only mutates status and archive flags - so appended entries stay + * valid for the life of the process. + */ + transferIndex: new OrderedIndex({ sortKeyOf: (transfer) => transfer.createdAt }), }; /** Remove all records from the store. Primarily used in tests/seeding. */ function reset() { store.users.clear(); store.transfers.clear(); + store.transferIndex.reset(); auditService.reset(); } diff --git a/src/utils/cursor.js b/src/utils/cursor.js new file mode 100644 index 0000000..4c41727 --- /dev/null +++ b/src/utils/cursor.js @@ -0,0 +1,202 @@ +'use strict'; + +const crypto = require('crypto'); +const ApiError = require('./ApiError'); + +/** + * Opaque, tamper-evident pagination cursors. + * + * A cursor is `base64url(payload) + "." + base64url(HMAC-SHA256(payload))`. + * It is opaque to clients: the payload is an implementation detail and the + * signature means a client cannot craft or edit one. + * + * The payload binds a cursor to the query that produced it: + * v cursor format version + * o sort order the page was produced with + * k timestamp of the record the cursor points at (validated on resume) + * s sequence number of that record (the authoritative position) + * f fingerprint of the filter set + * a fingerprint of the calling actor + * + * Binding matters because a cursor is a position inside one specific ordered + * result set. Replaying it against a different filter, a different sort order, + * or - most importantly - a different API token would silently return a page + * from a result set the cursor was never computed against. Every mismatch is + * rejected rather than reinterpreted. + * + * The signing key comes from PAGINATION_CURSOR_SECRET. Without it a random + * per-process key is generated, which is correct for the in-memory store + * (cursors are meaningless across restarts anyway) but must be set explicitly + * before running more than one instance behind a load balancer. + */ + +const CURSOR_VERSION = 1; + +const SECRET = process.env.PAGINATION_CURSOR_SECRET + || crypto.randomBytes(32).toString('hex'); + +/** Longest cursor string accepted, to bound work on hostile input. */ +const MAX_CURSOR_LENGTH = 512; + +/** + * Stable fingerprint of an arbitrary value. + * Object keys are sorted so that fingerprints do not depend on insertion order. + * @param {*} value + * @returns {string} 16 hex characters. + */ +function fingerprint(value) { + return crypto.createHash('sha256').update(canonicalize(value)).digest('hex').slice(0, 16); +} + +/** + * Deterministic JSON encoding used as fingerprint input. + * @param {*} value + * @returns {string} + */ +function canonicalize(value) { + if (value === null || typeof value !== 'object') return JSON.stringify(value) ?? 'null'; + if (Array.isArray(value)) return `[${value.map(canonicalize).join(',')}]`; + const keys = Object.keys(value).filter((key) => value[key] !== undefined).sort(); + return `{${keys.map((key) => `${JSON.stringify(key)}:${canonicalize(value[key])}`).join(',')}}`; +} + +/** + * Fingerprint identifying the caller a cursor belongs to. + * + * Derived from the API token so a cursor minted for one token is rejected when + * presented with another, even when both tokens can read the collection. The + * raw token is never stored in the cursor. + * + * Keyed with the signing secret rather than plainly hashed: a cursor is handed + * to the client, and an unkeyed hash of a token would let anyone who captured + * one brute-force the token offline. + * + * @param {import('express').Request} req + * @returns {string} + */ +function actorFingerprint(req) { + const subject = req && req.token ? `token:${req.token}` : 'anonymous'; + return crypto.createHmac('sha256', SECRET).update(subject).digest('hex').slice(0, 16); +} + +/** + * Encode a signed cursor. + * @param {object} params + * @param {'asc'|'desc'} params.order + * @param {string} params.key - timestamp of the record pointed at. + * @param {number} params.seq - sequence number of that record. + * @param {string} params.filter - filter fingerprint. + * @param {string} params.actor - actor fingerprint. + * @returns {string} + */ +function encodeCursor({ order, key, seq, filter, actor }) { + const payload = Buffer.from( + JSON.stringify({ v: CURSOR_VERSION, o: order, k: key, s: seq, f: filter, a: actor }), + 'utf8' + ).toString('base64url'); + + return `${payload}.${signPayload(payload)}`; +} + +/** + * Decode and fully validate a cursor. + * + * @param {string} raw + * @param {object} expected + * @param {'asc'|'desc'} expected.order + * @param {string} expected.filter + * @param {string} expected.actor + * @returns {{ order: 'asc'|'desc', key: string, seq: number }} + * @throws {ApiError} 400 for malformed, forged, stale-format or cross-filter + * cursors; 403 for a cursor belonging to a different actor. + */ +function decodeCursor(raw, expected) { + if (typeof raw !== 'string' || raw.length === 0 || raw.length > MAX_CURSOR_LENGTH) { + throw invalidCursor('Cursor is malformed'); + } + + const separator = raw.indexOf('.'); + if (separator <= 0 || separator === raw.length - 1) { + throw invalidCursor('Cursor is malformed'); + } + + const payload = raw.slice(0, separator); + const signature = raw.slice(separator + 1); + + if (!constantTimeEquals(signature, signPayload(payload))) { + // Same error as a malformed cursor: a caller probing the endpoint learns + // nothing about whether their edit was structurally valid. + throw invalidCursor('Cursor is malformed'); + } + + let decoded; + try { + decoded = JSON.parse(Buffer.from(payload, 'base64url').toString('utf8')); + } catch { + throw invalidCursor('Cursor is malformed'); + } + + if (!decoded || typeof decoded !== 'object' || decoded.v !== CURSOR_VERSION) { + throw invalidCursor('Cursor was issued by an incompatible API version'); + } + if (!Number.isSafeInteger(decoded.s) || decoded.s < 0 || typeof decoded.k !== 'string') { + throw invalidCursor('Cursor is malformed'); + } + + // Actor first: crossing an actor boundary is the security-relevant failure. + if (decoded.a !== expected.actor) { + throw new ApiError(403, 'Cursor was issued to a different API token', { + code: 'CURSOR_ACTOR_MISMATCH', + }); + } + if (decoded.o !== expected.order) { + throw invalidCursor('Cursor was issued for a different sort order', 'CURSOR_ORDER_MISMATCH'); + } + if (decoded.f !== expected.filter) { + throw invalidCursor( + 'Cursor was issued for a different set of filters; restart paging without a cursor', + 'CURSOR_FILTER_MISMATCH' + ); + } + + return { order: decoded.o, key: decoded.k, seq: decoded.s }; +} + +/** + * @param {string} payload + * @returns {string} + */ +function signPayload(payload) { + return crypto.createHmac('sha256', SECRET).update(payload).digest('base64url'); +} + +/** + * Length-safe constant-time string comparison. + * @param {string} a + * @param {string} b + * @returns {boolean} + */ +function constantTimeEquals(a, b) { + const left = Buffer.from(a, 'utf8'); + const right = Buffer.from(b, 'utf8'); + if (left.length !== right.length) return false; + return crypto.timingSafeEqual(left, right); +} + +/** + * @param {string} message + * @param {string} [code] + * @returns {ApiError} + */ +function invalidCursor(message, code = 'INVALID_CURSOR') { + return ApiError.badRequest(message, { code }); +} + +module.exports = { + CURSOR_VERSION, + MAX_CURSOR_LENGTH, + actorFingerprint, + decodeCursor, + encodeCursor, + fingerprint, +}; diff --git a/src/utils/historyPage.js b/src/utils/historyPage.js new file mode 100644 index 0000000..0826d93 --- /dev/null +++ b/src/utils/historyPage.js @@ -0,0 +1,114 @@ +'use strict'; + +const config = require('../config'); +const ApiError = require('./ApiError'); +const { actorFingerprint, decodeCursor, encodeCursor, fingerprint } = require('./cursor'); +const { parseHistoryPagination } = require('./pagination'); + +/** + * Shared request handling for cursor-paginated history collections. + * + * Both history endpoints need the same six steps - parse and bound the + * pagination parameters, fingerprint the actor and the filters, validate any + * supplied cursor against both, run the indexed scan, mint the next cursor, and + * shape the envelope. Keeping that in one place is what guarantees the two + * endpoints cannot drift into subtly different cursor semantics. + * + * @param {object} params + * @param {import('express').Request} params.req + * @param {string} params.collection - fingerprint namespace, so a transfers + * cursor can never validate against the audit endpoint even under identical filters. + * @param {object} params.filters - canonical (normalised) filter description. + * @param {'asc'|'desc'} params.defaultOrder + * @param {(args: { order: 'asc'|'desc', limit: number, afterSeq: number|null, + * skip: number, maxScan: number }) => object} params.query - indexed scan. + * @param {() => number} params.countTotal - total matching records; only called + * in legacy offset mode, where the response contract promises a total. + * @param {(seq: number) => (string|null)} params.resolvePosition - timestamp of the + * record at an index position, used to reject cursors that no longer describe + * the position they were minted for. + * @returns {{ items: object[], envelope: object }} + */ +function buildHistoryPage({ + req, + collection, + filters, + defaultOrder, + query, + countTotal, + resolvePosition, +}) { + const { maxScan } = config.pagination; + const { mode, limit, order, cursor: rawCursor, offset } = parseHistoryPagination( + req.query, + { defaultOrder } + ); + + const actor = actorFingerprint(req); + const filterPrint = fingerprint([collection, filters]); + + let afterSeq = null; + let skip = 0; + + if (mode === 'cursor') { + const position = decodeCursor(rawCursor, { order, filter: filterPrint, actor }); + + // The cursor carries the timestamp of the record it was minted at. If the + // position now holds a different record - the in-memory store was reset and + // sequence numbers were reissued - resuming would silently return an + // unrelated slice, so the cursor is rejected instead. + if (resolvePosition(position.seq) !== position.key) { + throw ApiError.badRequest( + 'Cursor no longer refers to a valid position; restart paging without a cursor', + { code: 'STALE_CURSOR' } + ); + } + + afterSeq = position.seq; + } else if (offset > maxScan) { + // Deep offsets are the failure this endpoint exists to fix: the server would + // have to walk every skipped record on every request, and the window shifts + // whenever a record is inserted. Refuse rather than do unbounded work. + throw ApiError.badRequest( + `offset may not exceed ${maxScan}; use cursor pagination for deep pages`, + { code: 'OFFSET_TOO_DEEP', maxOffset: maxScan } + ); + } else { + skip = offset; + } + + const page = query({ order, limit, afterSeq, skip, maxScan }); + + // Position of the last record this scan examined. Present even when the page + // is the last one, so an `order=asc` client can park here and pick up records + // appended later without re-reading anything. + const endCursor = page.last + ? encodeCursor({ order, key: page.last.key, seq: page.last.seq, filter: filterPrint, actor }) + : rawCursor; + + const envelope = { + count: page.items.length, + limit, + order, + pageInfo: { + hasMore: page.hasMore, + nextCursor: page.hasMore ? endCursor : null, + endCursor, + scanned: page.scanned, + // True when the work budget, not the end of the collection, ended the + // page. The page is still gap-free: follow nextCursor to continue. + scanTruncated: page.scanTruncated, + }, + }; + + if (mode === 'offset') { + // Legacy fields. `total` costs a full filtered pass, which is precisely the + // cost cursor mode avoids, so it is not offered there. + envelope.total = countTotal(); + envelope.offset = offset; + } + + return { items: page.items, envelope }; +} + +module.exports = { buildHistoryPage }; diff --git a/src/utils/orderedIndex.js b/src/utils/orderedIndex.js new file mode 100644 index 0000000..3acdfc7 --- /dev/null +++ b/src/utils/orderedIndex.js @@ -0,0 +1,250 @@ +'use strict'; + +/** + * Append-only ordered index. + * + * Backs cursor pagination for the history collections (transfers, audit log). + * Records are appended in creation order and never moved or removed, which + * gives every record a dense, immutable, strictly increasing sequence number. + * + * Why sequence numbers rather than raw timestamps: + * - `createdAt` / `at` only have millisecond resolution, so several records + * routinely share a timestamp. A cursor built on the timestamp alone + * cannot resume deterministically inside such a tie. + * - The sequence number is assigned in insertion order, so ordering by + * `seq` is identical to ordering by `(timestamp, insertion order)` without + * depending on the wall clock being monotonic. + * + * The timestamp is still carried on each record so a cursor can be validated + * against the position it claims to point at. + * + * Cost model (n = records in the index, m = records in a group): + * append O(1) + * seek to a cursor position O(1) ungrouped, O(log m) grouped + * scan O(records examined), hard-capped by `maxScan` + */ +class OrderedIndex { + /** + * @param {object} options + * @param {(item: object) => string} options.sortKeyOf - extracts the timestamp sort key. + * @param {(item: object) => (string|null)} [options.groupKeyOf] - optional secondary + * index key, so an equality filter on that field can be paged without + * scanning unrelated records. + */ + constructor({ sortKeyOf, groupKeyOf = null } = {}) { + if (typeof sortKeyOf !== 'function') { + throw new TypeError('OrderedIndex: sortKeyOf must be a function'); + } + this.sortKeyOf = sortKeyOf; + this.groupKeyOf = groupKeyOf; + /** @type {Array<{ seq: number, key: string, item: object }>} */ + this.records = []; + /** @type {Map>} */ + this.groups = new Map(); + this.nextSeq = 0; + } + + /** Number of records held by the index. */ + get size() { + return this.records.length; + } + + /** + * Append an item, assigning it the next sequence number. + * @param {object} item + * @returns {{ seq: number, key: string, item: object }} the stored record. + */ + append(item) { + const record = { seq: this.nextSeq++, key: String(this.sortKeyOf(item)), item }; + this.records.push(record); + + if (this.groupKeyOf) { + const groupKey = this.groupKeyOf(item); + if (groupKey != null) { + const bucket = this.groups.get(groupKey); + if (bucket) { + bucket.push(record); + } else { + this.groups.set(groupKey, [record]); + } + } + } + + return record; + } + + /** Drop every record. Used by store resets and tests. */ + reset() { + this.records.length = 0; + this.groups.clear(); + this.nextSeq = 0; + } + + /** + * The record array a query should walk: the whole index, or one group. + * @param {string|null} group + * @returns {Array<{ seq: number, key: string, item: object }>} + */ + recordsFor(group) { + if (group == null) return this.records; + return this.groups.get(group) || []; + } + + /** + * Look up the record carrying a given sequence number. + * @param {number} seq + * @param {string|null} [group] + * @returns {{ seq: number, key: string, item: object }|null} + */ + recordAt(seq, group = null) { + const records = this.recordsFor(group); + if (group == null) { + // The ungrouped index is dense, so seq is the array position. + return records[seq] || null; + } + const position = lowerBound(records, seq); + const record = records[position]; + return record && record.seq === seq ? record : null; + } + + /** + * Page through the index. + * + * Walks from the position just past `afterSeq` in the requested direction, + * returning at most `limit` items that satisfy `match`. The walk examines at + * most `maxScan` records, so a highly selective filter over a large index + * costs a bounded amount of work per request instead of a full table scan. + * + * @param {object} options + * @param {number|null} [options.afterSeq] - exclusive start position; null starts at the edge. + * @param {'asc'|'desc'} [options.order] + * @param {string|null} [options.group] - restrict to a secondary-index group. + * @param {number} options.limit - maximum items to return. + * @param {number} options.maxScan - maximum records to examine. + * @param {(item: object) => boolean} [options.match] - residual filter predicate. + * @param {number} [options.skip] - drop this many matching items before collecting + * (used only by the legacy offset path; counts against `maxScan`). + * @returns {{ + * items: object[], + * last: ({ seq: number, key: string, item: object }|null), + * hasMore: boolean, + * scanned: number, + * scanTruncated: boolean, + * skipped: number + * }} + */ + scan({ + afterSeq = null, + order = 'desc', + group = null, + limit, + maxScan, + match = null, + skip = 0, + }) { + const records = this.recordsFor(group); + const step = order === 'asc' ? 1 : -1; + + let position = startPosition(records, afterSeq, order, group); + + const items = []; + let scanned = 0; + let skipped = 0; + let hasMore = false; + let scanTruncated = false; + // The record the next cursor should point at: the last one examined, so a + // budget-truncated page resumes exactly where this one stopped. + let frontier = null; + + while (position >= 0 && position < records.length) { + if (scanned >= maxScan) { + // Out of budget before reaching the end of the index. Report more data + // is available and, when the page came up short, that the shortfall is + // a budget artefact rather than the end of the collection. + scanTruncated = items.length < limit; + hasMore = true; + break; + } + + const record = records[position]; + position += step; + scanned += 1; + + if (match && !match(record.item)) { + // A skipped non-match still advances the frontier: resuming from it + // cannot lose a matching record, because non-matches are stable for a + // fixed filter. + frontier = record; + continue; + } + + if (skipped < skip) { + skipped += 1; + frontier = record; + continue; + } + + if (items.length === limit) { + // One matching record beyond the page proves there is a next page. + hasMore = true; + break; + } + + items.push(record.item); + frontier = record; + } + + return { items, last: frontier, hasMore, scanned, scanTruncated, skipped }; + } +} + +/** + * First array position whose record has `seq >= target`. + * @param {Array<{ seq: number }>} records + * @param {number} target + * @returns {number} + */ +function lowerBound(records, target) { + let low = 0; + let high = records.length; + while (low < high) { + const mid = (low + high) >>> 1; + if (records[mid].seq < target) { + low = mid + 1; + } else { + high = mid; + } + } + return low; +} + +/** + * Resolve the array position a scan should start at, exclusive of `afterSeq`. + * @param {Array<{ seq: number }>} records + * @param {number|null} afterSeq + * @param {'asc'|'desc'} order + * @param {string|null} group + * @returns {number} + */ +function startPosition(records, afterSeq, order, group) { + if (afterSeq == null) { + return order === 'asc' ? 0 : records.length - 1; + } + + // Ascending resumes at the first record after the cursor; descending resumes + // at the last record before it. Both are exclusive of the cursor itself, which + // is what makes pages non-overlapping. + // The ungrouped index is dense (seq === array position), so the binary search + // is only needed for group buckets. + if (order === 'asc') { + return group == null + ? Math.min(Math.max(afterSeq, -1) + 1, records.length) + : lowerBound(records, afterSeq + 1); + } + + return group == null + ? Math.min(afterSeq, records.length) - 1 + : lowerBound(records, afterSeq) - 1; +} + +module.exports = { OrderedIndex }; diff --git a/src/utils/pagination.js b/src/utils/pagination.js index de73eb2..6771b7c 100644 --- a/src/utils/pagination.js +++ b/src/utils/pagination.js @@ -1,10 +1,18 @@ 'use strict'; +const config = require('../config'); +const ApiError = require('./ApiError'); + const DEFAULT_LIMIT = 50; const MAX_LIMIT = 200; /** * Parse and clamp pagination query parameters. + * + * Legacy lenient parser: unparseable values silently fall back to defaults and + * oversized limits are clamped. Retained for the collections that still expose + * only offset pagination. + * * @param {object} query - typically req.query. * @returns {{ limit: number, offset: number }} */ @@ -24,8 +32,118 @@ function parsePagination(query = {}) { return { limit, offset }; } +/** + * Parse pagination parameters for a history collection. + * + * Unlike {@link parsePagination} this rejects out-of-range input instead of + * quietly correcting it: a client that asks for 10 000 rows and receives 200 + * has no way to tell it did not receive everything, which is exactly the class + * of bug an unbounded history query causes downstream. + * + * @param {object} query - typically req.query. + * @param {object} [options] + * @param {'asc'|'desc'} [options.defaultOrder] + * @param {number} [options.defaultLimit] + * @param {number} [options.maxLimit] + * @returns {{ mode: 'cursor'|'offset', limit: number, order: 'asc'|'desc', cursor: string|null, offset: number }} + * @throws {ApiError} 400 on invalid limit, offset, order, or a cursor combined + * with an offset. + */ +function parseHistoryPagination(query = {}, options = {}) { + const { + defaultOrder = 'desc', + defaultLimit = config.pagination.defaultLimit, + maxLimit = config.pagination.maxLimit, + } = options; + + const limit = parseLimit(query.limit, defaultLimit, maxLimit); + const order = parseOrder(query.order, defaultOrder); + const cursor = query.cursor == null || query.cursor === '' ? null : String(query.cursor); + const offset = parseOffset(query.offset); + + if (cursor !== null && offset > 0) { + throw ApiError.badRequest( + 'Provide either cursor or offset, not both', + { code: 'CONFLICTING_PAGINATION' } + ); + } + + return { mode: cursor !== null ? 'cursor' : 'offset', limit, order, cursor, offset }; +} + +/** + * @param {*} raw + * @param {number} defaultLimit + * @param {number} maxLimit + * @returns {number} + */ +function parseLimit(raw, defaultLimit, maxLimit) { + if (raw == null || raw === '') return defaultLimit; + + const limit = toInteger(raw); + if (limit === null || limit < 1) { + throw ApiError.badRequest('limit must be a positive integer', { + code: 'INVALID_LIMIT', + maxLimit, + }); + } + if (limit > maxLimit) { + throw ApiError.badRequest(`limit may not exceed ${maxLimit}`, { + code: 'LIMIT_TOO_LARGE', + maxLimit, + }); + } + return limit; +} + +/** + * @param {*} raw + * @returns {number} + */ +function parseOffset(raw) { + if (raw == null || raw === '') return 0; + + const offset = toInteger(raw); + if (offset === null || offset < 0) { + throw ApiError.badRequest('offset must be a non-negative integer', { + code: 'INVALID_OFFSET', + }); + } + return offset; +} + +/** + * @param {*} raw + * @param {'asc'|'desc'} defaultOrder + * @returns {'asc'|'desc'} + */ +function parseOrder(raw, defaultOrder) { + if (raw == null || raw === '') return defaultOrder; + if (raw !== 'asc' && raw !== 'desc') { + throw ApiError.badRequest('order must be "asc" or "desc"', { + code: 'INVALID_ORDER', + allowed: ['asc', 'desc'], + }); + } + return raw; +} + +/** + * Strict integer parse: rejects "12abc", "1.5", "1e3" and other values that + * parseInt would happily truncate. + * @param {*} raw + * @returns {number|null} + */ +function toInteger(raw) { + const text = String(raw).trim(); + if (!/^[+-]?\d+$/.test(text)) return null; + const value = Number(text); + return Number.isSafeInteger(value) ? value : null; +} + module.exports = { DEFAULT_LIMIT, MAX_LIMIT, + parseHistoryPagination, parsePagination, }; diff --git a/test/paginationApi.test.js b/test/paginationApi.test.js new file mode 100644 index 0000000..ef146f1 --- /dev/null +++ b/test/paginationApi.test.js @@ -0,0 +1,682 @@ +'use strict'; + +const { test, before, after, beforeEach } = require('node:test'); +const assert = require('node:assert/strict'); + +// Must be set before any module reads config at require-time. +process.env.NODE_ENV = 'test'; +process.env.PAGINATION_CURSOR_SECRET = 'api-test-cursor-secret'; +// The suite walks many pages; the default 100 req/min limiter would trip. +process.env.RATE_LIMIT_MAX = '100000'; + +const createApp = require('../src/app'); +const { reset: resetStore } = require('../src/store'); +const transferService = require('../src/services/transferService'); +const auditService = require('../src/services/auditService'); + +const ADMIN = 'test-token-admin'; +const READONLY = 'test-token-readonly'; + +let server; +let baseUrl; + +before(() => { + const app = createApp(); + return new Promise((resolve) => { + server = app.listen(0, () => { + baseUrl = `http://127.0.0.1:${server.address().port}`; + resolve(); + }); + }); +}); + +after(() => { + if (server) server.close(); +}); + +beforeEach(() => { + resetStore(); +}); + +/** + * GET a JSON endpoint with a bearer token. + * @param {string} path + * @param {string} [token] + */ +async function get(path, token = ADMIN) { + const res = await fetch(`${baseUrl}${path}`, { + headers: { Authorization: `Bearer ${token}` }, + }); + return { status: res.status, body: await res.json() }; +} + +/** + * Create `count` transfers named T0..T(count-1) in order. + * Console output is suppressed because each creation emits a mock-Stellar debug line. + * @param {number} count + * @returns {object[]} + */ +function seedTransfers(count) { + const original = console.log; + console.log = () => {}; + try { + const created = []; + for (let i = 0; i < count; i += 1) { + created.push(transferService.createTransfer({ + senderName: `T${i}`, + recipientName: `R${i}`, + amount: 10, + from: 'USD', + to: 'EUR', + })); + } + return created; + } finally { + console.log = original; + } +} + +/** Sender names of the transfers in a response body. */ +const names = (body) => body.transfers.map((t) => t.senderName); + +/** + * Walk every page of a collection with cursor pagination. + * @param {string} path - endpoint plus filters, without cursor. + * @param {object} [options] + * @returns {Promise<{ items: object[], pages: number, requests: number }>} + */ +async function drainCursor(path, { token = ADMIN, key = 'transfers', onPage = null } = {}) { + const items = []; + let cursor = null; + let pages = 0; + + for (;;) { + const url = cursor ? `${path}&cursor=${encodeURIComponent(cursor)}` : path; + const { status, body } = await get(url, token); + assert.equal(status, 200, `page ${pages} failed: ${JSON.stringify(body)}`); + + items.push(...body[key]); + pages += 1; + assert.ok(pages < 500, 'pagination did not terminate'); + + if (onPage) await onPage(pages, body); + if (!body.pageInfo.hasMore) { + assert.equal(body.pageInfo.nextCursor, null); + return { items, pages, last: body }; + } + cursor = body.pageInfo.nextCursor; + assert.ok(cursor, 'hasMore was true but no nextCursor was issued'); + } +} + +// ─── Pagination contract ────────────────────────────────────────────────────── + +test('cursor traversal returns every transfer exactly once, in order', async () => { + seedTransfers(25); + + const { items, pages } = await drainCursor('/api/transfers?limit=4'); + + assert.deepEqual(items.map((t) => t.senderName), [...Array(25).keys()].map((i) => `T${i}`)); + assert.equal(new Set(items.map((t) => t.id)).size, 25, 'no duplicates'); + assert.equal(pages, 7); +}); + +test('descending traversal is the exact reverse of ascending', async () => { + seedTransfers(17); + + const ascending = await drainCursor('/api/transfers?limit=5&order=asc'); + const descending = await drainCursor('/api/transfers?limit=5&order=desc'); + + assert.deepEqual( + descending.items.map((t) => t.id), + ascending.items.map((t) => t.id).reverse() + ); +}); + +test('a page smaller than the limit terminates the traversal', async () => { + seedTransfers(3); + const { status, body } = await get('/api/transfers?limit=10'); + + assert.equal(status, 200); + assert.equal(body.count, 3); + assert.equal(body.limit, 10); + assert.equal(body.order, 'asc'); + assert.equal(body.pageInfo.hasMore, false); + assert.equal(body.pageInfo.nextCursor, null); + assert.ok(body.pageInfo.endCursor, 'a terminal page still exposes a resume position'); +}); + +test('an exactly-filled final page does not produce a trailing empty page', async () => { + seedTransfers(8); + const { pages, items } = await drainCursor('/api/transfers?limit=4'); + + assert.equal(pages, 2); + assert.equal(items.length, 8); +}); + +test('cursor pagination over an empty collection is well formed', async () => { + const { status, body } = await get('/api/transfers?limit=5'); + + assert.equal(status, 200); + assert.deepEqual(body.transfers, []); + assert.equal(body.count, 0); + assert.equal(body.pageInfo.hasMore, false); + assert.equal(body.pageInfo.nextCursor, null); +}); + +test('audit cursor traversal returns every entry exactly once, newest first', async () => { + seedTransfers(12); // one transfer.created entry each + + const { items } = await drainCursor('/api/audit?limit=5', { key: 'entries' }); + + assert.equal(items.length, 12); + assert.equal(new Set(items.map((e) => e.id)).size, 12); + assert.deepEqual( + items.map((e) => e.resourceId), + auditService.getEntries().map((e) => e.resourceId) + ); +}); + +// ─── Regression: the failure mode offset pagination has ─────────────────────── + +test('REGRESSION: offset paging duplicates a row when a transfer is inserted mid-walk', async () => { + // This is the original defect, asserted directly so the fix cannot silently + // regress into the offset behaviour. + const seeded = seedTransfers(6); + + const first = await get('/api/transfers?order=desc&limit=3&offset=0'); + assert.deepEqual(names(first.body), ['T5', 'T4', 'T3']); + + seedTransfers(1); // a transfer arrives while the client is paging + + const second = await get('/api/transfers?order=desc&limit=3&offset=3'); + + const overlap = names(second.body).filter((n) => names(first.body).includes(n)); + assert.deepEqual(overlap, ['T3'], 'offset paging is expected to repeat the boundary row'); + assert.equal(seeded.length, 6); +}); + +test('cursor paging shows no duplicate when a transfer is inserted mid-walk', async () => { + seedTransfers(6); + + const first = await get('/api/transfers?order=desc&limit=3'); + assert.deepEqual(names(first.body), ['T5', 'T4', 'T3']); + + seedTransfers(1); + + const second = await get( + `/api/transfers?order=desc&limit=3&cursor=${encodeURIComponent(first.body.pageInfo.nextCursor)}` + ); + + assert.deepEqual(names(second.body), ['T2', 'T1', 'T0']); + assert.equal(second.body.pageInfo.hasMore, false); +}); + +test('REGRESSION: offset paging skips a row when one leaves the filtered set mid-walk', async () => { + const seeded = seedTransfers(6); + + const first = await get('/api/transfers?limit=3&offset=0'); + assert.deepEqual(names(first.body), ['T0', 'T1', 'T2']); + + // Archiving removes T1 from the default result set, shifting the window left. + transferService.archiveTransfer(seeded[1].id); + + const second = await get('/api/transfers?limit=3&offset=3'); + + assert.ok(!names(second.body).includes('T3'), 'offset paging is expected to skip T3'); + assert.deepEqual(names(second.body), ['T4', 'T5']); +}); + +test('cursor paging skips nothing when a transfer leaves the filtered set mid-walk', async () => { + const seeded = seedTransfers(6); + + const first = await get('/api/transfers?limit=3'); + assert.deepEqual(names(first.body), ['T0', 'T1', 'T2']); + + transferService.archiveTransfer(seeded[1].id); + + const second = await get( + `/api/transfers?limit=3&cursor=${encodeURIComponent(first.body.pageInfo.nextCursor)}` + ); + + assert.deepEqual(names(second.body), ['T3', 'T4', 'T5']); +}); + +// ─── Concurrent inserts ─────────────────────────────────────────────────────── + +test('a full cursor walk with a write before every page has no duplicates or gaps', async () => { + const snapshot = seedTransfers(30).map((t) => t.id); + + const { items } = await drainCursor('/api/transfers?limit=4', { + onPage: async () => { seedTransfers(1); }, + }); + + const ids = items.map((t) => t.id); + assert.equal(new Set(ids).size, ids.length, 'walk returned a duplicate'); + // Ascending order: every transfer present when the walk began must appear, + // and the transfers created during the walk may appear at the tail. + assert.deepEqual(ids.slice(0, 30), snapshot, 'walk lost or reordered a pre-existing transfer'); +}); + +test('a descending cursor walk is unaffected by transfers created during the walk', async () => { + const snapshot = seedTransfers(30).map((t) => t.id).reverse(); + + const { items } = await drainCursor('/api/transfers?limit=4&order=desc', { + onPage: async () => { seedTransfers(1); }, + }); + + // Descending from a fixed starting position: newer records sort before the + // start of the walk, so the walk sees exactly the original 30. + assert.deepEqual(items.map((t) => t.id), snapshot); +}); + +test('an audit walk with entries appended between pages has no duplicates', async () => { + seedTransfers(20); + + const { items } = await drainCursor('/api/audit?limit=3', { + key: 'entries', + onPage: async () => { auditService.addEntry({ action: 'noise.created', resourceId: 'noise' }); }, + }); + + const ids = items.map((e) => e.id); + assert.equal(new Set(ids).size, ids.length, 'audit walk returned a duplicate'); + assert.ok(items.every((e) => e.action === 'transfer.created'), + 'a descending audit walk must not pick up entries appended after it started'); + assert.equal(items.length, 20); +}); + +test('a client can tail an ascending audit feed from endCursor without re-reading', async () => { + seedTransfers(4); + + const first = await drainCursor('/api/audit?limit=10&order=asc', { key: 'entries' }); + assert.equal(first.items.length, 4); + + seedTransfers(3); + + const { status, body } = await get( + `/api/audit?limit=10&order=asc&cursor=${encodeURIComponent(first.last.pageInfo.endCursor)}` + ); + + assert.equal(status, 200); + assert.equal(body.count, 3, 'tailing returns only entries appended since the last page'); + const seen = new Set(first.items.map((e) => e.id)); + assert.ok(body.entries.every((e) => !seen.has(e.id))); +}); + +// ─── Actor scope binding ────────────────────────────────────────────────────── + +test('a cursor minted for one token is rejected for another with 403', async () => { + seedTransfers(10); + + const first = await get('/api/transfers?limit=3', ADMIN); + const { status, body } = await get( + `/api/transfers?limit=3&cursor=${encodeURIComponent(first.body.pageInfo.nextCursor)}`, + READONLY + ); + + assert.equal(status, 403); + assert.equal(body.error.details.code, 'CURSOR_ACTOR_MISMATCH'); +}); + +test('cursor rejection is symmetric between tokens', async () => { + seedTransfers(10); + + const first = await get('/api/transfers?limit=3', READONLY); + const { status } = await get( + `/api/transfers?limit=3&cursor=${encodeURIComponent(first.body.pageInfo.nextCursor)}`, + ADMIN + ); + + assert.equal(status, 403); +}); + +test('each token can page the same collection with its own cursor', async () => { + seedTransfers(10); + + const adminFirst = await get('/api/transfers?limit=3', ADMIN); + const readonlyFirst = await get('/api/transfers?limit=3', READONLY); + + const adminSecond = await get( + `/api/transfers?limit=3&cursor=${encodeURIComponent(adminFirst.body.pageInfo.nextCursor)}`, + ADMIN + ); + const readonlySecond = await get( + `/api/transfers?limit=3&cursor=${encodeURIComponent(readonlyFirst.body.pageInfo.nextCursor)}`, + READONLY + ); + + assert.equal(adminSecond.status, 200); + assert.equal(readonlySecond.status, 200); + assert.deepEqual(names(adminSecond.body), names(readonlySecond.body)); +}); + +test('a transfers cursor is rejected by the audit endpoint', async () => { + seedTransfers(10); + + const first = await get('/api/transfers?limit=3&order=desc'); + const { status, body } = await get( + `/api/audit?limit=3&cursor=${encodeURIComponent(first.body.pageInfo.nextCursor)}` + ); + + assert.equal(status, 400); + assert.equal(body.error.details.code, 'CURSOR_FILTER_MISMATCH'); +}); + +test('a tampered cursor is rejected without revealing what was wrong', async () => { + seedTransfers(10); + + const first = await get('/api/transfers?limit=3'); + const cursor = first.body.pageInfo.nextCursor; + const [payload, signature] = cursor.split('.'); + const edited = JSON.parse(Buffer.from(payload, 'base64url').toString('utf8')); + edited.s = 0; // rewind to the start of the collection + const forged = `${Buffer.from(JSON.stringify(edited)).toString('base64url')}.${signature}`; + + const { status, body } = await get(`/api/transfers?limit=3&cursor=${encodeURIComponent(forged)}`); + + assert.equal(status, 400); + assert.equal(body.error.details.code, 'INVALID_CURSOR'); +}); + +test('a cursor from before a store reset is rejected as stale', async () => { + seedTransfers(10); + const first = await get('/api/transfers?limit=3'); + const cursor = first.body.pageInfo.nextCursor; + + resetStore(); + seedTransfers(10); // sequence numbers are reissued to different transfers + + const { status, body } = await get(`/api/transfers?limit=3&cursor=${encodeURIComponent(cursor)}`); + + assert.equal(status, 400); + assert.equal(body.error.details.code, 'STALE_CURSOR'); +}); + +test('reading a collection still requires the matching scope', async () => { + seedTransfers(3); + const { status } = await get('/api/audit?limit=3', 'test-token-transfers'); + assert.equal(status, 403); +}); + +// ─── Filter binding ─────────────────────────────────────────────────────────── + +test('a cursor is rejected when the status filter changes', async () => { + const seeded = seedTransfers(10); + transferService.claimTransfer(seeded[0].id); + + const first = await get('/api/transfers?limit=3&status=pending'); + const { status, body } = await get( + `/api/transfers?limit=3&status=claimed&cursor=${encodeURIComponent(first.body.pageInfo.nextCursor)}` + ); + + assert.equal(status, 400); + assert.equal(body.error.details.code, 'CURSOR_FILTER_MISMATCH'); +}); + +test('a cursor is rejected when the search filter changes', async () => { + seedTransfers(10); + + // Both needles match every seeded transfer, so the two queries differ only in + // their filter fingerprint - not in the rows they would return. + const first = await get('/api/transfers?limit=2&q=T'); + const { status, body } = await get( + `/api/transfers?limit=2&q=R&cursor=${encodeURIComponent(first.body.pageInfo.nextCursor)}` + ); + + assert.equal(status, 400); + assert.equal(body.error.details.code, 'CURSOR_FILTER_MISMATCH'); +}); + +test('a cursor is rejected when the archived filter changes', async () => { + const seeded = seedTransfers(10); + transferService.archiveTransfer(seeded[9].id); + + const first = await get('/api/transfers?limit=3'); + const { status, body } = await get( + `/api/transfers?limit=3&archived=all&cursor=${encodeURIComponent(first.body.pageInfo.nextCursor)}` + ); + + assert.equal(status, 400); + assert.equal(body.error.details.code, 'CURSOR_FILTER_MISMATCH'); +}); + +test('a cursor is rejected when the sort order changes', async () => { + seedTransfers(10); + + const first = await get('/api/transfers?limit=3&order=asc'); + const { status, body } = await get( + `/api/transfers?limit=3&order=desc&cursor=${encodeURIComponent(first.body.pageInfo.nextCursor)}` + ); + + assert.equal(status, 400); + assert.equal(body.error.details.code, 'CURSOR_ORDER_MISMATCH'); +}); + +test('filters that normalise to the same query share cursors', async () => { + seedTransfers(10); + + // archived is false by default, and search is trimmed and lowercased. + const first = await get('/api/transfers?limit=3&q=T'); + assert.ok(first.body.pageInfo.nextCursor, 'expected more than one page'); + + const { status, body } = await get( + `/api/transfers?limit=3&q=%20t%20&archived=false&cursor=${encodeURIComponent(first.body.pageInfo.nextCursor)}` + ); + + assert.equal(status, 200, JSON.stringify(body)); + assert.deepEqual(names(body), ['T3', 'T4', 'T5']); +}); + +test('a filtered cursor walk returns only matching rows, with no gaps', async () => { + const seeded = seedTransfers(30); + for (let i = 0; i < 30; i += 3) transferService.claimTransfer(seeded[i].id); + + const { items } = await drainCursor('/api/transfers?limit=4&status=claimed'); + + assert.equal(items.length, 10); + assert.ok(items.every((t) => t.status === 'claimed')); + assert.deepEqual( + items.map((t) => t.senderName), + [0, 3, 6, 9, 12, 15, 18, 21, 24, 27].map((i) => `T${i}`) + ); +}); + +test('the audit resourceId filter pages through its own index', async () => { + const [first] = seedTransfers(5); + transferService.claimTransfer(first.id); + seedTransfers(5); + + const { items } = await drainCursor( + `/api/audit?limit=1&resourceId=${encodeURIComponent(first.id)}`, + { key: 'entries' } + ); + + assert.equal(items.length, 2); + assert.ok(items.every((e) => e.resourceId === first.id)); + assert.deepEqual(items.map((e) => e.action), ['transfer.claimed', 'transfer.created']); +}); + +test('an audit cursor is rejected when the resourceId filter changes', async () => { + const seeded = seedTransfers(4); + for (const t of seeded) transferService.claimTransfer(t.id); + + const first = await get(`/api/audit?limit=1&resourceId=${encodeURIComponent(seeded[0].id)}`); + const { status, body } = await get( + `/api/audit?limit=1&resourceId=${encodeURIComponent(seeded[1].id)}` + + `&cursor=${encodeURIComponent(first.body.pageInfo.nextCursor)}` + ); + + assert.equal(status, 400); + assert.equal(body.error.details.code, 'CURSOR_FILTER_MISMATCH'); +}); + +test('an invalid status filter is still rejected', async () => { + const { status, body } = await get('/api/transfers?status=teleported'); + assert.equal(status, 400); + assert.match(body.error.message, /Invalid status filter/); +}); + +// ─── Bounded queries ────────────────────────────────────────────────────────── + +test('an oversized limit is rejected rather than silently truncated', async () => { + seedTransfers(5); + + for (const path of ['/api/transfers?limit=5000', '/api/audit?limit=5000']) { + const { status, body } = await get(path); + assert.equal(status, 400, path); + assert.equal(body.error.details.code, 'LIMIT_TOO_LARGE'); + assert.equal(body.error.details.maxLimit, 200); + } +}); + +test('malformed limit, offset and order values are rejected', async () => { + for (const query of ['limit=0', 'limit=-5', 'limit=abc', 'offset=-1', 'offset=x', 'order=up']) { + const { status } = await get(`/api/transfers?${query}`); + assert.equal(status, 400, `accepted ${query}`); + } +}); + +test('a deep offset is refused and points the caller at cursors', async () => { + const { status, body } = await get('/api/transfers?offset=999999'); + + assert.equal(status, 400); + assert.equal(body.error.details.code, 'OFFSET_TOO_DEEP'); + assert.match(body.error.message, /cursor pagination/); +}); + +test('cursor and offset cannot be combined', async () => { + seedTransfers(5); + const first = await get('/api/transfers?limit=2'); + const { status, body } = await get( + `/api/transfers?limit=2&offset=2&cursor=${encodeURIComponent(first.body.pageInfo.nextCursor)}` + ); + + assert.equal(status, 400); + assert.equal(body.error.details.code, 'CONFLICTING_PAGINATION'); +}); + +// ─── Large fixture: cost of a page does not grow with history ──────────────── + +test('a deep cursor page costs the same as a shallow one over a large history', async () => { + seedTransfers(20000); + + const shallow = await get('/api/transfers?limit=50'); + assert.equal(shallow.status, 200); + assert.equal(shallow.body.count, 50); + // 50 returned plus one lookahead record: no dependence on collection size. + assert.equal(shallow.body.pageInfo.scanned, 51); + + // Jump ~19 000 records deep by chaining cursors at a large page size. + let cursor = shallow.body.pageInfo.nextCursor; + let deep = shallow; + for (let i = 0; i < 95; i += 1) { + deep = await get(`/api/transfers?limit=200&cursor=${encodeURIComponent(cursor)}`); + assert.equal(deep.status, 200); + cursor = deep.body.pageInfo.nextCursor; + } + + const started = process.hrtime.bigint(); + const deepPage = await get(`/api/transfers?limit=50&cursor=${encodeURIComponent(cursor)}`); + const elapsedMs = Number(process.hrtime.bigint() - started) / 1e6; + + assert.equal(deepPage.status, 200); + assert.equal(deepPage.body.count, 50); + assert.equal(deepPage.body.pageInfo.scanned, 51, + 'a page 19 000 records deep must examine no more records than a first page'); + assert.ok(elapsedMs < 250, `deep page took ${elapsedMs.toFixed(1)}ms`); +}); + +test('a selective filter over a large history is capped by the scan budget', async () => { + const seeded = seedTransfers(20000); + // Exactly one match, sitting at the far end of the index. + transferService.claimTransfer(seeded[19999].id); + + const { status, body } = await get('/api/transfers?limit=10&status=claimed'); + + assert.equal(status, 200); + assert.equal(body.pageInfo.scanned, 10000, 'the scan must stop at the configured budget'); + assert.equal(body.pageInfo.scanTruncated, true); + assert.equal(body.pageInfo.hasMore, true, + 'a budget-truncated page must not report the collection as exhausted'); + + // Following the cursor still reaches the match: bounded, not lossy. + const { items } = await drainCursor('/api/transfers?limit=10&status=claimed'); + assert.equal(items.length, 1); + assert.equal(items[0].id, seeded[19999].id); +}); + +test('a full cursor walk of a large history returns every record exactly once', async () => { + const seeded = seedTransfers(5000); + + const { items } = await drainCursor('/api/transfers?limit=200'); + + assert.equal(items.length, 5000); + assert.deepEqual(items.map((t) => t.id), seeded.map((t) => t.id)); +}); + +// ─── Backwards compatibility ────────────────────────────────────────────────── + +test('the legacy offset response contract is unchanged', async () => { + seedTransfers(5); + + const { status, body } = await get('/api/transfers?limit=2&offset=1'); + + assert.equal(status, 200); + assert.equal(body.total, 5); + assert.equal(body.count, 2); + assert.equal(body.limit, 2); + assert.equal(body.offset, 1); + assert.deepEqual(names(body), ['T1', 'T2']); +}); + +test('the legacy audit offset response contract is unchanged', async () => { + seedTransfers(5); + + const { status, body } = await get('/api/audit?limit=2&offset=1'); + + assert.equal(status, 200); + assert.equal(body.total, 5); + assert.equal(body.count, 2); + assert.equal(body.offset, 1); + assert.equal(body.entries.length, 2); +}); + +test('offset mode still issues a cursor so clients can migrate mid-walk', async () => { + seedTransfers(9); + + const offsetPage = await get('/api/transfers?limit=3&offset=3'); + assert.deepEqual(names(offsetPage.body), ['T3', 'T4', 'T5']); + + const { status, body } = await get( + `/api/transfers?limit=3&cursor=${encodeURIComponent(offsetPage.body.pageInfo.nextCursor)}` + ); + + assert.equal(status, 200); + assert.deepEqual(names(body), ['T6', 'T7', 'T8']); +}); + +test('cursor mode omits total, which offset mode still pays for', async () => { + seedTransfers(5); + + const offsetPage = await get('/api/transfers?limit=2'); + assert.equal(offsetPage.body.total, 5); + + const cursorPage = await get( + `/api/transfers?limit=2&cursor=${encodeURIComponent(offsetPage.body.pageInfo.nextCursor)}` + ); + assert.equal(cursorPage.body.total, undefined); + assert.equal(cursorPage.body.offset, undefined); +}); + +test('default listing behaviour is untouched for callers that pass no parameters', async () => { + const seeded = seedTransfers(3); + transferService.archiveTransfer(seeded[1].id); + + const { status, body } = await get('/api/transfers'); + + assert.equal(status, 200); + assert.equal(body.limit, 50); + assert.equal(body.offset, 0); + assert.equal(body.total, 2); + assert.deepEqual(names(body), ['T0', 'T2']); +}); diff --git a/test/paginationCursor.test.js b/test/paginationCursor.test.js new file mode 100644 index 0000000..b559749 --- /dev/null +++ b/test/paginationCursor.test.js @@ -0,0 +1,371 @@ +'use strict'; + +const { test } = require('node:test'); +const assert = require('node:assert/strict'); + +// A fixed signing key keeps encoded cursors reproducible within the file and +// documents the production requirement to pin one across instances. +process.env.PAGINATION_CURSOR_SECRET = 'unit-test-cursor-secret'; + +const { OrderedIndex } = require('../src/utils/orderedIndex'); +const { + actorFingerprint, + decodeCursor, + encodeCursor, + fingerprint, + MAX_CURSOR_LENGTH, +} = require('../src/utils/cursor'); +const { parseHistoryPagination, parsePagination } = require('../src/utils/pagination'); + +// ─── OrderedIndex ───────────────────────────────────────────────────────────── + +/** Build an index of `count` items, all sharing one timestamp by default. */ +function buildIndex(count, { at = '2024-01-01T00:00:00.000Z', groupOf = null } = {}) { + const index = new OrderedIndex({ + sortKeyOf: (item) => item.at, + groupKeyOf: groupOf ? (item) => groupOf(item) : null, + }); + for (let i = 0; i < count; i += 1) { + index.append({ n: i, at: typeof at === 'function' ? at(i) : at }); + } + return index; +} + +/** Walk every page of an index, returning the items and the page sizes. */ +function drain(index, { order, limit, group = null, match = null, maxScan = 1e9 }) { + const items = []; + const pages = []; + let afterSeq = null; + let guard = 0; + + for (;;) { + const page = index.scan({ afterSeq, order, limit, group, match, maxScan }); + items.push(...page.items); + pages.push(page.items.length); + if (!page.hasMore) break; + assert.ok(page.last, 'a page reporting hasMore must expose a resume position'); + afterSeq = page.last.seq; + guard += 1; + assert.ok(guard < 10000, 'pagination did not terminate'); + } + + return { items, pages }; +} + +test('OrderedIndex assigns dense, strictly increasing sequence numbers', () => { + const index = buildIndex(5); + assert.deepEqual(index.records.map((r) => r.seq), [0, 1, 2, 3, 4]); + assert.equal(index.size, 5); +}); + +test('OrderedIndex orders records that share a timestamp deterministically', () => { + // Every record has the same millisecond, so the timestamp alone cannot order + // them. The sequence tie-breaker still yields insertion order both ways. + const index = buildIndex(20); + + const ascending = drain(index, { order: 'asc', limit: 3 }); + const descending = drain(index, { order: 'desc', limit: 3 }); + + assert.deepEqual(ascending.items.map((i) => i.n), [...Array(20).keys()]); + assert.deepEqual(descending.items.map((i) => i.n), [...Array(20).keys()].reverse()); +}); + +test('OrderedIndex pages are exclusive of the cursor record in both directions', () => { + const index = buildIndex(10); + + for (const order of ['asc', 'desc']) { + const first = index.scan({ order, limit: 4, maxScan: 1e9 }); + const second = index.scan({ afterSeq: first.last.seq, order, limit: 4, maxScan: 1e9 }); + + const firstSeqs = first.items.map((i) => i.n); + const secondSeqs = second.items.map((i) => i.n); + const overlap = firstSeqs.filter((n) => secondSeqs.includes(n)); + + assert.deepEqual(overlap, [], `pages overlapped for order=${order}`); + } +}); + +test('OrderedIndex reports hasMore only while records remain', () => { + const index = buildIndex(9); + const { pages } = drain(index, { order: 'asc', limit: 4 }); + assert.deepEqual(pages, [4, 4, 1]); + + const exact = drain(buildIndex(8), { order: 'asc', limit: 4 }); + assert.deepEqual(exact.pages, [4, 4], 'an exactly-filled last page must not yield an empty page'); +}); + +test('OrderedIndex applies a residual filter without losing or repeating records', () => { + const index = buildIndex(50); + const match = (item) => item.n % 7 === 0; + + const { items } = drain(index, { order: 'asc', limit: 2, match }); + + assert.deepEqual(items.map((i) => i.n), [0, 7, 14, 21, 28, 35, 42, 49]); +}); + +test('OrderedIndex pages a secondary-index group without scanning other groups', () => { + const index = buildIndex(300, { groupOf: (item) => `g${item.n % 3}` }); + + const page = index.scan({ group: 'g1', order: 'asc', limit: 5, maxScan: 1e9 }); + + assert.deepEqual(page.items.map((i) => i.n), [1, 4, 7, 10, 13]); + // 5 returned plus 1 lookahead: the 200 records in the other groups are never touched. + assert.equal(page.scanned, 6); +}); + +test('OrderedIndex group paging is exclusive and complete', () => { + const index = buildIndex(300, { groupOf: (item) => `g${item.n % 3}` }); + + const { items } = drain(index, { order: 'desc', limit: 7, group: 'g2' }); + const expected = [...Array(300).keys()].filter((n) => n % 3 === 2).reverse(); + + assert.deepEqual(items.map((i) => i.n), expected); + assert.equal(new Set(items.map((i) => i.n)).size, items.length, 'no duplicates'); +}); + +test('OrderedIndex caps work at maxScan and stays resumable', () => { + const index = buildIndex(1000); + // Matches only the very last record, so an uncapped scan would walk all 1000. + const match = (item) => item.n === 999; + + const page = index.scan({ order: 'asc', limit: 10, match, maxScan: 100 }); + + assert.deepEqual(page.items, []); + assert.equal(page.scanned, 100); + assert.equal(page.scanTruncated, true); + assert.equal(page.hasMore, true, 'a truncated scan must not claim the collection is exhausted'); + + // Resuming from the truncated frontier eventually reaches the record. + const { items } = drain(index, { order: 'asc', limit: 10, match, maxScan: 100 }); + assert.deepEqual(items.map((i) => i.n), [999]); +}); + +test('OrderedIndex maxScan does not flag truncation when the page is full', () => { + const index = buildIndex(1000); + const page = index.scan({ order: 'asc', limit: 10, maxScan: 10 }); + + assert.equal(page.items.length, 10); + assert.equal(page.hasMore, true); + assert.equal(page.scanTruncated, false); +}); + +test('OrderedIndex recordAt resolves positions in the index and in a group', () => { + const index = buildIndex(30, { + at: (i) => `2024-01-01T00:00:00.${String(i).padStart(3, '0')}Z`, + groupOf: (item) => `g${item.n % 2}`, + }); + + assert.equal(index.recordAt(7).item.n, 7); + assert.equal(index.recordAt(7, 'g1').item.n, 7); + assert.equal(index.recordAt(7, 'g0'), null, 'seq 7 is not in group g0'); + assert.equal(index.recordAt(999), null); +}); + +test('OrderedIndex reset clears records, groups and the sequence counter', () => { + const index = buildIndex(5, { groupOf: () => 'g' }); + index.reset(); + + assert.equal(index.size, 0); + assert.equal(index.nextSeq, 0); + assert.deepEqual(index.recordsFor('g'), []); +}); + +test('OrderedIndex scanning an empty index yields an empty terminal page', () => { + const page = buildIndex(0).scan({ order: 'desc', limit: 10, maxScan: 100 }); + assert.deepEqual(page.items, []); + assert.equal(page.hasMore, false); + assert.equal(page.last, null); +}); + +// ─── Cursor codec ───────────────────────────────────────────────────────────── + +const BOUND = { order: 'desc', filter: 'filter-a', actor: 'actor-a' }; + +function mint(overrides = {}) { + return encodeCursor({ key: '2024-01-01T00:00:00.000Z', seq: 42, ...BOUND, ...overrides }); +} + +test('encodeCursor round-trips through decodeCursor', () => { + const decoded = decodeCursor(mint(), BOUND); + assert.deepEqual(decoded, { order: 'desc', key: '2024-01-01T00:00:00.000Z', seq: 42 }); +}); + +test('cursors are opaque and URL-safe', () => { + const cursor = mint(); + assert.match(cursor, /^[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+$/); + assert.equal(encodeURIComponent(cursor), cursor); +}); + +test('decodeCursor rejects a cursor whose payload was edited', () => { + const cursor = mint(); + const [payload, signature] = cursor.split('.'); + const decoded = JSON.parse(Buffer.from(payload, 'base64url').toString('utf8')); + decoded.s = 0; + const forged = `${Buffer.from(JSON.stringify(decoded)).toString('base64url')}.${signature}`; + + assert.throws(() => decodeCursor(forged, BOUND), (err) => { + assert.equal(err.statusCode, 400); + assert.equal(err.details.code, 'INVALID_CURSOR'); + return true; + }); +}); + +test('decodeCursor rejects a cursor with a re-signed payload from an unknown key', () => { + const crypto = require('crypto'); + const payload = Buffer.from( + JSON.stringify({ v: 1, o: 'desc', k: 'x', s: 0, f: 'filter-a', a: 'actor-a' }) + ).toString('base64url'); + const forged = `${payload}.${crypto.createHmac('sha256', 'wrong-key').update(payload).digest('base64url')}`; + + assert.throws(() => decodeCursor(forged, BOUND), /malformed/); +}); + +test('decodeCursor rejects structurally invalid input', () => { + for (const bad of [null, undefined, '', 'nodot', '.sig', 'payload.', 'x'.repeat(MAX_CURSOR_LENGTH + 1)]) { + assert.throws(() => decodeCursor(bad, BOUND), /malformed/, `accepted ${JSON.stringify(bad)}`); + } +}); + +test('decodeCursor rejects a cursor issued to a different actor with 403', () => { + assert.throws( + () => decodeCursor(mint(), { ...BOUND, actor: 'actor-b' }), + (err) => { + assert.equal(err.statusCode, 403); + assert.equal(err.details.code, 'CURSOR_ACTOR_MISMATCH'); + return true; + } + ); +}); + +test('decodeCursor checks the actor before the filter', () => { + // A caller must not be able to learn anything about another actor's cursor by + // varying filters until the error message changes. + assert.throws( + () => decodeCursor(mint(), { order: 'asc', filter: 'filter-z', actor: 'actor-b' }), + (err) => { + assert.equal(err.details.code, 'CURSOR_ACTOR_MISMATCH'); + return true; + } + ); +}); + +test('decodeCursor rejects a cursor issued for a different filter set', () => { + assert.throws(() => decodeCursor(mint(), { ...BOUND, filter: 'filter-b' }), (err) => { + assert.equal(err.statusCode, 400); + assert.equal(err.details.code, 'CURSOR_FILTER_MISMATCH'); + return true; + }); +}); + +test('decodeCursor rejects a cursor issued for a different sort order', () => { + assert.throws(() => decodeCursor(mint(), { ...BOUND, order: 'asc' }), (err) => { + assert.equal(err.details.code, 'CURSOR_ORDER_MISMATCH'); + return true; + }); +}); + +test('decodeCursor rejects a cursor from an incompatible format version', () => { + const crypto = require('crypto'); + const payload = Buffer.from( + JSON.stringify({ v: 99, o: 'desc', k: 'x', s: 0, f: 'filter-a', a: 'actor-a' }) + ).toString('base64url'); + const signature = crypto + .createHmac('sha256', process.env.PAGINATION_CURSOR_SECRET) + .update(payload) + .digest('base64url'); + + assert.throws(() => decodeCursor(`${payload}.${signature}`, BOUND), /incompatible API version/); +}); + +test('fingerprint is stable across key ordering and distinguishes different filters', () => { + assert.equal( + fingerprint({ status: 'pending', archived: false }), + fingerprint({ archived: false, status: 'pending' }) + ); + assert.notEqual( + fingerprint({ status: 'pending', archived: false }), + fingerprint({ status: 'pending', archived: true }) + ); + assert.notEqual(fingerprint(['transfers', {}]), fingerprint(['audit', {}])); +}); + +test('actorFingerprint derives from the token and never contains it', () => { + const print = actorFingerprint({ token: 'test-token-admin' }); + assert.notEqual(print, actorFingerprint({ token: 'test-token-readonly' })); + assert.equal(print, actorFingerprint({ token: 'test-token-admin' })); + assert.notEqual(print, actorFingerprint({}), 'a token must not fingerprint as anonymous'); + assert.ok(!print.includes('test-token')); +}); + +test('actorFingerprint is keyed, not a plain hash of the token', () => { + // A leaked cursor must not let an attacker confirm a guessed token offline. + const crypto = require('crypto'); + const plainHash = crypto.createHash('sha256').update('token:test-token-admin').digest('hex'); + assert.notEqual(actorFingerprint({ token: 'test-token-admin' }), plainHash.slice(0, 16)); +}); + +test('getEntriesForResource does not treat a missing id as "all resources"', () => { + const auditService = require('../src/services/auditService'); + auditService.reset(); + auditService.addEntry({ action: 'transfer.created', resourceId: 'txn-1' }); + + assert.deepEqual(auditService.getEntriesForResource(undefined), []); + assert.deepEqual(auditService.getEntriesForResource(null), []); + assert.deepEqual(auditService.getEntriesForResource(''), []); + assert.equal(auditService.getEntriesForResource('txn-1').length, 1); +}); + +// ─── Query parameter parsing ────────────────────────────────────────────────── + +test('parseHistoryPagination applies defaults', () => { + assert.deepEqual(parseHistoryPagination({}, { defaultOrder: 'asc' }), { + mode: 'offset', + limit: 50, + order: 'asc', + cursor: null, + offset: 0, + }); +}); + +test('parseHistoryPagination rejects an oversized limit rather than clamping it', () => { + assert.throws(() => parseHistoryPagination({ limit: '5000' }), (err) => { + assert.equal(err.statusCode, 400); + assert.equal(err.details.code, 'LIMIT_TOO_LARGE'); + assert.equal(err.details.maxLimit, 200); + return true; + }); +}); + +test('parseHistoryPagination rejects limits that are not positive integers', () => { + for (const limit of ['0', '-1', 'abc', '1.5', '1e3', ' ']) { + assert.throws(() => parseHistoryPagination({ limit }), /limit/, `accepted limit=${limit}`); + } +}); + +test('parseHistoryPagination rejects negative and non-integer offsets', () => { + for (const offset of ['-1', 'abc', '2.5']) { + assert.throws(() => parseHistoryPagination({ offset }), /offset/, `accepted offset=${offset}`); + } + assert.equal(parseHistoryPagination({ offset: '10' }).offset, 10); +}); + +test('parseHistoryPagination rejects an unknown order', () => { + assert.throws(() => parseHistoryPagination({ order: 'sideways' }), (err) => { + assert.equal(err.details.code, 'INVALID_ORDER'); + return true; + }); +}); + +test('parseHistoryPagination rejects cursor and offset used together', () => { + assert.throws(() => parseHistoryPagination({ cursor: 'abc', offset: '10' }), (err) => { + assert.equal(err.details.code, 'CONFLICTING_PAGINATION'); + return true; + }); + // offset=0 is the default and does not conflict. + assert.equal(parseHistoryPagination({ cursor: 'abc', offset: '0' }).mode, 'cursor'); +}); + +test('parsePagination remains lenient for the collections still using it', () => { + assert.deepEqual(parsePagination({ limit: '5000', offset: '-3' }), { limit: 200, offset: 0 }); + assert.deepEqual(parsePagination({}), { limit: 50, offset: 0 }); +});