feat: cursor pagination for transfer and audit history

.env.example

@@ -39,6 +39,18 @@ DB_POOL_CONNECTION_TIMEOUT_MS=2000
 CACHE_DEFAULT_POLICY=no-store
 CACHE_RATES_MAX_AGE_SECONDS=10
 
+# History pagination (GET /api/transfers, GET /api/audit)
+PAGINATION_DEFAULT_LIMIT=50
+# Requests above this limit are rejected with 400, not clamped.
+PAGINATION_MAX_LIMIT=200
+# Max records a single history query may examine before returning a
+# resumable, truncated page.
+PAGINATION_MAX_SCAN=10000
+# HMAC key used to sign pagination cursors. Left unset a random key is
+# generated per process; set it explicitly when running more than one instance,
+# or cursors minted by one will be rejected by another.
+# PAGINATION_CURSOR_SECRET=
+
 # API Token authentication
 # JSON map of token → array of scopes. If omitted, hardcoded demo tokens are used.
 # Valid scopes: transfers:read, transfers:write, users:read, users:write, audit:read

CHANGELOG.md

@@ -18,6 +18,51 @@ When preparing a new release:
 
 ### Added
 
+- Cursor pagination for `GET /api/transfers` and `GET /api/audit`. Pass
+  `?cursor=` (with optional `?order=asc|desc`) to page by an indexed position
+  instead of a row offset; responses carry a `pageInfo` block with
+  `hasMore`, `nextCursor`, `endCursor`, `scanned` and `scanTruncated`.
+  Cursors are HMAC-signed and bound to the API token, filter set and sort
+  order they were issued for, so they cannot be replayed across actor scopes
+  or filters. Signing key: `PAGINATION_CURSOR_SECRET`.
+- Append-only ordered indexes behind transfer and audit history
+  (`src/utils/orderedIndex.js`), giving both collections a deterministic total
+  order and constant-cost page seeks regardless of how deep the page is. The
+  audit log also gains a secondary index by `resourceId`.
+- Per-request work budget for history queries (`PAGINATION_MAX_SCAN`,
+  default `10000`), so a highly selective filter over a large history costs a
+  bounded amount of work. A budget-truncated page is gap-free and resumable
+  via its `nextCursor`.
+- `PAGINATION_DEFAULT_LIMIT` and `PAGINATION_MAX_LIMIT` configuration.
+
+### Changed
+
+- **Breaking:** `GET /api/transfers` and `GET /api/audit` now reject a `limit`
+  above `PAGINATION_MAX_LIMIT` (200) with `400 LIMIT_TOO_LARGE` instead of
+  silently clamping it, and reject malformed `limit`/`offset`/`order` values
+  instead of falling back to defaults. Silent clamping left callers unable to
+  tell a truncated page from a complete one. Other collections
+  (`GET /api/users`) keep the previous lenient behaviour.
+- **Breaking:** `?offset=` beyond `PAGINATION_MAX_SCAN` is rejected with
+  `400 OFFSET_TOO_DEEP`; deep pages must use `?cursor=`. `?cursor=` and
+  `?offset=` cannot be combined.
+- `?offset=` is deprecated on both history endpoints but otherwise unchanged:
+  the `total`, `count`, `limit` and `offset` fields and the default ordering
+  are all preserved, and offset responses now also carry a `nextCursor` so
+  clients can migrate mid-walk. `total` is not returned in cursor mode,
+  because computing it requires the full-collection pass that cursor
+  pagination exists to avoid.
+
+### Fixed
+
+- Offset pagination over transfer and audit history repeated or skipped rows
+  when records were written while a client was paging, because the window was
+  defined by a row count rather than a position. Cursor pagination anchors to
+  an immutable creation position instead, so concurrent writes cannot shift a
+  page boundary.
+
+### Added
+
 - Error tracking integration hook (`src/services/errorTrackingService.js`)
   that captures every error through a replaceable transport (console by
   default) and enriches it with request context (id, method, url).

README.md

@@ -45,6 +45,10 @@ The application is configured using environment variables (typically defined in
 | `DB_POOL_CONNECTION_TIMEOUT_MS` | Time to wait for a connection before timing out (ms) | `2000` |
 | `CACHE_DEFAULT_POLICY` | Default cache policy for endpoints (`no-store`, `public`, `private`) | `no-store` |
 | `CACHE_RATES_MAX_AGE_SECONDS` | Cache duration for rates endpoints (seconds) | `10` |
+| `PAGINATION_DEFAULT_LIMIT` | Page size used when a request omits `limit` | `50` |
+| `PAGINATION_MAX_LIMIT` | Largest accepted `limit`; bigger requests are rejected | `200` |
+| `PAGINATION_MAX_SCAN` | Max records a single history query may examine | `10000` |
+| `PAGINATION_CURSOR_SECRET` | HMAC key used to sign pagination cursors | *(random per process)* |
 | `API_TOKENS` | JSON object mapping API tokens to their allowed scopes (see [Authentication](#authentication)) | *(demo tokens)* |
 
 
@@ -109,7 +113,7 @@ src/
   routes/       Express routers
   services/     business logic (rates, quotes, transfers, users, error tracking)
   store/        in-memory store and seed data
-  utils/        logger, ids, money, ApiError, asyncHandler
+  utils/        logger, ids, money, ApiError, asyncHandler, pagination/cursors
   validators/   request validators
   app.js        Express app assembly
   index.js      server bootstrap
@@ -127,6 +131,91 @@ consistent envelope:
 Every response carries an `X-Request-Id` header (echoed from the request when
 supplied) so logs and errors can be correlated.
 
+### Pagination
+
+`GET /api/transfers` and `GET /api/audit` page with **opaque cursors**. A cursor
+names a position in the collection rather than a row count, so pages stay
+correct while records are being written.
+
+| Parameter | Description |
+|-----------|-------------|
+| `limit` | Page size. Defaults to `PAGINATION_DEFAULT_LIMIT`; values above `PAGINATION_MAX_LIMIT` are **rejected with 400**, not clamped. |
+| `order` | `asc` or `desc`. Transfers default to `asc`, audit entries to `desc`. |
+| `cursor` | Opaque cursor from a previous response's `pageInfo`. |
+| `offset` | Legacy, deprecated. Cannot be combined with `cursor`. |
+
+Every response carries a `pageInfo` block:
+
+```json
+{
+  "count": 50,
+  "limit": 50,
+  "order": "asc",
+  "pageInfo": {
+    "hasMore": true,
+    "nextCursor": "eyJ2Ijox...",
+    "endCursor": "eyJ2Ijox...",
+    "scanned": 51,
+    "scanTruncated": false
+  },
+  "transfers": []
+}
+```
+
+- `nextCursor` — pass as `?cursor=` to fetch the next page; `null` on the last page.
+- `endCursor` — the position this page ended at, present even on the last page.
+  With `order=asc` a client can park here and poll for records appended later
+  without re-reading anything.
+- `scanned` / `scanTruncated` — how many records the query examined, and whether
+  it stopped at `PAGINATION_MAX_SCAN` rather than at the end of the collection.
+  A truncated page is still gap-free: follow `nextCursor` to continue.
+
+To page a collection, walk it:
+
+```bash
+curl -H "Authorization: Bearer $TOKEN" \
+  "http://localhost:3000/api/transfers?limit=50"
+# then, with pageInfo.nextCursor from the previous response:
+curl -H "Authorization: Bearer $TOKEN" \
+  "http://localhost:3000/api/transfers?limit=50&cursor=$CURSOR"
+```
+
+Cursors are signed and bound to the query that produced them. Reusing one
+against a different filter set, a different sort order, or a different API
+token is rejected rather than reinterpreted:
+
+| Condition | Status | `details.code` |
+|-----------|--------|----------------|
+| Cursor presented with a different API token | `403` | `CURSOR_ACTOR_MISMATCH` |
+| Filters changed mid-walk | `400` | `CURSOR_FILTER_MISMATCH` |
+| Sort order changed mid-walk | `400` | `CURSOR_ORDER_MISMATCH` |
+| Cursor edited, forged or truncated | `400` | `INVALID_CURSOR` |
+| Cursor predates a store restart | `400` | `STALE_CURSOR` |
+| `limit` above the maximum | `400` | `LIMIT_TOO_LARGE` |
+| `offset` beyond `PAGINATION_MAX_SCAN` | `400` | `OFFSET_TOO_DEEP` |
+| `cursor` and `offset` sent together | `400` | `CONFLICTING_PAGINATION` |
+
+Cursors are signed with `PAGINATION_CURSOR_SECRET`. Without it a random key is
+generated per process, which is fine for the in-memory store — cursors do not
+outlive a restart anyway — but **set it explicitly before running more than one
+instance behind a load balancer**, or cursors minted by one instance will be
+rejected by another.
+
+#### Deprecated: offset pagination
+
+`?offset=` still works and still returns the original `total`/`offset` fields,
+so existing clients keep working. It has two problems cursors fix:
+
+- **Instability.** The offset window is defined by a row count, so a record
+  inserted (or archived) while a client is paging shifts every later page: rows
+  get repeated or skipped entirely.
+- **Cost.** The server must walk every skipped record on each request, and
+  `total` costs a full pass over the filtered collection. Offsets beyond
+  `PAGINATION_MAX_SCAN` are refused for that reason.
+
+Offset responses include `pageInfo.nextCursor` as well, so a client can start on
+offsets and switch to cursors mid-walk.
+
 ### Caching and headers
 
 The API implements Cache-Control response headers for security and efficiency:
@@ -160,8 +249,9 @@ rounded away.
 - `POST /api/transfers` — create a transfer.
   Body: `{ senderName, recipientName, amount, from, to }`
 - `GET /api/transfers` — list transfers. Supports `?status=`, `?q=` (name
-  search), `?archived=` (true/false/all), and `?limit=`/`?offset=` pagination.
-  Archived transfers are excluded from results by default.
+  search), `?archived=` (true/false/all), and [cursor pagination](#pagination)
+  via `?cursor=`/`?limit=`/`?order=`. Archived transfers are excluded from
+  results by default; the default order is `asc` (oldest first).
 - `GET /api/transfers/stats` — aggregate counts and volume by currency.
 - `GET /api/transfers/:id` — fetch one transfer.
 - `POST /api/transfers/:id/claim` — recipient claims the transfer.
@@ -230,4 +320,8 @@ curl -H "Authorization: Bearer $TOKEN" "http://localhost:3000/api/users"
 
 # View audit log  (requires audit:read)
 curl -H "Authorization: Bearer $TOKEN" "http://localhost:3000/api/audit"
+
+# Page the audit log for one resource, newest first  (requires audit:read)
+curl -H "Authorization: Bearer $TOKEN" \
+  "http://localhost:3000/api/audit?resourceId=<id>&limit=20"
 ```

src/config/index.js

@@ -59,6 +59,16 @@ const config = {
     ratesMaxAge: parseInt(process.env.CACHE_RATES_MAX_AGE_SECONDS, 10) || 10,
   },
 
+  pagination: {
+    // Page size used when a request does not ask for one.
+    defaultLimit: parseInt(process.env.PAGINATION_DEFAULT_LIMIT, 10) || 50,
+    // Hard ceiling on a single page. Larger requests are rejected, not clamped.
+    maxLimit: parseInt(process.env.PAGINATION_MAX_LIMIT, 10) || 200,
+    // Ceiling on records a single history query may examine. Bounds the cost of
+    // a highly selective filter (or a deep offset) over a large history.
+    maxScan: parseInt(process.env.PAGINATION_MAX_SCAN, 10) || 10000,
+  },
+
   apiTokens: (() => {
     try {
       if (process.env.API_TOKENS) {

src/controllers/auditController.js

@@ -1,34 +1,37 @@
 'use strict';
 
 const auditService = require('../services/auditService');
-const { parsePagination } = require('../utils/pagination');
+const { buildHistoryPage } = require('../utils/historyPage');
 
 /**
  * Audit log controllers.
  */
 
 /**
  * GET /api/audit
- * Return all audit log entries, newest first, with limit/offset pagination.
- * Supports optional ?resourceId= query param to filter by resource.
+ * Return audit log entries, newest first by default.
+ *
+ * Supports ?resourceId= to filter by resource, ?order=asc|desc, ?limit=, and
+ * either ?cursor= (stable under concurrent writes) or the legacy ?offset=.
  */
 function listAuditEntries(req, res) {
-  const { resourceId } = req.query;
+  const resourceId = req.query.resourceId == null || req.query.resourceId === ''
+    ? null
+    : String(req.query.resourceId);
 
-  const all = resourceId
-    ? auditService.getEntriesForResource(resourceId)
-    : auditService.getEntries();
+  const filters = { resourceId };
 
-  const { limit, offset } = parsePagination(req.query);
-  const entries = all.slice(offset, offset + limit);
-
-  res.json({
-    total: all.length,
-    count: entries.length,
-    limit,
-    offset,
-    entries,
+  const { items, envelope } = buildHistoryPage({
+    req,
+    collection: 'audit',
+    filters,
+    defaultOrder: 'desc',
+    query: (args) => auditService.queryEntries({ resourceId, ...args }),
+    countTotal: () => auditService.countEntries(resourceId),
+    resolvePosition: (seq) => auditService.positionKeyAt(seq, resourceId),
   });
+
+  res.json({ ...envelope, entries: items });
 }
 
 module.exports = {

src/controllers/transferController.js

@@ -1,7 +1,7 @@
 'use strict';
 
 const transferService = require('../services/transferService');
-const { parsePagination } = require('../utils/pagination');
+const { buildHistoryPage } = require('../utils/historyPage');
 
 /**
  * Transfer controllers.
@@ -21,6 +21,10 @@ function createTransfer(req, res) {
  * List transfers, optionally filtered by ?status= and/or ?q= (name search).
  * Archived transfers are excluded by default; pass ?archived=true to see only archived,
  * or ?archived=all to include both archived and non-archived.
+ *
+ * Pagination: pass ?cursor= to page by the creation-order index (stable while
+ * transfers are being created), or the legacy ?offset=. ?order= selects asc
+ * (oldest first, the default) or desc. ?limit= is capped by config.pagination.maxLimit.
  */
 function listTransfers(req, res) {
   const archivedParam = req.query.archived;
@@ -33,14 +37,23 @@ function listTransfers(req, res) {
     archived = false;
   }
 
-  const all = transferService.listTransfers({
+  const filters = transferService.normaliseTransferFilters({
     status: req.query.status,
     search: req.query.q,
     archived,
   });
-  const { limit, offset } = parsePagination(req.query);
-  const transfers = all.slice(offset, offset + limit);
-  res.json({ total: all.length, count: transfers.length, limit, offset, transfers });
+
+  const { items, envelope } = buildHistoryPage({
+    req,
+    collection: 'transfers',
+    filters,
+    defaultOrder: 'asc',
+    query: (args) => transferService.queryTransfers({ ...filters, ...args }),
+    countTotal: () => transferService.listTransfers(filters).length,
+    resolvePosition: (seq) => transferService.positionKeyAt(seq),
+  });
+
+  res.json({ ...envelope, transfers: items });
 }
 
 /**