diff --git a/core/src/main/java/io/questdb/client/Sender.java b/core/src/main/java/io/questdb/client/Sender.java index 5106da96..4e04e4d5 100644 --- a/core/src/main/java/io/questdb/client/Sender.java +++ b/core/src/main/java/io/questdb/client/Sender.java @@ -38,6 +38,13 @@ import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender; import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine; import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorWebSocketSendLoop; +import io.questdb.client.cutlass.qwp.client.sf.cursor.OrphanScanner; +import io.questdb.client.cutlass.qwp.client.sf.cursor.PersistedSymbolDict; +import io.questdb.client.cutlass.qwp.client.sf.cursor.SlotLock; +import io.questdb.client.cutlass.qwp.client.sf.cursor.MmapSegmentCorruptionException; +import io.questdb.client.cutlass.qwp.client.sf.cursor.SfRecoveryException; +import io.questdb.client.cutlass.qwp.client.sf.cursor.SfSanitizedResidueException; +import io.questdb.client.cutlass.qwp.client.sf.cursor.UnreplayableSlotException; import io.questdb.client.impl.ConfStringParser; import io.questdb.client.impl.ConfigString; import io.questdb.client.impl.ConfigView; @@ -48,6 +55,7 @@ import io.questdb.client.std.Decimal256; import io.questdb.client.std.Decimal64; import io.questdb.client.std.Files; +import io.questdb.client.std.FilesFacade; import io.questdb.client.std.IntList; import io.questdb.client.std.Numbers; import io.questdb.client.std.NumericException; @@ -790,12 +798,16 @@ default Sender uuidColumn(CharSequence name, long lo, long hi) { * unconnected sender; the I/O thread runs the same retry loop in * the background. The user thread can call {@code at()} / * {@code flush()} immediately; rows accumulate in the cursor SF - * engine until the wire is up. Connect failures are retried - * indefinitely in the background; a terminal upgrade failure - * (auth reject, capability mismatch) is delivered to the async - * error inbox as a {@link io.questdb.client.SenderError} (no - * synchronous throw on the user call site). Wire - * {@code error_handler=...} to observe these. + * engine until the wire is up. Transport failures (unreachable or + * dropped server) retry indefinitely and are never surfaced -- the + * buffered rows are safe in SF and the server may still appear. A + * terminal auth, upgrade or capability rejection on the initial + * connect (before the wire has ever come up) has no caller left to + * throw at, so it is delivered to the async error inbox as a + * {@link io.questdb.client.SenderError}; wire {@code error_handler=...} + * to observe it. Once the wire has come up even once, SF owns the + * buffered data and the same rejections (e.g. a credential rotation) + * become transients retried indefinitely. * *

* Default resolution when the caller does not pick a value: @@ -995,6 +1007,12 @@ final class LineSenderBuilder { // build() time. 0 or negative is a documented "disable" value, so // a Long.MIN_VALUE sentinel keeps it distinguishable from "unset". private static final long DURABLE_ACK_KEEPALIVE_NOT_SET = Long.MIN_VALUE; + private static final org.slf4j.Logger LOG = org.slf4j.LoggerFactory.getLogger(LineSenderBuilder.class); + // How many quarantined copies of one slot may pile up under sf_dir before build() + // refuses to set aside another. Each is an unreplayable slot a human still has to + // look at; accumulating them without bound would turn a disk-space problem into a + // second incident. + private static final int MAX_QUARANTINE_SLOT_ATTEMPTS = 64; private static final int MIN_BUFFER_SIZE = AuthUtils.CHALLENGE_LEN + 1; // challenge size + 1; // sf-client.md section 4.4: the inbox capacity must accommodate the // distinct error categories in a bursty error stream so that drop-oldest @@ -1010,6 +1028,16 @@ final class LineSenderBuilder { private static final int PROTOCOL_TCP = 0; private static final int PROTOCOL_UDP = 3; private static final int PROTOCOL_WEBSOCKET = 2; + @TestOnly + private static volatile Runnable quarantineAfterCloseHook; + @TestOnly + private static volatile FilesFacade quarantineFilesFacade = FilesFacade.INSTANCE; + // Suffix for a slot set aside by quarantineTornSlot. Deliberately NOT the + // sender's own slot name, so a restarted sender does not re-adopt it as its own; + // quarantineTornSlot then marks it .failed, so the orphan drainer skips it too and + // the bytes stay put for a human to inspect and resend. + private static final String QUARANTINE_SLOT_SUFFIX = + OrphanScanner.QUARANTINE_SLOT_INFIX; private final ObjList hosts = new ObjList<>(); private final IntList ports = new IntList(); private long authTimeoutMillis = QwpWebSocketSender.DEFAULT_AUTH_TIMEOUT_MS; @@ -1058,6 +1086,7 @@ final class LineSenderBuilder { private int errorInboxCapacity = PARAMETER_NOT_SET_EXPLICITLY; private int maxFrameRejections = PARAMETER_NOT_SET_EXPLICITLY; private long poisonMinEscalationWindowMillis = PARAMETER_NOT_SET_EXPLICITLY; + private long catchUpCapGapMinEscalationWindowMillis = PARAMETER_NOT_SET_EXPLICITLY; private String httpPath; private String httpSettingsPath; private int httpTimeout = PARAMETER_NOT_SET_EXPLICITLY; @@ -1066,8 +1095,8 @@ final class LineSenderBuilder { // explicitly", which build() resolves to SYNC when any reconnect_* // knob was tuned by the user, otherwise OFF. SYNC retries on the // user thread up to the reconnect cap. ASYNC returns immediately - // and lets the I/O thread retry in the background, surfacing - // terminal failures via the error inbox. + // and lets the I/O thread retry every endpoint/transport failure in + // the background without surfacing it to the producer. private InitialConnectMode initialConnectMode = null; private String keyId; private int maxBackgroundDrainers = DEFAULT_MAX_BACKGROUND_DRAINERS; @@ -1512,6 +1541,10 @@ public Sender build() { long actualPoisonMinEscalationWindowMillis = poisonMinEscalationWindowMillis != PARAMETER_NOT_SET_EXPLICITLY ? poisonMinEscalationWindowMillis : CursorWebSocketSendLoop.DEFAULT_POISON_MIN_ESCALATION_WINDOW_MILLIS; + long actualCatchUpCapGapMinEscalationWindowMillis = + catchUpCapGapMinEscalationWindowMillis != PARAMETER_NOT_SET_EXPLICITLY + ? catchUpCapGapMinEscalationWindowMillis + : CursorWebSocketSendLoop.DEFAULT_CATCHUP_CAP_GAP_MIN_ESCALATION_WINDOW_MILLIS; // sfDir is the parent (group root); the actual slot lives // under sfDir/senderId. This is what the engine sees — the @@ -1556,56 +1589,167 @@ public Sender build() { ? (sfSyncIntervalMillis == PARAMETER_NOT_SET_EXPLICITLY ? DEFAULT_SF_SYNC_INTERVAL_MILLIS : sfSyncIntervalMillis) * 1_000_000L : 0L; - CursorSendEngine cursorEngine = new CursorSendEngine( - slotPath, actualSfMaxSegmentBytes, - actualSfMaxTotalBytes, actualSfAppendDeadlineNanos, - actualSfSyncIntervalNanos); - int actualErrorInboxCapacity = errorInboxCapacity != PARAMETER_NOT_SET_EXPLICITLY - ? errorInboxCapacity - : io.questdb.client.cutlass.qwp.client.sf.cursor.SenderErrorDispatcher.DEFAULT_CAPACITY; - int actualConnectionListenerInboxCapacity = connectionListenerInboxCapacity != PARAMETER_NOT_SET_EXPLICITLY - ? connectionListenerInboxCapacity - : io.questdb.client.cutlass.qwp.client.sf.cursor.SenderConnectionDispatcher.DEFAULT_CAPACITY; - List wsEndpoints = - new ArrayList<>(hosts.size()); - for (int i = 0, n = hosts.size(); i < n; i++) { - wsEndpoints.add(new QwpWebSocketSender.Endpoint(hosts.getQuick(i), ports.getQuick(i))); - } - QwpWebSocketSender connected; - try { - connected = QwpWebSocketSender.connect( - wsEndpoints, - wsTlsConfig, - actualAutoFlushRows, - actualAutoFlushBytes, - actualAutoFlushIntervalNanos, - wsAuthHeader, - requestDurableAck, - cursorEngine, - actualCloseFlushTimeoutMillis, - actualReconnectMaxDurationMillis, - actualReconnectInitialBackoffMillis, - actualReconnectMaxBackoffMillis, - actualInitialConnectMode, - errorHandler, - actualErrorInboxCapacity, - actualDurableAckKeepaliveIntervalMillis, - authTimeoutMillis, - connectTimeoutMillis == PARAMETER_NOT_SET_EXPLICITLY ? 0 : connectTimeoutMillis, - connectionListener, - actualConnectionListenerInboxCapacity, - actualMaxFrameRejections, - actualPoisonMinEscalationWindowMillis - ); - } catch (Throwable t) { - // connect() failed before ownership of cursorEngine - // transferred — close it ourselves. + QwpWebSocketSender connected = null; + // The parent-anchored logical lock is stable across a slot rename. Keep it + // from before the directory-local lock is acquired until connect() has either + // adopted that engine or quarantine has closed, renamed and recreated it. + // This closes the inode-swap window in which an already-queued orphan drainer + // could otherwise acquire the renamed directory's old .lock and later operate + // on the fresh slot through the original pathname. + try (SlotLock logicalSlotLock = slotPath == null + ? null + : SlotLock.acquireLogical(slotPath)) { + // The constructor's own recovery seed can throw UnreplayableSlotException + // too (SegmentRing.openExisting refuses a slot when it had to skip an + // unreadable segment -- see C5): its frame range cannot be shown + // already-acked, so recovery sets the slot aside rather than risk seeding + // the ack cursor past frames that were never delivered. That verdict gets + // the exact same quarantine-and-continue treatment as the connect()-time + // verdict below -- constructing cursorEngine is not inside the loop below, + // so a throw here would otherwise escape build() entirely, uncaught. + // quarantineTornSlot(null, ...) renames the WHOLE slot directory aside + // (not just the unreadable segment file) before building the replacement + // at the original slotPath, so the replacement starts on a genuinely empty + // directory with nothing left to skip -- it cannot throw the same way + // twice, which is what makes looping unnecessary here. + boolean quarantined = false; + CursorSendEngine cursorEngine; try { - cursorEngine.close(); - } catch (Throwable ignored) { - // best-effort + try { + cursorEngine = new CursorSendEngine( + slotPath, actualSfMaxSegmentBytes, + actualSfMaxTotalBytes, actualSfAppendDeadlineNanos, + actualSfSyncIntervalNanos); + } catch (SfSanitizedResidueException first) { + // NOT terminal, and it must be intercepted ahead of its + // SfRecoveryException parent below. Recovery durably zeroed + // proven-dead sealed residue BEFORE failing closed, so the + // chain on disk is already healed: quarantining here would + // set aside a slot whose backlog replays perfectly. Retry + // once over the healed chain; a repeat is genuine and takes + // the terminal arm. + LOG.info("sf slot {}: sealed residue sanitized during recovery ({}); " + + "retrying over the healed chain", + slotPath, first.getMessage()); + cursorEngine = new CursorSendEngine( + slotPath, actualSfMaxSegmentBytes, + actualSfMaxTotalBytes, actualSfAppendDeadlineNanos, + actualSfSyncIntervalNanos); + } + } catch (UnreplayableSlotException | SfRecoveryException + | MmapSegmentCorruptionException e) { + // The terminal recovery verdicts, and the only ones build() + // sets a slot aside for. UnreplayableSlotException says the + // symbol dictionary cannot be rebuilt from any source; + // SfRecoveryException and MmapSegmentCorruptionException say + // the durable chain itself is proven corrupt or incomplete. + // None of the three clears on a retry, and senderId is stable + // with a not-fully-drained slot retained on close -- so + // without this arm every restart re-recovers the same slot and + // throws again, and the application cannot construct a Sender + // at all, not even to BUFFER new rows. + // + // Deliberately NOT catching plain MmapSegmentException or + // SfOperationalException: those are operational (EMFILE, + // ENOMEM, an unreadable-but-possibly-intact file). Aborting + // startup on them is correct; quarantining on them would + // convert a transient into the permanent loss of a healthy + // slot's durable frames. + if (slotPath == null) { + throw e; + } + quarantined = true; + cursorEngine = quarantineTornSlot( + null, e, sfDir, senderId, slotPath, actualSfMaxSegmentBytes, + actualSfMaxTotalBytes, actualSfAppendDeadlineNanos, + actualSfSyncIntervalNanos, errorHandler); + } + int actualErrorInboxCapacity = errorInboxCapacity != PARAMETER_NOT_SET_EXPLICITLY + ? errorInboxCapacity + : io.questdb.client.cutlass.qwp.client.sf.cursor.SenderErrorDispatcher.DEFAULT_CAPACITY; + int actualConnectionListenerInboxCapacity = connectionListenerInboxCapacity != PARAMETER_NOT_SET_EXPLICITLY + ? connectionListenerInboxCapacity + : io.questdb.client.cutlass.qwp.client.sf.cursor.SenderConnectionDispatcher.DEFAULT_CAPACITY; + List wsEndpoints = + new ArrayList<>(hosts.size()); + for (int i = 0, n = hosts.size(); i < n; i++) { + wsEndpoints.add(new QwpWebSocketSender.Endpoint(hosts.getQuick(i), ports.getQuick(i))); + } + // The recovery seed inside connect() is the authority on whether a recovered + // slot can be replayed: it rebuilds the dictionary from its intact prefix and + // then from the surviving frames' own delta sections, and throws + // UnreplayableSlotException only once neither source holds the missing ids. + // Quarantining on anything weaker would set aside slots that recovery can + // still rescue, so build() waits for that verdict rather than pre-judging it. + while (connected == null) { + try { + connected = QwpWebSocketSender.connect( + wsEndpoints, + wsTlsConfig, + actualAutoFlushRows, + actualAutoFlushBytes, + actualAutoFlushIntervalNanos, + wsAuthHeader, + requestDurableAck, + cursorEngine, + actualCloseFlushTimeoutMillis, + actualReconnectMaxDurationMillis, + actualReconnectInitialBackoffMillis, + actualReconnectMaxBackoffMillis, + actualInitialConnectMode, + errorHandler, + actualErrorInboxCapacity, + actualDurableAckKeepaliveIntervalMillis, + authTimeoutMillis, + connectTimeoutMillis == PARAMETER_NOT_SET_EXPLICITLY ? 0 : connectTimeoutMillis, + connectionListener, + actualConnectionListenerInboxCapacity, + actualMaxFrameRejections, + actualPoisonMinEscalationWindowMillis, + actualCatchUpCapGapMinEscalationWindowMillis + ); + } catch (UnreplayableSlotException e) { + // The one failure build() recovers from. The slot's frames reference ids + // that nothing still holds, so they can never go on the wire -- but that is + // no reason to take the producer down with them. Before this, the throw + // escaped build() and, because senderId is stable and a not-fully-drained + // slot is retained on close, every retry re-recovered the same slot and + // threw again: the application could not construct a Sender at all, so it + // could not even BUFFER new rows. An already-lost batch became an unbounded + // outage of everything after it. + // + // Set the slot aside instead, keep its bytes for forensics and resend, and + // start the producer on a clean one. Once only: a second such failure would + // mean the FRESH slot is unreplayable, which cannot happen, so let it out + // rather than loop. + if (quarantined || slotPath == null) { + try { + // close(false): we still hold the logical slot lock. + cursorEngine.close(false); + } catch (Throwable ignored) { + // best-effort + } + throw e; + } + quarantined = true; + cursorEngine = quarantineTornSlot( + cursorEngine, e, sfDir, senderId, slotPath, actualSfMaxSegmentBytes, + actualSfMaxTotalBytes, actualSfAppendDeadlineNanos, + actualSfSyncIntervalNanos, errorHandler); + } catch (Throwable t) { + // connect() failed before ownership of cursorEngine + // transferred — close it ourselves. close(false) + // because logicalSlotLock is still held here: a fresh + // slot is fully drained, so the default close would + // unlink the very lock file this scope holds. + try { + cursorEngine.close(false); + } catch (Throwable ignored) { + // best-effort + } + throw t; + } } - throw t; } // connect() succeeded — `connected` now owns cursorEngine // via setCursorEngine(engine, true). From here on, ANY @@ -1721,6 +1865,35 @@ public Sender build() { return sender; } + /** + * Minimum wall-clock time (millis) a symbol-dictionary catch-up CAP GAP must + * persist before an orphan store-and-forward drainer quarantines its slot. + *

+ * A cap gap means a symbol already accepted by one node is too large to + * re-register on the node the sender just failed over to, because that node + * advertises a smaller maximum batch size. On a homogeneous cluster this cannot + * happen; it takes a heterogeneous or mid-roll cluster, or an operator lowering + * the cap below existing data. + *

+ * A foreground sender retries such a gap indefinitely because the larger-cap node + * may simply be away. An orphan drainer may quarantine only once the gap has BOTH + * recurred many times AND persisted for this long. Raise it for a cluster whose + * rolling restarts take longer than the 5-minute default; set it to {@code 0} to + * quarantine an orphan slot as soon as the retry count is exhausted. + *

+ * WebSocket transport only. + */ + public LineSenderBuilder catchUpCapGapMinEscalationWindowMillis(long millis) { + if (protocol != PARAMETER_NOT_SET_EXPLICITLY && protocol != PROTOCOL_WEBSOCKET) { + throw new LineSenderException("catch_up_cap_gap_min_escalation_window_millis is only supported for WebSocket transport"); + } + if (millis < 0) { + throw new LineSenderException("catch_up_cap_gap_min_escalation_window_millis must be >= 0: ").put(millis); + } + this.catchUpCapGapMinEscalationWindowMillis = millis; + return this; + } + /** * close() drain timeout in milliseconds. The sender's {@code close()} * method blocks up to this many millis waiting for the server to ACK @@ -2167,11 +2340,10 @@ public LineSenderBuilder initialConnectMode(InitialConnectMode mode) { } /** - * Opt in to retrying the initial connect with the same backoff / - * cap / auth-terminal policy as in-flight reconnect. Set true if - * your deployment expects the server to come up shortly after the - * sender. Auth failures (HTTP 401/403/non-101) stay terminal in - * either mode. + * Opt in to retrying the initial connect with backoff on the calling + * thread, bounded by the configured reconnect cap. Set true if your + * deployment expects the server to come up shortly after the sender. + * Auth failures (HTTP 401/403/non-101) stay terminal in this SYNC mode. *

* When this method is not called, the resolution rule documented * on {@link InitialConnectMode} applies: SYNC implicitly when any @@ -2476,7 +2648,7 @@ public LineSenderBuilder maxFrameRejections(int rejections) { * alone can turn a transient outage into a producer-fatal terminal. * This window guarantees a brief outage a chance to clear (an OK * at/beyond the suspect resets the detector) first. {@code 0} disables - * the dwell (legacy immediate escalation at the strike threshold). + * the dwell (immediate escalation at the strike threshold). * Default {@code 5_000} (5 s). WebSocket only. */ public LineSenderBuilder poisonMinEscalationWindowMillis(long millis) { @@ -2512,8 +2684,8 @@ public LineSenderBuilder reconnectMaxBackoffMillis(long millis) { * with exponential backoff until connect succeeds or this many * millis elapse, then throws. The background reconnect loop * (mid-stream outages and async initial connect) does NOT consult - * this value: it retries indefinitely and halts only on a terminal - * auth/upgrade error or {@code close()}. + * this value: endpoint and transport failures are retried indefinitely + * until {@code close()}. *

* Default {@code 300_000} (5 minutes). Lower for fail-fast startup; * higher for tolerating a slow server boot. Must be positive: a zero @@ -2954,6 +3126,140 @@ private static long parseSizeValue(@NotNull StringSink value, @NotNull String na } } + /** + * Sets a slot aside that either connect() (a symbol dictionary that cannot cover its + * surviving frames) or the {@code CursorSendEngine} constructor itself (recovery had + * to skip an unreadable segment -- see C5) declared unreplayable, and returns a fresh + * engine on an empty slot so the producer can keep producing. + *

+ * {@code torn} is the live engine to release, or {@code null} when the verdict came + * from the constructor and no engine was ever built -- there is nothing to + * {@link CursorSendEngine#close(boolean)} in that case, only the directory to rename. + *

+ * Such a slot is unreplayable BY THIS PRODUCER: either its frames reference symbol ids + * the recovered dictionary lost (a host/power crash tore the unsynced side-file), so a + * producer seeded from the short dictionary would hand those ids to different symbols + * and silently misattribute values -- or recovery could not show a skipped segment's + * frames were already acked, so replaying would risk seeding the ack cursor past data + * that was never delivered. Detecting either is correct and load-bearing -- but simply + * THROWING is not a safe response. {@code senderId} defaults to a stable name, so a + * restarted process re-adopts the same slot; the engine's close retains a slot that is + * not fully drained; and so every subsequent {@code build()} would re-recover the same + * slot and throw again -- forever, until an operator deleted the directory by hand. The + * application could not construct a Sender at all, and so could not even BUFFER new + * rows. That trades a bounded, already-lost batch for an unbounded outage of everything + * after it, which inverts the one guarantee store-and-forward exists to give. + *

+ * So: rename the slot aside instead, and mark it {@code .failed}. The verdict is + * authoritative -- the recovery seed already tried every source of truth (the + * persisted prefix AND the surviving frames' own deltas, or the per-segment scan for + * the C5 case), and the orphan drainer's own replay guard uses that same walk, so there + * is nothing a drainer could rebuild that the seed did not. {@code markFailed} (below) + * therefore quarantines the copy for a human rather than leaving a drainer to retry an + * unreplayable slot forever; a full-dictionary-fallback slot never reaches here, because + * its dictionary is discarded at recovery and it never throws. The bytes are preserved + * on disk for forensics and a manual resend, and the new name -- NOT the sender's own + * slot name -- keeps a restarted sender from re-adopting it (and, for the C5 case, also + * keeps a later recovery from ever re-scanning the individually-renamed {@code .corrupt} + * segment inside it). The producer, meanwhile, starts on a clean empty slot and never + * notices. + *

+ * If the rename fails (a Windows share lock, a read-only mount) there is no way to + * free the slot name without destroying data, so fall back to the old behaviour and + * throw -- loudly, and never silently dropping bytes. + */ + private static CursorSendEngine quarantineTornSlot( + CursorSendEngine torn, RuntimeException cause, String sfDir, + String senderId, String slotPath, + long sfMaxSegmentBytes, long sfMaxTotalBytes, long sfAppendDeadlineNanos, + long sfSyncIntervalNanos, + io.questdb.client.SenderErrorHandler errorHandler + ) { + // The verdict, and the reason, come from the recovery seed -- the only code that + // has tried every source of truth. Recomputing them here would mean a second, + // independently-drifting notion of "unreplayable". + final String detail = cause.getMessage(); + // Release the slot lock and the dictionary fd before renaming. connect()'s failure + // path already closed the engine; close() is idempotent, so make it explicit rather + // than depend on that. close(false): build() holds the logical slot lock across this + // whole transition -- that is precisely what serialises the rename against a queued + // orphan drainer -- so the engine must not unlink it. torn is null when the verdict + // came from the constructor itself: no engine was ever built, so there is nothing to + // close. + if (torn != null) { + torn.close(false); + } + Runnable hook = quarantineAfterCloseHook; + if (hook != null) { + hook.run(); + } + + FilesFacade ff = quarantineFilesFacade; + String quarantinePath = null; + for (int i = 0; i < MAX_QUARANTINE_SLOT_ATTEMPTS; i++) { + String candidate = sfDir + "/" + senderId + QUARANTINE_SLOT_SUFFIX + i; + if (!ff.exists(candidate)) { + quarantinePath = candidate; + break; + } + } + if (quarantinePath == null || ff.rename(slotPath, quarantinePath) != 0) { + throw new LineSenderException( + detail + "; the affected data must be resent. The slot could not be set aside " + + "automatically (" + (quarantinePath == null + ? "too many quarantined slots already under " + sfDir + : "rename to " + quarantinePath + " failed") + + "), so this sender cannot start until " + + slotPath + " is moved or removed by hand"); + } + // Mark the quarantined copy so the orphan drainer treats it as a + // human-in-the-loop slot rather than silently retrying it forever. + OrphanScanner.markFailed(quarantinePath, detail); + LOG.error("{} -- the slot has been set aside at {} and the affected data must be resent; " + + "this sender continues on a fresh, empty slot at {}", + detail, quarantinePath, slotPath); + // build() no longer throws for this -- it starts the producer on a fresh slot so + // the outage stays bounded. But abandoning buffered rows is precisely the event a + // caller must be able to act on, and LOG.error alone cannot carry it: this client + // ships slf4j-api with no binding, so an embedding app with no provider gets a NOP + // logger and the loss is announced nowhere. Deliver it programmatically too, so an + // errorHandler can alert / page / record it. Dispatched synchronously here because + // the async SenderErrorDispatcher belongs to the connected sender, which does not + // exist yet at build time. A throwing handler must not turn a contained outage back + // into a failed build, so swallow anything it raises. + if (errorHandler != null) { + try { + errorHandler.onError(new SenderError( + SenderError.Category.PROTOCOL_VIOLATION, + SenderError.Policy.TERMINAL, + SenderError.NO_STATUS_BYTE, + detail + " [slot set aside at " + quarantinePath + + "; sender continues on a fresh slot at " + slotPath + ']', + SenderError.NO_MESSAGE_SEQUENCE, + SenderError.NO_MESSAGE_SEQUENCE, + SenderError.NO_MESSAGE_SEQUENCE, + null, + System.nanoTime() + )); + } catch (Throwable handlerFailure) { + LOG.error("sender error handler threw while reporting a quarantined slot: {}", + String.valueOf(handlerFailure)); + } + } + return new CursorSendEngine(slotPath, sfMaxSegmentBytes, sfMaxTotalBytes, + sfAppendDeadlineNanos, sfSyncIntervalNanos); + } + + @TestOnly + public static void setQuarantineAfterCloseHookForTest(Runnable hook) { + quarantineAfterCloseHook = hook; + } + + @TestOnly + public static void setQuarantineFilesFacadeForTest(FilesFacade ff) { + quarantineFilesFacade = ff == null ? FilesFacade.INSTANCE : ff; + } + private static int resolveIPv4(String host) { try { byte[] addr = InetAddress.getByName(host).getAddress(); @@ -3457,6 +3763,12 @@ private LineSenderBuilder fromConfig(CharSequence configurationString) { } pos = getValue(configurationString, pos, sink, "poison_min_escalation_window_millis"); poisonMinEscalationWindowMillis(parseLongValue(sink, "poison_min_escalation_window_millis")); + } else if (Chars.equals("catch_up_cap_gap_min_escalation_window_millis", sink)) { + if (protocol != PROTOCOL_WEBSOCKET) { + throw new LineSenderException("catch_up_cap_gap_min_escalation_window_millis is only supported for WebSocket transport"); + } + pos = getValue(configurationString, pos, sink, "catch_up_cap_gap_min_escalation_window_millis"); + catchUpCapGapMinEscalationWindowMillis(parseLongValue(sink, "catch_up_cap_gap_min_escalation_window_millis")); } else if (Chars.equals("initial_connect_retry", sink)) { if (protocol != PROTOCOL_WEBSOCKET) { throw new LineSenderException("initial_connect_retry is only supported for WebSocket transport"); @@ -3729,6 +4041,9 @@ private LineSenderBuilder fromConfigWebSocket(CharSequence configurationString) if (view.has("poison_min_escalation_window_millis")) { poisonMinEscalationWindowMillis(wsLong(view, v, "poison_min_escalation_window_millis")); } + if (view.has("catch_up_cap_gap_min_escalation_window_millis")) { + catchUpCapGapMinEscalationWindowMillis(wsLong(view, v, "catch_up_cap_gap_min_escalation_window_millis")); + } if (view.has("sf_append_deadline_millis")) { sfAppendDeadlineMillis(wsLong(view, v, "sf_append_deadline_millis")); } @@ -3912,6 +4227,7 @@ public java.util.Map wsConfigSnapshotForTest() { m.put("max_background_drainers", maxBackgroundDrainers); m.put("max_frame_rejections", maxFrameRejections); m.put("poison_min_escalation_window_millis", poisonMinEscalationWindowMillis); + m.put("catch_up_cap_gap_min_escalation_window_millis", catchUpCapGapMinEscalationWindowMillis); m.put("error_inbox_capacity", errorInboxCapacity); m.put("connection_listener_inbox_capacity", connectionListenerInboxCapacity); m.put("token", httpToken); diff --git a/core/src/main/java/io/questdb/client/SenderError.java b/core/src/main/java/io/questdb/client/SenderError.java index d8e78503..5c713685 100644 --- a/core/src/main/java/io/questdb/client/SenderError.java +++ b/core/src/main/java/io/questdb/client/SenderError.java @@ -202,6 +202,13 @@ public enum Category { * mapped for forward compatibility with servers that NACK it explicitly. */ NOT_WRITABLE, + /** + * A delta symbol dictionary began above the server's per-connection dictionary. + * Wire {@code 0x0D}. Unlike {@link #PARSE_ERROR} this is a function of server + * state, not of the frame's bytes, so the same frame succeeds after the + * connection's dictionary catch-up has run. + */ + DICTIONARY_GAP, /** * A frame the server (or an intermediary) deterministically rejects: the * poison-frame detector observed the same head-of-line frame fail diff --git a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java index 94018fff..1290fcab 100644 --- a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java +++ b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java @@ -469,6 +469,35 @@ public void sendBinary(long dataPtr, int length, int timeout) { sendBuffer.reset(); } + /** + * Sends two native-memory slices as one WebSocket binary frame. The slices + * are copied directly into the WebSocket send buffer and masked there, so a + * caller assembling a small protocol prefix around a large immutable body + * does not need a second contiguous staging buffer. + * + * @param firstPtr pointer to the first payload slice + * @param firstLength first payload-slice length + * @param secondPtr pointer to the second payload slice + * @param secondLength second payload-slice length + * @param timeout timeout in milliseconds + */ + public void sendBinary( + long firstPtr, + int firstLength, + long secondPtr, + int secondLength, + int timeout + ) { + checkConnected(); + sendBuffer.reset(); + sendBuffer.beginFrame(); + sendBuffer.putBlockOfBytes(firstPtr, firstLength); + sendBuffer.putBlockOfBytes(secondPtr, secondLength); + WebSocketSendBuffer.FrameInfo frame = sendBuffer.endBinaryFrame(); + doSend(sendBuffer.getBufferPtr() + frame.offset, frame.length, timeout); + sendBuffer.reset(); + } + /** * Sends binary data with default timeout. */ @@ -476,6 +505,13 @@ public void sendBinary(long dataPtr, int length) { sendBinary(dataPtr, length, defaultTimeout); } + /** + * Sends two native-memory slices as one binary frame with the default timeout. + */ + public void sendBinary(long firstPtr, int firstLength, long secondPtr, int secondLength) { + sendBinary(firstPtr, firstLength, secondPtr, secondLength, defaultTimeout); + } + /** * Sends a close frame. */ diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/BatchTooLargeForCapException.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/BatchTooLargeForCapException.java new file mode 100644 index 00000000..adc62007 --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/BatchTooLargeForCapException.java @@ -0,0 +1,41 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.qwp.client; + +import io.questdb.client.cutlass.line.LineSenderException; + +/** + * A batch that cannot fit the server's advertised cap however it is split. Distinct from + * a plain {@link LineSenderException} because the batch is RETAINED for a later flush + * against a larger-cap node, so {@code close()} has to recognise it and discard the batch + * rather than abandon every row an earlier flush already published. Matching on the + * message text would be the alternative, and would silently swallow unrelated failures. + */ +public class BatchTooLargeForCapException extends LineSenderException { + + public BatchTooLargeForCapException(CharSequence message) { + super(message); + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/GlobalSymbolDictionary.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/GlobalSymbolDictionary.java index 3b4c9a90..d1b0f13e 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/GlobalSymbolDictionary.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/GlobalSymbolDictionary.java @@ -50,6 +50,37 @@ public GlobalSymbolDictionary(int initialCapacity) { this.idToSymbol = new ObjList<>(initialCapacity); } + /** + * Appends {@code symbol} at the next sequential id, matching a recovered / + * persisted dictionary's dense id order, WITHOUT de-duplicating. + *

+ * Recovery ({@code QwpWebSocketSender.seedGlobalDictionaryFromPersisted}) + * replays the persisted entries in id order to rebuild this dictionary. It must + * NOT collapse two source strings that decode to the same characters, because + * the persisted {@code .symbol-dict}, the on-wire delta and the I/O-thread + * catch-up mirror all key on the entry POSITION (id), not on the string. The + * only strings that collide this way are malformed lone UTF-16 surrogates, + * which the UTF-8 encoder maps to {@code '?'}: {@link #getOrAddSymbol} would + * de-dup them and leave this dictionary SHORTER than the persisted entry count, + * desyncing the producer's delta baseline from the catch-up mirror (which uses + * {@code pd.size()}) and silently misattributing later symbols. Appending + * unconditionally keeps {@link #size()} equal to that count. The reverse lookup + * keeps the highest id for a colliding string, which is harmless: both ids + * encode to the same bytes, so resolving either is equivalent. + * + * @param symbol the recovered symbol string (must not be null) + * @return the id assigned (the previous {@link #size()}) + */ + public int addRecoveredSymbol(String symbol) { + if (symbol == null) { + throw new IllegalArgumentException("symbol cannot be null"); + } + int newId = idToSymbol.size(); + symbolToId.put(symbol, newId); + idToSymbol.add(symbol); + return newId; + } + /** * Clears all symbols from the dictionary. *

diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/NativeBufferWriter.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/NativeBufferWriter.java index aad95f3c..86343772 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/NativeBufferWriter.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/NativeBufferWriter.java @@ -76,6 +76,27 @@ public static int varintSize(long value) { return (64 - Long.numberOfLeadingZeros(value) + 6) / 7; } + /** + * Writes {@code value} as an unsigned LEB128 varint directly at native address + * {@code addr} and returns the address just past the last byte. The canonical + * raw-address varint writer shared by the SF cursor's persisted dictionary and + * catch-up frame builder. + *

+ * {@code value} must be non-negative: the signed {@code value > 0x7F} loop emits + * a SINGLE truncated byte for a negative long, whereas {@link #varintSize} + * returns 10 for it -- a size/write mismatch that would corrupt the stream. All + * callers pass ids/lengths/counts (non-negative); the assert pins that contract. + */ + public static long writeVarint(long addr, long value) { + assert value >= 0 : "unsigned LEB128 varint requires a non-negative value: " + value; + while (value > 0x7F) { + Unsafe.getUnsafe().putByte(addr++, (byte) ((value & 0x7F) | 0x80)); + value >>>= 7; + } + Unsafe.getUnsafe().putByte(addr++, (byte) value); + return addr; + } + @Override public void close() { if (bufferPtr != 0) { @@ -305,6 +326,7 @@ public void putUtf8(CharSequence value) { */ @Override public void putVarint(long value) { + assert value >= 0 : "unsigned LEB128 varint requires a non-negative value: " + value; ensureCapacity(10); // max varint bytes long addr = bufferPtr + position; while (value > 0x7F) { @@ -336,11 +358,7 @@ public void skip(int bytes) { } private static void writeVarintDirect(long addr, long value) { - while (value > 0x7F) { - Unsafe.getUnsafe().putByte(addr++, (byte) ((value & 0x7F) | 0x80)); - value >>>= 7; - } - Unsafe.getUnsafe().putByte(addr, (byte) value); + writeVarint(addr, value); } private void encodeUtf8(CharSequence value, int utf8Len) { diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java index ced1a1b5..f38d9e93 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java @@ -26,6 +26,8 @@ import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; import io.questdb.client.std.QuietCloseable; +import io.questdb.client.std.Unsafe; +import io.questdb.client.std.Vect; import static io.questdb.client.cutlass.qwp.protocol.QwpConstants.*; @@ -39,6 +41,16 @@ public class QwpWebSocketEncoder implements QuietCloseable { private final QwpColumnWriter columnWriter = new QwpColumnWriter(); private NativeBufferWriter buffer; + // Byte offsets, within the buffer, of the symbol-dict delta ENTRY region + // ([len][utf8]... only, without the two section varints) that beginMessage + // last wrote. Let the producer persist those bytes straight to the slot's + // .symbol-dict instead of re-encoding the same symbols (see + // QwpWebSocketSender.persistNewSymbolsBeforePublish). Valid until the next + // beginMessage; stored as offsets so they survive a buffer realloc. + private int deltaCount; + private int deltaEntriesEnd; + private int deltaEntriesStart; + private int deltaStart; // QWP ingress always advertises Gorilla timestamp encoding. The column // writer still emits a per-column encoding byte and falls back to raw // values when delta-of-delta overflows int32. @@ -65,8 +77,8 @@ public void beginMessage( int batchMaxId ) { buffer.reset(); - int deltaStart = confirmedMaxId + 1; - int deltaCount = Math.max(0, batchMaxId - confirmedMaxId); + deltaStart = confirmedMaxId + 1; + deltaCount = Math.max(0, batchMaxId - confirmedMaxId); byte headerFlags = (byte) (flags | FLAG_DELTA_SYMBOL_DICT); byte origFlags = flags; flags = headerFlags; @@ -75,10 +87,12 @@ public void beginMessage( payloadStart = buffer.getPosition(); buffer.putVarint(deltaStart); buffer.putVarint(deltaCount); + deltaEntriesStart = buffer.getPosition(); for (int id = deltaStart; id < deltaStart + deltaCount; id++) { String symbol = globalDict.getSymbol(id); buffer.putString(symbol); } + deltaEntriesEnd = buffer.getPosition(); columnWriter.setBuffer(buffer); } @@ -90,6 +104,64 @@ public void close() { } } + /** + * Copies one single-table split message from the combined message currently + * staged in this encoder. The table body is copied byte-for-byte from its + * recorded offset; columns and rows are not encoded again. + */ + public int copySplitMessage( + MicrobatchBuffer target, + int tableBodyOffset, + int tableBodyLength, + boolean deferCommit, + int confirmedMaxId, + int batchMaxId + ) { + if (target.getBufferPos() != 0) { + throw new IllegalStateException("split message target is not empty"); + } + if (tableBodyOffset < deltaEntriesEnd + || tableBodyLength < 0 + || (long) tableBodyOffset + tableBodyLength > buffer.getPosition()) { + throw new IllegalArgumentException("table body slice is outside the staged message"); + } + + int splitDeltaStart = confirmedMaxId + 1; + int splitDeltaCount = Math.max(0, batchMaxId - confirmedMaxId); + int deltaEntriesLength = splitDeltaEntriesLength(splitDeltaStart, splitDeltaCount); + int messageSize = splitMessageSize( + tableBodyLength, splitDeltaStart, splitDeltaCount, deltaEntriesLength); + target.ensureCapacity(messageSize); + + long source = buffer.getBufferPtr(); + long destination = target.getBufferPtr(); + Vect.memcpy(destination, source, HEADER_SIZE); + + byte splitFlags = Unsafe.getUnsafe().getByte(source + HEADER_OFFSET_FLAGS); + if (deferCommit) { + splitFlags |= FLAG_DEFER_COMMIT; + } else { + splitFlags &= ~FLAG_DEFER_COMMIT; + } + Unsafe.getUnsafe().putByte(destination + HEADER_OFFSET_FLAGS, splitFlags); + Unsafe.getUnsafe().putShort(destination + 6, (short) 1); + Unsafe.getUnsafe().putInt(destination + 8, messageSize - HEADER_SIZE); + + long writeAddress = destination + HEADER_SIZE; + writeAddress = NativeBufferWriter.writeVarint(writeAddress, splitDeltaStart); + writeAddress = NativeBufferWriter.writeVarint(writeAddress, splitDeltaCount); + if (deltaEntriesLength > 0) { + Vect.memcpy(writeAddress, source + deltaEntriesStart, deltaEntriesLength); + writeAddress += deltaEntriesLength; + } + Vect.memcpy(writeAddress, source + tableBodyOffset, tableBodyLength); + writeAddress += tableBodyLength; + assert writeAddress == destination + messageSize; + + target.setBufferPos(messageSize); + return messageSize; + } + public int encode(QwpTableBuffer tableBuffer) { buffer.reset(); writeHeader(1, 0); @@ -122,6 +194,33 @@ public QwpBufferWriter getBuffer() { return buffer; } + /** + * Byte length of the symbol-dict delta ENTRY region ({@code [len][utf8]...}, + * excluding the two section varints) that {@link #beginMessage} last wrote. + */ + public int getDeltaEntriesLen() { + return deltaEntriesEnd - deltaEntriesStart; + } + + /** + * Byte offset, within {@link #getBuffer()}, of the symbol-dict delta ENTRY + * region {@link #beginMessage} last wrote. + */ + public int getDeltaEntriesStart() { + return deltaEntriesStart; + } + + public int getSplitMessageSize(int tableBodyLength, int confirmedMaxId, int batchMaxId) { + if (tableBodyLength < 0) { + throw new IllegalArgumentException("tableBodyLength must be non-negative"); + } + int splitDeltaStart = confirmedMaxId + 1; + int splitDeltaCount = Math.max(0, batchMaxId - confirmedMaxId); + int deltaEntriesLength = splitDeltaEntriesLength(splitDeltaStart, splitDeltaCount); + return splitMessageSize( + tableBodyLength, splitDeltaStart, splitDeltaCount, deltaEntriesLength); + } + public void setDeferCommit(boolean defer) { if (defer) { flags |= FLAG_DEFER_COMMIT; @@ -144,4 +243,35 @@ public void writeHeader(int tableCount, int payloadLength) { buffer.putShort((short) tableCount); buffer.putInt(payloadLength); } + + private int splitDeltaEntriesLength(int splitDeltaStart, int splitDeltaCount) { + if (splitDeltaCount == 0) { + return 0; + } + if (splitDeltaStart != deltaStart || splitDeltaCount != deltaCount) { + throw new IllegalStateException("split delta does not match the staged message" + + " [stagedStart=" + deltaStart + + ", stagedCount=" + deltaCount + + ", splitStart=" + splitDeltaStart + + ", splitCount=" + splitDeltaCount + ']'); + } + return deltaEntriesEnd - deltaEntriesStart; + } + + private int splitMessageSize( + int tableBodyLength, + int splitDeltaStart, + int splitDeltaCount, + int deltaEntriesLength + ) { + long messageSize = (long) HEADER_SIZE + + NativeBufferWriter.varintSize(splitDeltaStart) + + NativeBufferWriter.varintSize(splitDeltaCount) + + deltaEntriesLength + + tableBodyLength; + if (messageSize > Integer.MAX_VALUE) { + throw new OutOfMemoryError("split QWP message size overflow: " + messageSize); + } + return (int) messageSize; + } } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index 92de0565..a6a33d69 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -47,9 +47,12 @@ import io.questdb.client.cutlass.qwp.client.sf.cursor.DefaultSenderConnectionListener; import io.questdb.client.cutlass.qwp.client.sf.cursor.DefaultSenderErrorHandler; import io.questdb.client.cutlass.qwp.client.sf.cursor.DefaultSenderProgressHandler; +import io.questdb.client.cutlass.qwp.client.sf.cursor.MmapSegment; +import io.questdb.client.cutlass.qwp.client.sf.cursor.PersistedSymbolDict; import io.questdb.client.cutlass.qwp.client.sf.cursor.SenderConnectionDispatcher; import io.questdb.client.cutlass.qwp.client.sf.cursor.SenderErrorDispatcher; import io.questdb.client.cutlass.qwp.client.sf.cursor.SenderProgressDispatcher; +import io.questdb.client.cutlass.qwp.client.sf.cursor.UnreplayableSlotException; import io.questdb.client.cutlass.qwp.protocol.QwpConstants; import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; import io.questdb.client.std.CharSequenceObjHashMap; @@ -57,6 +60,7 @@ import io.questdb.client.std.Decimal128; import io.questdb.client.std.Decimal256; import io.questdb.client.std.Decimal64; +import io.questdb.client.std.IntList; import io.questdb.client.std.Misc; import io.questdb.client.std.Numbers; import io.questdb.client.std.NumericException; @@ -153,7 +157,11 @@ public class QwpWebSocketSender implements Sender { private final QwpWebSocketEncoder encoder; private final List endpoints; // Global symbol dictionary for delta encoding - private final GlobalSymbolDictionary globalSymbolDictionary; + // Not final: seedGlobalDictionaryFromPersisted replaces it with a pre-sized instance + // before anything can observe the original. Nothing retains a reference -- the encoder + // and the persisted dictionary both take it as a per-call parameter -- and the swap + // happens during construction, before the producer or the I/O thread exist. + private GlobalSymbolDictionary globalSymbolDictionary; // Serializes FOREGROUND connect walks only (see buildAndConnect): the // shared-round state in hostTracker (pickNext/beginRound/attempted // bits), roundSeq, roundConnectAttemptSeq, and the foreground lifecycle @@ -169,6 +177,19 @@ public class QwpWebSocketSender implements Sender { // behind a drainer's endpoint walk. private final ReentrantLock connectWalkLock = new ReentrantLock(); private final QwpHostHealthTracker hostTracker; + // Per-table encoded body byte counts captured during flushPendingRows' combined + // encode. flushPendingRowsSplit uses them both for preflight sizing and to walk + // the staged body slices without encoding the batch a second time. Cleared and + // repopulated on every flush; reused to stay zero-GC. + // The non-empty tables of the batch currently being flushed, collected ONCE per + // flush and then iterated by every pass. Each pass used to re-walk tableBuffers.keys() + // and re-probe the hash map per table -- 3 probes per table on a plain flush and 5 on + // a split, on the producer's thread. Reused across flushes to stay zero-GC. Held in + // lockstep so index i names the same table in both, and in the same order as + // splitFrameBodyBytes, which is what lets the split passes index them directly. + private final ObjList flushTableBuffers = new ObjList<>(); + private final ObjList flushTableNames = new ObjList<>(); + private final IntList splitFrameBodyBytes = new IntList(); private final CharSequenceObjHashMap tableBuffers; // null means plain text (no TLS) private final ClientTlsConfiguration tlsConfig; @@ -230,6 +251,17 @@ public class QwpWebSocketSender implements Sender { private CursorSendEngine cursorEngine; private CursorWebSocketSendLoop cursorSendLoop; private boolean deferCommit; + // True when the sender emits incremental (delta) symbol dictionaries: each + // message carries only symbol ids not yet sent on the wire, rather than the + // full dictionary from id 0. Enabled in memory-mode (a reconnect replays from + // the in-process ring) and in file-mode store-and-forward when the per-slot + // persisted dictionary opened. In both, the I/O thread re-registers the whole + // dictionary via a catch-up frame before replaying, so a non-self-sufficient + // delta frame never dangles an id on a fresh server. Falls back to full + // self-sufficient frames only when the persisted dictionary is unavailable in + // file-mode (recovery/orphan-drain would then have nothing to rebuild the + // deltas from). Set in setCursorEngine. + private boolean deltaDictEnabled; // User-supplied observer for background orphan-slot drainer events. // Volatile: written by setDrainerListener (any thread, before or after // startOrphanDrainers) and read at pool-creation time. Null -> drainers @@ -280,11 +312,21 @@ public class QwpWebSocketSender implements Sender { // still terminal. // ASYNC → user thread does not connect at all. The I/O thread runs // the reconnect loop in the background, indefinitely - // (Invariant B); terminal failures (auth/upgrade reject) - // are delivered to the SenderError dispatcher rather than - // thrown from the constructor. + // (Invariant B); endpoint-policy and transport failures stay + // contained in that loop and never reach the producer. private Sender.InitialConnectMode initialConnectMode = Sender.InitialConnectMode.OFF; private boolean ownsCursorEngine; + // Whether close() may let the engine reclaim the parent-anchored LOGICAL slot lock. + // False only while an outer frame holds it: Sender.build() acquires it for the whole + // construct -> connect -> quarantine transition, and connect()'s own rollback closes + // this sender from INSIDE that scope. A fresh slot is "fully drained" by definition + // (publishedFsn < 0), so the default close(true) took the reclaim branch and unlinked + // the very lock file build() was holding -- on POSIX that frees the pathname without + // releasing the flock, so the next acquireLogical creates a SECOND inode and locks it. + // build()'s own careful close(false) calls could not prevent it, because connect() + // closed the engine first. Reset to true once connect() hands ownership back, so a + // later user-initiated close() still retires the lock normally. + private boolean reclaimLogicalSlotLockOnClose = true; private long pendingBytes; // Set true by close() once the SF slot flock has been released (the normal // teardown path). Stays false if an I/O or manager worker did not stop and @@ -316,6 +358,12 @@ public class QwpWebSocketSender implements Sender { // maxFrameRejections (connect-string key poison_min_escalation_window_millis). private long poisonMinEscalationWindowMillis = CursorWebSocketSendLoop.DEFAULT_POISON_MIN_ESCALATION_WINDOW_MILLIS; + // Minimum wall-clock dwell a symbol-dict catch-up cap gap must persist before an + // orphan drainer may quarantine its slot (connect-string key + // catch_up_cap_gap_min_escalation_window_millis). Foreground senders retry forever. See + // CursorWebSocketSendLoop.DEFAULT_CATCHUP_CAP_GAP_MIN_ESCALATION_WINDOW_MILLIS. + private long catchUpCapGapMinEscalationWindowMillis = + CursorWebSocketSendLoop.DEFAULT_CATCHUP_CAP_GAP_MIN_ESCALATION_WINDOW_MILLIS; private long reconnectInitialBackoffMillis = CursorWebSocketSendLoop.DEFAULT_RECONNECT_INITIAL_BACKOFF_MILLIS; private long reconnectMaxBackoffMillis = @@ -334,6 +382,12 @@ public class QwpWebSocketSender implements Sender { // beginRound(true) call. roundSeq=1 is the first round; CONNECTED in the // first round indicates the initial connect. private long roundSeq; + // Highest global symbol id the producer has baked into a frame so far, or -1. + // Lifetime-monotonic in delta mode -- it is NOT reset on reconnect, because + // the I/O thread re-registers the full dictionary via a catch-up frame before + // replaying, so the producer's delta baseline stays valid across the wire + // boundary. Used only when deltaDictEnabled; ignored in full-dict mode. + private int sentMaxSymbolId = -1; // When true, auto-flush sends messages with FLAG_DEFER_COMMIT and only // explicit flush() triggers the server-side commit. Enables accumulating // arbitrarily large datasets that exceed the server's recv buffer. @@ -688,7 +742,8 @@ public static QwpWebSocketSender connect( durableAckKeepaliveIntervalMillis, authTimeoutMs, connectTimeoutMs, connectionListener, connectionListenerInboxCapacity, CursorWebSocketSendLoop.DEFAULT_MAX_HEAD_FRAME_REJECTIONS, - CursorWebSocketSendLoop.DEFAULT_POISON_MIN_ESCALATION_WINDOW_MILLIS); + CursorWebSocketSendLoop.DEFAULT_POISON_MIN_ESCALATION_WINDOW_MILLIS, + CursorWebSocketSendLoop.DEFAULT_CATCHUP_CAP_GAP_MIN_ESCALATION_WINDOW_MILLIS); } /** @@ -719,7 +774,8 @@ public static QwpWebSocketSender connect( SenderConnectionListener connectionListener, int connectionListenerInboxCapacity, int maxFrameRejections, - long poisonMinEscalationWindowMillis + long poisonMinEscalationWindowMillis, + long catchUpCapGapMinEscalationWindowMillis ) { QwpWebSocketSender sender = new QwpWebSocketSender( endpoints, tlsConfig, @@ -737,6 +793,7 @@ public static QwpWebSocketSender connect( sender.durableAckKeepaliveIntervalMillis = durableAckKeepaliveIntervalMillis; sender.maxFrameRejections = maxFrameRejections; sender.poisonMinEscalationWindowMillis = poisonMinEscalationWindowMillis; + sender.catchUpCapGapMinEscalationWindowMillis = catchUpCapGapMinEscalationWindowMillis; sender.initialConnectMode = initialConnectMode == null ? Sender.InitialConnectMode.OFF : initialConnectMode; @@ -753,7 +810,20 @@ public static QwpWebSocketSender connect( } sender.ensureConnected(); } catch (Throwable t) { - sender.close(); + // Preserve t's IDENTITY through the rollback. Sender.build() routes on the + // exception type -- only UnreplayableSlotException reaches its quarantine + // handler -- and close() accumulates cleanup errors and ends in + // rethrowTerminal, so letting a close failure propagate here would REPLACE t + // and silently demote a recoverable slot back to the permanent build() brick. + // This rollback always runs inside Sender.build()'s acquireLogical scope, so + // the logical slot lock is held one frame up. Closing the engine with the + // default reclaim would unlink the lock file build() is still holding. + sender.reclaimLogicalSlotLockOnClose = false; + try { + sender.close(); + } catch (Throwable closeFailure) { + t.addSuppressed(closeFailure); + } throw t; } return sender; @@ -1115,7 +1185,24 @@ public void close() { // rows -> mmap'd / malloc'd ring). After this, the // cursor engine's publishedFsn reflects the final // target the I/O loop must drive ackedFsn up to. - flushPendingRows(deferCommit); + // A pre-flight rejection means this batch cannot fit + // the current cap however it is split. It is + // RETAINED by design so it can go out once a + // larger-cap node is reached -- but on close there is + // no later flush, and letting the throw escape here + // skips sendCommitMessage, sealAndSwapBuffer and + // drainOnClose, abandoning every row an earlier + // successful flush already published. The message + // that path emits tells the caller to close the + // sender to discard the batch, so honour that: + // discard it, remember the error, and let the rest of + // close() run. rethrowTerminal below still surfaces it. + try { + flushPendingRows(deferCommit); + } catch (BatchTooLargeForCapException e) { + resetTableBuffersAfterFlush(); + terminalError = captureCloseError(terminalError, e); + } if (!deferCommit && hasDeferredMessages) { sendCommitMessage(); } @@ -1811,6 +1898,38 @@ public long getPendingBytes() { return pendingBytes; } + /** + * Snapshot of the producer's symbol prefix whose persisted-dictionary chunks + * have committed. The persisted size advances only after the chunk CRC and + * payload have been written, so this observes the write-ahead boundary without + * reopening the live mmap-backed dictionary (which would attempt recovery-tail + * truncation and is not supported while the producer owns the file on Windows). + * Returns {@code null} in memory mode or when the persisted dictionary is + * unavailable. + */ + @TestOnly + public ObjList getPersistedSymbolsForTest() { + CursorSendEngine engine = cursorEngine; + if (engine == null) { + return null; + } + PersistedSymbolDict persisted = engine.getPersistedSymbolDict(); + if (persisted == null) { + return null; + } + int persistedSize = persisted.size(); + int globalSize = globalSymbolDictionary.size(); + if (persistedSize > globalSize) { + throw new IllegalStateException("persisted symbol dictionary exceeds producer dictionary" + + " [persisted=" + persistedSize + ", producer=" + globalSize + ']'); + } + ObjList snapshot = new ObjList<>(persistedSize); + for (int i = 0; i < persistedSize; i++) { + snapshot.add(globalSymbolDictionary.getSymbol(i)); + } + return snapshot; + } + /** * Server-advertised cap on the per-batch raw byte size. Zero before the * first connect; updated by every successful reconnect via @@ -1834,6 +1953,17 @@ public QwpTableBuffer getTableBuffer(String tableName) { return buffer; } + /** + * Whether this sender is still in delta-encoded mode. Flips to {@code false} + * permanently once {@link #disableDeltaDict} fires (a persisted-dictionary + * write failure, including a recognised mmap access fault) -- every later + * flush then ships full self-sufficient frames instead. + */ + @TestOnly + public boolean isDeltaDictEnabledForTest() { + return deltaDictEnabled; + } + /** * Total binary frames whose ACKs have been received and applied. */ @@ -2299,6 +2429,22 @@ public void setCursorEngine(CursorSendEngine engine, boolean takeOwnership) { } this.cursorEngine = engine; this.ownsCursorEngine = takeOwnership && engine != null; + // Delta encoding is available in memory-mode (in-process catch-up) and in + // file-mode when the persisted dictionary opened (recovery / orphan-drain + // rebuild the dictionary from it). Otherwise fall back to full self- + // sufficient frames. See CursorSendEngine.isDeltaDictEnabled. + this.deltaDictEnabled = engine != null && engine.isDeltaDictEnabled(); + // Recovery: repopulate the producer's global dictionary from the slot's + // persisted dictionary so newly ingested symbols continue from the + // recovered ids (rather than colliding with them at 0), and the delta + // baseline resumes where the crashed session left off. + // NOT gated on deltaDictEnabled. That flag is false exactly when the slot's dictionary + // failed to open -- which is precisely when the frames on disk are the only surviving + // copy of the symbols and the rebuild matters most. Gating the seed on it made the + // rebuild dead code for the very case it was written for. + if (engine != null && engine.wasRecoveredFromDisk()) { + seedGlobalDictionaryFromPersisted(engine.getPersistedSymbolDict()); + } if (engine != null) { engine.setSlotLockReleaseListener(this::onSlotLockReleased); } @@ -2504,7 +2650,8 @@ public synchronized void startOrphanDrainers( requestDurableAck, durableAckKeepaliveIntervalMillis, maxFrameRejections, - poisonMinEscalationWindowMillis); + poisonMinEscalationWindowMillis, + catchUpCapGapMinEscalationWindowMillis); ref[0] = drainer; drainerPool.submit(drainer); } @@ -3283,7 +3430,11 @@ private synchronized Throwable closeRemainingResources(Throwable terminalError) if (ownsCursorEngine && cursorEngine != null) { CursorSendEngine engine = cursorEngine; try { - engine.close(); + // reclaimLogicalSlotLockOnClose is false when a caller above us + // (Sender.build()) still HOLDS the parent-anchored logical lock: + // unlinking it here would free the pathname without releasing the + // flock, letting the next acquireLogical create a second inode. + engine.close(reclaimLogicalSlotLockOnClose); } catch (Throwable t) { LOG.error("Error closing owned CursorSendEngine: {}", String.valueOf(t)); terminalError = captureCloseError(terminalError, t); @@ -3330,8 +3481,14 @@ private synchronized Throwable closeRemainingResources(Throwable terminalError) } } - private int countNonEmptyTables(ObjList keys) { - int tableCount = 0; + /** + * Collects the batch's non-empty tables into {@link #flushTableNames} / + * {@link #flushTableBuffers} and returns how many there are. One walk of the key + * list and one hash probe per table, for the whole flush. + */ + private int collectNonEmptyTables(ObjList keys) { + flushTableNames.clear(); + flushTableBuffers.clear(); for (int i = 0, n = keys.size(); i < n; i++) { CharSequence tableName = keys.getQuick(i); if (tableName == null) { @@ -3339,10 +3496,11 @@ private int countNonEmptyTables(ObjList keys) { } QwpTableBuffer tableBuffer = tableBuffers.get(tableName); if (tableBuffer != null && tableBuffer.getRowCount() > 0) { - tableCount++; + flushTableNames.add(tableName); + flushTableBuffers.add(tableBuffer); } } - return tableCount; + return flushTableBuffers.size(); } private Endpoint currentEndpoint() { @@ -3454,14 +3612,24 @@ private void drainOnClose(boolean errorOwnedByCustomHandler) { } if (System.nanoTime() >= deadlineNanos) { long acked = cursorEngine.ackedFsn(); - LOG.warn("close() drain timed out after {}ms [target={} acked={}], pending data may be lost", - closeFlushTimeoutMillis, target, acked); + // Name the outage the I/O thread is riding out, when there is one. A + // foreground sender now retries endpoint-policy rejections indefinitely, + // so a revoked token reaches the operator HERE -- and blaming timeout + // tuning for what is actually an auth failure is how the review found + // this path misdirecting. + CursorWebSocketSendLoop loop = cursorSendLoop; + Throwable outage = loop == null ? null : loop.lastReconnectError(); + LOG.warn("close() drain timed out after {}ms [target={} acked={}], pending data may be lost{}", + closeFlushTimeoutMillis, target, acked, + outage == null ? "" : "; wire is not draining: " + outage.getMessage()); throw new LineSenderException("close() drain timed out after ") .put(closeFlushTimeoutMillis).put(" ms [targetFsn=") .put(target).put(", ackedFsn=").put(acked) .put("] - server did not acknowledge ") .put(target - acked) - .put(" pending batches; data may be lost (use larger closeFlushTimeoutMillis or smaller batches)"); + .put(outage == null + ? " pending batches; data may be lost (use larger closeFlushTimeoutMillis or smaller batches)" + : " pending batches; the wire is not draining: " + outage.getMessage()); } java.util.concurrent.locks.LockSupport.parkNanos(50_000L); } @@ -3533,10 +3701,12 @@ private void ensureConnected() { // Encoder stays at its default (V1 -- the only supported wire // version today). Frames written before the first successful // connect commit to V1 because cursor segments are immutable; - // a future version bump must account for that. Auth/upgrade - // rejects are surfaced via the error inbox by the I/O - // thread, not thrown here; plain connect failures retry - // indefinitely (Invariant B). + // a future version bump must account for that. Transport + // failures retry indefinitely on the I/O thread (Invariant B). + // But a terminal auth, upgrade or capability rejection on this + // initial connect -- before the wire is ever up -- is surfaced + // to the async SenderErrorHandler and latched for a close() + // rethrow, not retried. client = null; break; case OFF: @@ -3556,13 +3726,14 @@ private void ensureConnected() { client, cursorEngine, 0L, CursorWebSocketSendLoop.DEFAULT_PARK_NANOS, reconnectFactory, - reconnectMaxDurationMillis, reconnectInitialBackoffMillis, reconnectMaxBackoffMillis, requestDurableAck, durableAckKeepaliveIntervalMillis, maxFrameRejections, - poisonMinEscalationWindowMillis); + poisonMinEscalationWindowMillis, + catchUpCapGapMinEscalationWindowMillis, + CursorWebSocketSendLoop.ReconnectPolicy.FOREGROUND); // Plug the async-delivery sink before start() so the I/O thread // never observes a null dispatcher between recordFatal and // notification — the test for null in dispatchError handles @@ -3588,6 +3759,17 @@ private void ensureConnected() { cursorSendLoop.setConnectionDispatcher(connectionDispatcher); cursorSendLoop.start(); } catch (Throwable t) { + // start() (or dispatcher construction) failed after cursorSendLoop was + // assigned. Close it so a caller that retries -- re-entering + // ensureConnected and reassigning cursorSendLoop above -- cannot orphan + // a recovered slot's ctor-seeded native mirror (freed only by close() + // or the I/O loop, neither of which has run). close() is idempotent and + // frees the mirror via its loopNeverRan path; it also closes the shared + // client, so the client.close() below is a safe idempotent no-op. + if (cursorSendLoop != null) { + cursorSendLoop.close(); + cursorSendLoop = null; + } if (client != null) { client.close(); client = null; @@ -3615,18 +3797,18 @@ private void ensureConnected() { host, port, client.getServerQwpVersion(), serverMaxBatchSize, effectiveAutoFlushBytes); } else { // Async mode: I/O thread will drive the connect. Encoder uses - // its default version (V1). The symbol-dict watermark still gets - // reset for consistency with the sync path; the post-connect replay - // path does not need a producer-side reset signal because every - // cursor frame is self-sufficient. + // its default version (V1). The per-batch symbol-dict watermark still + // gets reset for consistency with the sync path; the post-connect + // replay path needs no producer-side reset signal (see below). Endpoint ep = endpoints.get(0); LOG.info("Async initial connect deferred to I/O thread [firstHost={}, firstPort={}, endpointCount={}]", ep.host, ep.port, endpoints.size()); } - // Server starts fresh on each connection, so reset the symbol-dict - // watermark. Cursor frames are self-sufficient (every frame carries its - // full inline schema + a symbol-dict delta from id 0), so post-reconnect - // replay needs no producer-side reset signal. + // Server starts fresh on each connection, so reset the per-batch + // symbol-dict watermark. Every frame still carries its full inline schema, + // and the fresh server's dictionary is re-established either by a full-dict + // frame (full-dict mode) or by an I/O-thread catch-up frame before replay + // (delta mode), so post-reconnect replay needs no producer-side reset signal. resetSymbolDictStateForNewConnection(); connectionError.set(null); @@ -3664,7 +3846,7 @@ private void flushPendingRows(boolean deferCommit) { cachedTimestampNanosColumn = null; ObjList keys = tableBuffers.keys(); - int tableCount = countNonEmptyTables(keys); + int tableCount = collectNonEmptyTables(keys); if (tableCount == 0) { pendingBytes = 0; currentTableBufferSnapshotBytes = currentTableBuffer == null @@ -3679,50 +3861,73 @@ private void flushPendingRows(boolean deferCommit) { } ensureActiveBufferReady(); - // Cursor SF requires every on-disk frame to be self-sufficient: - // recorded frames replay to fresh server connections (orphan-slot - // drainers and post-reconnect replay), so always emit the full - // symbol-dict delta from id=0 and the full column schema inline, - // never a back-reference the target server may not have seen. + // In full-dict mode every frame is self-sufficient: it carries the whole + // symbol dictionary from id 0 so orphan-drain / recovery replay to a fresh + // server never dangles a symbol id. In delta mode (memory-mode, and + // file-mode store-and-forward once the persisted dictionary opened) each + // frame carries only ids above sentMaxSymbolId; a reconnect re-registers + // the dictionary via an I/O-thread catch-up frame before replay, so the + // producer's monotonic baseline stays valid across the wire boundary. encoder.setDeferCommit(deferCommit); encoder.beginMessage(tableCount, globalSymbolDictionary, - /*confirmedMaxId=*/ -1, currentBatchMaxSymbolId); - for (int i = 0, n = keys.size(); i < n; i++) { - CharSequence tableName = keys.getQuick(i); - if (tableName == null) { - continue; - } - QwpTableBuffer tableBuffer = tableBuffers.get(tableName); - if (tableBuffer == null || tableBuffer.getRowCount() == 0) { - continue; - } + symbolDeltaBaseline(), currentBatchMaxSymbolId); + // Record each table's encoded body size (position delta across addTable). + // When the batch needs splitting, these lengths delimit immutable body + // slices in the combined encoder buffer for direct frame assembly. The + // capture is a couple of int ops per table on the common path. + splitFrameBodyBytes.clear(); + int combinedBodyStart = encoder.getBuffer().getPosition(); + int bodyStart = combinedBodyStart; + for (int i = 0; i < tableCount; i++) { + QwpTableBuffer tableBuffer = flushTableBuffers.getQuick(i); if (LOG.isDebugEnabled()) { LOG.debug("Encoding table [name={}, rows={}, batchMaxId={}]", - tableName, tableBuffer.getRowCount(), currentBatchMaxSymbolId); + flushTableNames.getQuick(i), tableBuffer.getRowCount(), currentBatchMaxSymbolId); } encoder.addTable(tableBuffer); + int bodyEnd = encoder.getBuffer().getPosition(); + splitFrameBodyBytes.add(bodyEnd - bodyStart); + bodyStart = bodyEnd; } int messageSize = encoder.finishMessage(); QwpBufferWriter buffer = encoder.getBuffer(); - if (serverMaxBatchSize > 0 && messageSize > serverMaxBatchSize) { - flushPendingRowsSplit(keys, deferCommit); + // Snapshot the volatile cap ONCE for this whole flush. The I/O thread lowers + // serverMaxBatchSize on a mid-stream failover to a smaller-cap node + // (applyServerBatchSizeLimit); if the split pre-flight and the publish loop + // re-read the field independently, a failover between them would size frames + // against two different caps -- breaking the all-or-nothing guarantee and + // firing the publish-loop assert on a legitimate race. Both use this snapshot; + // the next flush picks up the new cap. + int cap = serverMaxBatchSize; + if (cap > 0 && messageSize > cap) { + // Keep the completed combined frame staged in the encoder while the + // split path copies its delta entries and table-body slices. + flushPendingRowsSplit(deferCommit, combinedBodyStart, cap); return; } + // Write-ahead: durably persist this frame's new symbols BEFORE it is + // published, so a recovered/orphan-drained slot can always rebuild the + // dictionary the (non-self-sufficient) delta frame references. No-op in + // memory mode and when the frame introduces no new symbols. + persistNewSymbolsBeforePublish(); activeBuffer.ensureCapacity(messageSize); activeBuffer.write(buffer.getBufferPtr(), messageSize); activeBuffer.incrementRowCount(); sealAndSwapBuffer(); + // The frame carrying ids up to currentBatchMaxSymbolId is now on the ring; + // advance the delta baseline so the next frame ships only newer ids. + advanceSentMaxSymbolId(); hasDeferredMessages = deferCommit; if (!deferCommit) { lastCommitBoundaryFsn = cursorEngine.publishedFsn(); } - resetTableBuffersAfterFlush(keys); + resetTableBuffersAfterFlush(); } /** @@ -3731,64 +3936,127 @@ private void flushPendingRows(boolean deferCommit) { * own message. All messages except the last carry FLAG_DEFER_COMMIT * so the server appends rows without committing until the final * message arrives. + *

+ * Not atomic across frames. The frames publish one at a time, so a + * publish failure partway through -- {@link #sealAndSwapBuffer()} throwing on + * frame k>1, e.g. a backpressure deadline or the buffer-recycle timeout -- + * leaves frames 1..k-1 already on the ring as deferred (appended, not yet + * committed). The throw propagates past the {@code resetTableBuffersAfterFlush} + * at the end of the loop, so the source rows survive in their table buffers + * and the NEXT flush re-emits the whole batch; the eventual commit then + * commits the already-published prefix alongside the re-sent copies, + * delivering those rows at-least-once (duplicated), not exactly-once. This is + * within store-and-forward's at-least-once contract -- a DEDUP table or a + * durable-ack await absorbs the duplicate, and the symbol-dict state stays + * consistent on the retry (the re-sent frames carry empty deltas and the + * write-ahead persist is a {@code pd.size()} no-op). Making the split atomic + * (rolling back the published prefix, or skipping it on retry) would be a + * larger change. * * @param deferCommit when true, ALL messages (including the last) * carry FLAG_DEFER_COMMIT. When false, only the * last message omits the flag. */ - private void flushPendingRowsSplit(ObjList keys, boolean deferCommit) { + private void flushPendingRowsSplit( + boolean deferCommit, + int combinedBodyStart, + int cap + ) { if (LOG.isDebugEnabled()) { - LOG.debug("Splitting flush across multiple messages [serverMaxBatchSize={}, defer={}]", serverMaxBatchSize, deferCommit); - } - - // Collect non-empty table indices so we know which is last. - int nonEmptyCount = 0; - for (int i = 0, n = keys.size(); i < n; i++) { - CharSequence tableName = keys.getQuick(i); - if (tableName == null) { - continue; + LOG.debug("Splitting flush across multiple messages [serverMaxBatchSize={}, defer={}]", cap, deferCommit); + } + + // Collect non-empty table indices so we know which is last, AND pre-flight + // every split frame's size BEFORE publishing any of them. The split hands + // frames to the ring one at a time (all but the last deferred -- appended but + // uncommitted); if a later table's frame were only found oversized + // mid-publish, the already-published prefix would strand on the ring, a + // subsequent commit would deliver it as a partial batch, and + // resetTableBuffersAfterFlush would discard every source row -- a partial + // commit the caller was told (by the throw) had failed. Checking all sizes up + // front makes the split all-or-nothing: either every frame fits and all + // publish, or none publish and we throw with nothing stranded. + // + // Each split frame's size is derived from the combined encode flushPendingRows + // already performed. simBaseline mirrors the publish loop's baseline advance + // (advanceSentMaxSymbolId), so each size equals the frame the staged-slice + // assembler will build; this pass mutates no delta/persist state. + int nonEmptyCount = flushTableBuffers.size(); + int simBaseline = symbolDeltaBaseline(); + for (int i = 0; i < nonEmptyCount; i++) { + CharSequence tableName = flushTableNames.getQuick(i); + int messageSize = encoder.getSplitMessageSize( + splitFrameBodyBytes.getQuick(i), simBaseline, currentBatchMaxSymbolId); + if (messageSize > cap) { + // The batch stays BUFFERED: this throw precedes every publish, and a + // rejected flush must not silently discard the caller's rows (see + // SelfSufficientFramesTest#testOversizedTableSplitStrandsNothing). So the + // next flush() re-encodes and re-rejects the same batch until either the + // reachable cap grows -- a failover to a larger-cap node, which is the + // case retaining the rows exists to survive -- or the sender is closed, + // which discards them. It cannot drain against a cap this table will + // never fit, so say that here rather than let a caller read the repeat + // rejections as a transient and keep appending to a batch that only + // grows. + throw new BatchTooLargeForCapException("single table batch too large for server batch cap") + .put(" [table=").put(tableName) + .put(", messageSize=").put(messageSize) + .put(", serverMaxBatchSize=").put(cap).put(']') + .put("; the batch is retained for retry and every flush() will " + + "reject it again until a larger-cap node is reached -- " + + "close the sender to discard it, or produce smaller batches"); } - QwpTableBuffer tableBuffer = tableBuffers.get(tableName); - if (tableBuffer != null && tableBuffer.getRowCount() > 0) { - nonEmptyCount++; + // Mirror advanceSentMaxSymbolId: once the first frame ships the batch's + // new ids, the remaining frames carry an empty delta above the baseline. + if (deltaDictEnabled && currentBatchMaxSymbolId > simBaseline) { + simBaseline = currentBatchMaxSymbolId; } } - int sent = 0; - for (int i = 0, n = keys.size(); i < n; i++) { - CharSequence tableName = keys.getQuick(i); - if (tableName == null) { - continue; - } - QwpTableBuffer tableBuffer = tableBuffers.get(tableName); - if (tableBuffer == null || tableBuffer.getRowCount() == 0) { - continue; - } + int tableBodyOffset = combinedBodyStart; + for (int i = 0; i < nonEmptyCount; i++) { + CharSequence tableName = flushTableNames.getQuick(i); - sent++; - boolean isLast = (sent == nonEmptyCount); + boolean isLast = (i == nonEmptyCount - 1); boolean deferThis = deferCommit || !isLast; - encoder.setDeferCommit(deferThis); - encoder.beginMessage(1, globalSymbolDictionary, - /*confirmedMaxId=*/ -1, currentBatchMaxSymbolId); - encoder.addTable(tableBuffer); - int messageSize = encoder.finishMessage(); - QwpBufferWriter buffer = encoder.getBuffer(); - - if (messageSize > serverMaxBatchSize) { - resetTableBuffersAfterFlush(keys); - throw new LineSenderException("single table batch too large for server batch cap") - .put(" [table=").put(tableName) - .put(", messageSize=").put(messageSize) - .put(", serverMaxBatchSize=").put(serverMaxBatchSize).put(']'); - } - + int tableBodyLength = splitFrameBodyBytes.getQuick(i); + // Persist before touching activeBuffer. If the write-ahead fails, the + // caller can retry with both the source rows and active microbatch + // unchanged. The first frame carries the batch's new symbols; later + // frames are no-ops once the baseline has advanced. + persistNewSymbolsBeforePublish(); ensureActiveBufferReady(); - activeBuffer.ensureCapacity(messageSize); - activeBuffer.write(buffer.getBufferPtr(), messageSize); + // The combined encoder buffer remains immutable for the whole split. + // Assemble this frame directly into the active microbatch: patched + // header + staged delta bytes + the staged table-body slice. No row or + // column is encoded a second time. + int messageSize = encoder.copySplitMessage( + activeBuffer, + tableBodyOffset, + tableBodyLength, + deferThis, + symbolDeltaBaseline(), + currentBatchMaxSymbolId + ); + tableBodyOffset += tableBodyLength; + // The pre-flight pass above already verified every split frame fits the + // cap, so none can be found oversized here -- which is what keeps this + // loop from publishing (and stranding) a deferred prefix before an + // oversized table. Both passes size against the SAME snapshot cap, so a + // mid-flush failover cannot make them disagree; the assert therefore only + // catches a genuine divergence between the pre-flight arithmetic and the + // staged assembler (a future bug), not a cap race. It deliberately does NOT + // reset+throw here, because by this point a prefix may already be on the ring. + assert messageSize <= cap + : "split frame exceeded serverMaxBatchSize after pre-flight [table=" + tableName + + ", messageSize=" + messageSize + ", serverMaxBatchSize=" + cap + ']'; + activeBuffer.incrementRowCount(); sealAndSwapBuffer(); + // Frame queued: advance so the next split frame's delta starts above + // the ids this one just registered. + advanceSentMaxSymbolId(); } encoder.setDeferCommit(false); @@ -3798,7 +4066,7 @@ private void flushPendingRowsSplit(ObjList keys, boolean deferComm // committed the whole group. lastCommitBoundaryFsn = cursorEngine.publishedFsn(); } - resetTableBuffersAfterFlush(keys); + resetTableBuffersAfterFlush(); } private void onSlotLockReleased() { @@ -3809,18 +4077,13 @@ private void onSlotLockReleased() { } } - private void resetTableBuffersAfterFlush(ObjList keys) { - for (int i = 0, n = keys.size(); i < n; i++) { - CharSequence tableName = keys.getQuick(i); - if (tableName == null) { - continue; - } - QwpTableBuffer tableBuffer = tableBuffers.get(tableName); - if (tableBuffer == null || tableBuffer.getRowCount() == 0) { - continue; - } - tableBuffer.reset(); + private void resetTableBuffersAfterFlush() { + for (int i = 0, n = flushTableBuffers.size(); i < n; i++) { + flushTableBuffers.getQuick(i).reset(); } + // Drop the references; the next flush re-collects. + flushTableNames.clear(); + flushTableBuffers.clear(); currentBatchMaxSymbolId = -1; pendingBytes = 0; currentTableBufferSnapshotBytes = 0; @@ -3837,8 +4100,25 @@ private void sendCommitMessage() { LOG.debug("Sending commit message for deferred batch"); } encoder.setDeferCommit(false); + // A commit carries no rows, and it must also carry NO new symbols. Unlike + // the flush paths, sendCommitMessage does NOT write-ahead-persist the + // dictionary, so shipping a symbol here would put an id on the wire that a + // recovered slot cannot rebuild from the persisted .symbol-dict, diverging + // the producer dictionary from the surviving frames and silently + // misattributing reused ids after a crash. currentBatchMaxSymbolId can sit + // ABOVE sentMaxSymbolId (e.g. a cancelled row: cancelRow does not roll back + // currentBatchMaxSymbolId or unregister the symbol), so bound the delta at + // what has already been sent -- and therefore already persisted. In delta + // mode pass sentMaxSymbolId, yielding an empty delta + // [sentMaxSymbolId+1 .. sentMaxSymbolId]; in full-dict mode keep + // currentBatchMaxSymbolId (reset to -1 by the prior flush, so the commit frame + // carries an empty delta too -- harmless, since the preceding full-dict data + // frames already registered the dictionary on this connection). Any symbol a + // cancelled row leaked is picked up (and persisted) by the next real flush, + // whose persistNewSymbolsBeforePublish resumes from pd.size(). + int commitBatchMaxId = deltaDictEnabled ? sentMaxSymbolId : currentBatchMaxSymbolId; encoder.beginMessage(0, globalSymbolDictionary, - /*confirmedMaxId=*/ -1, currentBatchMaxSymbolId); + symbolDeltaBaseline(), commitBatchMaxId); int messageSize = encoder.finishMessage(); QwpBufferWriter buffer = encoder.getBuffer(); ensureActiveBufferReady(); @@ -3850,15 +4130,334 @@ private void sendCommitMessage() { lastCommitBoundaryFsn = cursorEngine.publishedFsn(); } + /** + * Advances the delta baseline once a frame carrying the current batch's + * symbols has been queued onto the ring. No-op in full-dict mode. Only ever + * moves the baseline forward, so a batch that used no new symbols leaves it + * unchanged. + */ + private void advanceSentMaxSymbolId() { + if (deltaDictEnabled && currentBatchMaxSymbolId > sentMaxSymbolId) { + sentMaxSymbolId = currentBatchMaxSymbolId; + } + } + + /** + * Stops emitting delta dictionaries for the rest of this sender's life, after the + * per-slot {@code .symbol-dict} has proved unwritable. + *

+ * The side-file can stop accepting appends mid-run -- a full disk or an exhausted + * quota, where SF's own segments stay writable because they are pre-allocated mmap + * files while the dictionary is the one thing still growing. Without a way back, + * {@code deltaDictEnabled} is written once at {@code setCursorEngine} and every + * later {@code flush()} re-throws forever: a condition store-and-forward is built + * to survive becomes total, permanent ingestion loss. + *

+ * Full self-sufficient frames need no side file at all -- each carries the whole + * dictionary from id 0, which is exactly what recovery and orphan-drain replay + * against a fresh server. So degrade instead of dying. The producer's monotonic + * baseline stops being consulted ({@link #symbolDeltaBaseline()} returns -1), and + * the write-ahead persist becomes a no-op. + *

+ * Producer-thread only, like every other reader of {@code deltaDictEnabled}. + */ + private void disableDeltaDict(Throwable cause) { + if (!deltaDictEnabled) { + return; + } + deltaDictEnabled = false; + LOG.warn("symbol dictionary persistence failed; this sender has switched to full " + + "self-sufficient frames for the rest of its life (bandwidth cost only -- " + + "no data is at risk, and recovery replays such frames without a side file)", + cause); + } + + /** + * Writes the ids the surviving frames contributed above the persisted prefix back + * into {@code .symbol-dict}, immediately, before any new frame can be published. + *

+ * {@link #seedGlobalDictionaryFromPersisted} can rebuild the producer dictionary from + * TWO sources -- the side-file's intact prefix and the surviving frames' own delta + * sections -- and then resumes {@code sentMaxSymbolId} at the combined tip. When the + * frames contributed anything, that tip is ABOVE {@code pd.size()}, so every frame + * published from here on carries a {@code deltaStart} the side-file cannot describe. + * That breaks the write-ahead invariant the whole design rests on: the persisted + * dictionary must be a superset of every recoverable frame's references. + *

+ * The steady-state write-ahead does NOT close that gap on its own. It persists + * {@code [pd.size() .. currentBatchMaxSymbolId]} and returns early when the batch's + * highest id is below {@code pd.size()}, so it heals only if -- and only as far as -- + * a later batch happens to reference the recovered high ids. Meanwhile the frames + * that carry those ids are the oldest unacked, so they are the FIRST to be acked and + * trimmed. Once they are gone, an ordinary process crash (which store-and-forward + * promises to survive; only the original tear needs a host crash) leaves a slot whose + * frames reference ids nothing holds: recovery marks a gap and {@code build()} + * quarantines it with "resend the affected data". + *

+ * Healing here, eagerly and in full, restores the invariant before the window opens. + */ + private void healPersistedDictionary(PersistedSymbolDict pd) { + if (pd == null || !deltaDictEnabled) { + return; + } + int from = pd.size(); + int to = globalSymbolDictionary.size() - 1; + if (to < from) { + return; // the side-file already covers everything the frames defined + } + try { + pd.appendSymbols(globalSymbolDictionary, from, to); + } catch (Throwable t) { + // A recognised mmap access fault is NOT a JVM failure to propagate: it is how + // an unbacked or sparse append page surfaces, and commitMappedChunk uses + // Crc32c.updateUnsafe precisely so it arrives catchable here. Rethrowing it + // raw skips disableDeltaDict, and from healPersistedDictionary it escapes + // Sender.build() as neither UnreplayableSlotException nor LineSenderException, + // so the slot is neither quarantined nor reported and every restart re-faults + // the same page. + if (t instanceof Error && !MmapSegment.isMmapAccessFault(t)) { + throw (Error) t; + } + // Do NOT fail recovery: the surviving frames still carry these ids in their + // own deltas, so THIS session replays correctly either way. Only a future + // recovery, after those frames are trimmed, would be affected -- and the + // degrade below removes even that exposure by dropping back to frames that + // need no side file. + disableDeltaDict(t); + } + } + + /** + * Appends the symbols this frame introduces ({@code [sentMaxSymbolId+1 .. + * currentBatchMaxSymbolId]}) to the slot's persisted dictionary BEFORE the + * frame is published to the ring. This write-ahead ordering keeps the + * persisted dictionary a superset of every process-crash-recoverable frame's + * references, so recovery and orphan-drain can re-register it on a fresh + * server. Not fsync'd (see PersistedSymbolDict) -- a host crash that tears it + * is caught by the send loop's replay guard. No-op in memory mode (no + * persisted dictionary) and when the frame introduces no new symbols. + */ + private void persistNewSymbolsBeforePublish() { + if (!deltaDictEnabled || cursorEngine == null) { + return; + } + PersistedSymbolDict pd = cursorEngine.getPersistedSymbolDict(); + if (pd == null) { + return; + } + // Persist [pd.size() .. currentBatchMaxSymbolId] as ONE write, BEFORE the + // frame is published. + // + // Resume from the dictionary's own durable size, NOT sentMaxSymbolId+1: + // the persist advances pd.size() only after a full write, whereas + // sentMaxSymbolId only advances after the WHOLE frame is published (via + // advanceSentMaxSymbolId, after activeBuffer.write). If a prior persist + // threw (short write -- disk full/quota) or the publish threw, the frame + // was not published and sentMaxSymbolId stayed put, while the symbols + // before the failure are already on disk. Keying the resume point off + // sentMaxSymbolId+1 would re-append that persisted prefix on the retry, + // duplicating entries and corrupting the dense id->symbol mapping recovery + // relies on (position i must be symbol id i). pd.size() resumes exactly + // past what is already durable, so the write-ahead is idempotent. + int from = pd.size(); + if (currentBatchMaxSymbolId < from) { + return; // nothing new to persist (warm batch, or an idempotent retry) + } + // Fast path: the frame the encoder just built already holds these symbols + // in its delta section as [len][utf8]... -- byte-identical to what + // PersistedSymbolDict stores. In the common case pd.size() equals the + // frame's delta start id (sentMaxSymbolId+1), so persist those bytes + // straight from the frame instead of re-encoding the symbols. After a + // failed publish the durable size has run ahead of the wire baseline, so + // the frame's delta covers MORE than remains to persist; then re-encode + // just the [from .. currentBatchMaxSymbolId] suffix. + try { + if (from == sentMaxSymbolId + 1) { + QwpBufferWriter buffer = encoder.getBuffer(); + pd.appendRawEntries( + buffer.getBufferPtr() + encoder.getDeltaEntriesStart(), + encoder.getDeltaEntriesLen(), + currentBatchMaxSymbolId - from + 1); + } else { + pd.appendSymbols(globalSymbolDictionary, from, currentBatchMaxSymbolId); + } + } catch (Throwable t) { + // A failed write to the persisted dictionary throws a low-level + // IllegalStateException: in production that is ff.allocate refusing to grow + // the mmap append window (how a full disk / exhausted quota surfaces there), + // and behind an injected facade a short positioned write. Surface it as a + // LineSenderException -- like every other flush-path failure, e.g. the cursor + // append in sealAndSwapBuffer -- so a caller catching LineSenderException + // around flush() also catches a disk-full during the write-ahead persist. The + // persist ran before publish and pd.size() did not advance on the failure, so + // the still-buffered rows re-persist the same range idempotently on retry. + // A JVM Error is never a persist failure; let it propagate -- except a + // recognised mmap access fault, which is NOT a JVM failure to propagate: it + // is how an unbacked or sparse append page surfaces, and commitMappedChunk + // uses Crc32c.updateUnsafe precisely so it arrives catchable here. Rethrowing + // it raw skips disableDeltaDict, and from healPersistedDictionary it escapes + // Sender.build() as neither UnreplayableSlotException nor LineSenderException, + // so the slot is neither quarantined nor reported and every restart re-faults + // the same page. + if (t instanceof Error && !MmapSegment.isMmapAccessFault(t)) { + throw (Error) t; + } + // Degrade before throwing, so this failure is survivable rather than terminal: + // every LATER flush emits full self-sufficient frames, which need no side file + // (see disableDeltaDict). This one flush still has to fail -- beginMessage has + // already baked a delta deltaStart into the staged frame, and publishing it + // would put ids on the ring that the side-file cannot describe. The throw + // precedes every publish, so the caller's rows stay buffered and the next + // flush() re-encodes them from id 0. + disableDeltaDict(t); + throw new LineSenderException("failed to persist symbol dictionary before publish; " + + "this sender has switched to full self-sufficient frames -- retry the flush", t); + } + } + private void resetSymbolDictStateForNewConnection() { - // The new server has an empty symbol dictionary, so the next batch - // must ship a delta starting at id 0. beginMessage() always passes - // confirmedMaxId = -1; resetting the batch watermark here keeps a - // stale value from suppressing re-emission of symbol ids the new - // server has never seen. + // Runs on the foreground (initial) connect only -- NOT on the I/O thread's + // reconnect/failover path. The per-batch watermark is drained state, so + // clearing it here is harmless. sentMaxSymbolId is deliberately left + // untouched: in delta mode the I/O thread re-registers the whole + // dictionary with a catch-up frame on reconnect, so the producer's + // monotonic baseline must survive the wire boundary; resetting it would + // desync the producer from the I/O thread's sent-dictionary count. currentBatchMaxSymbolId = -1; } + /** + * On recovery, repopulates the producer's {@link GlobalSymbolDictionary} so that newly + * ingested symbols continue ABOVE every id the surviving frames already define, and + * resumes the delta baseline at that tip. + *

+ * Seeds from TWO sources, in this order: + *

    + *
  1. the slot's persisted {@code .symbol-dict} -- its intact prefix; then
    2. + *
  2. the surviving frames' OWN delta sections, for every id above that prefix + * ({@link CursorSendEngine#addRecoveredSymbolsTo}).
    3. + *
+ * Those are exactly the two sources, in exactly the order, that the send loop's mirror + * is built from: its constructor seeds {@code sentDictCount} from the same dictionary, + * and {@code accumulateSentDict} then extends it from the same frames as they replay. So + * the producer's {@code sentMaxSymbolId + 1} and the loop's {@code sentDictCount} land on + * the same number BY CONSTRUCTION -- the invariant the torn-dictionary guard rests on -- + * rather than by the two happening to agree. + *

+ * Uses {@link GlobalSymbolDictionary#addRecoveredSymbol} (append, NOT de-dup): the + * persisted dictionary, the on-wire delta and the mirror all key on the entry POSITION + * (id), so the producer's id space must match the recovered entry count exactly. + * {@code getOrAddSymbol} would collapse two source strings that decode to the same + * characters -- only malformed lone UTF-16 surrogates, which UTF-8-encode to {@code '?'} + * -- leaving this dictionary SHORTER than the count and silently misattributing later + * symbols. + *

+ * Why seeding from the frames matters. The dictionary is not fsync'd (see + * {@code PersistedSymbolDict}), so a host/power crash can tear off its newest entries + * while the segment frames that introduced those ids survive -- and those newest frames, + * being the least likely to be acked, are exactly the ones that replay. Seeded from the + * short dictionary alone, this producer would hand its next new symbol an id those frames + * already define, putting two symbols on one id and silently misattributing values. The + * old code detected that and threw, which was safe but far too blunt: it bricked + * {@code build()} for slots the background drainer replays PERFECTLY, because the frames + * carry the torn-off symbols in their own deltas and {@code accumulateSentDict} rebuilds + * the dictionary from them. This method now rebuilds the producer from the same bytes, + * so a torn -- or entirely lost -- dictionary is recoverable whenever the surviving + * frames define the ids themselves. The next flush's write-ahead persist then re-writes + * those ids (it resumes from {@code pd.size()}), healing the side-file on disk. + *

+ * What still fails clean. A genuine GAP: the ids below a surviving frame's delta + * start were introduced by frames that were acked and TRIMMED away, so they lived only in + * the lost dictionary and nothing can rebuild them. + * {@code addRecoveredSymbolsTo} returns -1 for that and we throw. It is the same + * condition the send loop's replay guard ({@code deltaStart > sentDictCount}) trips on, so + * producer and drainer now agree on exactly which slots are recoverable, instead of the + * producer rejecting slots the drainer drains. + */ + private void seedGlobalDictionaryFromPersisted(PersistedSymbolDict pd) { + if (cursorEngine == null) { + return; + } + // 1. The dictionary's intact prefix. addRecoveredSymbol appends without de-dup, so + // the producer's size tracks pd.size() exactly -- which is what the send loop's + // mirror also seeds sentDictCount from. + // Pre-size before pouring the recovered symbols in. The default capacity is 64, + // so rebuilding a large dictionary rehashed the map ~log2(n/64) times, each pass + // O(current size) -- roughly doubling the rebuild and touching a growing table + // the whole way. recoveredMaxSymbolId + 1 is the upper bound the seed can reach. + long expected = Math.max(pd == null ? 0L : pd.recoveredSize(), + cursorEngine.recoveredMaxSymbolId() + 1L); + if (expected > globalSymbolDictionary.size() && expected <= Integer.MAX_VALUE) { + globalSymbolDictionary = new GlobalSymbolDictionary((int) expected); + } + int baseline = 0; + if (pd != null && pd.size() > 0) { + pd.addLoadedSymbolsTo(globalSymbolDictionary); + baseline = globalSymbolDictionary.size(); + } + // 2. Everything the surviving frames define above that prefix, straight out of their + // own delta sections -- the same bytes, in the same order, accumulateSentDict will + // feed the mirror as those frames go back on the wire. + long coverage = cursorEngine.addRecoveredSymbolsTo(baseline, globalSymbolDictionary); + if (coverage < 0) { + // A gap: the surviving frames reference ids below their own delta start, + // introduced by frames since acked and trimmed away, and the persisted + // dictionary no longer holds them (a host crash tore its unsynced tail, or it + // could not be opened). That gap only matters for frames that will REPLAY. + // When every recovered committed frame is already acked + // (ackedFsn >= recoveredCommitBoundaryFsn), NOTHING replays: the gap is in + // data the server already has, and the retired orphan-deferred tail above the + // commit boundary is never transmitted. Throwing here would raise a false + // "resend required" for delivered data AND -- because such a slot is fully + // drained -- let build()'s connect-path close unlink the (already-delivered) + // bytes the quarantine claims to preserve. So DON'T throw: seed the intact + // prefix only; addRecoveredSymbolsTo adds nothing on a -1 exactly as the + // send loop's mirror does, so the producer baseline and the mirror's + // sentDictCount still agree by construction. The producer resumes above the + // prefix and the fully-drained slot is cleaned up on close. + long ackedFsn = cursorEngine.ackedFsn(); + long commitBoundaryFsn = cursorEngine.recoveredCommitBoundaryFsn(); + if (ackedFsn >= commitBoundaryFsn) { + sentMaxSymbolId = globalSymbolDictionary.size() - 1; + LOG.info("recovered store-and-forward slot has a torn/incomplete symbol dictionary, " + + "but every committed frame was already acked so nothing needs replaying; " + + "resuming on the intact prefix without quarantine and without data loss " + + "[recoveredPrefixSize={}, ackedFsn={}, commitBoundaryFsn={}]", + baseline, ackedFsn, commitBoundaryFsn); + return; + } + // Genuine loss: unacked committed frames reference ids nothing still holds. + // Typed, because Sender.build() sets such a slot aside instead of failing: this + // is the point at which every source of truth has been tried and none of them + // holds the missing ids. See UnreplayableSlotException. + throw new UnreplayableSlotException( + "recovered store-and-forward symbol dictionary is incomplete and cannot be " + + "rebuilt from the surviving frames (likely a host crash tore its unsynced " + + "tail): the frames reference symbol ids below their own delta start, which " + + "were introduced by frames since acked and trimmed away, so nothing still " + + "holds them; the recovered dictionary holds only " + + (pd == null ? 0 : pd.size()) + " id(s) -- resend the affected data"); + } + // Producer baseline == the coverage the replay will establish == the mirror's + // sentDictCount once those frames have gone out. The first new frame therefore + // starts its delta exactly at the tip, and the replay guard passes. + sentMaxSymbolId = globalSymbolDictionary.size() - 1; + // ...but the baseline now sits ABOVE pd.size() whenever the frames contributed + // ids, so restore the write-ahead invariant right now rather than hoping a later + // batch reaches high enough to do it. See healPersistedDictionary. + healPersistedDictionary(pd); + } + + /** + * The symbol id below which the server already holds every dictionary entry, + * used as {@code confirmedMaxId} when encoding a frame. In delta mode this is + * the producer's monotonic sent watermark; in full-dict mode it is -1 so every + * frame re-ships the dictionary from id 0. + */ + private int symbolDeltaBaseline() { + return deltaDictEnabled ? sentMaxSymbolId : -1; + } + private void rollbackRow() { if (currentTableBuffer != null) { currentTableBuffer.cancelCurrentRow(); @@ -3920,7 +4519,7 @@ private void sealAndSwapBuffer() { // back to it; flushPendingRows aborts its post-enqueue state // updates after this throw, so the source rows stay intact and the // next batch re-emits the same rows along with the full inline - // schema and symbol-dict delta from id 0. + // schema and the symbol-dict delta the batch requires. if (toSend.isSending()) { toSend.markRecycled(); } else if (toSend.isSealed()) { @@ -3949,12 +4548,18 @@ private void sendRow() { // batch stay intact. (The check ignores the null-padding bytes // nextRow() will add; that's bounded by numColumns * elemSize and // far below any realistic cap.) - if (serverMaxBatchSize > 0) { + // Snapshot the volatile cap ONCE, as flushPendingRows does. The I/O thread + // lowers serverMaxBatchSize -- or clears it to 0 on a failover to a node that + // advertises no cap -- mid-stream via applyServerBatchSizeLimit. Re-reading the + // field across the guard and the throw could observe it drop to 0 between reads + // and reject the row against a "cap" of 0, which actually means "no cap". + int cap = serverMaxBatchSize; + if (cap > 0) { long rowBytes = currentTableBuffer.getBufferedBytes() - currentTableBufferSnapshotBytes; - if (rowBytes > serverMaxBatchSize) { + if (rowBytes > cap) { throw new LineSenderException("row too large for server batch cap") .put(" [rowBytes=").put(rowBytes) - .put(", serverMaxBatchSize=").put(serverMaxBatchSize).put(']'); + .put(", serverMaxBatchSize=").put(cap).put(']'); } } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketResponse.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketResponse.java index 015b25ef..81d59d28 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketResponse.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketResponse.java @@ -67,6 +67,12 @@ public class WebSocketResponse { public static final int MIN_DURABLE_ACK_SIZE = 3; // status + tableCount public static final int MIN_ERROR_RESPONSE_SIZE = 11; // status + sequence + error length public static final int MIN_OK_RESPONSE_SIZE = 11; // status + sequence + tableCount + /** + * A delta symbol dictionary began above the server's connection dictionary. Wire + * {@code 0x0D}. Server state decides this, not the frame bytes, so it is retriable: + * recycling the wire re-runs the catch-up from id 0 and the same frames then land. + */ + public static final byte STATUS_DICTIONARY_GAP = 0x0D; /** * Per-table durable-upload acknowledgment. Emitted by servers where * primary replication is enabled and the connection opted in via @@ -224,6 +230,8 @@ public String getStatusName() { return "INTERNAL_ERROR"; case STATUS_NOT_WRITABLE: return "NOT_WRITABLE"; + case STATUS_DICTIONARY_GAP: + return "DICTIONARY_GAP"; default: return "UNKNOWN(" + (status & 0xFF) + ")"; } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/BackgroundDrainer.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/BackgroundDrainer.java index 6f6d7eff..ce54c4b1 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/BackgroundDrainer.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/BackgroundDrainer.java @@ -36,6 +36,7 @@ import org.slf4j.LoggerFactory; import java.util.concurrent.ThreadLocalRandom; +import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.LockSupport; /** @@ -44,7 +45,9 @@ *

* Lifecycle: *

    - *
  1. Acquire the slot's {@code .lock}; skip silently on contention.
    2. + *
  2. Acquire the parent-anchored logical-slot lock and revalidate the + * scanner snapshot; skip silently on contention or a stale snapshot.
    3. + *
  3. Acquire the slot's {@code .lock}, then release the logical lock.
    4. *
  4. Open a {@link CursorSendEngine} on the slot — recovery picks up * every {@code .sfa} file already on disk.
    5. *
  5. Open a fresh {@link WebSocketClient} via the supplied factory @@ -78,12 +81,10 @@ public final class BackgroundDrainer implements Runnable { * wall-clock budget {@code reconnectMaxDurationMillis} also caps this * capability-gap loop; whichever is hit first triggers escalation. Both * halves of the budget measure a capability-gap episode: the - * wall clock accumulates only across uninterrupted gap-to-gap intervals - * (never before the first gap is observed, and never across an - * intervening transport window -- an unreachable cluster is not - * "failing to settle"), and an intervening role reject restarts the - * episode -- it proves the topology changed, so the next capability-gap - * error is a fresh episode against a newly promoted node. 16 + * wall clock accumulates only across uninterrupted gap-to-gap intervals. + * Any intervening transport or role state restarts the episode: capability + * gaps separated by an unrelated transient are not consecutive evidence + * of a persistent cluster capability mismatch. 16 * attempts gives the cluster room to settle through a rolling upgrade * (each attempt walks every endpoint internally) without letting a genuine * cluster-wide misconfig hang the drainer forever. @@ -142,6 +143,11 @@ public final class BackgroundDrainer implements Runnable { // Minimum wall-clock dwell before poison escalation, forwarded to every // drain loop; mirrors the owner sender's poison_min_escalation_window_millis. private final long poisonMinEscalationWindowMillis; + // Minimum wall-clock dwell a symbol-dict catch-up cap gap must persist before this + // orphan drainer's send loop latches a terminal. Foreground senders retry forever; + // this bounded policy is permitted only so a persistent orphan slot can be + // quarantined for operator intervention. + private final long catchUpCapGapMinEscalationWindowMillis; public BackgroundDrainer( String slotPath, @@ -159,7 +165,8 @@ public BackgroundDrainer( reconnectMaxBackoffMillis, requestDurableAck, durableAckKeepaliveIntervalMillis, CursorWebSocketSendLoop.DEFAULT_MAX_HEAD_FRAME_REJECTIONS, - CursorWebSocketSendLoop.DEFAULT_POISON_MIN_ESCALATION_WINDOW_MILLIS); + CursorWebSocketSendLoop.DEFAULT_POISON_MIN_ESCALATION_WINDOW_MILLIS, + CursorWebSocketSendLoop.DEFAULT_CATCHUP_CAP_GAP_MIN_ESCALATION_WINDOW_MILLIS); } /** @@ -179,13 +186,15 @@ public BackgroundDrainer( boolean requestDurableAck, long durableAckKeepaliveIntervalMillis, int maxHeadFrameRejections, - long poisonMinEscalationWindowMillis + long poisonMinEscalationWindowMillis, + long catchUpCapGapMinEscalationWindowMillis ) { this(slotPath, segmentSizeBytes, sfMaxTotalBytes, 0L, clientFactory, reconnectMaxDurationMillis, reconnectInitialBackoffMillis, reconnectMaxBackoffMillis, requestDurableAck, durableAckKeepaliveIntervalMillis, maxHeadFrameRejections, - poisonMinEscalationWindowMillis); + poisonMinEscalationWindowMillis, + catchUpCapGapMinEscalationWindowMillis); } /** @@ -204,7 +213,8 @@ public BackgroundDrainer( boolean requestDurableAck, long durableAckKeepaliveIntervalMillis, int maxHeadFrameRejections, - long poisonMinEscalationWindowMillis + long poisonMinEscalationWindowMillis, + long catchUpCapGapMinEscalationWindowMillis ) { this.slotPath = slotPath; this.segmentSizeBytes = segmentSizeBytes; @@ -218,6 +228,7 @@ public BackgroundDrainer( this.durableAckKeepaliveIntervalMillis = durableAckKeepaliveIntervalMillis; this.maxHeadFrameRejections = maxHeadFrameRejections; this.poisonMinEscalationWindowMillis = poisonMinEscalationWindowMillis; + this.catchUpCapGapMinEscalationWindowMillis = catchUpCapGapMinEscalationWindowMillis; } /** @@ -230,7 +241,7 @@ public BackgroundDrainer( @TestOnly public BackgroundDrainer() { this(null, 0L, 0L, null, 0L, 0L, 0L, false, 0L, - CursorWebSocketSendLoop.DEFAULT_MAX_HEAD_FRAME_REJECTIONS, 0L); + CursorWebSocketSendLoop.DEFAULT_MAX_HEAD_FRAME_REJECTIONS, 0L, 0L); } /** @@ -244,8 +255,9 @@ public BackgroundDrainer() { * durable ack -- i.e. the symptom of a misconfigured cluster or a * rolling-upgrade transient. *

    - * For the foreground sender that condition is loud-fail: the producer - * is actively pushing data. The drainer is asymmetric: source data is + * Blocking foreground startup keeps its fail-fast policy, while asynchronous + * foreground startup and reconnect keep buffering and retrying through a + * rolling capability change. The orphan drainer is asymmetric: source data is * pinned (durable-ack-mode trims only on STATUS_DURABLE_ACK frames, * which the offending endpoints by definition do not send), so we * give the cluster a budget to settle before quarantining the slot. @@ -258,21 +270,17 @@ public BackgroundDrainer() { * {@link QwpDurableAckMismatchException} sweeps only. Transient * conditions -- an all-replica failover window (role reject) or a * transport error -- are retried indefinitely (Invariant B) and never - * consume the budget: the wall-clock half accumulates only across - * uninterrupted gap-to-gap intervals, so a mid-episode transport window - * pauses the clock (without touching the attempt count), and a role - * reject additionally restarts the episode, because it proves the - * topology changed under the rolling upgrade. + * consume the budget. Either transient restarts the attempt count and wall + * clock so only uninterrupted capability-gap sweeps can escalate. * Genuine terminals (auth failure, non-421 upgrade reject) preserve * the original behavior: mark failed, exit. * * @return a fresh durable-ack-capable client, or {@code null} if * {@link #outcome} has been set to FAILED or STOPPED */ - @TestOnly public WebSocketClient connectWithDurableAckRetry() { // run() already set runnerThread; setting it again here is a no-op - // on that path but wires up direct @TestOnly calls so requestStop() + // on that path but wires up direct callers so requestStop() // can unpark them too. runnerThread = Thread.currentThread(); long backoffMillis = reconnectInitialBackoffMillis; @@ -280,25 +288,29 @@ public WebSocketClient connectWithDurableAckRetry() { // QwpDurableAckMismatchException sweeps; the wall-clock half // accumulates ONLY across uninterrupted gap-to-gap intervals, so // transient churn (role reject, transport) can never burn the budget - // -- neither before the first gap is observed nor mid-episode (a - // cluster unreachable for longer than the whole budget that comes - // back still gapped has consumed none of it). An intervening role - // reject resets the episode (topology churn: the offending node is - // gone); a transport error neither increments nor resets the attempt - // count -- a dropped socket does not prove promotion churn, and - // resetting on it would let a flaky-but-misconfigured cluster evade - // the cap forever -- it only pauses the wall clock: the gap-to-gap - // interval spanning the transport window is not charged. + // -- neither before the first gap is observed nor mid-episode. Any + // intervening role or transport state resets the episode: after the + // cluster leaves the capability-gap state, later gaps must establish a + // fresh consecutive run before quarantine is permitted. int capabilityGapAttempts = 0; // Wall-clock time accumulated across uninterrupted gap-to-gap // intervals of the current episode; escalates once it reaches // capabilityGapBudgetNanos (or the attempt cap fires first). long capabilityGapElapsedNanos = 0L; // Timestamp of the previous capability-gap sweep; 0 = the next gap - // charges nothing (episode start, post-role-reject restart, or the - // interval was interrupted by a transport window). + // charges nothing because a fresh episode is starting. long lastCapabilityGapNanos = 0L; - final long capabilityGapBudgetNanos = reconnectMaxDurationMillis * 1_000_000L; + // Saturate rather than multiply: reconnect_max_duration_millis is validated only + // as > 0, so a large value (Long.MAX_VALUE is the natural way to ask for "never + // give up") wraps a raw multiply NEGATIVE. capabilityGapElapsedNanos then clears + // the budget on the FIRST capability-gap sweep -- 0 >= a negative -- and, because + // this gate is an OR with the attempt cap, nothing else holds it back: the slot + // quarantines immediately, skipping the whole 16-sweep settle budget. Asking for + // more tolerance would buy exactly none. TimeUnit clamps at Long.MAX_VALUE, which + // is the intended "effectively unbounded". CursorWebSocketSendLoop's dwell + // conversion guards the same way for the same reason. + final long capabilityGapBudgetNanos = + TimeUnit.MILLISECONDS.toNanos(reconnectMaxDurationMillis); // Observability-only counter for the transient all-replica window; // never consulted for escalation (Invariant B). int roleRejectAttempts = 0; @@ -319,8 +331,7 @@ public WebSocketClient connectWithDurableAckRetry() { } catch (QwpAuthFailedException | WebSocketUpgradeException e) { // Genuinely non-retriable across the cluster (auth 401/403, or a // non-421 upgrade reject): waiting will not fix it, so quarantine - // immediately -- exactly as the live sender's background loop - // (CursorWebSocketSendLoop.connectLoop) halts on these errors. + // immediately under the orphan reconnect policy. String msg = e.getMessage(); LOG.error("drainer terminal upgrade/auth error for slot {}: {}", slotPath, msg); lastErrorMessage = msg; @@ -433,10 +444,11 @@ public WebSocketClient connectWithDurableAckRetry() { // Invariant B -- but it is NOT a transport outage, so log it // truthfully below rather than mislabelling it "cluster unreachable". lastErrorMessage = t.getMessage(); - // Pause the episode wall clock: the gap-to-gap interval this - // window interrupts is never charged. Attempts and elapsed - // already accumulated are preserved (anti-evasion: see the - // budget comment above). + // This unrelated state breaks the consecutive capability-gap + // run. Restart both halves of the settle budget so a later gap + // must establish a fresh episode before quarantine. + capabilityGapAttempts = 0; + capabilityGapElapsedNanos = 0L; lastCapabilityGapNanos = 0L; long nowWarn = System.nanoTime(); if (nowWarn - lastTransportWarnNanos >= 5_000_000_000L) { @@ -576,14 +588,50 @@ private boolean stopRequestedOrInterrupted() { @Override public void run() { runnerThread = Thread.currentThread(); + SlotLock logicalSlotLock = null; CursorSendEngine engine = null; WebSocketClient client = null; CursorWebSocketSendLoop loop = null; try { - // The engine acquires the slot's .lock itself — we don't need - // (and must not) double-lock it. If another sender or drainer - // holds it, the engine constructor throws and we exit silently - // (no .failed sentinel — contention is expected, not an error). + // Scanner results are only snapshots. Serialize adoption against + // a producer's close -> quarantine rename -> fresh-slot recreate + // transition, then revalidate while that stable parent-anchored + // lock is held. The slot's own .lock inode moves with a rename and + // cannot provide this guarantee by itself. + if (slotPath != null) { + try { + logicalSlotLock = SlotLock.acquireLogical(slotPath); + } catch (SlotLockContentionException t) { + LOG.info("orphan logical slot already locked, skipping: {} ({})", + slotPath, t.getMessage()); + outcome = DrainOutcome.LOCKED_BY_OTHER; + return; + } catch (Exception t) { + // Everything else here is LOCAL and pre-adoption: a permission + // problem on the shared .slot-locks directory, momentary fd + // exhaustion, an unwritable parent. It must NOT reach the outer + // catch, which writes the .failed sentinel unconditionally -- + // OrphanScanner.isCandidateOrphan treats that sentinel as + // disqualifying and nothing ever removes it, so a healthy slot + // whose real owner later dies would be stranded until an + // operator intervened. Report FAILED and let the next scan retry. + String msg = t.toString(); + LOG.warn("drainer could not take the logical slot lock for {}: {}", + slotPath, msg, t); + lastErrorMessage = msg; + outcome = DrainOutcome.FAILED; + return; + } + if (!OrphanScanner.isCandidateOrphan(slotPath)) { + LOG.info("orphan candidate changed before adoption, skipping: {}", slotPath); + outcome = DrainOutcome.SUCCESS; + return; + } + } + + // The engine acquires the directory-local .lock itself. Keep the + // lock order logical -> local, and release the short-lived logical + // lock only after the engine has secured stable ownership. try { try { engine = new CursorSendEngine(slotPath, segmentSizeBytes, @@ -618,21 +666,43 @@ public void run() { // Repeated scans cannot repair it, so preserve the terminal // quarantine path in the outer catch below. throw t; + } catch (UnreplayableSlotException t) { + // The constructor assigns this.slotLock (CursorSendEngine's own + // .lock, taken under SlotLock.acquire) before the try that opens + // the ring, so this throw proves the ring was opened under that + // lock -- adoption happened, even though the local `engine` + // reference here is still null. Quarantine explicitly instead of + // falling into the retryable catch below: the verdict this type + // carries is that the slot's symbol dictionary cannot be rebuilt + // from ANY source, which no number of retries changes. Placed + // after the SfSanitizedResidueException retry and before the + // catch-all so an operational failure -- which must NOT + // permanently quarantine an orphan slot -- still lands on the + // retryable arm. + String msg = t.getMessage(); + LOG.error("drainer slot {} is unreplayable, quarantining: {}", slotPath, msg); + lastErrorMessage = msg; + OrphanScanner.markFailed(slotPath, "unreplayable: " + msg); + outcome = DrainOutcome.FAILED; + return; } catch (Exception t) { // Every other pre-publication construction exception is // retryable: setup I/O, resource pressure, unexpected setup // faults, and future operational failures do not prove durable - // data corruption. The constructor has closed partial - // resources; SlotLock retains and retries any unconfirmed - // unlock. Leave no .failed sentinel for the next orphan scan. - // Error deliberately escapes to the outer Error path: it is - // observable after teardown but cannot quarantine intact data. - // This path retries on every orphan scan, so the log line is - // the ONLY diagnostic a deterministic bug (e.g. an unexpected - // NPE, whose getMessage() is null) ever produces: attach the - // throwable for the stack trace and carry class+message into - // the telemetry surface, mirroring the outer setup-failure - // catch below. + // data corruption. In particular SfOperationalException lands + // here (it extends IllegalStateException), which is exactly + // where it must land -- an EMFILE/ENOMEM during setup must + // never permanently quarantine an orphan slot. The constructor + // has closed partial resources; SlotLock retains and retries any + // unconfirmed unlock. Leave no .failed sentinel for the next + // orphan scan. Error deliberately escapes to the outer Error + // path: it is observable after teardown but cannot quarantine + // intact data. This path retries on every orphan scan, so the + // log line is the ONLY diagnostic a deterministic bug (e.g. an + // unexpected NPE, whose getMessage() is null) ever produces: + // attach the throwable for the stack trace and carry + // class+message into the telemetry surface, mirroring the outer + // setup-failure catch below. String msg = t.toString(); LOG.warn("drainer setup temporarily unavailable for slot {}: {}", slotPath, msg, t); @@ -641,6 +711,15 @@ public void run() { return; } engineForTesting = engine; + if (logicalSlotLock != null) { + logicalSlotLock.close(); + logicalSlotLock = null; + } + // A recovered deferred-only tail is an aborted transaction and can + // be retired locally once everything below it is already ACKed. + // Do this before opening a socket: auth/upgrade failures must not + // quarantine a slot that has no wire-visible work left. + engine.retireRecoveredOrphanTailIfReady(); long target = engine.publishedFsn(); if (engine.ackedFsn() >= target) { LOG.info("orphan slot already drained: {} (acked={} target={})", @@ -668,13 +747,14 @@ public void run() { client, engine, 0L, CursorWebSocketSendLoop.DEFAULT_PARK_NANOS, clientFactory, - reconnectMaxDurationMillis, reconnectInitialBackoffMillis, reconnectMaxBackoffMillis, requestDurableAck, durableAckKeepaliveIntervalMillis, maxHeadFrameRejections, - poisonMinEscalationWindowMillis); + poisonMinEscalationWindowMillis, + catchUpCapGapMinEscalationWindowMillis, + CursorWebSocketSendLoop.ReconnectPolicy.ORPHAN); loop.start(); while (!stopRequestedOrInterrupted()) { @@ -781,6 +861,21 @@ public void run() { LOG.debug("drainer setup failed for slot {}: {}", slotPath, msg, t); } lastErrorMessage = msg; + // Everything reaching here is terminal by construction, so the + // sentinel is unconditional -- notably SfRecoveryException and + // MmapSegmentCorruptionException, which the construction catch + // deliberately rethrows for exactly this treatment. A null engine + // does NOT mean "pre-adoption" for those: CursorSendEngine assigns + // its slotLock field before the try that opens the ring, so the + // throw proves the ring was opened under the slot lock. + // + // The retryable, pre-adoption failures never get this far: lock + // contention, operational setup errors and the logical-lock + // acquisition above all return on their own typed arms without a + // sentinel. That matters because OrphanScanner.isCandidateOrphan + // treats .failed as disqualifying and nothing ever removes it, so a + // sentinel on a transient failure would strand a healthy slot's + // unacked data once its real owner died. try { OrphanScanner.markFailed(slotPath, "setup: " + msg); } catch (Throwable ignored) { @@ -835,6 +930,9 @@ public void run() { + "slot lock releases when it exits", slotPath); } } + if (logicalSlotLock != null) { + logicalSlotLock.close(); + } // Don't let a later requestStop() unpark an unrelated task that // the pool's executor may have scheduled onto this same thread. runnerThread = null; diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java index 80f54486..ec9aba41 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java @@ -24,6 +24,7 @@ package io.questdb.client.cutlass.qwp.client.sf.cursor; +import io.questdb.client.cutlass.qwp.client.GlobalSymbolDictionary; import io.questdb.client.std.Compat; import io.questdb.client.std.Files; import io.questdb.client.std.FilesFacade; @@ -93,6 +94,11 @@ public final class CursorSendEngine implements QuietCloseable { // callback allocation failure cannot orphan manager resources. A timed-out // close can then hand it to either manager path without allocating. private final Runnable deferredClose; + // Facade for the persisted symbol dictionary only. Separate from + // filesFacade (which the manager supplies for segments, watermark and + // manifest) so a test can fault-inject a dictionary write without also + // breaking the segment chain underneath it. + private final FilesFacade dictFf; private final FilesFacade filesFacade; private final SegmentManager manager; // We own the manager iff the user constructed us with no manager — in that @@ -109,9 +115,9 @@ public final class CursorSendEngine implements QuietCloseable { private final SlotLock slotLock; // True when the constructor recovered an existing on-disk slot rather // than starting fresh. Diagnostic accessor for tests and observability; - // cursor frames are self-sufficient (every frame carries full schema + - // full symbol-dict delta), so producer-side schema reset on recovery - // is not required. + // every frame carries its full inline schema, so producer-side schema reset + // on recovery is not required (the symbol dictionary, which delta frames do + // NOT carry in full, is re-registered by an I/O-thread catch-up instead). private final boolean wasRecoveredFromDisk; // FSN of the last commit-bearing (non-FLAG_DEFER_COMMIT) frame found in a // ring recovered from disk, or -1 for fresh/memory rings and recovered @@ -121,6 +127,27 @@ public final class CursorSendEngine implements QuietCloseable { // covers them. Read by the sender's close-time drain to avoid waiting on // acks that cannot arrive. private long recoveredCommitBoundaryFsn = -1L; + // Highest symbol id any recovered delta frame references, or -1 for + // fresh/memory rings (and recovered rings with no symbol-bearing frame). A + // resuming producer seeds its dictionary baseline from the persisted + // .symbol-dict; if that dictionary was torn below this id by a host crash + // (the side-file is not fsync'd), the producer would re-use ids the surviving + // frames already define. seedGlobalDictionaryFromPersisted compares this + // against the recovered dictionary size to fail clean instead. Computed once + // in the constructor's recovery branch; -1 elsewhere. + private long recoveredMaxSymbolId = -1L; + // How many times the constructor folded the recovered ring. Observable because + // recoveryFramesVisited() cannot be: the full-dict-fallback re-fold REPLACES the + // analysis instance, resetting that counter, so a second fold is invisible through it. + private int recoveryFoldCount; + // Highest deltaStart across the recovered COMMITTED frames; 0 when none carries a symbol + // dictionary. ZERO means every surviving frame is SELF-SUFFICIENT -- it re-registers its + // dictionary from id 0 -- so the slot replays with no dictionary at all and the send loop + // needs no catch-up. ABOVE zero means at least one frame is a true delta whose ids depend + // on registrations it does not itself carry, so the loop must seed its mirror (and ship a + // catch-up) before replaying. Both the full-dict-fallback discard below and the send + // loop's mirror seeding key off this. + private long recoveredMaxSymbolDeltaStart; // FSN of the last frame of a recovered orphaned deferred tail, or -1 when // the recovered ring has no such tail. When >= 0, frames // [recoveredCommitBoundaryFsn + 1 .. recoveredOrphanTipFsn] all carry @@ -136,6 +163,17 @@ public final class CursorSendEngine implements QuietCloseable { // constructor, closed by {@link #close()}. The segment manager writes // through this on every tick where ackedFsn has advanced. private final AckWatermark watermark; + // Engine-owned per-slot symbol dictionary file (disk mode only; {@code null} + // in memory mode and if open() failed). Enables delta-encoded SF frames: + // recovery / orphan-drain load it to re-register the dictionary on the fresh + // server before replaying non-self-sufficient frames. Opened in the + // constructor, closed by {@link #close()}. When null in disk mode the engine + // reports delta encoding as unavailable and the sender keeps full-dict frames. + private final PersistedSymbolDict persistedSymbolDict; + // Engine-owned output of the single ordered recovery walk. It is retained + // because both producer seeding and every recycled send loop need the same + // frame-rebuilt symbol suffix. Null for fresh and memory-only engines. + private final RecoveredFrameAnalysis recoveredFrameAnalysis; // close() is publicly callable from any thread (Sender.close from a user // thread, JVM shutdown hooks, test cleanup). volatile + synchronized // close() makes the check-and-set atomic and gives readers a fence. @@ -168,6 +206,14 @@ public final class CursorSendEngine implements QuietCloseable { // Ensures this engine has at most one entry in the shared flock-release // retry driver. The error path only: ordinary closes never enqueue work. private final AtomicBoolean flockReleaseRetryStarted = new AtomicBoolean(); + // False only when a caller ABOVE this engine holds the parent-anchored + // logical slot lock (Sender.build() across construct -> connect -> + // quarantine). finishClose then skips the fully-drained unlink of that + // lock file, which would otherwise free the pathname while the caller + // still holds the flock on it. Latched by close(boolean) before any + // cleanup runs; volatile because finishClose can run later on the manager + // worker's exit thread or the shared flock-release retry driver. + private volatile boolean reclaimLogicalSlotLock = true; // Published before deferredClose is registered. The manager lock provides // the callback handoff fence; volatile also covers a direct test/retry read. private volatile boolean fullyDrainedForDeferredClose; @@ -216,6 +262,29 @@ public CursorSendEngine(String sfDir, long segmentSizeBytes, this(sfDir, segmentSizeBytes, maxTotalBytes, appendDeadlineNanos, 0L); } + /** + * As {@link #CursorSendEngine(String, long, long, long)}, but with an explicit + * {@link FilesFacade} for the persisted symbol dictionary. + *

    + * The seam exists so a test can drive a dictionary I/O fault -- a short write from + * a full disk or an exhausted quota -- through the REAL producer path + * ({@code flush()} -> the write-ahead persist), and assert it surfaces as a + * {@code LineSenderException} like every other flush-path failure rather than as a + * raw {@code IllegalStateException} that would sail past every caller's + * {@code catch (LineSenderException)}. Nothing else could reach that translation: + * {@code PersistedSymbolDict} has facade-aware overloads, but the engine used to + * call only the {@code FilesFacade.INSTANCE} ones, so no fault could be injected + * from outside. Deliberately narrower than the manager's facade: the segment + * chain stays on the production one, so a dictionary fault cannot be confused + * with a recovery failure. + */ + @TestOnly + public CursorSendEngine(String sfDir, long segmentSizeBytes, + long maxTotalBytes, long appendDeadlineNanos, FilesFacade dictFf) { + this(sfDir, segmentSizeBytes, null, true, maxTotalBytes, + appendDeadlineNanos, 0L, dictFf); + } + /** * Creates an engine with an optional periodic data-checkpoint interval. * A positive interval requires disk-backed store-and-forward mode. @@ -257,6 +326,14 @@ private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager man private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager manager, boolean ownsManager, long maxTotalBytes, long appendDeadlineNanos, long syncIntervalNanos) { + this(sfDir, segmentSizeBytes, manager, ownsManager, maxTotalBytes, + appendDeadlineNanos, syncIntervalNanos, FilesFacade.INSTANCE); + } + + private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager manager, + boolean ownsManager, long maxTotalBytes, long appendDeadlineNanos, + long syncIntervalNanos, FilesFacade dictFf) { + this.dictFf = dictFf; // Allocate the bound callback before constructing an owned manager. // Field initializers have completed, but no engine-owned native/disk // resource exists yet. If callback allocation throws, construction @@ -320,11 +397,21 @@ private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager man // reference instead of orphaning the mmap'd segments + fds. SegmentRing ringInProgress = null; AckWatermark watermarkInProgress = null; + PersistedSymbolDict persistedDictInProgress = null; + RecoveredFrameAnalysis recoveredFrameAnalysisInProgress = null; try { // Disk mode: try to recover any *.sfa files left behind by a prior // session before deciding to start fresh. Without this the engine // would create a new sf-initial.sfa at baseSeq=0, overlapping FSNs // already on disk and corrupting ACK translation, trim, and replay. + // May throw UnreplayableSlotException or SfRecoveryException / + // MmapSegmentCorruptionException (the durable chain is proven corrupt + // or incomplete) -- the catch (Throwable) below cleans up and lets + // them propagate so the constructor's two callers, Sender.build() and + // BackgroundDrainer, can quarantine the slot instead of seeding the + // ack past frames that were never sent. A plain MmapSegmentException + // is operational: it aborts startup without quarantining, so a + // transient EMFILE/ENOMEM can never silently drop durable frames. SegmentRing.Recovery recovery = memoryMode ? null : SegmentRing.recover(filesFacade, sfDir, segmentSizeBytes); SegmentRing recovered = recovery == null ? null : recovery.ring(); @@ -386,6 +473,12 @@ private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager man throw new SfOperationalException( "could not open required ack watermark for SF slot " + sfDir); } + // Load the persisted symbol dictionary so delta-encoded frames + // in this recovered slot can be re-registered on the fresh + // server before replay. Returns null on any I/O or parse + // failure, in which case the sender degrades to full + // self-sufficient frames for this session. + persistedDictInProgress = PersistedSymbolDict.open(dictFf, sfDir); long baseSeed = lowestBase - 1; long watermarkFsn = watermarkInProgress.read(); // Reject watermarks past publishedFsn: a correctly @@ -403,6 +496,13 @@ private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager man if (seed >= 0) { recovered.acknowledge(seed); } + // Fold the whole recovered ring once. The result checkpoints all + // running metadata and raw symbol bytes at each commit-bearing + // frame, so its final snapshot excludes an orphan deferred tail + // without requiring a second bounded scan. + recoveredFrameAnalysisInProgress = recovered.analyzeRecovery( + persistedDictInProgress == null ? 0 : persistedDictInProgress.size()); + recoveryFoldCount++; // Locate the last commit-bearing frame below a potentially // orphaned FLAG_DEFER_COMMIT tail. A producer that crashed (or // closed) mid-transaction leaves deferred frames with no @@ -412,12 +512,7 @@ private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager man // drainOnClose), and (b) replaying them into a NEW session's // commit would resurrect half a transaction -- see the WARN // below. Computed before the I/O loop or producer append. - this.recoveredCommitBoundaryFsn = recovered.findLastFsnWithoutPayloadFlag( - io.questdb.client.cutlass.qwp.protocol.QwpConstants.HEADER_OFFSET_FLAGS, - io.questdb.client.cutlass.qwp.protocol.QwpConstants.FLAG_DEFER_COMMIT, - io.questdb.client.cutlass.qwp.protocol.QwpConstants.MAGIC_MESSAGE, - io.questdb.client.cutlass.qwp.protocol.QwpConstants.HEADER_SIZE - ); + this.recoveredCommitBoundaryFsn = recoveredFrameAnalysisInProgress.commitBoundaryFsn(); if (publishedFsn >= 0 && recoveredCommitBoundaryFsn < publishedFsn) { this.recoveredOrphanTipFsn = publishedFsn; LOG.warn("recovered SF log ends with {} deferred frame(s) whose transaction was never " @@ -427,6 +522,84 @@ private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager man publishedFsn - Math.max(recoveredCommitBoundaryFsn, -1L), recoveredCommitBoundaryFsn, publishedFsn); } + // Highest symbol id the surviving COMMITTED frames reference. A + // resuming producer compares this against its recovered dictionary + // size (seedGlobalDictionaryFromPersisted) to detect a host-crash + // tear: if a committed frame references an id the (unsynced, torn) + // .symbol-dict no longer holds, resuming would re-use it. The walk is + // bounded to recoveredCommitBoundaryFsn so the aborted orphan-deferred + // tail -- retired without ever being transmitted -- does not inflate + // this and over-reject an otherwise-recoverable slot. maxDeltaEnd() + // returns 0 when no such frame carries a symbol, yielding -1 here. + // Computed before the I/O loop or producer append; single-threaded. + this.recoveredMaxSymbolId = recoveredFrameAnalysisInProgress.maxDeltaEnd() - 1L; + // Full-dict-fallback recovery. When the persisted .symbol-dict is a + // SUBSET of the ids the surviving frames reference + // (recoveredMaxSymbolId >= its size) YET every such frame is + // self-sufficient (maxDeltaStart() == 0 -- a full-dict frame that + // re-registers its dictionary from id 0), the slot was written in + // full-dict fallback: the dictionary never opened when writing, so no + // side-file exists and this recovery opened a FRESH EMPTY one. Those + // frames replay with no dictionary, so discard the empty side-file and + // recover in full-dict mode -- isDeltaDictEnabled() then reports false + // and the producer + send loop both run full-dict, exactly as the slot + // was written. Without this the sender's seed-time guard would treat the + // empty dictionary as a host-crash tear and brick build(), even though + // the orphan drainer drains the same frames fine. A genuine torn DELTA + // dictionary keeps a frame with deltaStart > 0 (maxDeltaStart() > 0) + // and is NOT discarded here: it still fails clean at seed time, since + // the ids its delta frames reference cannot be rebuilt without the lost + // dictionary. The recoveredMaxSymbolId >= size guard means this never + // fires for a slot whose dictionary is intact, nor for an empty slot + // (recoveredMaxSymbolId == -1). Single-threaded; before the I/O loop. + this.recoveredMaxSymbolDeltaStart = recoveredFrameAnalysisInProgress.maxDeltaStart(); + if (persistedDictInProgress != null + && recoveredMaxSymbolId >= persistedDictInProgress.size() + && recoveredMaxSymbolDeltaStart == 0L) { + // Only a NON-EMPTY discarded dictionary can desynchronise the fold. The + // first fold was keyed to this very size, so when it is 0 the re-fold + // below recomputes the identical baseline-0 result -- a no-op. The guard + // exists for the case below where discardedSize is genuinely > 0: the + // re-fold there is a second O(payload bytes) walk -- with a + // byte-at-a-time varint decode over full-dict frames, which carry the + // whole dictionary -- so it is worth paying only when the mismatch it + // resolves is real. + int discardedSize = persistedDictInProgress.size(); + persistedDictInProgress.close(); + persistedDictInProgress = null; + // Re-fold at baseline 0. The analysis above was keyed to the + // dictionary's size, and every consumer of it presents the baseline it + // derived the same way -- seedGlobalDictionaryFromPersisted computes + // baseline 0 once pd is gone, and checkedRecoveryAnalysis rejects a + // baseline that disagrees with the fold. Discarding a dictionary that + // held entries (size > 0) therefore desynchronised the two and threw + // "recovery symbol baseline mismatch" out of build(). checkedRecoveryAnalysis + // now raises that as an UnreplayableSlotException, so build() would at least + // set the slot aside rather than rethrow forever -- but quarantining is + // still the wrong outcome HERE, because this slot is fully recoverable + // (its frames carry their whole dictionary inline). Re-folding avoids the + // mismatch entirely instead of trading a permanent brick for a needless + // quarantine plus a "resend the affected data" the operator does not owe. + // + // Reachable on one transient plus one crash: a session whose + // .symbol-dict fails to open (EIO, fd exhaustion, a Windows share + // lock) falls back to full-dict frames and, per the never-recreate + // contract, leaves the previous session's populated side-file intact; + // if that session then crashes, this recovery opens a dictionary with + // size > 0 next to self-sufficient frames that out-reach it. + // + // Re-folding rather than keeping the dictionary preserves the discard's + // whole point -- the slot recovers in full-dict mode, exactly as it was + // written, with producer, mirror and replay guard all anchored at 0. + if (discardedSize > 0) { + recoveredFrameAnalysisInProgress.close(); + recoveredFrameAnalysisInProgress = recovered.analyzeRecovery(0); + recoveryFoldCount++; + this.recoveredCommitBoundaryFsn = recoveredFrameAnalysisInProgress.commitBoundaryFsn(); + this.recoveredMaxSymbolId = recoveredFrameAnalysisInProgress.maxDeltaEnd() - 1L; + this.recoveredMaxSymbolDeltaStart = recoveredFrameAnalysisInProgress.maxDeltaStart(); + } + } } else { // Fresh start with no recovered segments. Any stale // watermark from a prior fully-drained session refers @@ -440,6 +613,28 @@ private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager man throw new SfOperationalException( "could not open required ack watermark for SF slot " + sfDir); } + // A fresh slot MUST start with an EMPTY symbol dictionary. + // Unlike the ack watermark above -- a discardable optimization a + // max() clamp protects -- the dictionary is load-bearing: a + // delta frame referencing an id missing from it is unrecoverable, + // and a STALE dictionary inherited here (the segments are gone, so + // the producer is NOT seeded from it) shifts the dense id->symbol + // mapping and silently misattributes symbols on the next + // reconnect. openClean() truncates any survivor to empty rather + // than trusting a best-effort delete that may have failed (e.g. a + // Windows share lock). If the file is simply ABSENT and creating it + // fails, persistedSymbolDict stays null and the sender falls back to + // full self-sufficient frames -- safe for this session, since there + // is no prior generation's id space to lose. But if a dictionary + // DOES survive here and cannot be truncated, that fallback is only + // safe for THIS session: the next recovery would seed its catch-up + // from the stale file and replay this generation's frames on top, + // misattributing symbols with no detectable gap. So openClean() + // refuses that case outright (throws UnreplayableSlotException) + // instead of degrading to null; the catch (Throwable) below cleans + // up and lets it propagate so Sender.build() can quarantine this + // fresh, dataless slot instead of bricking every restart. + persistedDictInProgress = PersistedSymbolDict.openClean(dictFf, sfDir); } MmapSegment initial; String initialPath = null; @@ -503,10 +698,12 @@ private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager man manager.start(); } manager.register(ringInProgress, sfDir, watermarkInProgress, syncIntervalNanos); - // All construction succeeded — commit the ring and - // watermark references. + // All construction succeeded -- commit the ring, watermark, and + // symbol-dictionary references. this.ring = ringInProgress; this.watermark = watermarkInProgress; + this.persistedSymbolDict = persistedDictInProgress; + this.recoveredFrameAnalysis = recoveredFrameAnalysisInProgress; } catch (Throwable t) { // Stop an owned manager before freeing the ring and watermark it may // touch, then release the slot lock. Each cleanup is in its own @@ -532,6 +729,18 @@ private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager man } catch (Throwable ignored) { } } + if (persistedDictInProgress != null) { + try { + persistedDictInProgress.close(); + } catch (Throwable ignored) { + } + } + if (recoveredFrameAnalysisInProgress != null) { + try { + recoveredFrameAnalysisInProgress.close(); + } catch (Throwable ignored) { + } + } if (acquiredLock != null) { try { acquiredLock.close(); @@ -659,7 +868,27 @@ public void checkDurability() { } @Override - public synchronized void close() { + public void close() { + close(true); + } + + /** + * As {@link #close()}, but {@code reclaimLogicalSlotLock == false} skips the + * parent-anchored logical slot-lock unlink at fully-drained retirement. + *

    + * A caller that HOLDS that lock must pass {@code false}. {@code Sender.build()} + * keeps it across engine construction and connect, and closes the engine from + * inside that scope when connect fails -- and a fresh slot is "fully drained" by + * definition ({@code publishedFsn() < 0}), so the default path would unlink the + * lock file while build() still holds the flock on it. On POSIX that frees the + * pathname without releasing the lock, so the next {@code acquireLogical} creates + * a SECOND inode and locks it successfully: two parties owning a lock whose only + * job is serialising the quarantine close->rename->recreate window. + */ + public synchronized void close(boolean reclaimLogicalSlotLock) { + // Latch before the early return: a retried close() must not widen a + // previous caller's narrower reclaim decision. + this.reclaimLogicalSlotLock &= reclaimLogicalSlotLock; if (closed && closeCompleted) return; closed = true; // Capture drain state BEFORE closing the ring — once the ring is @@ -905,6 +1134,18 @@ private void finishClose(boolean fullyDrained) { } catch (Throwable ignored) { } } + if (persistedSymbolDict != null) { + try { + persistedSymbolDict.close(); + } catch (Throwable ignored) { + } + } + if (recoveredFrameAnalysis != null) { + try { + recoveredFrameAnalysis.close(); + } catch (Throwable ignored) { + } + } if (fullyDrained && watermark != null) { boolean segmentsRemoved = false; try { @@ -928,6 +1169,26 @@ private void finishClose(boolean fullyDrained) { + "engine on this slot recovers them as fully acked and retries the " + "unlink on its own close", sfDir); } + try { + // Slot fully drained: the dictionary has no frames behind it. + PersistedSymbolDict.removeOrphan(sfDir); + } catch (Throwable ignored) { + } + if (reclaimLogicalSlotLock) { + try { + // The logical slot lock lives OUTSIDE the slot dir (in the + // shared .slot-locks dir) so it survives a slot rename; the + // fully-drained retirement that removes this slot's other + // side-files must remove it too, or .slot-locks accumulates a + // dead lock+pid pair per distinct slot name for the lifetime of + // sf_dir. Holding the directory-local lock is NOT sufficient to + // make this safe -- it says nothing about the LOGICAL lock, which + // a caller higher in the same stack may hold. Only a caller that + // knows it does not hold it may reclaim, hence the flag. + SlotLock.removeOrphanLogical(sfDir); + } catch (Throwable ignored) { + } + } } if (durabilityFailure != null) { throw durabilityFailure; @@ -1265,6 +1526,79 @@ public void ensureFlockReleaseRetryScheduled() { } } + /** + * Decodes the cached recovery suffix directly into the producer's global + * dictionary. Recovery always builds the analysis with the persisted + * prefix size as its baseline, so no intermediate cardinality-sized list is + * needed on the production path. + */ + public long addRecoveredSymbolsTo(int baseline, GlobalSymbolDictionary target) { + if (recoveredFrameAnalysis == null) { + return baseline; + } + RecoveredFrameAnalysis analysis = checkedRecoveryAnalysis(baseline); + long coverage = analysis.coverage(); + if (coverage >= 0L) { + analysis.addDecodedSymbolsTo(target); + } + return coverage; + } + + long recoveredSymbolCoverage(int baseline) { + return checkedRecoveryAnalysis(baseline).coverage(); + } + + int recoveredSymbolSuffixCount(int baseline) { + return checkedRecoveryAnalysis(baseline).rawCount(); + } + + int recoveredSymbolSuffixLen(int baseline) { + return checkedRecoveryAnalysis(baseline).rawLen(); + } + + void copyRecoveredSymbolSuffix(int baseline, long target) { + RecoveredFrameAnalysis analysis = checkedRecoveryAnalysis(baseline); + int len = analysis.rawLen(); + if (len > 0) { + io.questdb.client.std.Unsafe.getUnsafe().copyMemory(analysis.rawAddr(), target, len); + } + } + + @TestOnly + public int recoveryFoldCount() { + return recoveryFoldCount; + } + + @TestOnly + public long recoveryFramesVisited() { + return recoveredFrameAnalysis == null ? 0L : recoveredFrameAnalysis.framesVisited(); + } + + @TestOnly + public long recoverySymbolEntriesVisited() { + return recoveredFrameAnalysis == null ? 0L : recoveredFrameAnalysis.symbolEntriesVisited(); + } + + @TestOnly + public int recoverySymbolNativeCapacity() { + return recoveredFrameAnalysis == null ? 0 : recoveredFrameAnalysis.rawCapacity(); + } + + private RecoveredFrameAnalysis checkedRecoveryAnalysis(int baseline) { + if (recoveredFrameAnalysis == null || recoveredFrameAnalysis.baseline() != baseline) { + // UnreplayableSlotException, NOT IllegalStateException: Sender.build() routes + // on the type, and only this type reaches its quarantine handler. A raw + // IllegalStateException escapes build() instead, and because senderId is + // stable and a not-fully-drained slot is retained on close, every restart + // re-recovers the same slot and rethrows -- the application can never + // construct a Sender, so it cannot even BUFFER new rows. + throw new UnreplayableSlotException("recovery symbol baseline mismatch [expected=" + + (recoveredFrameAnalysis == null ? "none" : recoveredFrameAnalysis.baseline()) + + ", actual=" + baseline + ']'); + } + return recoveredFrameAnalysis; + } + /** * Pass-through to {@link SegmentRing#findSegmentContaining(long)}. */ @@ -1283,6 +1617,15 @@ public MmapSegment firstSealed() { return ring.firstSealed(); } + /** + * The engine's persisted symbol dictionary, or {@code null} in memory mode + * (and in disk mode if it failed to open). The producer appends new symbols + * to it; recovery / orphan-drain read its loaded entries to seed catch-up. + */ + public PersistedSymbolDict getPersistedSymbolDict() { + return persistedSymbolDict; + } + /** * Number of times {@link #appendBlocking} hit * {@link SegmentRing#BACKPRESSURE_NO_SPARE} on its first attempt and @@ -1293,6 +1636,18 @@ public long getTotalBackpressureStalls() { return backpressureStallCount.get(); } + /** + * Whether the sender may delta-encode symbol dictionaries on this engine. + * Always true in memory mode (the send loop keeps an in-process catch-up + * mirror). In disk mode it requires the persisted dictionary to have opened, + * since delta frames are not self-sufficient and recovery / orphan-drain must + * be able to rebuild the dictionary from disk. When false in disk mode the + * sender falls back to full self-sufficient frames. + */ + public boolean isDeltaDictEnabled() { + return sfDir == null || persistedSymbolDict != null; + } + /** * Pass-through to {@link SegmentRing#nextSealedAfter(MmapSegment)}. */ @@ -1365,6 +1720,31 @@ public long recoveredCommitBoundaryFsn() { return recoveredCommitBoundaryFsn; } + /** + * Highest symbol id any recovered delta frame references, or {@code -1} for + * fresh/memory rings (and recovered rings with no symbol-bearing frame). A + * resuming producer compares this against its recovered dictionary size to + * detect a host-crash tear of the persisted {@code .symbol-dict}. + */ + public long recoveredMaxSymbolId() { + return recoveredMaxSymbolId; + } + + /** + * Highest {@code deltaStart} across the recovered committed frames; {@code 0} when every + * surviving frame is self-sufficient (or none carries a dictionary at all). + *

    + * The send loop uses this to decide whether it needs a catch-up: at zero, every frame + * re-registers its dictionary from id 0 as it replays, so seeding the mirror -- and + * shipping a catch-up frame off it -- would be pure redundancy. Above zero, at least one + * frame's delta starts above ids it does not itself carry, so the mirror must hold those + * ids before the replay begins, or the server rejects the frame with + * STATUS_DICTIONARY_GAP. + */ + public long recoveredMaxSymbolDeltaStart() { + return recoveredMaxSymbolDeltaStart; + } + /** * FSN of the last frame of a recovered orphaned deferred tail, or * {@code -1} when none. See {@link #recoveredCommitBoundaryFsn()}: the @@ -1374,6 +1754,31 @@ public long recoveredOrphanTipFsn() { return recoveredOrphanTipFsn; } + /** + * Retires a recovered deferred tail once every frame below it is ACKed. + * The operation is local and idempotent: no wire sequence ever referred + * to these aborted-transaction frames. + * + * @return true if no orphan tail remains, false if lower frames still need + * server ACKs + */ + public boolean retireRecoveredOrphanTailIfReady() { + long orphanTip = recoveredOrphanTipFsn; + if (orphanTip < 0L) { + return true; + } + long orphanStart = recoveredCommitBoundaryFsn + 1L; + if (ackedFsn() < orphanStart - 1L) { + return false; + } + LOG.warn("retiring orphaned deferred tail: {} frame(s) [fsn {}..{}] belong to a transaction " + + "whose commit was never published; aborting them (never transmitted, slots trimmed)", + orphanTip - orphanStart + 1L, orphanStart, orphanTip); + acknowledge(orphanTip); + recoveredOrphanTipFsn = -1L; + return true; + } + /** * Ascending removal rank of a segment file name for * {@link #unlinkAllSegmentFiles(String)}. {@code sf-initial.sfa} is diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java index 3f58fa35..de1df55e 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java @@ -31,14 +31,17 @@ import io.questdb.client.cutlass.http.client.WebSocketFrameHandler; import io.questdb.client.cutlass.http.client.WebSocketUpgradeException; import io.questdb.client.cutlass.line.LineSenderException; +import io.questdb.client.cutlass.qwp.client.NativeBufferWriter; import io.questdb.client.cutlass.qwp.client.QwpAuthFailedException; import io.questdb.client.cutlass.qwp.client.QwpDurableAckMismatchException; import io.questdb.client.cutlass.qwp.client.QwpIngressRoleRejectedException; import io.questdb.client.cutlass.qwp.client.QwpRoleMismatchException; import io.questdb.client.cutlass.qwp.client.QwpVersionMismatchException; import io.questdb.client.cutlass.qwp.client.WebSocketResponse; +import io.questdb.client.cutlass.qwp.protocol.QwpConstants; import io.questdb.client.cutlass.qwp.websocket.WebSocketCloseCode; import io.questdb.client.std.CharSequenceLongHashMap; +import io.questdb.client.std.MemoryTag; import io.questdb.client.std.QuietCloseable; import io.questdb.client.std.Unsafe; import org.jetbrains.annotations.TestOnly; @@ -65,9 +68,13 @@ * can trim fully-acked segments.

    6. *
  6. On wire failure, runs the configured reconnect policy: capped * exponential backoff with jitter, retried indefinitely (Invariant B -- - * a store-and-forward drainer never gives up on a wall-clock budget), - * with only auth-style failures (401/403/non-101 upgrade reject) treated - * as terminal. On reconnect success, repositions the cursor at + * a store-and-forward drainer never gives up on a wall-clock budget). + * Once a foreground sender has connected even once, endpoint-policy + * failures (auth, upgrade, durable-ack capability) become transients it + * keeps retrying, so credential and rolling-capability changes stay + * contained by store-and-forward; they are terminal only for an orphan + * drainer or on a foreground sender's initial connect (before the wire + * has ever come up). On reconnect success, repositions the cursor at * {@code ackedFsn+1} and replays.
    7. *
* No locks on the steady-state path. The producer thread (user) writes @@ -147,17 +154,97 @@ public final class CursorWebSocketSendLoop implements QuietCloseable { * briefly down), so a strike count alone can false-positive a transient into * a producer-fatal terminal. Requiring the suspect to survive this window * gives a brief outage time to clear (an OK at/beyond the suspect resets the - * detector) before escalation. {@code 0} = legacy immediate escalation at the + * detector) before escalation. {@code 0} = immediate escalation at the * strike threshold. Configurable per sender via the * {@code poison_min_escalation_window_millis} connect-string key or * {@code LineSenderBuilder.poisonMinEscalationWindowMillis(long)}. */ public static final long DEFAULT_POISON_MIN_ESCALATION_WINDOW_MILLIS = 5_000L; + /** + * Default minimum wall-clock dwell (millis) a symbol-dict catch-up CAP GAP must + * persist before an orphan drainer may latch a terminal, even once + * {@link #MAX_CATCHUP_CAP_GAP_ATTEMPTS} cap gaps have accrued. Same idea as + * {@link #DEFAULT_POISON_MIN_ESCALATION_WINDOW_MILLIS}, for the same reason: a + * strike count measures "how many times did we look", not "how long has this been + * true", and only the latter distinguishes a permanent cluster capability gap from + * a node that is briefly away. + *

+ * It matters here because the cap gap's trigger IS a failover. With the reconnect + * backoff capped at {@link #DEFAULT_RECONNECT_MAX_BACKOFF_MILLIS} (5 s), 16 cap + * gaps accrue in roughly two minutes -- comfortably less than an ordinary rolling + * restart of the larger-cap node. A count-only budget could therefore quarantine an + * otherwise drainable orphan slot during a routine cluster operation. Requiring the + * dwell as well means the orphan terminal needs BOTH "we have looked many times" AND + * "it has stayed true for a long time". Foreground senders never apply this terminal + * policy: they retry indefinitely until a larger-cap node returns. + *

+ * 5 minutes -- the same figure as {@link #DEFAULT_RECONNECT_MAX_DURATION_MILLIS}, + * this codebase's existing notion of how long a transient outage may plausibly + * last. Configurable per sender via the + * {@code catch_up_cap_gap_min_escalation_window_millis} connect-string key or + * {@code LineSenderBuilder.catchUpCapGapMinEscalationWindowMillis(long)}; {@code 0} + * restores count-only escalation for orphan drainers. + */ + public static final long DEFAULT_CATCHUP_CAP_GAP_MIN_ESCALATION_WINDOW_MILLIS = 300_000L; + /** + * Packing ceiling for a dictionary catch-up when the server advertises no batch cap + * (an older build that omits X-QWP-Max-Batch-Size, or one whose derived cap collapsed + * to zero). "Not advertised" is not "unbounded": the transport still closes anything + * larger than the server's receive buffer with 1009, and a catch-up-only close is + * deliberately non-terminal, so an unchunked catch-up reconnects into the identical + * frame forever. Deliberately well below the 131072 default receive buffer, since the + * real value cannot be known when it is not advertised. + *

+ * This bounds MULTI-ENTRY packing only. The cap-gap terminal is measured against a + * separate, generous limit -- see sendDictCatchUp -- so that a single entry which + * already shipped inside a data frame is never reclassified as unsendable. + */ + static final int UNCAPPED_CATCHUP_PACKING_LIMIT = 64 * 1024; private static final Logger LOG = LoggerFactory.getLogger(CursorWebSocketSendLoop.class); + // Settle budget for the symbol-dict catch-up cap gap: how many cap-gap attempts + // -- catch-ups that reached a fresh server and found a single dictionary entry + // too large for its advertised batch cap -- may occur consecutively before an + // orphan drainer latches a terminal. This is a + // SANCTIONED orphan-only terminal (a genuine cluster batch-size capability gap), + // the connect-time analog of the orphan drainer's durable-ack capability gap + // (DEFAULT_MAX_DURABLE_ACK_MISMATCH_ATTEMPTS). A homogeneous cluster never trips + // it -- an entry that fit its data frame under a cap always fits its bare catch-up + // frame under that same cap -- so it only affects a heterogeneous / rolling-cap + // cluster, where a failover to a smaller-cap node can hit it for an entry an + // earlier node accepted. A foreground sender retries forever. An orphan drainer + // rides out the transient window until a larger-cap node returns; only a persistent + // gap (this many consecutive cap gaps with no successful catch-up or unrelated + // reconnect state in between) latches. + // + // Budget accounting (satisfies "a transient must never burn the terminal + // budget"): catchUpCapGapAttempts increments ONLY inside sendDictCatchUp when a + // node is reached and an entry is oversized. A successful catch-up ends the + // episode, as does any unrelated reconnect state (connect refusal, catch-up send + // failure, upgrade/role rejection): otherwise its downtime would count toward the + // wall-clock dwell even though no cap gap was observed. The cap-gap exception + // itself does NOT reset the episode, so consecutive small-cap nodes still prove a + // persistent cluster capability gap and can exhaust the orphan policy. + private static final int MAX_CATCHUP_CAP_GAP_ATTEMPTS = 16; + // Hard ceiling for the lifetime-monotonic sent-dictionary mirror. The mirror + // fields are int, so it cannot exceed Integer.MAX_VALUE bytes; reaching even + // this needs ~200M+ distinct symbols on a single connection, far past any real + // workload. The guard exists so that pathological growth fails loudly instead + // of overflowing the int capacity math into a heap-corrupting copyMemory. + private static final int MAX_SENT_DICT_BYTES = Integer.MAX_VALUE - 8; /** * Throttle "reconnect attempt N failed" WARN logs to one per 5 s. */ private static final long RECONNECT_LOG_THROTTLE_NANOS = 5_000_000_000L; + // Test seam: when true, recovery mirror seeding throws immediately AFTER + // ensureSentDictCapacity has grown (and therefore taken ownership of) the mirror, + // standing in for the copyRecoveredSymbolSuffix-adjacent failure that leaves a + // freshly malloc'd mirror with no owner. It sits after the grow deliberately: the + // constructor's cleanup only frees an OWNED mirror, so a seam before the grow + // would leave nothing to free and the leak guard would pass with the cleanup + // deleted. Production never sets it; volatile only so a test thread's write is + // visible to the loop under test. + @TestOnly + public static volatile boolean forceMirrorSeedFailureForTest; // Pre-converted to nanos for the comparison in sendDurableAckKeepaliveIfDue. // Zero or negative disables the keepalive entirely. private final long durableAckKeepaliveIntervalNanos; @@ -175,8 +262,14 @@ public final class CursorWebSocketSendLoop implements QuietCloseable { // by the server -- holding stale watermarks across the wire boundary // would falsely advance trim before re-confirmation. private final CharSequenceLongHashMap durableTableWatermarks = new CharSequenceLongHashMap(); + // Pre-converted to nanos. Consulted only by the orphan terminal policy. Zero disables + // the dwell entirely (count-only escalation at MAX_CATCHUP_CAP_GAP_ATTEMPTS); the + // user-facing 5-minute default is applied at the config layer. + private final long catchUpCapGapMinEscalationWindowNanos; + private final CatchUpCapGapPolicy catchUpCapGapPolicy; private final CursorSendEngine engine; private final long parkNanos; + private final ReconnectPolicy reconnectPolicy; // FIFO of OK-acked batches awaiting durable-upload confirmation. Used only // when durableAckMode is true. Each entry binds a wireSeq to the per-table // (name, seqTxn) pairs the server reported on the OK frame. The queue is @@ -198,12 +291,6 @@ public final class CursorWebSocketSendLoop implements QuietCloseable { private final ReconnectFactory reconnectFactory; private final long reconnectInitialBackoffMillis; private final long reconnectMaxBackoffMillis; - // Retained for constructor symmetry and passed in by callers, but NOT - // consulted by the background loop: Invariant B removed the wall-clock - // give-up from connectLoop. The budget still bounds the blocking (non-lazy) - // initial connect via QwpWebSocketSender -> connectWithRetry, which takes it - // as an explicit argument rather than reading this field. - private final long reconnectMaxDurationMillis; private final WebSocketResponse response = new WebSocketResponse(); private final ResponseHandler responseHandler = new ResponseHandler(); private final CountDownLatch shutdownLatch = new CountDownLatch(1); @@ -229,6 +316,80 @@ public final class CursorWebSocketSendLoop implements QuietCloseable { // by category. Includes both retriable and terminal outcomes — i.e. every // server-side rejection observed regardless of how the loop reacted. private final AtomicLong totalServerErrors = new AtomicLong(); + // Delta symbol dictionary catch-up state (see swapClient). + // ALWAYS active -- in memory mode, in disk mode, and (critically) even when the + // per-slot persisted dictionary failed to open. sentDictCount is this loop's model + // of how many ids the CURRENT server has been told about, and the torn-dict guard + // in trySendOne reads it, so it must track the wire in every mode; gating it on + // engine.isDeltaDictEnabled() is what once froze it at 0 and terminal'd a slot + // whose frames replay perfectly from id 0 (see the constructor). On a recovered / + // orphan-drained slot the constructor SEEDS sentDict* from the persisted + // dictionary when there is one; otherwise the mirror starts empty and grows from + // the frames themselves. The loop mirrors, in sentDict*, every symbol it has ever + // sent -- the concatenated [len varint][utf8] bytes in global-id order + // (sentDictBytes*) plus the count (sentDictCount) -- so that on reconnect it can + // re-register the whole dictionary on the fresh server (which discards its + // dictionary on every disconnect) before replaying frames whose deltas start above + // id 0. All of this is touched only by the I/O thread. + // Footprint note: this mirror is a SECOND copy of the dictionary -- the same + // symbols the producer's GlobalSymbolDictionary already holds as Java Strings -- + // kept as native UTF-8 bytes for the reconnect-catch-up capability. So a + // memory-mode connection's steady-state dictionary footprint is ~2x the symbol + // set. It is bounded by distinct-symbol count (not per-row) and never trimmed for + // the connection's lifetime (a reconnect may need the whole dictionary at any + // moment), so it cannot be dropped; it is an intentional cost of the feature. + private long sentDictBytesAddr; + private int sentDictBytesCapacity; + private int sentDictBytesLen; + // False while the mirror is unallocated, and while a loop of EITHER policy borrows + // the engine's persisted prefix -- both borrow now; neither consumes it. Any growth + // copy-on-writes into loop-owned memory and sets this true; only owned buffers are + // freed. + private boolean sentDictBytesOwned; + private int sentDictCount; + // True when replay frames can start above dictionary id zero and therefore + // depend on a catch-up on a fresh connection. Delta-enabled live engines + // always have this dependency. A recovered delta slot whose dictionary + // failed to open is identified by its non-zero recovered delta start. The + // remaining false case is full-dictionary fallback: every frame re-registers + // from id zero, so sending the mirror first would duplicate the dictionary. + private final boolean hasReplayDictionaryDependency; + // Reusable staging buffer for the small QWP catch-up prefix (fixed header plus + // delta range varints). Symbol bytes stay in sentDictBytesAddr and are passed as a + // second WebSocket payload slice, avoiding a full-dictionary staging copy. + private long catchUpFrameAddr; + private int catchUpFrameCapacity; + private int catchUpFrameGrowthCount; + // Orphan-policy cap-gap attempts -- catch-ups that reached a node and found an entry + // too large for its batch cap -- with no intervening successful catch-up or unrelated + // reconnect state (see MAX_CATCHUP_CAP_GAP_ATTEMPTS for the full budget accounting). + // Foreground retries never increment it. I/O-thread-only. + private int catchUpCapGapAttempts; + // System.nanoTime() of the FIRST cap gap of the current orphan-policy episode. + // Anchors the escalation dwell. Reset together with catchUpCapGapAttempts on a + // successful catch-up or an unrelated reconnect state, so only an uninterrupted run + // of cap-gap observations contributes wall-clock dwell. Anchoring at the FIRST gap + // (not refreshed per attempt) lets consecutive small-cap nodes satisfy the dwell; + // resetting after transport/role states prevents their downtime from doing so. + // I/O-thread-only, like catchUpCapGapAttempts. + // + // -1 marks "no episode open" for a debugger or a heap dump, but it is NOT the test + // for one -- catchUpCapGapAttempts == 0 is (see sendDictCatchUp). A nanoTime instant + // is only meaningful as a difference: its origin is arbitrary and the spec permits + // negative values, so no state may ride on this field's sign. + private long catchUpCapGapFirstNanos = -1L; + // True once a real ring frame (data or commit) has been sent on the CURRENT + // connection, as opposed to only the dictionary catch-up. The catch-up consumes + // wire sequences (nextWireSeq), so nextWireSeq > 0 no longer implies "the head + // frame was sent": onClose's poison-strike gate and handleServerRejection's + // pre-send gate key off THIS instead. Without it, a transient outage AFTER the + // catch-up but BEFORE the first data frame (a flapping LB/middlebox that accepts + // the upgrade + catch-up then closes) would be mistaken for a deterministic + // head-frame rejection and escalate to a PROTOCOL_VIOLATION terminal -- breaking + // the store-and-forward "retry a transient outage forever" contract. Reset per + // connection in setWireBaselineWithCatchUp; set in trySendOne after a successful + // send. + private boolean dataFrameSentThisConnection; private volatile WebSocketClient client; // Optional: when non-null, every server-rejection error (retriable and // terminal alike) is offered to the dispatcher for async delivery to the user's @@ -262,11 +423,19 @@ public final class CursorWebSocketSendLoop implements QuietCloseable { private long shutdownAwaitTimeoutMillis = DEFAULT_CLOSE_SHUTDOWN_AWAIT_MILLIS; // Sticky flag: false until the very first time a live client is installed // (either via the constructor in SYNC/OFF mode or via swapClient on a - // successful connect attempt in any mode). Once true, stays true. Used to - // distinguish a "never reached the server" terminal failure (looks like a - // config typo or firewall block) from "lost connection after we were - // up" (looks transient). + // successful connect attempt in any mode). Once true, stays true. + // LOAD-BEARING, not observability: it is half of endpointPolicyFailureIsTerminal(), + // which latches an auth / upgrade / durable-ack rejection only for an ORPHAN drainer + // or a sender that has never connected. Relocate or lazily initialise the write and a + // foreground sender's transient auth blip becomes a producer-fatal terminal -- the + // exact failure this policy exists to prevent. private volatile boolean hasEverConnected; + // Cause of the outage the reconnect loop is currently riding out, or null once a + // connect succeeds. Written by the I/O thread, read by the producer thread so a + // backpressure or drain-timeout failure can NAME the reason the wire is not draining + // instead of blaming disk sizing. Was a connectLoop local that never escaped, which + // is why a revoked token surfaced to operators as "sf_max_total_bytes too small". + private volatile Throwable lastReconnectError; private volatile Thread ioThread; // Typed marker for a durable-ack CAPABILITY-GAP terminal: set (before the // terminalError latch, so a checkError() caller that observes the latch is @@ -274,8 +443,8 @@ public final class CursorWebSocketSendLoop implements QuietCloseable { // QwpDurableAckMismatchException. The orphan drainer consults it to route // a mid-drain capability gap into its budgeted settle-retry // (BackgroundDrainer.connectWithDurableAckRetry) instead of quarantining - // the slot on the first sweep; the foreground sender ignores it and keeps - // its spec'd loud-fail (sf-client.md section 8.1). Write-once alongside + // the slot on the first sweep. Foreground reconnects never set this marker; + // they keep retrying after a successful initial connection. Write-once alongside // terminalError: the only writer runs on the I/O thread under the same // first-writer-wins latch. private volatile QwpDurableAckMismatchException capabilityGapTerminal; @@ -379,7 +548,7 @@ public final class CursorWebSocketSendLoop implements QuietCloseable { private final int maxHeadFrameRejections; // Minimum wall-clock dwell (nanos) a suspect frame must stay poisoned before // escalation, even after the strike threshold is hit (connect-string key - // poison_min_escalation_window_millis). 0 = legacy immediate escalation. + // poison_min_escalation_window_millis). 0 = immediate. private final long poisonMinEscalationWindowNanos; private volatile boolean running; // sendOffset: byte offset inside sendingSegment of the first not-yet-sent @@ -406,19 +575,18 @@ public final class CursorWebSocketSendLoop implements QuietCloseable { * {@code client} may be {@code null} only if {@code reconnectFactory} * is non-null — this is the async-initial-connect path: the I/O thread * runs the same retry loop on its first iteration to obtain a live - * client, and a terminal failure (auth/upgrade reject) is delivered - * through the dispatcher rather than thrown to the constructor's - * caller; plain connect failures are retried indefinitely - * (Invariant B: no wall-clock budget give-up). + * client. Every endpoint-policy and transport failure stays inside that + * background retry loop; it is never delivered to the foreground producer + * (Invariant B: no wall-clock budget give-up). Blocking OFF/SYNC startup + * performs its fail-fast policy before constructing this loop. */ public CursorWebSocketSendLoop(WebSocketClient client, CursorSendEngine engine, long fsnAtZero, long parkNanos, ReconnectFactory reconnectFactory, - long reconnectMaxDurationMillis, long reconnectInitialBackoffMillis, long reconnectMaxBackoffMillis) { this(client, engine, fsnAtZero, parkNanos, reconnectFactory, - reconnectMaxDurationMillis, reconnectInitialBackoffMillis, + reconnectInitialBackoffMillis, reconnectMaxBackoffMillis, false); } @@ -434,12 +602,11 @@ public CursorWebSocketSendLoop(WebSocketClient client, CursorSendEngine engine, public CursorWebSocketSendLoop(WebSocketClient client, CursorSendEngine engine, long fsnAtZero, long parkNanos, ReconnectFactory reconnectFactory, - long reconnectMaxDurationMillis, long reconnectInitialBackoffMillis, long reconnectMaxBackoffMillis, boolean durableAckMode) { this(client, engine, fsnAtZero, parkNanos, reconnectFactory, - reconnectMaxDurationMillis, reconnectInitialBackoffMillis, + reconnectInitialBackoffMillis, reconnectMaxBackoffMillis, durableAckMode, DEFAULT_DURABLE_ACK_KEEPALIVE_INTERVAL_MILLIS); } @@ -454,19 +621,18 @@ public CursorWebSocketSendLoop(WebSocketClient client, CursorSendEngine engine, public CursorWebSocketSendLoop(WebSocketClient client, CursorSendEngine engine, long fsnAtZero, long parkNanos, ReconnectFactory reconnectFactory, - long reconnectMaxDurationMillis, long reconnectInitialBackoffMillis, long reconnectMaxBackoffMillis, boolean durableAckMode, long durableAckKeepaliveIntervalMillis) { this(client, engine, fsnAtZero, parkNanos, reconnectFactory, - reconnectMaxDurationMillis, reconnectInitialBackoffMillis, + reconnectInitialBackoffMillis, reconnectMaxBackoffMillis, durableAckMode, durableAckKeepaliveIntervalMillis, DEFAULT_MAX_HEAD_FRAME_REJECTIONS); } /** - * Eleven-arg overload — omits the poison-escalation dwell window, which + * Ten-arg overload — omits the poison-escalation dwell window, which * defaults to {@code 0} (legacy: escalate as soon as maxHeadFrameRejections * strikes accrue, with no minimum wall-clock). The user-facing 5s default * ({@link #DEFAULT_POISON_MIN_ESCALATION_WINDOW_MILLIS}) is applied at the @@ -477,37 +643,90 @@ public CursorWebSocketSendLoop(WebSocketClient client, CursorSendEngine engine, public CursorWebSocketSendLoop(WebSocketClient client, CursorSendEngine engine, long fsnAtZero, long parkNanos, ReconnectFactory reconnectFactory, - long reconnectMaxDurationMillis, long reconnectInitialBackoffMillis, long reconnectMaxBackoffMillis, boolean durableAckMode, long durableAckKeepaliveIntervalMillis, int maxHeadFrameRejections) { this(client, engine, fsnAtZero, parkNanos, reconnectFactory, - reconnectMaxDurationMillis, reconnectInitialBackoffMillis, + reconnectInitialBackoffMillis, reconnectMaxBackoffMillis, durableAckMode, durableAckKeepaliveIntervalMillis, maxHeadFrameRejections, 0L); } /** - * Master constructor — also accepts the poison-frame detector threshold + * Eleven-arg overload — omits the symbol-dict cap-gap escalation dwell, which + * defaults to {@code 0}. This and the twelve-arg overload use the foreground-safe + * retry-forever policy; orphan drainers opt into count-and-dwell escalation through + * the policy-aware master constructor. + */ + public CursorWebSocketSendLoop(WebSocketClient client, CursorSendEngine engine, + long fsnAtZero, long parkNanos, + ReconnectFactory reconnectFactory, + long reconnectInitialBackoffMillis, + long reconnectMaxBackoffMillis, + boolean durableAckMode, + long durableAckKeepaliveIntervalMillis, + int maxHeadFrameRejections, + long poisonMinEscalationWindowMillis) { + this(client, engine, fsnAtZero, parkNanos, reconnectFactory, + reconnectInitialBackoffMillis, + reconnectMaxBackoffMillis, durableAckMode, + durableAckKeepaliveIntervalMillis, maxHeadFrameRejections, + poisonMinEscalationWindowMillis, 0L); + } + + /** + * Twelve-arg overload. Catch-up cap gaps are retried indefinitely, which is the + * required policy for a foreground store-and-forward sender. Orphan drainers use the + * policy-aware overload below to opt into bounded terminal escalation. + *

+ * Also accepts the poison-frame detector threshold * ({@code max_frame_rejections}): consecutive server-active rejections of * the same head-of-line frame, with no ack progress in between, before the - * loop escalates to a typed terminal. Must be {@code >= 1}. The final - * argument is the minimum wall-clock dwell (millis) the suspect must stay - * poisoned before escalation ({@code poison_min_escalation_window_millis}; - * {@code >= 0}, where 0 = legacy immediate escalation at the threshold). + * loop escalates to a typed terminal. Must be {@code >= 1}. Then the minimum + * wall-clock dwell (millis) the suspect must stay poisoned before escalation + * ({@code poison_min_escalation_window_millis}; {@code >= 0}, where 0 = + * immediate escalation at the threshold). + *

+ * The final argument is the analogous dwell for the symbol-dict catch-up cap gap; it + * is consulted only when the policy-aware overload selects + * {@link CatchUpCapGapPolicy#TERMINAL_AFTER_SETTLE_BUDGET}. */ public CursorWebSocketSendLoop(WebSocketClient client, CursorSendEngine engine, long fsnAtZero, long parkNanos, ReconnectFactory reconnectFactory, - long reconnectMaxDurationMillis, long reconnectInitialBackoffMillis, long reconnectMaxBackoffMillis, boolean durableAckMode, long durableAckKeepaliveIntervalMillis, int maxHeadFrameRejections, - long poisonMinEscalationWindowMillis) { + long poisonMinEscalationWindowMillis, + long catchUpCapGapMinEscalationWindowMillis) { + this(client, engine, fsnAtZero, parkNanos, reconnectFactory, + reconnectInitialBackoffMillis, + reconnectMaxBackoffMillis, durableAckMode, + durableAckKeepaliveIntervalMillis, maxHeadFrameRejections, + poisonMinEscalationWindowMillis, catchUpCapGapMinEscalationWindowMillis, + CatchUpCapGapPolicy.RETRY_FOREVER); + } + + /** + * Implementation constructor. The {@link ReconnectPolicy} overload is the public + * entry point; it names the foreground versus orphan ownership distinction directly + * and maps it onto the {@link CatchUpCapGapPolicy} this constructor takes. + */ + private CursorWebSocketSendLoop(WebSocketClient client, CursorSendEngine engine, + long fsnAtZero, long parkNanos, + ReconnectFactory reconnectFactory, + long reconnectInitialBackoffMillis, + long reconnectMaxBackoffMillis, + boolean durableAckMode, + long durableAckKeepaliveIntervalMillis, + int maxHeadFrameRejections, + long poisonMinEscalationWindowMillis, + long catchUpCapGapMinEscalationWindowMillis, + CatchUpCapGapPolicy catchUpCapGapPolicy) { if (maxHeadFrameRejections < 1) { throw new IllegalArgumentException( "maxHeadFrameRejections must be >= 1: " + maxHeadFrameRejections); @@ -516,6 +735,24 @@ public CursorWebSocketSendLoop(WebSocketClient client, CursorSendEngine engine, throw new IllegalArgumentException( "poisonMinEscalationWindowMillis must be >= 0: " + poisonMinEscalationWindowMillis); } + if (catchUpCapGapMinEscalationWindowMillis < 0) { + throw new IllegalArgumentException( + "catchUpCapGapMinEscalationWindowMillis must be >= 0: " + + catchUpCapGapMinEscalationWindowMillis); + } + // TimeUnit conversion saturates at Long.MAX_VALUE. A raw multiply + // wraps large valid millisecond values negative, which makes the + // elapsed-time gate appear satisfied as soon as the strike threshold + // is reached and can quarantine a recoverable orphan slot prematurely. + this.catchUpCapGapMinEscalationWindowNanos = + TimeUnit.MILLISECONDS.toNanos(catchUpCapGapMinEscalationWindowMillis); + if (catchUpCapGapPolicy == null) { + throw new IllegalArgumentException("catchUpCapGapPolicy must be non-null"); + } + this.catchUpCapGapPolicy = catchUpCapGapPolicy; + this.reconnectPolicy = catchUpCapGapPolicy == CatchUpCapGapPolicy.RETRY_FOREVER + ? ReconnectPolicy.FOREGROUND + : ReconnectPolicy.ORPHAN; if (engine == null) { throw new IllegalArgumentException("engine must be non-null"); } @@ -525,19 +762,141 @@ public CursorWebSocketSendLoop(WebSocketClient client, CursorSendEngine engine, } this.client = client; this.engine = engine; + this.hasReplayDictionaryDependency = engine.isDeltaDictEnabled() + || engine.recoveredMaxSymbolDeltaStart() > 0L; + // Recovery / orphan-drain: the loop starts with a fresh in-memory mirror, + // so seed it from the slot's persisted dictionary. That way the very first + // connection re-registers the whole dictionary (via a catch-up frame) + // before replaying the recovered delta frames. + // + // Deliberately NOT gated on engine.isDeltaDictEnabled(). The mirror, the + // catch-up and the replay guard together are this loop's model of what the + // SERVER's dictionary holds, and that model must track every mode. Gating + // them on that flag is what once bricked a recoverable slot: an unopenable + // .symbol-dict reports isDeltaDictEnabled()=false, yet the frames already on + // disk are still DELTA frames (deltaStart 0,1,2,...). With the mirror gated + // off, sentDictCount froze at 0 while the ungated guard kept comparing + // against it, so the frame at deltaStart=1 tripped a terminal -- even though + // replaying the whole sequence from id 0 rebuilds the dictionary on the + // server contiguously and drains perfectly. isDeltaDictEnabled() decides + // what the PRODUCER emits and persists; it must never decide what this loop + // tracks. When the dictionary could not be opened, pd is null, the mirror + // simply starts empty and grows from the frames themselves. + PersistedSymbolDict pd = engine.getPersistedSymbolDict(); + if (pd != null && pd.recoveredSize() > 0) { + int len = pd.loadedEntriesLen(); + if (len > 0) { + // Seed by reference. The foreground loop takes ownership after + // construction succeeds; orphan-drainer loops keep borrowing because + // one engine can create several sessions during capability-gap + // recycling. A borrowed mirror performs copy-on-write if a recovered + // frame suffix must extend it. + sentDictBytesAddr = pd.loadedEntriesAddr(); + sentDictBytesCapacity = len; + sentDictBytesLen = len; + // Set the count only alongside the bytes so sentDictCount can + // never claim symbols the mirror does not hold -- and take it from + // recoveredSize(), NOT the live size(). They differ: the recovery + // heal (QwpWebSocketSender.healPersistedDictionary) appends the + // frame-contributed suffix to the file before this loop is built, so + // size() would already have run past the loaded byte region. + sentDictCount = pd.recoveredSize(); + } + } + // ...and then from the surviving frames' own delta sections, exactly as the producer + // seeds its dictionary. The mirror is the loop's model of what the SERVER holds and it + // is what the catch-up frame ships, so when it stops at the persisted prefix while the + // producer's baseline runs past it, the loop condemns (deltaStart > sentDictCount) a + // slot the producer just seeded successfully. Same two sources, same order, so the two + // land on the same number by construction. + // ...but ONLY when a surviving frame actually depends on ids it does not carry + // (recoveredMaxSymbolDeltaStart > 0). At zero every frame is self-sufficient and + // re-registers its dictionary from id 0 as it replays, so seeding the mirror would buy + // nothing and cost a catch-up frame on every connection -- full-dict slots must stay + // catch-up-free. + // The prefix seed above may still be borrowed from PersistedSymbolDict. + // Extending it inside this cleanup boundary is what makes that safe: + // ensureSentDictCapacity copy-on-writes a borrowed prefix into loop-OWNED + // native memory, so from that point a failure would leak it -- and an + // allocation failure here propagates out of the constructor before any + // caller can own the half-built loop, leaving nothing else able to + // release it. releaseSentDictBytes() in the catch frees exactly that + // mirror, and only when this loop owns it. + try { + if (engine.recoveredMaxSymbolDeltaStart() > 0L) { + int baseline = sentDictCount; + if (engine.recoveredSymbolCoverage(baseline) >= 0L) { + int suffixLen = engine.recoveredSymbolSuffixLen(baseline); + int suffixCount = engine.recoveredSymbolSuffixCount(baseline); + if (suffixLen > 0) { + ensureSentDictCapacity((long) sentDictBytesLen + suffixLen); + if (forceMirrorSeedFailureForTest) { + // Throw AFTER the grow, never before it. ensureSentDictCapacity + // is what copy-on-writes a borrowed prefix into a loop-OWNED + // allocation (sentDictBytesOwned = true), and the catch's + // releaseSentDictBytes() is gated on exactly that flag. A seam + // in front of the grow fires while the mirror is still borrowed + // from PersistedSymbolDict, so the cleanup frees nothing and the + // guard below passes even with the cleanup deleted -- it would + // stop pinning the leak it exists for. + throw new LineSenderException( + "simulated mirror seed allocation failure (test only)"); + } + engine.copyRecoveredSymbolSuffix( + baseline, sentDictBytesAddr + sentDictBytesLen); + sentDictBytesLen += suffixLen; + sentDictCount += suffixCount; + } + } + } + // No per-entry offset index is derived here, and none is kept anywhere: + // the mirror carries its own framing ([len varint][utf8] per entry), so + // sendDictCatchUp -- its only reader -- recovers each entry's span with a + // running pointer as it walks. A recovered slot that drains without ever + // reconnecting, the normal case, therefore never pays that O(n) walk and + // carries no index memory for it. + } catch (Throwable t) { + releaseSentDictBytes(); + throw t; + } + // BOTH policies borrow the engine-owned recovery sources; neither consumes them. + // + // The foreground path used to take ownership here -- pd.takeLoadedEntries() plus + // engine.releaseRecoveredSymbolStorage() -- which made the hand-off ONE-SHOT while + // leaving a second construction reachable: QwpWebSocketSender.ensureConnected's + // catch closes and nulls cursorSendLoop precisely so a caller can retry, and + // `connected` is only set at the very end. takeLoadedEntries zeroes addr/len but + // NOT size, and releaseRecoveredSymbolStorage zeroes the raw buffer but NOT + // baseline, so the retry saw pd.recoveredSize() > 0 with loadedEntriesLen() == 0, + // left sentDictCount at 0, and then either tripped checkedRecoveryAnalysis' + // baseline mismatch or -- when recoveredMaxSymbolDeltaStart == 0 -- latched the + // torn-dictionary terminal ("resend required") on a perfectly healthy slot. + // + // Borrowing removes the state machine instead of trying to make it idempotent. + // Lifetime is safe in both directions: QwpWebSocketSender.close() closes the loop + // before the engine, and BackgroundDrainer's finally does the same, so the buffers + // always outlive their borrower. Any growth copy-on-writes into loop-owned memory + // (ensureSentDictCapacity), and releaseSentDictBytes frees only what the loop owns. this.fsnAtZero = fsnAtZero; this.parkNanos = parkNanos; this.reconnectFactory = reconnectFactory; - this.reconnectMaxDurationMillis = reconnectMaxDurationMillis; this.reconnectInitialBackoffMillis = reconnectInitialBackoffMillis; this.reconnectMaxBackoffMillis = reconnectMaxBackoffMillis; this.durableAckMode = durableAckMode; + // Saturate, never multiply raw -- the same hazard the cap-gap dwell above + // guards. A raw multiply wraps a large millisecond value NEGATIVE, and both of + // these read as "elapsed >= window", so a negative makes the gate trivially + // true: the keepalive would ping on every loop pass, and the poison detector + // would escalate on the strike count alone -- exactly the transient-to-terminal + // false positive the dwell exists to stop (Invariant B). Both keys are + // user-supplied and validated only as >= 0, so asking for a very long window is + // legal, and asking for a longer one must never buy a shorter one. this.durableAckKeepaliveIntervalNanos = durableAckKeepaliveIntervalMillis > 0 - ? durableAckKeepaliveIntervalMillis * 1_000_000L + ? TimeUnit.MILLISECONDS.toNanos(durableAckKeepaliveIntervalMillis) : 0L; this.maxHeadFrameRejections = maxHeadFrameRejections; this.poisonMinEscalationWindowNanos = poisonMinEscalationWindowMillis > 0 - ? poisonMinEscalationWindowMillis * 1_000_000L + ? TimeUnit.MILLISECONDS.toNanos(poisonMinEscalationWindowMillis) : 0L; // SYNC/OFF startup hands a live client to the constructor, so we // already know we reached the server at least once. ASYNC startup @@ -555,6 +914,40 @@ public CursorWebSocketSendLoop(WebSocketClient client, CursorSendEngine engine, } } + /** + * Policy-aware master constructor. A foreground sender fails fast while + * establishing its first connection, then retries endpoint-policy failures + * indefinitely after it has been live. An orphan drainer returns such failures + * to its owner so the slot can follow its settle/quarantine policy. + */ + public CursorWebSocketSendLoop(WebSocketClient client, CursorSendEngine engine, + long fsnAtZero, long parkNanos, + ReconnectFactory reconnectFactory, + long reconnectInitialBackoffMillis, + long reconnectMaxBackoffMillis, + boolean durableAckMode, + long durableAckKeepaliveIntervalMillis, + int maxHeadFrameRejections, + long poisonMinEscalationWindowMillis, + long catchUpCapGapMinEscalationWindowMillis, + ReconnectPolicy reconnectPolicy) { + this(client, engine, fsnAtZero, parkNanos, reconnectFactory, + reconnectInitialBackoffMillis, + reconnectMaxBackoffMillis, durableAckMode, + durableAckKeepaliveIntervalMillis, maxHeadFrameRejections, + poisonMinEscalationWindowMillis, catchUpCapGapMinEscalationWindowMillis, + catchUpPolicyFor(reconnectPolicy)); + } + + private static CatchUpCapGapPolicy catchUpPolicyFor(ReconnectPolicy reconnectPolicy) { + if (reconnectPolicy == null) { + throw new IllegalArgumentException("reconnectPolicy must be non-null"); + } + return reconnectPolicy == ReconnectPolicy.FOREGROUND + ? CatchUpCapGapPolicy.RETRY_FOREVER + : CatchUpCapGapPolicy.TERMINAL_AFTER_SETTLE_BUDGET; + } + /** * Maps a server status byte to a {@link SenderError.Category}. Exposed for unit tests. */ @@ -573,6 +966,8 @@ public static SenderError.Category classify(byte status) { return SenderError.Category.WRITE_ERROR; case WebSocketResponse.STATUS_NOT_WRITABLE: return SenderError.Category.NOT_WRITABLE; + case WebSocketResponse.STATUS_DICTIONARY_GAP: + return SenderError.Category.DICTIONARY_GAP; default: return SenderError.Category.UNKNOWN; } @@ -601,12 +996,21 @@ public static WebSocketClient connectWithRetry( String contextLabel ) { long startNanos = System.nanoTime(); - long deadlineNanos = startNanos + maxDurationMillis * 1_000_000L; + // Saturating budget, compared against ELAPSED rather than an absolute + // startNanos+budget deadline. A large reconnect_max_duration_millis (e.g. + // Long.MAX_VALUE, the natural "retry until the server boots") must not collapse + // the budget to zero: a raw maxDurationMillis*1_000_000L wraps NEGATIVE -- the + // same overflow the drainer's capabilityGapBudgetNanos and the loop dwells were + // hardened against -- so the pre-condition loop below would make ZERO attempts. + // Even TimeUnit.toNanos saturated to Long.MAX_VALUE would overflow an absolute + // startNanos+budget sum; comparing the bounded nanoTime difference against the + // budget stays correct at saturation. + long budgetNanos = TimeUnit.MILLISECONDS.toNanos(maxDurationMillis); long backoffMillis = initialBackoffMillis; int attempts = 0; long lastLogNanos = 0L; Throwable lastError = null; - while (System.nanoTime() < deadlineNanos) { + while (System.nanoTime() - startNanos < budgetNanos) { attempts++; try { WebSocketClient c = factory.reconnect(); @@ -661,7 +1065,7 @@ public static WebSocketClient connectWithRetry( // this > 0. Mirrors BackgroundDrainer's sweep-loop jitter guard. long jitter = ThreadLocalRandom.current().nextLong(Math.max(1L, backoffMillis)); long sleepMillis = backoffMillis + jitter; - long remainingMillis = (deadlineNanos - System.nanoTime()) / 1_000_000L; + long remainingMillis = (budgetNanos - (System.nanoTime() - startNanos)) / 1_000_000L; if (remainingMillis <= 0) { break; } @@ -693,9 +1097,10 @@ public static WebSocketClient connectWithRetry( @TestOnly public static SenderError.Policy defaultPolicyFor(SenderError.Category category) { switch (category) { - case WRITE_ERROR: // transient server state (disk pressure, suspended table) - case INTERNAL_ERROR: // transient by definition; deterministic repeats poison-escalate - case UNKNOWN: // fail open: status byte from a newer server + case WRITE_ERROR: // transient server state (disk pressure, suspended table) + case INTERNAL_ERROR: // transient by definition; deterministic repeats poison-escalate + case DICTIONARY_GAP: // server state, not frame bytes: re-register and replay + case UNKNOWN: // fail open: status byte from a newer server return SenderError.Policy.RETRIABLE; case NOT_WRITABLE: // read-only replica / demoting primary: rotate endpoints return SenderError.Policy.RETRIABLE_OTHER; @@ -732,6 +1137,21 @@ public static boolean isTerminalCloseCode(int code) { } } + @TestOnly + public static int maxCatchUpCapGapAttempts() { + return MAX_CATCHUP_CAP_GAP_ATTEMPTS; + } + + @TestOnly + public static int maxSentDictBytes() { + return MAX_SENT_DICT_BYTES; + } + + @TestOnly + public static int uncappedCatchUpPackingLimit() { + return UNCAPPED_CATCHUP_PACKING_LIMIT; + } + /** * Surfaces any error the I/O thread recorded. Called by the producer * thread (typically from inside its append wrapper) so failures don't @@ -790,6 +1210,12 @@ public synchronized void close() { // after — the latch await is only skipped when the loop never ran. running = false; Thread t = ioThread; + // The symbol-dict mirror (sentDictBytesAddr) is I/O-thread-owned and gets + // freed on ioLoop's exit path. When t == null the loop never ran (start() + // was never called, or t.start() failed before committing ioThread), so + // that free never happens and a seeded (recovery / orphan-drain) mirror + // would leak. Capture that here; free it below, after client teardown. + boolean loopNeverRan = t == null; if (t != null) { LockSupport.unpark(t); // Only await the shutdown latch if the I/O thread actually ran. @@ -917,6 +1343,17 @@ public synchronized void close() { } client = null; } + // Free the I/O-thread-owned symbol-dict mirror ONLY when the loop never + // ran (see loopNeverRan). If it ran, ioLoop's exit already freed it -- and + // on the failed-stop path (interrupted latch await, ioThread left set) the + // thread may still be mid-send, so touching the mirror here would race. + // A duplicate close observes sentDictBytesAddr == 0 and skips. + if (loopNeverRan && sentDictBytesAddr != 0) { + releaseSentDictBytes(); + } + if (loopNeverRan) { + freeCatchUpFrameBuffer(); + } } /** @@ -1107,6 +1544,25 @@ public synchronized void start() { // straight to the active and orphan everything in sealed. try { positionCursorForStart(); + } catch (CatchUpSendException e) { + // A recovered sender re-registers its dictionary with a catch-up on + // the very first connect. Here that runs on the CALLER thread (sync + // start), so we must NOT let it drive connectLoop -- that would block + // Sender construction forever on a transient outage. Drop the dead + // client instead: the I/O thread then reconnects via + // attemptInitialConnect -> swapClient and re-sends the catch-up off + // this thread. If the failure was already terminal (recordFatal set + // running=false, e.g. an entry too large for the batch cap), the I/O + // thread simply winds down and checkError() surfaces it. + WebSocketClient dead = client; + client = null; + if (dead != null) { + try { + dead.close(); + } catch (Throwable ignored) { + // best-effort + } + } } catch (Throwable t) { running = false; releaseSendingSegment(); @@ -1177,10 +1633,12 @@ private void applyDurableAck() { /** * Drives the very first connect attempt on the I/O thread, used in the * async-initial-connect mode (constructed with {@code client == null}). - * Reuses the same retry+backoff machinery as {@link #fail(Throwable)} — - * connect failures are retried indefinitely (Invariant B), and a - * terminal upgrade reject is delivered through the dispatcher, not - * thrown to the producer. + * Reuses the same retry+backoff machinery as {@link #fail(Throwable)}. + * Transport failures retry indefinitely here (Invariant B). But this path + * runs with {@code !hasEverConnected}, so an auth, upgrade or capability + * rejection is terminal (see {@link #endpointPolicyFailureIsTerminal()}): + * it is dispatched to the async {@code SenderErrorHandler} and latched for + * a {@code close()} rethrow, rather than retried. */ private void attemptInitialConnect() { connectLoop(new LineSenderException( @@ -1202,6 +1660,15 @@ private void clearDurableAckTracking() { lastFrameOrPingNanos = 0L; } + private static boolean isCatchUpCapGap(Throwable t) { + return t instanceof CatchUpSendException && ((CatchUpSendException) t).isCapGap; + } + + private void resetCatchUpCapGapEpisode() { + catchUpCapGapAttempts = 0; + catchUpCapGapFirstNanos = -1L; + } + /** * Shared per-outage retry loop. Used by {@link #fail(Throwable)} for * mid-flight wire failures (phase="reconnect") and by @@ -1219,21 +1686,32 @@ private void connectLoop(Throwable initial, String phase, long paceFirstAttemptM recordFatal(initial); return; } + // A wire/role state that interrupted an orphan cap-gap run is not evidence that + // the cluster's batch cap stayed incompatible during the outage. Start a fresh + // episode; the cap-gap exception itself is the one reconnect cause that preserves + // the existing attempt count and dwell anchor. + if (!isCatchUpCapGap(initial)) { + resetCatchUpCapGapEpisode(); + } snapshotReplayTarget(); LOG.warn("cursor I/O loop entering {} loop: {}", phase, initial.getMessage()); long outageStartNanos = System.nanoTime(); - // INVARIANT B: a store-and-forward drainer must NEVER terminate on a + // INVARIANT B: a store-and-forward loop must NEVER terminate on a // wall-clock reconnect budget. A replica-only / all-endpoints-replica // window is TRANSIENT -- a replica gets promoted, a primary reappears -- // so this background loop retries for as long as it is running, backing - // off between attempts. The ONLY terminal conditions are a genuinely - // non-retriable upgrade (auth / non-421 upgrade / durable-ack capability - // gap), which return directly below, or the sender being stopped. SF + // off between attempts. Endpoint-policy failures (auth / non-421 + // upgrade / durable-ack capability gap) are terminal only for orphan + // drainers. Foreground senders retry them from asynchronous startup onward + // so a credential or cluster capability rotation cannot stop the producer. SF // exhaustion is surfaced to the PRODUCER as append backpressure, never - // here. reconnect_max_duration_millis is intentionally NOT consulted: it - // bounds only the blocking (non-lazy) initial connect in - // QwpWebSocketSender.buildAndConnect, never this background loop. + // here. reconnect_max_duration_millis is intentionally NOT consulted by + // THIS loop. Its holders pass it explicitly where it does apply: the + // blocking (non-lazy) initial connect hands it to connectWithRetry (see + // QwpWebSocketSender.ensureConnected, the SYNC branch), and + // BackgroundDrainer converts it into the durable-ack capability-gap + // budget. Neither bounds this loop's steady-state reconnect. long backoffMillis = reconnectInitialBackoffMillis; if (paceFirstAttemptMillis > 0 && running) { // NACK-initiated recycle against a reachable server: pace the @@ -1253,7 +1731,7 @@ private void connectLoop(Throwable initial, String phase, long paceFirstAttemptM } int attempts = 0; long lastLogNanos = 0L; - Throwable lastReconnectError = initial; + lastReconnectError = initial; while (running) { attempts++; totalReconnectAttempts.incrementAndGet(); @@ -1289,66 +1767,86 @@ private void connectLoop(Throwable initial, String phase, long paceFirstAttemptM phase, elapsedMs, attempts, fsnAtZero); return; } + // A null factory result is an unsuccessful connect state, not a cap-gap + // observation. Do not let the time spent retrying it satisfy orphan dwell. + resetCatchUpCapGapEpisode(); } catch (QwpAuthFailedException | WebSocketUpgradeException e) { - // Terminal across all configured endpoints per spec sf-client.md - // section 13.3: auth (401/403) bypasses reconnect and surfaces as - // SECURITY_ERROR. WebSocketUpgradeException reaching here is always - // non-421: QwpUpgradeFailures.classify upstream converts a - // 421-with-X-QuestDB-Role to QwpIngressRoleRejectedException, and a - // 421 without that header walks the transport-error path in - // buildAndConnect and lands as a LineSenderException, falling into - // the Throwable branch below. - LOG.error("terminal upgrade error during {} -- won't retry: {}", - phase, e.getMessage()); - long fromFsn = engine.ackedFsn() + 1L; - long toFsn = Math.max(fromFsn, engine.publishedFsn()); - SenderError err = new SenderError( - SenderError.Category.SECURITY_ERROR, - SenderError.Policy.TERMINAL, - SenderError.NO_STATUS_BYTE, - "ws-upgrade-failed: " + e.getMessage(), - SenderError.NO_MESSAGE_SEQUENCE, - fromFsn, - toFsn, - null, - System.nanoTime() - ); - totalServerErrors.incrementAndGet(); - recordFatal(new LineSenderServerException(err)); - dispatchError(err); - return; + if (endpointPolicyFailureIsTerminal()) { + // Orphans return control to their quarantine owner. + // WebSocketUpgradeException reaching here is always non-421: + // role rejects are classified into the transient branch below. + LOG.error("terminal upgrade error during {} -- won't retry: {}", + phase, e.getMessage()); + long fromFsn = engine.ackedFsn() + 1L; + long toFsn = Math.max(fromFsn, engine.publishedFsn()); + SenderError err = new SenderError( + SenderError.Category.SECURITY_ERROR, + SenderError.Policy.TERMINAL, + SenderError.NO_STATUS_BYTE, + "ws-upgrade-failed: " + e.getMessage(), + SenderError.NO_MESSAGE_SEQUENCE, + fromFsn, + toFsn, + null, + System.nanoTime() + ); + totalServerErrors.incrementAndGet(); + recordFatal(new LineSenderServerException(err)); + dispatchError(err); + return; + } + resetCatchUpCapGapEpisode(); + lastReconnectError = e; + dispatchRetriedEndpointPolicyFailure( + SenderError.Category.SECURITY_ERROR, "ws-upgrade-failed: " + e.getMessage()); + long now = System.nanoTime(); + if (now - lastLogNanos >= RECONNECT_LOG_THROTTLE_NANOS) { + LOG.warn("{} attempt {}: foreground auth/upgrade policy rejected the connection; " + + "retrying while store-and-forward owns the buffered data -- {}", + phase, attempts, e.getMessage()); + lastLogNanos = now; + } } catch (QwpDurableAckMismatchException e) { - // Per spec sf-client.md section 8.1: the client opted into durable - // ack but the cluster cannot honour it. Loud fail at connect rather - // than silently waiting for ack frames that will never arrive. - // Classified as PROTOCOL_VIOLATION (config/capability mismatch), - // not SECURITY_ERROR -- this is not an auth failure. - LOG.error("durable-ack mismatch during {} -- won't retry: {}", - phase, e.getMessage()); - if (terminalError == null) { - // Mirror recordFatal's first-writer-wins latch: only the - // sweep that owns the terminal may mark the gap, and the - // marker must be visible before the terminalError volatile - // write that checkError() keys on. - capabilityGapTerminal = e; + if (endpointPolicyFailureIsTerminal()) { + // Orphans hand a capability gap back to BackgroundDrainer's + // settle budget. + LOG.error("durable-ack mismatch during {} -- won't retry: {}", + phase, e.getMessage()); + if (terminalError == null) { + // Publish the marker before terminalError, which is the + // volatile first-writer-wins latch observed by the owner. + capabilityGapTerminal = e; + } + long fromFsn = engine.ackedFsn() + 1L; + long toFsn = Math.max(fromFsn, engine.publishedFsn()); + SenderError err = new SenderError( + SenderError.Category.PROTOCOL_VIOLATION, + SenderError.Policy.TERMINAL, + SenderError.NO_STATUS_BYTE, + "durable-ack-mismatch: " + e.getMessage(), + SenderError.NO_MESSAGE_SEQUENCE, + fromFsn, + toFsn, + null, + System.nanoTime() + ); + totalServerErrors.incrementAndGet(); + recordFatal(new LineSenderServerException(err)); + dispatchError(err); + return; } - long fromFsn = engine.ackedFsn() + 1L; - long toFsn = Math.max(fromFsn, engine.publishedFsn()); - SenderError err = new SenderError( + resetCatchUpCapGapEpisode(); + lastReconnectError = e; + dispatchRetriedEndpointPolicyFailure( SenderError.Category.PROTOCOL_VIOLATION, - SenderError.Policy.TERMINAL, - SenderError.NO_STATUS_BYTE, - "durable-ack-mismatch: " + e.getMessage(), - SenderError.NO_MESSAGE_SEQUENCE, - fromFsn, - toFsn, - null, - System.nanoTime() - ); - totalServerErrors.incrementAndGet(); - recordFatal(new LineSenderServerException(err)); - dispatchError(err); - return; + "durable-ack-mismatch: " + e.getMessage()); + long now = System.nanoTime(); + if (now - lastLogNanos >= RECONNECT_LOG_THROTTLE_NANOS) { + LOG.warn("{} attempt {}: foreground durable-ack capability is temporarily " + + "unavailable; retrying while store-and-forward owns the buffered data -- {}", + phase, attempts, e.getMessage()); + lastLogNanos = now; + } } catch (QwpRoleMismatchException | QwpIngressRoleRejectedException e) { // Role mismatch: every reachable endpoint role-rejected the // upgrade -- right now they are all replicas / primary-catchup. @@ -1365,6 +1863,7 @@ private void connectLoop(Throwable initial, String phase, long paceFirstAttemptM // endpoint, forever. Growing to reconnectMaxBackoffMillis // mirrors the orphan drainer's role-reject path and honours the // documented capped-exponential-backoff contract. + resetCatchUpCapGapEpisode(); lastReconnectError = e; long now = System.nanoTime(); if (now - lastLogNanos >= RECONNECT_LOG_THROTTLE_NANOS) { @@ -1388,6 +1887,9 @@ private void connectLoop(Throwable initial, String phase, long paceFirstAttemptM recordFatal(e); throw (Error) e; } + if (!isCatchUpCapGap(e)) { + resetCatchUpCapGapEpisode(); + } lastReconnectError = e; long now = System.nanoTime(); if (now - lastLogNanos >= RECONNECT_LOG_THROTTLE_NANOS) { @@ -1433,6 +1935,72 @@ private void connectLoop(Throwable initial, String phase, long paceFirstAttemptM phase, elapsedMs, attempts, lastMsg); } + /** + * Whether an endpoint-policy failure (auth, non-421 upgrade, durable-ack + * capability gap) latches a terminal instead of retrying under Invariant B. + *

+ * Two callers own a terminal, for different reasons: + *

+ * Once a FOREGROUND sender has reached the server even once, initialization is + * over and store-and-forward owns the buffered data: every endpoint-policy + * failure is then a transient the drainer must ride out, never a producer-fatal + * terminal (Invariant B). + */ + /** + * Reports an endpoint-policy rejection a FOREGROUND sender is riding out. + *

+ * Retrying is what the store-and-forward contract demands, but until now the retry was + * programmatically invisible: dispatchError ran only in the terminal branches, so a + * revoked token produced nothing but a throttled slf4j WARN -- in a library that ships + * embedded in user applications, frequently with no binding configured. flush() kept + * returning success until SF filled, and the failure then surfaced as "cursor ring + * backpressured ... sf_max_total_bytes too small", pointing the operator at disk + * sizing. That is the same complaint this PR makes to justify keeping STARTUP + * terminal; post-start it was merely delayed by SF capacity, not avoided. + *

+ * RETRIABLE, not TERMINAL: the handler learns the wire is being rejected while the + * producer stays alive and no data is at risk. + */ + private void dispatchRetriedEndpointPolicyFailure(SenderError.Category category, String message) { + long fromFsn = engine.ackedFsn() + 1L; + dispatchError(new SenderError( + category, + SenderError.Policy.RETRIABLE, + SenderError.NO_STATUS_BYTE, + message, + SenderError.NO_MESSAGE_SEQUENCE, + fromFsn, + Math.max(fromFsn, engine.publishedFsn()), + null, + System.nanoTime() + )); + } + + /** + * Cause of the outage the reconnect loop is riding out, or {@code null} when the wire + * is up. Lets a producer-side failure name the real reason rather than blaming disk + * sizing -- see the field. + */ + public Throwable lastReconnectError() { + return lastReconnectError; + } + + private boolean endpointPolicyFailureIsTerminal() { + return reconnectPolicy == ReconnectPolicy.ORPHAN || !hasEverConnected; + } + /** * Send {@code err} to the async-delivery dispatcher if one is configured. * Producer-side typed throw (TERMINAL) goes through {@code recordFatal} + @@ -1477,7 +2045,7 @@ private boolean recordHeadRejectionStrike(long rejectedFsn) { // middlebox, or a fast NACK loop), so the count alone would false- // positive a brief outage into a producer-fatal terminal. Any OK at/ // beyond the suspect during the window resets the detector (see the - // STATUS_OK handler). Window 0 = legacy immediate escalation. + // STATUS_OK handler). Window 0 = immediate escalation. return (now - poisonFirstStrikeNanos) >= poisonMinEscalationWindowNanos; } @@ -1582,8 +2150,8 @@ private void enqueuePendingOk(long wireSeq) { * listener both non-null), enters the per-outage retry loop: capped * exponential backoff with jitter, retried for as long as the loop is * running -- there is NO wall-clock give-up (Invariant B: a store-and- - * forward drainer only terminates on SF exhaustion or a genuinely non- - * retriable auth/upgrade reject). On the first successful reconnect the + * forward drainer does not give up on endpoint or transport state). On the + * first successful reconnect the * I/O loop resumes with reset wire state and replays from * {@code engine.ackedFsn() + 1}. *

@@ -1689,11 +2257,10 @@ private void ioLoop() { // Async-initial-connect path: ctor accepted a null client because // a reconnect factory is wired. Drive the very first connect on // this thread so the producer thread never blocks on it. - // attemptInitialConnect either sets `client` (success) or records - // a terminal failure and clears `running` (auth/upgrade reject; - // plain connect failures retry indefinitely under Invariant B). - // Either way, the main loop below sees the outcome via the - // `running` and `client` fields. + // attemptInitialConnect retries every endpoint/transport failure + // indefinitely under Invariant B and returns only after it installs + // a client or close() clears `running`. The main loop below sees the + // outcome via the `running` and `client` fields. if (client == null && running) { attemptInitialConnect(); } @@ -1774,6 +2341,12 @@ private void ioLoop() { // best-effort } } + // The symbol-dict mirror is I/O-thread-owned; free it here, on the + // owning thread's exit path, after the last send that could touch it. + if (sentDictBytesAddr != 0) { + releaseSentDictBytes(); + } + freeCatchUpFrameBuffer(); shutdownLatch.countDown(); Runnable closeCallback = delegatedClose; if (closeCallback != null) { @@ -1795,8 +2368,11 @@ private void ioLoop() { /** * Walk the engine's segments to find the one containing {@code targetFsn}, * and set {@code sendOffset} to the byte offset of that frame within it. - * This is called at startup and after every reconnect, after fsnAtZero has - * already been reset to {@code targetFsn} and nextWireSeq to 0. + * This is called at startup and after every reconnect, once + * {@link #setWireBaselineWithCatchUp} has anchored the wire baseline + * ({@code fsnAtZero} / {@code nextWireSeq}) -- which may leave {@code nextWireSeq} + * past the catch-up frames it emitted. This method only positions the byte + * cursor at {@code targetFsn}; it does not touch the wire mapping. *

* If {@code targetFsn} is already published, the method positions the byte * cursor exactly at that frame. If {@code targetFsn} is not published yet, @@ -1961,10 +2537,7 @@ private void snapshotReplayTarget() { private void swapClient(WebSocketClient newClient) { WebSocketClient old = this.client; this.client = newClient; - // Sticky: once the wire is up, we've reached the server at least - // once for this sender's lifetime. Used downstream to classify a - // subsequent terminal failure as transient vs config-likely. - this.hasEverConnected = true; + this.lastReconnectError = null; if (old != null) { try { old.close(); @@ -1992,7 +2565,674 @@ private void swapClient(WebSocketClient newClient) { // carrying stale state across the wire boundary would either // double-trim or starve the queue. clearDurableAckTracking(); + setWireBaselineWithCatchUp(replayStart); positionCursorAt(replayStart); + // Sticky: once the wire is up, we've reached the server at least once + // for this sender's lifetime. Exposed to the owning sender for + // connection-state observability, and it ends initialization: from here + // on endpointPolicyFailureIsTerminal() stops latching endpoint-policy + // failures on a foreground sender and rides them out instead, because + // store-and-forward now owns the buffered data (Invariant B). + // Deliberately LAST: a first connection that completes the upgrade but + // dies inside the dictionary catch-up above was never fully + // established, so the sender stays INITIALIZING and a subsequent + // endpoint-policy rejection still latches the startup terminal. + this.hasEverConnected = true; + } + + /** + * Sets the wire-sequence baseline for a fresh connection and, when the symbol + * dictionary mirror is non-empty, emits a full-dictionary catch-up frame first + * so the fresh server (whose dictionary starts empty) can resolve the + * non-self-sufficient delta frames that replay next. + *

+ * The catch-up occupies wire seq 0, which maps to the already-acked FSN just + * below {@code replayStart} (a harmless re-ack); real replay frames then follow + * from wire seq 1. With nothing to catch up (fresh sender, or full-dict mode), + * or before a client exists (async initial connect), keep the plain 1:1 + * {@code fsnAtZero == replayStart} mapping; the catch-up then happens on the + * first real connection via swapClient. + */ + private void setWireBaselineWithCatchUp(long replayStart) { + // Fresh connection: no data frame has been sent on it yet. Reset before the + // catch-up (which sends only dictionary frames) so onClose / + // handleServerRejection can tell "only the catch-up went out" from "the + // head data frame went out". + dataFrameSentThisConnection = false; + // Do not gate solely on isDeltaDictEnabled(): a recovered delta slot can + // lose access to its side-file while its on-disk frames still start above + // zero. hasReplayDictionaryDependency preserves that distinction while + // keeping full-dictionary fallback catch-up-free across every reconnect. + if (client != null && sentDictCount > 0 && hasReplayDictionaryDependency) { + this.nextWireSeq = 0L; + // The catch-up may span several frames when the dictionary exceeds the + // server's batch cap; each consumes a wire sequence (0 .. n-1) that maps + // to an already-acked FSN, so the first real frame still lands on + // replayStart. + int catchUpFrames = sendDictCatchUp(); + this.fsnAtZero = replayStart - catchUpFrames; + } else { + this.fsnAtZero = replayStart; + this.nextWireSeq = 0L; + } + } + + /** + * Returns the symbol-dictionary delta start id of a frame, or -1 when the + * frame carries no delta section. Used by the pre-send torn-dictionary guard. + */ + private int frameDeltaStart(long payloadAddr, int payloadLen) { + if (!isDeltaFrame(payloadAddr, payloadLen)) { + return -1; + } + long encoded = readVarintAt(payloadAddr + QwpConstants.HEADER_SIZE, payloadAddr + payloadLen); + // A malformed start id reads as "no delta section", which the caller's + // `deltaStart >= 0` gate already handles. + return encoded < 0L ? -1 : (int) (encoded >>> 3); + } + + // True only for a well-formed QWP frame this encoder produced that carries a + // delta symbol-dict section. The magic check keeps the dict logic from + // misreading non-QWP payloads (e.g. synthetic frames injected by tests) whose + // bytes happen to set the delta flag. + private static boolean isDeltaFrame(long payloadAddr, int payloadLen) { + if (payloadLen < QwpConstants.HEADER_SIZE + || Unsafe.getUnsafe().getInt(payloadAddr) != QwpConstants.MAGIC_MESSAGE) { + return false; + } + byte flags = Unsafe.getUnsafe().getByte(payloadAddr + QwpConstants.HEADER_OFFSET_FLAGS); + return (flags & QwpConstants.FLAG_DELTA_SYMBOL_DICT) != 0; + } + + /** + * Copies the symbol-dictionary delta a just-sent frame carries into the + * loop-local mirror ({@link #sentDictBytesAddr}) so a future reconnect can + * re-register it. Frames are sent in FSN order carrying monotonically + * extending deltas, so a frame whose delta starts exactly at + * {@link #sentDictCount} extends the mirror; a replayed or empty-delta frame + * (nothing new) is skipped. Only ever called in delta mode, for a frame the + * pre-send guard already classified as a delta frame. + * + * @param payloadAddr address of the QWP message (12-byte header first) + * @param payloadLen message length in bytes + * @param deltaStart the frame's delta start id, already decoded by the + * pre-send guard ({@link #frameDeltaStart}) -- passed in so + * the magic/flags and start-id varint are not re-parsed + */ + private void accumulateSentDict(long payloadAddr, int payloadLen, int deltaStart) { + long limit = payloadAddr + payloadLen; + // deltaStart is known (the guard decoded it); locate deltaCount just past + // its canonical LEB128 encoding rather than re-reading the header and the + // start-id varint. + long p = payloadAddr + QwpConstants.HEADER_SIZE + NativeBufferWriter.varintSize(deltaStart); + long encodedCount = readVarintAt(p, limit); + if (encodedCount < 0L) { + return; // malformed -- never happens for frames we encoded + } + long deltaCount = encodedCount >>> 3; + p += encodedCount & 7L; + // The mirror holds ids [0, sentDictCount). Accumulate ONLY the part of this + // frame's delta [deltaStart, deltaStart+deltaCount) that extends past the + // tip -- ids [sentDictCount, deltaStart+deltaCount). Cases: + // - deltaStart > sentDictCount: a gap. trySendOne's torn-dictionary guard + // rejects it before send, so it never reaches here; bail defensively + // rather than accumulate past a hole. + // - deltaEnd <= sentDictCount: a pure replay/overlap we already hold -- + // nothing new. + // - deltaStart <= sentDictCount < deltaEnd: extend the mirror by the tail. + // deltaStart == sentDictCount is the steady-state case (skip == 0). + // Handling the partial overlap explicitly -- rather than dropping the whole + // frame whenever deltaStart != sentDictCount -- keeps the mirror complete + // even if a future producer ever emits a delta that overlaps the tip; + // silently dropping the new tail would leave the reconnect catch-up + // incomplete and shift server-side ids. + long deltaEnd = deltaStart + deltaCount; + if (deltaCount <= 0 || deltaStart > sentDictCount || deltaEnd <= sentDictCount) { + return; + } + // Keeps no per-entry offset index: the mirror stores each entry as + // [len varint][utf8], so the reconnect catch-up (sendDictCatchUp) walks it with + // a running pointer on demand. Maintaining an index here would put a store per + // new symbol on the per-frame I/O path -- and on the workload this feature + // targets (a new symbol per ROW) that is a store per row -- to serve a reconnect + // that may never happen; the catch-up's single walk is dwarfed by shipping the + // same n bytes over the wire. + // + // Walk past the already-held prefix [deltaStart, sentDictCount), then copy + // the new tail [sentDictCount, deltaEnd). + int skip = sentDictCount - deltaStart; + for (int i = 0; i < skip; i++) { + long encoded = readVarintAt(p, limit); + if (encoded < 0L) { + return; // malformed -- bail rather than corrupt the mirror + } + p += (encoded & 7L) + (encoded >>> 3); + if (p > limit) { + return; + } + } + long regionStart = p; + long newCount = deltaEnd - sentDictCount; + // Walk the new tail only to find where it ends; the entry boundaries are not + // recorded here (see above -- the catch-up re-walks the mirror when it needs them). + for (long i = 0; i < newCount; i++) { + long encoded = readVarintAt(p, limit); + if (encoded < 0L) { + // Malformed -- never happens for frames we encoded; bail rather + // than corrupt the mirror. + return; + } + p += (encoded & 7L) + (encoded >>> 3); + if (p > limit) { + return; + } + } + int regionBytes = (int) (p - regionStart); + // long sum: sentDictBytesLen + regionBytes can exceed Integer.MAX_VALUE on + // a very-high-cardinality lifetime connection; ensureSentDictCapacity then + // fails loudly rather than overflowing to a negative int (which would make + // the capacity check pass and copyMemory scribble past the buffer). + ensureSentDictCapacity((long) sentDictBytesLen + regionBytes); + Unsafe.getUnsafe().copyMemory(regionStart, sentDictBytesAddr + sentDictBytesLen, regionBytes); + sentDictBytesLen += regionBytes; + sentDictCount += (int) newCount; + } + + private void ensureSentDictCapacity(long required) { + if (sentDictBytesCapacity >= required) { + return; + } + if (required > MAX_SENT_DICT_BYTES) { + // Latch a terminal, do NOT just throw: accumulateSentDict runs AFTER + // the frame's sendBinary, so a bare throw unwinds to ioLoop -> fail() + // -> connectLoop, which (running still true) reconnects and replays the + // same frame, which re-overflows the never-shrinking mirror -- an + // unbounded reconnect livelock rather than the "fails loudly" the + // MAX_SENT_DICT_BYTES ceiling promises. recordFatal flips running=false + // so connectLoop's !running guard winds the loop down and checkError() + // surfaces the terminal, matching sendCatchUpChunk's guard for the same + // ceiling. The throw still unwinds past the pending copyMemory. + LineSenderException err = new LineSenderException("symbol dictionary mirror exceeds the maximum size [" + + "required=" + required + ", max=" + MAX_SENT_DICT_BYTES + ']'); + recordFatal(err); + throw err; + } + // Grow in long to avoid the capacity*2 int overflow (negative) that would + // otherwise degrade the doubling near 1 GB; clamp to the int ceiling. + long newCap = Math.max((long) sentDictBytesCapacity * 2, Math.max(4096L, required)); + if (newCap > MAX_SENT_DICT_BYTES) { + newCap = MAX_SENT_DICT_BYTES; + } + try { + if (sentDictBytesOwned) { + sentDictBytesAddr = Unsafe.realloc( + sentDictBytesAddr, + sentDictBytesCapacity, + (int) newCap, + MemoryTag.NATIVE_DEFAULT); + } else { + long newAddr = Unsafe.malloc((int) newCap, MemoryTag.NATIVE_DEFAULT); + if (sentDictBytesLen > 0) { + Unsafe.getUnsafe().copyMemory(sentDictBytesAddr, newAddr, sentDictBytesLen); + } + sentDictBytesAddr = newAddr; + sentDictBytesOwned = true; + } + } catch (Throwable t) { + // Latch for exactly the reason the MAX_SENT_DICT_BYTES arm above does, and it + // was asymmetric until now: accumulateSentDict runs AFTER the frame's + // sendBinary, so a bare throw unwinds to ioLoop -> fail() -> connectLoop, + // which (running still true) reconnects and replays the same frame, which + // re-attempts the same allocation. A genuine out-of-memory therefore became an + // unbounded reconnect livelock instead of the loud failure the ceiling arm + // promises. recordFatal flips running=false so connectLoop winds down and + // checkError() surfaces it; the throw still unwinds past the pending copy. + recordFatal(t); + throw t; + } + sentDictBytesCapacity = (int) newCap; + } + + private void releaseSentDictBytes() { + if (sentDictBytesAddr != 0 && sentDictBytesOwned) { + Unsafe.free(sentDictBytesAddr, sentDictBytesCapacity, MemoryTag.NATIVE_DEFAULT); + } + sentDictBytesAddr = 0; + sentDictBytesCapacity = 0; + sentDictBytesLen = 0; + sentDictBytesOwned = false; + sentDictCount = 0; + } + + /** + * Decodes the varint at {@code [p, limit)} and returns {@code (value << 3) | bytes}, + * or {@code -1} when it is truncated or runs past a canonical length. + *

+ * Packing the consumed length into the return value -- the shape + * {@code RecoveredFrameAnalysis.readVarint} already uses -- drops a store to an + * instance field on every call, and this runs once per SYMBOL: per frame in + * accumulateSentDict, and once per mirror entry in sendDictCatchUp. + *

+ * The {@code -1} also makes truncation DETECTABLE. The field version returned 0 with + * {@code varintEnd == p} when {@code p >= limit}, so a caller computing + * {@code p = varintEnd + len} got {@code p == limit} and its {@code p > limit} + * bail-out could not fire at an exact boundary -- guards that could not guard. + */ + private static long readVarintAt(long p, long limit) { + long value = 0; + int shift = 0; + int bytes = 0; + while (p < limit && bytes < 6) { + byte b = Unsafe.getUnsafe().getByte(p++); + value |= (long) (b & 0x7F) << shift; + bytes++; + if ((b & 0x80) == 0) { + return (value << 3) | bytes; + } + shift += 7; + } + return -1L; + } + + /** + * Re-registers the whole symbol dictionary on a fresh connection, split into + * as many table-less frames as the server's advertised batch cap requires so + * no single frame exceeds it (a large dictionary would otherwise be rejected). + * Each chunk carries a contiguous id range {@code [start .. start+count)}, in + * order, so the server accumulates them exactly as it would the original + * per-frame deltas. Returns the number of frames sent (each consumed a wire + * sequence), so the caller can align {@code fsnAtZero}. Throws {@link + * CatchUpSendException} on a send error (retriable -- the caller reconnects); + * a single entry too large for the cap is non-retriable, so it latches a + * terminal before throwing. + */ + private int sendDictCatchUp() { + int cap = client.getServerMaxBatchSize(); + // Two limits, deliberately different when the server advertises no cap. + // + // packingLimit bounds a multi-entry chunk. With no advertised cap the receive + // buffer is unknown, so this stays conservative (UNCAPPED_CATCHUP_PACKING_LIMIT, + // well under the 131072 default receive buffer) and a dictionary of ordinary + // symbols always chunks into frames a server will accept. + // + // soloFrameLimit bounds the CAP-GAP terminal below and must not be tightened the + // same way: an entry that already went out inside a data frame was bounded by + // effectiveAutoFlushBytes, not by our fallback, so tightening it would invent a + // new terminal for data the producer sent successfully. When a cap IS advertised + // the two coincide, which is what the homogeneous-cluster argument below rests on. + // + // No unsplit fast path: when the dictionary fits, the walk below emits exactly + // one frame anyway, so the special case only ever removed the bound. + int packingLimit = cap > 0 ? cap : UNCAPPED_CATCHUP_PACKING_LIMIT; + int soloFrameLimit = cap > 0 ? cap : MAX_SENT_DICT_BYTES; + // Symbol-bytes budget for PACKING several entries into one chunk, leaving + // room for the 12-byte header and the two delta-section varints. Kept + // deliberately conservative (reserving 16 for the varints): it only makes a + // multi-entry chunk split marginally earlier, never over the limit. It must + // NOT gate the single-entry terminal -- that reserve is larger than the + // minimal data-frame overhead, so an entry the producer already shipped + // under this cap could exceed the reserve yet still fit its own catch-up + // frame; the terminal tests the real solo frame against soloFrameLimit instead. + int budget = Math.max(1, packingLimit - QwpConstants.HEADER_SIZE - 16); + int framesSent = 0; + int chunkStartId = 0; + long chunkStartAddr = sentDictBytesAddr; + int chunkSymbols = 0; + long chunkBytes = 0; + // Walk the mirror once with a running pointer, deriving each entry's byte span + // from its own [len varint][utf8] framing -- no separate offset index is kept + // (see accumulateSentDict). A mirror built from CRC-validated frames is always + // well-formed, so entryEnd can exceed the limit only under memory corruption; + // latch a terminal via recordFatal rather than let a bare throw unwind into + // connectLoop as a transient and reconnect-livelock. + long entryPtr = sentDictBytesAddr; + long mirrorLimit = sentDictBytesAddr + sentDictBytesLen; + for (int entryId = 0; entryId < sentDictCount; entryId++) { + long entryStart = entryPtr; + long encodedLen = readVarintAt(entryPtr, mirrorLimit); + long entryEnd = encodedLen < 0L + ? -1L + : entryPtr + (encodedLen & 7L) + (encodedLen >>> 3); + if (entryEnd < 0L || entryEnd > mirrorLimit) { + LineSenderException err = new LineSenderException( + "invalid symbol dictionary mirror during catch-up [entry=" + + entryId + ", count=" + sentDictCount + ']'); + recordFatal(err); + throw new CatchUpSendException(err); + } + entryPtr = entryEnd; + long entryBytes = entryEnd - entryStart; + // The exact table-less frame sendCatchUpChunk would build for THIS entry + // alone: header + deltaStart varint (the entry's own global id) + + // deltaCount varint (1) + the entry bytes. A cap gap exists only when even + // that solo frame exceeds the cap -- i.e. the entry genuinely cannot be + // re-registered. Testing the real solo frame (not the conservative + // packing budget above) is what keeps a HOMOGENEOUS cluster + // livelock-free: an entry the producer already shipped in a data frame + // under this cap (header + delta varints + entry + schema + >=1 row) is + // strictly larger than its bare catch-up frame, so it always fits here. + long soloFrameLen = QwpConstants.HEADER_SIZE + + NativeBufferWriter.varintSize(chunkStartId + chunkSymbols) + + NativeBufferWriter.varintSize(1) + + entryBytes; + if (soloFrameLen > soloFrameLimit) { + // Cap gap: this entry cannot be re-registered under the fresh + // server's advertised cap. A HOMOGENEOUS cluster never reaches here + // (an entry that fit its data frame under a cap always fits its bare + // catch-up frame under that same cap), so the only way in is a + // heterogeneous / rolling-cap failover to a smaller-cap node. + // + // A foreground sender retries indefinitely: store-and-forward must + // contain a transient heterogeneous-cluster window instead of surfacing + // it to the producer. Only an orphan drainer may apply the settle budget + // below and quarantine its slot after a persistent gap. Under budget the + // throw is RETRIABLE (no recordFatal) -- connectLoop reconnects with + // backoff and re-runs the catch-up, which resets the episode on a node + // that accepts it. An unrelated transport/upgrade/role state also resets + // the episode, so its downtime cannot satisfy the orphan dwell. On + // exhaustion latch via recordFatal, NOT fail() -- + // failing from inside the catch-up would re-enter connectLoop (see + // CatchUpSendException); the data must be resent after the cap is raised. + // Escalation needs BOTH the strike count AND a minimum wall-clock dwell + // (see DEFAULT_CATCHUP_CAP_GAP_MIN_ESCALATION_WINDOW_MILLIS). A count + // alone measures how often we looked, not how long the gap has held -- + // and 16 attempts at the capped backoff take only ~2 minutes, less than + // an ordinary rolling restart of the larger-cap node. Escalating on the + // count alone would therefore hard-fail a live store-and-forward + // producer during a routine cluster operation. + // + // The STRIKE COUNT -- never the anchor's sign -- says whether an episode is + // already open. A System.nanoTime() instant is only meaningful as a + // difference: its origin is arbitrary and the spec permits negative values, + // so a sign test cannot tell an unset anchor from a real one. Wherever it + // misreads a real anchor as unset it re-anchors to now on EVERY strike, + // pinning episodeNanos at ~0, and the dwell can then never be satisfied -- + // the terminal never latches, however long the gap truly persists. + // recordHeadRejectionStrike keys its episode off poisonStrikes for the same + // reason. + if (catchUpCapGapPolicy == CatchUpCapGapPolicy.RETRY_FOREVER) { + throw new CatchUpSendException(new LineSenderException( + "symbol dictionary entry too large for the server batch cap during catch-up [" + + "frameLen=" + soloFrameLen + ", cap=" + cap + ']' + + "; retrying indefinitely -- a larger-cap node may return"), true); + } + + long nowNanos = System.nanoTime(); + if (catchUpCapGapAttempts == 0) { + catchUpCapGapFirstNanos = nowNanos; + } + catchUpCapGapAttempts++; + long episodeNanos = nowNanos - catchUpCapGapFirstNanos; + boolean exhausted = catchUpCapGapAttempts >= MAX_CATCHUP_CAP_GAP_ATTEMPTS + && episodeNanos >= catchUpCapGapMinEscalationWindowNanos; + LineSenderException err = new LineSenderException( + "symbol dictionary entry too large for the server batch cap during catch-up [" + + "frameLen=" + soloFrameLen + ", cap=" + cap + ", attempt=" + + catchUpCapGapAttempts + '/' + MAX_CATCHUP_CAP_GAP_ATTEMPTS + + ", episodeMillis=" + (episodeNanos / 1_000_000L) + + '/' + (catchUpCapGapMinEscalationWindowNanos / 1_000_000L) + ']' + + (exhausted + ? "; the data must be resent after the cap is raised" + : "; retrying -- a larger-cap node may return")); + if (exhausted) { + recordFatal(err); + } + throw new CatchUpSendException(err, true); + } + if (chunkSymbols > 0 && chunkBytes + entryBytes > budget) { + sendCatchUpChunk(chunkStartId, chunkSymbols, chunkStartAddr, (int) chunkBytes); + framesSent++; + chunkStartId += chunkSymbols; + chunkStartAddr = entryStart; + chunkSymbols = 0; + chunkBytes = 0; + } + chunkSymbols++; + chunkBytes += entryBytes; + } + if (chunkSymbols > 0) { + sendCatchUpChunk(chunkStartId, chunkSymbols, chunkStartAddr, (int) chunkBytes); + framesSent++; + } + // The whole dictionary re-registered without a cap gap: this node accepts + // every entry, so the cap-gap episode (if any) is over -- reset BOTH the strike + // count and the wall-clock anchor. Resetting only the count would leave a stale + // anchor from an old episode, so the very first strike of a LATER episode would + // already satisfy the dwell. + resetCatchUpCapGapEpisode(); + return framesSent; + } + + /** + * Sends one table-less catch-up frame carrying dictionary ids + * {@code [deltaStart .. deltaStart+deltaCount)}. Throws {@link + * CatchUpSendException} on a send error instead of calling {@link #fail} + * (see that type for why the catch-up must not re-enter the reconnect loop); + * the caller turns it into a single, non-re-entrant reconnect. + */ + private void sendCatchUpChunk(int deltaStart, int deltaCount, long symbolsAddr, int symbolsLen) { + // Compute the frame size in long and fail loud if it would overflow the int + // size math into a negative Unsafe.malloc. sendDictCatchUp already caps each + // chunk's symbol bytes under the budget, so this is unreachable at real + // cardinality -- but the mirror-side ensureSentDictCapacity guards the same + // math, and a future caller must not be able to overflow this one silently. + long payloadLenL = (long) NativeBufferWriter.varintSize(deltaStart) + + NativeBufferWriter.varintSize(deltaCount) + + symbolsLen; + long frameLenL = QwpConstants.HEADER_SIZE + payloadLenL; + if (frameLenL > MAX_SENT_DICT_BYTES) { + LineSenderException err = new LineSenderException( + "symbol dictionary catch-up frame exceeds the maximum size [" + + "frameLen=" + frameLenL + ", max=" + MAX_SENT_DICT_BYTES + ']'); + recordFatal(err); + throw new CatchUpSendException(err); + } + int payloadLen = (int) payloadLenL; + int prefixLen = QwpConstants.HEADER_SIZE + + NativeBufferWriter.varintSize(deltaStart) + + NativeBufferWriter.varintSize(deltaCount); + ensureCatchUpFrameCapacity(prefixLen); + long frame = catchUpFrameAddr; + try { + Unsafe.getUnsafe().putByte(frame, (byte) 'Q'); + Unsafe.getUnsafe().putByte(frame + 1, (byte) 'W'); + Unsafe.getUnsafe().putByte(frame + 2, (byte) 'P'); + Unsafe.getUnsafe().putByte(frame + 3, (byte) '1'); + Unsafe.getUnsafe().putByte(frame + 4, (byte) client.getServerQwpVersion()); + // FLAG_DEFER_COMMIT: the catch-up carries dictionary entries but NO + // rows, so it must never trigger a server-side commit. Today it is + // always the first frame on a fresh (empty-transaction) connection, so + // committing nothing is a no-op -- but that invariant is load-bearing + // and unasserted. Deferring the (empty) commit removes the dependency: + // a future mid-stream catch-up cannot prematurely commit an in-flight + // deferred transaction. The dictionary delta still registers (as any + // deferred data frame's does); only the row commit is deferred, and the + // next real frame commits it. + Unsafe.getUnsafe().putByte(frame + QwpConstants.HEADER_OFFSET_FLAGS, + (byte) (QwpConstants.FLAG_GORILLA | QwpConstants.FLAG_DELTA_SYMBOL_DICT + | QwpConstants.FLAG_DEFER_COMMIT)); + Unsafe.getUnsafe().putShort(frame + 6, (short) 0); // tableCount + Unsafe.getUnsafe().putInt(frame + 8, payloadLen); + long q = NativeBufferWriter.writeVarint(frame + QwpConstants.HEADER_SIZE, deltaStart); + NativeBufferWriter.writeVarint(q, deltaCount); + client.sendBinary(frame, prefixLen, symbolsAddr, symbolsLen); + } catch (Throwable t) { + // Do NOT fail() here -- see CatchUpSendException. Signal the failure + // up so exactly one non-re-entrant reconnect follows. A JVM Error is + // never a transient reconnect case; let it propagate as-is so the + // I/O loop latches it terminal rather than looping on it. + if (t instanceof Error) { + throw (Error) t; + } + throw new CatchUpSendException(t); + } + nextWireSeq++; // this catch-up chunk consumed a wire sequence + lastFrameOrPingNanos = System.nanoTime(); + totalFramesSent.incrementAndGet(); + } + + @TestOnly + public int catchUpFrameGrowthCount() { + return catchUpFrameGrowthCount; + } + + @TestOnly + public int catchUpCapGapAttempts() { + return catchUpCapGapAttempts; + } + + @TestOnly + public long catchUpCapGapFirstNanos() { + return catchUpCapGapFirstNanos; + } + + @TestOnly + public long catchUpCapGapMinEscalationWindowNanos() { + return catchUpCapGapMinEscalationWindowNanos; + } + + @TestOnly + public long fsnAtZero() { + return fsnAtZero; + } + + @TestOnly + public long poisonFsn() { + return poisonFsn; + } + + @TestOnly + public int poisonStrikes() { + return poisonStrikes; + } + + @TestOnly + public long sentDictBytesAddr() { + return sentDictBytesAddr; + } + + @TestOnly + public int sentDictBytesLen() { + return sentDictBytesLen; + } + + @TestOnly + public boolean sentDictBytesOwned() { + return sentDictBytesOwned; + } + + @TestOnly + public int sentDictCount() { + return sentDictCount; + } + + @TestOnly + public int zeroProgressRecycles() { + return zeroProgressRecycles; + } + + @TestOnly + public void accumulateSentDictForTest(long payloadAddr, int payloadLen, int deltaStart) { + accumulateSentDict(payloadAddr, payloadLen, deltaStart); + } + + @TestOnly + public void connectLoopForTest(Throwable initial, String phase, long paceFirstAttemptMillis) { + connectLoop(initial, phase, paceFirstAttemptMillis); + } + + @TestOnly + public void deliverCloseForTest(int code, String reason) { + responseHandler.onClose(code, reason); + } + + @TestOnly + public void deliverResponseForTest(long payloadPtr, int payloadLen) { + responseHandler.onBinaryMessage(payloadPtr, payloadLen); + } + + @TestOnly + public void ensureSentDictCapacityForTest(long required) { + ensureSentDictCapacity(required); + } + + @TestOnly + public void positionCursorForStartForTest() { + positionCursorForStart(); + } + + @TestOnly + public void seedSentDictMirrorForTest(long addr, int bytes, int symbolCount) { + sentDictBytesAddr = addr; + sentDictBytesCapacity = bytes; + sentDictBytesLen = bytes; + sentDictCount = symbolCount; + sentDictBytesOwned = true; + } + + @TestOnly + public void sendCatchUpChunkForTest(int deltaStart, int deltaCount, long symbolsAddr, int symbolsLen) { + sendCatchUpChunk(deltaStart, deltaCount, symbolsAddr, symbolsLen); + } + + @TestOnly + public void setCatchUpCapGapFirstNanosForTest(long nanos) { + catchUpCapGapFirstNanos = nanos; + } + + @TestOnly + public void setDataFrameSentThisConnectionForTest(boolean value) { + dataFrameSentThisConnection = value; + } + + @TestOnly + public void setFsnAtZeroForTest(long value) { + fsnAtZero = value; + } + + @TestOnly + public void setNextWireSeqForTest(long value) { + nextWireSeq = value; + } + + @TestOnly + public void setRunningForTest(boolean value) { + running = value; + } + + @TestOnly + public void setWireBaselineWithCatchUpForTest(long replayStart) { + setWireBaselineWithCatchUp(replayStart); + } + + @TestOnly + public boolean trySendOneForTest() { + return trySendOne(); + } + + private void ensureCatchUpFrameCapacity(int required) { + if (catchUpFrameCapacity >= required) { + return; + } + long newCapacity = Math.max(required, Math.max(4096L, (long) catchUpFrameCapacity * 2L)); + if (newCapacity > MAX_SENT_DICT_BYTES) { + newCapacity = MAX_SENT_DICT_BYTES; + } + catchUpFrameAddr = Unsafe.realloc( + catchUpFrameAddr, + catchUpFrameCapacity, + (int) newCapacity, + MemoryTag.NATIVE_DEFAULT); + catchUpFrameCapacity = (int) newCapacity; + catchUpFrameGrowthCount++; + } + + private void freeCatchUpFrameBuffer() { + if (catchUpFrameAddr != 0L) { + Unsafe.free(catchUpFrameAddr, catchUpFrameCapacity, MemoryTag.NATIVE_DEFAULT); + catchUpFrameAddr = 0L; + catchUpFrameCapacity = 0; + } } private boolean tryReceiveAcks() { @@ -2025,10 +3265,32 @@ private boolean trySendOne() { return false; } if (nextWireSeq == 0) { - // Nothing sent on this connection yet: re-anchor in place past - // the retired tail. The wireSeq<->FSN mapping is untouched - // because no wire sequence has been consumed. - positionCursorForStart(); + // Reached only when the tail was NOT retirable at connection setup -- + // both setup sites (swapClient, positionCursorForStart) call + // tryRetireOrphanTail first -- which means frames below it still needed + // acks, which means they were SENT on this connection. So nextWireSeq is + // never 0 here in practice and this arm is effectively dead; it is kept + // as a cheap correctness guard rather than removed. + // + // NB the mapping is untouched only while NO wire sequence has been + // consumed. A dictionary catch-up consumes sequences 0..n-1 before any + // data frame, so after one this test is false and the recycle below -- + // which re-anchors the mapping from scratch -- is the correct path, not + // an avoidable cost. + try { + positionCursorForStart(); + } catch (CatchUpSendException e) { + // Re-anchor's catch-up send failed. fail() here is a fresh, + // non-re-entrant connectLoop entry from the I/O loop body -- + // the same recovery a normal trySendOne send failure takes. + // Preserve the wrapper on a cap gap so connectLoop's + // isCatchUpCapGap keeps the orphan settle episode (attempt count + // + dwell anchor) alive across the re-anchor recycle; an ordinary + // failure unwraps to the raw cause and restarts the episode, like + // any normal send failure. + fail(isCatchUpCapGap(e) ? e : e.getCause()); + return false; + } return true; } // Frames were already sent on this connection: the linear @@ -2079,12 +3341,55 @@ private boolean trySendOne() { if (frameEnd > pub) { return false; // payload not fully published yet } + long frameAddr = base + sendOffset + MmapSegment.FRAME_HEADER_SIZE; + // Torn-dictionary guard. sentDictCount is this loop's model of how many ids the + // CURRENT server has been told about. A frame whose delta starts ABOVE that + // coverage references ids the server was never given, and the server now rejects + // such a frame with STATUS_DICTIONARY_GAP rather than null-padding the hole. That + // rejection is retriable -- a wire recycle re-registers from id 0 -- but this + // frame cannot become sendable without one, and against a server old enough to + // null-pad it would land rows with silently NULL symbols. Fail terminally here, + // where the unreplayable range is still attributable. + // + // The guard, the mirror and the catch-up are ONE mechanism and are all + // ungated (see the constructor). A frame at deltaStart == sentDictCount is + // contiguous and extends the coverage, so a slot whose frames start at id 0 + // replays cleanly with no persisted dictionary at all -- which is exactly why + // the mirror below must keep advancing even when the dictionary is missing. + // A gap here therefore means genuine loss: a host/power crash tore the + // unsynced .symbol-dict below the ids the surviving frames still reference + // (SF is process-crash but not host-crash durable). + int deltaStart = frameDeltaStart(frameAddr, payloadLen); + if (deltaStart > sentDictCount) { + recordFatal(new LineSenderException( + "recovered store-and-forward symbol dictionary is incomplete (likely a host crash): " + + "frame delta start " + deltaStart + " exceeds recovered dictionary size " + + sentDictCount + "; cannot replay without corrupting data -- resend required")); + return false; + } try { - client.sendBinary(base + sendOffset + MmapSegment.FRAME_HEADER_SIZE, payloadLen); + client.sendBinary(frameAddr, payloadLen); } catch (Throwable t) { fail(t); return false; } + // A real ring frame (data or commit) has now gone out on this connection, + // as opposed to only the dictionary catch-up. onClose / handleServerRejection + // key their poison-strike vs pre-send decision off this, not off nextWireSeq + // (which the catch-up advances). + dataFrameSentThisConnection = true; + if (deltaStart >= 0) { + // Mirror the symbols this frame just registered on the server, so a later + // reconnect can rebuild the whole dictionary and the guard above keeps an + // accurate view of the server's coverage. Idempotent on replay: a frame + // whose delta we already hold advances nothing. + // + // Ungated (see the constructor): this is the ONLY thing that advances + // sentDictCount from the frames themselves, so gating it while leaving the + // guard ungated froze the coverage at 0 and terminal'd a slot that replays + // perfectly from id 0. + accumulateSentDict(frameAddr, payloadLen, deltaStart); + } lastFrameOrPingNanos = System.nanoTime(); sendOffset = frameEnd; long fsnSent = fsnAtZero + nextWireSeq; @@ -2114,8 +3419,11 @@ void positionCursorForStart() { // starts past it. Zero wire cost, no recycle. tryRetireOrphanTail(); long replayStart = engine.ackedFsn() + 1L; - this.fsnAtZero = replayStart; - this.nextWireSeq = 0L; + // Recovery / orphan-drain seed the dictionary mirror, so the initial + // connection may also need a catch-up (client is non-null in the + // sync-start and drainer paths; null in async-initial, where swapClient + // handles it on the first connect). + setWireBaselineWithCatchUp(replayStart); positionCursorAt(replayStart); } @@ -2141,25 +3449,40 @@ private boolean tryRetireOrphanTail() { if (orphanSkipTipFsn < 0) { return true; } - if (engine.ackedFsn() < orphanSkipStartFsn - 1L) { + if (!engine.retireRecoveredOrphanTailIfReady()) { return false; } - LOG.warn("retiring orphaned deferred tail: {} frame(s) [fsn {}..{}] belong to a transaction " - + "whose commit was never published; aborting them (never transmitted, slots trimmed)", - orphanSkipTipFsn - orphanSkipStartFsn + 1, orphanSkipStartFsn, orphanSkipTipFsn); - engine.acknowledge(orphanSkipTipFsn); orphanSkipStartFsn = -1L; orphanSkipTipFsn = -1L; return true; } + /** + * Determines whether an oversized symbol-dictionary catch-up entry is always + * retriable or may become terminal after the orphan settle budget. + */ + public enum CatchUpCapGapPolicy { + /** Keep reconnecting until a node with a compatible batch cap returns. */ + RETRY_FOREVER, + /** Quarantine an orphan slot after both the attempt and dwell limits expire. */ + TERMINAL_AFTER_SETTLE_BUDGET + } + + /** Identifies who owns data while the reconnect loop is unavailable. */ + public enum ReconnectPolicy { + /** A live producer keeps buffering and retries endpoint-policy failures. */ + FOREGROUND, + /** An orphan drainer returns terminal states to its quarantine owner. */ + ORPHAN + } + /** * Factory used by the I/O loop to build a fresh, connected, upgraded * {@link WebSocketClient} after a wire failure. Implementations close * the old client (if needed), build a new one with the same auth/TLS * config, connect, perform the WebSocket upgrade, and return it ready - * to send. Throw on a terminal failure (auth rejection, etc.) — the - * I/O loop will treat the throw as fatal. + * to send. The loop's {@link ReconnectPolicy} decides whether endpoint-policy + * failures are retried or returned as terminal. */ @FunctionalInterface public interface ReconnectFactory { @@ -2264,6 +3587,39 @@ void cancel() { } } + /** + * Signals that a symbol-dictionary catch-up frame could not be sent on the + * current connection. Thrown by {@link #sendDictCatchUp}/{@link + * #sendCatchUpChunk} instead of calling {@link #fail}: the catch-up runs + * inside {@link #connectLoop} (via {@link #swapClient}) and, on the initial + * connect, inside {@link #start()} / {@link #trySendOne} on the caller + * thread. Calling {@code fail()} from there would re-enter {@code + * connectLoop} -- corrupting the {@code fsnAtZero}/{@code nextWireSeq} wire + * mapping (a subsequent ACK then trims un-acked frames) and growing the + * stack until it overflows into a terminal, breaking the "retry a transient + * outage forever" invariant -- or run {@code connectLoop} on the caller + * thread and block {@code Sender} construction indefinitely. Each catch + * site instead turns it into ONE non-re-entrant reconnect: {@code + * connectLoop}'s own retry catch (swapClient path), a fresh {@code fail()} + * from the I/O loop body (trySendOne path), or dropping the dead client so + * the I/O thread reconnects (start path). A JVM {@code Error} is never + * wrapped -- it must stay terminal. The {@code capGap} marker lets the reconnect + * loop preserve only consecutive incompatible-cap observations; ordinary catch-up + * send failures restart the orphan settle episode. + */ + private static final class CatchUpSendException extends RuntimeException { + private final boolean isCapGap; + + CatchUpSendException(Throwable cause) { + this(cause, false); + } + + CatchUpSendException(Throwable cause, boolean isCapGap) { + super(cause); + this.isCapGap = isCapGap; + } + } + /** * One slot in the pendingDurable FIFO. Holds a wireSeq plus the per-table * (name, seqTxn) pairs from its OK frame. Empty entries (tableCount = 0) @@ -2401,7 +3757,7 @@ public void onClose(int code, String reason) { || code == WebSocketCloseCode.GOING_AWAY; LineSenderException cause = new LineSenderException( "WebSocket closed by server: code=" + code + " reason=" + reason); - if (!orderly && nextWireSeq > 0) { + if (!orderly && dataFrameSentThisConnection) { if (recordHeadRejectionStrike(Math.max(engine.ackedFsn(), highestOkFsn) + 1L)) { haltOnPoisonedFrame("ws-close[" + code + ' ' + WebSocketCloseCode.describe(code) + "]: " + reason, @@ -2508,17 +3864,21 @@ private void handleServerRejection(long wireSeq) { // value is only used to attribute an FSN to the error report -- // a rejection never advances the watermark. long highestSent = nextWireSeq - 1L; - if (highestSent < 0L) { - // Pre-send rejection: server emitted an error frame before - // we sent anything on this connection (typical after a - // fresh swapClient — auth failure, server-initiated halt, - // etc.). The server-named wireSeq does not correspond to - // any frame we sent, so clamping it to 0 and acknowledging - // fsnAtZero would silently advance ackedFsn past a real - // unsent batch (fsnAtZero == ackedFsn + 1 right after a - // swap). Skip the watermark advance entirely; still surface - // the error so the user's handler sees it and HALT errors - // remain producer-observable. + if (!dataFrameSentThisConnection) { + // Pre-send rejection: the server emitted an error frame before we + // sent any DATA frame on this connection (typical after a fresh + // swapClient -- auth failure, server-initiated halt, or a rejection + // of the dictionary catch-up itself). nextWireSeq may be > 0 here + // because the catch-up consumed wire sequences, so this keys off + // dataFrameSentThisConnection, not highestSent >= 0 -- otherwise a + // transient NACK of a catch-up frame would take the post-send + // poison-strike path and could escalate a transient outage to a + // terminal. The server-named wireSeq does not correspond to any + // data frame we sent, so clamping it to 0 and acknowledging + // fsnAtZero would silently advance ackedFsn past a real unsent + // batch. Skip the watermark advance entirely; still surface the + // error so the user's handler sees it and HALT errors remain + // producer-observable. handlePreSendRejection(wireSeq, status, category, policy); return; } @@ -2528,6 +3888,32 @@ private void handleServerRejection(long wireSeq) { wireSeq, highestSent); } long fsn = fsnAtZero + cappedSeq; + if (fsn <= engine.ackedFsn()) { + // The clamped wire seq maps at or below the replay head, so this + // NACK is for a dictionary catch-up frame -- which occupies the + // already-acked wire sequences below replayStart -- not a data frame + // this connection sent. (dataFrameSentThisConnection can be true here + // because trySendOne ships the head data frame before tryReceiveAcks + // reads the catch-up's NACK in the same loop iteration.) Attributing + // it a data FSN would key recordHeadRejectionStrike() off a + // below-baseline FSN -- negative when replayStart < catchUpFrames -- + // colliding with the poisonFsn == -1 "no suspect" sentinel and + // laundering a genuine poison run, and would report a bogus FSN. + // Treat it exactly like a pre-send rejection: surface + recycle, no + // poison strike, no watermark advance. Symmetric with the success + // path, where engine.acknowledge() no-ops at or below ackedFsn. A + // real replayed data frame is at fsn > ackedFsn, so it is never + // caught here. + // + // Catch-up frames therefore sit OUTSIDE the poison detector: a + // deterministically-NACKed catch-up recycles forever (paced, so no + // busy-loop). That is acceptable -- a catch-up only re-registers + // symbols the cluster already accepted, so a persistent NACK of one + // is a server bug, not a poison-frame the client can quarantine, and + // Invariant B's "retry a transient outage forever" applies. + handlePreSendRejection(wireSeq, status, category, policy); + return; + } // Best-effort table attribution: the parser populates // response.tableNames on error frames the same way it does on // STATUS_OK. If exactly one table was named, surface it; if diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java index d7d2b44d..779c9900 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java @@ -47,8 +47,8 @@ *

* On-disk layout — header and frame format: * 

- *   [u32 magic 'SF01'] [u8 ver=1] [u8 flags=0] [u16 reserved=0]
- *   [u64 baseSeq]       [u64 createdMicros]                       24-byte header
+ *   [u32 magic 'SF01'] [u8 ver=1] [u8 flags]   [u16 reserved=0]
+ *   [u64 baseSeq]      [u64 createdMicros]                        24-byte header
  *   frame, frame, ...                                              each frame:
  *                                                                  [u32 crc32c]
  *                                                                  [u32 payloadLen]
@@ -182,7 +182,8 @@ public static MmapSegment create(FilesFacade ff, String path, long baseSeq, long
         return create(ff, path, baseSeq, sizeBytes, false);
     }
 
-    static MmapSegment create(FilesFacade ff, String path, long baseSeq, long sizeBytes, boolean manifestRequired) {
+    static MmapSegment create(FilesFacade ff, String path, long baseSeq, long sizeBytes,
+                              boolean manifestRequired) {
         long pathPtr = ff.allocNativePath(path);
         try {
             return create(ff, pathPtr, path, baseSeq, sizeBytes, manifestRequired);
@@ -199,11 +200,13 @@ static MmapSegment create(FilesFacade ff, String path, long baseSeq, long sizeBy
      * {@code DirectUtf8Sink} by the rotation hot path so it does not incur a
      * per-call {@code byte[]} + native-malloc the way the String overload does.
      */
-    public static MmapSegment create(FilesFacade ff, long pathPtr, String displayPath, long baseSeq, long sizeBytes) {
+    public static MmapSegment create(FilesFacade ff, long pathPtr, String displayPath, long baseSeq,
+                                     long sizeBytes) {
         return create(ff, pathPtr, displayPath, baseSeq, sizeBytes, false);
     }
 
-    static MmapSegment create(FilesFacade ff, long pathPtr, String displayPath, long baseSeq, long sizeBytes, boolean manifestRequired) {
+    static MmapSegment create(FilesFacade ff, long pathPtr, String displayPath, long baseSeq, long sizeBytes,
+                              boolean manifestRequired) {
         if (sizeBytes < HEADER_SIZE + FRAME_HEADER_SIZE + 1) {
             throw new IllegalArgumentException(
                     "sizeBytes too small for header + one minimal frame: " + sizeBytes);
@@ -239,7 +242,8 @@ static MmapSegment create(FilesFacade ff, long pathPtr, String displayPath, long
             Unsafe.getUnsafe().putShort(addr + 6, (short) 0); // reserved
             Unsafe.getUnsafe().putLong(addr + 8, baseSeq);
             Unsafe.getUnsafe().putLong(addr + 16, Os.currentTimeMicros());
-            return new MmapSegment(ff, displayPath, fd, addr, sizeBytes, baseSeq, HEADER_SIZE, 0, false, 0L);
+            return new MmapSegment(ff, displayPath, fd, addr, sizeBytes, baseSeq,
+                    HEADER_SIZE, 0, false, 0L);
         } catch (Throwable t) {
             if (addr != Files.FAILED_MMAP_ADDRESS) {
                 ff.munmap(addr, sizeBytes, MemoryTag.MMAP_DEFAULT);
@@ -276,13 +280,59 @@ public static MmapSegment createInMemory(long baseSeq, long sizeBytes) {
             Unsafe.getUnsafe().putShort(addr + 6, (short) 0);
             Unsafe.getUnsafe().putLong(addr + 8, baseSeq);
             Unsafe.getUnsafe().putLong(addr + 16, Os.currentTimeMicros());
-            return new MmapSegment(null, null, -1, addr, sizeBytes, baseSeq, HEADER_SIZE, 0, true, 0L);
+            return new MmapSegment(null, null, -1, addr, sizeBytes, baseSeq,
+                    HEADER_SIZE, 0, true, 0L);
         } catch (Throwable t) {
             Unsafe.free(addr, sizeBytes, MemoryTag.NATIVE_DEFAULT);
             throw t;
         }
     }
 
+    /**
+     * True when {@code t} is the JVM's recoverable signal for a fault while
+     * accessing a memory-mapped region -- a SIGBUS/SIGSEGV that HotSpot
+     * translates into an {@code InternalError} at an {@code Unsafe} intrinsic
+     * site instead of aborting the process. It surfaces when a mapped page is
+     * not backed by real file blocks: a sparse {@code .sfa} tail on a
+     * filesystem whose pre-allocation leaves holes (e.g. ZFS, where a
+     * truncate-based {@code allocate} does not materialize blocks), or a file
+     * truncated under the mapping. Recovery treats this as an I/O boundary --
+     * the same way MappedByteBuffer readers do -- not a fatal VM error.
+     * 

+     * The message is matched on the fragment {@code "unsafe memory access
+     * operation"}, which is common to both HotSpot wordings and NOT
+     * version-stable as a whole: pre-21 JDKs (including the shipping/CI JDK 8)
+     * emit {@code "a fault occurred in a recent unsafe memory access operation
+     * in compiled Java code"}, while JDK 21+ shortened it to {@code "a fault
+     * occurred in an unsafe memory access operation"}. Matching the shared
+     * fragment keeps the guard effective on JDK 8/11/17 as well as 21+, while
+     * still being specific enough that a genuine VirtualMachineError (real OOM,
+     * StackOverflow) is never swallowed.
+     * 

+     * Delivery is NOT precise before JDK 21, so callers must guard too.
+     * Pre-21 HotSpot records the fault and raises the {@code InternalError} at
+     * the next return or safepoint check rather than at the faulting
+     * instruction, so it can surface in a CALLER's frame, past the handler that
+     * wraps the faulting read. JDK-8283699 made delivery precise in 21+.
+     * 

+     * Segment recovery no longer needs this predicate: {@link #openExisting}
+     * scans through positioned reads into a heap-free bounded buffer and only
+     * maps the file once the scan has validated it, so a sparse or unbacked
+     * page surfaces as an ordinary read failure rather than a SIGBUS. It stays
+     * public for the mappings that ARE still folded over
+     * {@code Crc32c.updateUnsafe}: {@code PersistedSymbolDict}'s append and
+     * heal paths, whose handlers in {@code QwpWebSocketSender} apply this
+     * exemption ahead of their own {@code Error} rethrow so a recognised fault
+     * degrades the sender instead of escaping as a raw {@code InternalError}.
+     */
+    public static boolean isMmapAccessFault(Throwable t) {
+        if (!(t instanceof InternalError)) {
+            return false;
+        }
+        String msg = t.getMessage();
+        return msg != null && msg.contains("unsafe memory access operation");
+    }
+
     /**
      * Opens an existing segment file for recovery. Validates the header and
      * scans frames through positioned reads, then mmaps the validated file RW.
@@ -679,44 +729,17 @@ public long tryAppend(long payloadAddr, int payloadLen) {
     }
 
     /**
-     * Walks every published frame in this segment and returns the FSN of the
-     * LAST frame whose payload does NOT carry the given flag bit, or {@code -1}
-     * when every frame carries it (or the segment is empty).
-     * 

-     * A frame counts as carrying the flag ONLY when it positively parses as a
-     * message of the expected protocol: payload at least {@code minPayloadLen}
-     * bytes AND the little-endian u32 at payload offset 0 equals
-     * {@code headerMagic} AND the byte at {@code flagsOffset} has
-     * {@code flagMask} set. Anything else -- short frames, foreign payloads,
-     * magic mismatches -- counts as NOT carrying the flag. This direction is
-     * deliberate: the caller retires (trims) frames ABOVE the returned FSN,
-     * so a frame we cannot positively identify must act as a retirement
-     * barrier, never as trimmable. Misclassifying an unknown frame as
-     * deferred would silently discard data that should replay.
-     * 

-     * Producer-thread only, and only meaningful before new appends race the
-     * walk (recovery time). Used to locate the last commit-bearing QWP frame
-     * below a potentially orphaned FLAG_DEFER_COMMIT tail: frames above the
-     * returned FSN all carry the flag, i.e. they belong to a transaction whose
-     * commit frame was never published.
+     * Feeds every recovered frame in this segment to the engine's single-pass
+     * recovery fold. Segments are visited oldest-first by {@link SegmentRing}.
      */
-    public long findLastFrameFsnWithoutPayloadFlag(int flagsOffset, int flagMask, int headerMagic, int minPayloadLen) {
-        long best = -1L;
+    void scanRecovery(RecoveredFrameAnalysis analysis) {
         long off = HEADER_SIZE;
         long frames = frameCount;
         for (long i = 0; i < frames; i++) {
             int payloadLen = Unsafe.getUnsafe().getInt(mmapAddress + off + 4);
-            long payload = mmapAddress + off + FRAME_HEADER_SIZE;
-            boolean flagSet = payloadLen >= minPayloadLen
-                    && payloadLen > flagsOffset
-                    && Unsafe.getUnsafe().getInt(payload) == headerMagic
-                    && (Unsafe.getUnsafe().getByte(payload + flagsOffset) & flagMask) != 0;
-            if (!flagSet) {
-                best = baseSeq + i;
-            }
+            analysis.accept(baseSeq + i, mmapAddress + off + FRAME_HEADER_SIZE, payloadLen);
             off += FRAME_HEADER_SIZE + payloadLen;
         }
-        return best;
     }
 
     /**
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/OrphanScanner.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/OrphanScanner.java
index 833960ec..a8ebaa83 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/OrphanScanner.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/OrphanScanner.java
@@ -61,6 +61,20 @@ public final class OrphanScanner {
 
     /** Name of the sentinel that disqualifies a slot from auto-drain. */
     public static final String FAILED_SENTINEL_NAME = ".failed";
+    /**
+     * Reserved infix for a slot {@code Sender.build()} set aside as unreplayable. Such a
+     * copy is kept for forensics and a manual resend; it is NEVER drainable, so the
+     * scanner excludes it BY NAME.
+     * 

+     * Excluding it by name rather than by the {@code .failed} sentinel alone is what makes
+     * the exclusion reliable. {@code markFailed} is best-effort and returns silently when
+     * it cannot open the sentinel -- and the condition that makes a slot unreplayable (a
+     * full or read-only disk) is exactly the condition that makes writing the sentinel
+     * fail. Without the name check, the SAME build() that quarantined the slot dispatches
+     * its orphan drainers twelve lines later and re-adopts it: unbounded churn against
+     * data explicitly declared human-in-the-loop, repeating on every restart.
+     */
+    public static final String QUARANTINE_SLOT_INFIX = ".unreplayable-";
 
     private OrphanScanner() {
     }
@@ -271,12 +285,29 @@ public static boolean isCandidateOrphan(String slotPath) {
         if (!Files.exists(slotPath)) {
             return false;
         }
+        if (isQuarantinedSlotName(slotPath)) {
+            return false;
+        }
         if (Files.exists(slotPath + "/" + FAILED_SENTINEL_NAME)) {
             return false;
         }
         return hasAnySegmentFile(slotPath);
     }
 
+    /**
+     * Whether {@code slotPath}'s last path element names a quarantined slot. Independent
+     * of the {@code .failed} sentinel, which is best-effort -- see
+     * {@link #QUARANTINE_SLOT_INFIX}.
+     */
+    public static boolean isQuarantinedSlotName(String slotPath) {
+        if (slotPath == null) {
+            return false;
+        }
+        int slash = Math.max(slotPath.lastIndexOf('/'), slotPath.lastIndexOf('\\'));
+        String name = slash < 0 ? slotPath : slotPath.substring(slash + 1);
+        return name.contains(QUARANTINE_SLOT_INFIX);
+    }
+
     /**
      * Drops a {@link #FAILED_SENTINEL_NAME} file in {@code slotPath}.
      * Idempotent — touching an existing sentinel is a no-op (its presence
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java
new file mode 100644
index 00000000..627d4c57
--- /dev/null
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java
@@ -0,0 +1,1276 @@
+/*******************************************************************************
+ *     ___                  _   ____  ____
+ *    / _ \ _   _  ___  ___| |_|  _ \| __ )
+ *   | | | | | | |/ _ \/ __| __| | | |  _ \
+ *   | |_| | |_| |  __/\__ \ |_| |_| | |_) |
+ *    \__\_\\__,_|\___||___/\__|____/|____/
+ *
+ *  Copyright (c) 2014-2019 Appsicle
+ *  Copyright (c) 2019-2026 QuestDB
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ ******************************************************************************/
+
+package io.questdb.client.cutlass.qwp.client.sf.cursor;
+
+import io.questdb.client.cutlass.qwp.client.GlobalSymbolDictionary;
+import io.questdb.client.cutlass.qwp.client.NativeBufferWriter;
+import io.questdb.client.std.Crc32c;
+import io.questdb.client.std.Files;
+import io.questdb.client.std.FilesFacade;
+import io.questdb.client.std.MemoryTag;
+import io.questdb.client.std.ObjList;
+import io.questdb.client.std.QuietCloseable;
+import io.questdb.client.std.Unsafe;
+import io.questdb.client.std.str.Utf8s;
+import org.jetbrains.annotations.TestOnly;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Append-only, per-slot persistence of the global symbol dictionary that a
+ * store-and-forward sender ships to the server with delta encoding. Lives at
+ * {@code /.symbol-dict} alongside the segment files, the slot lock and
+ * {@code .ack-watermark}.
+ * 

+ * Delta-encoded SF frames are NOT self-sufficient: a frame carries only the
+ * symbols it introduces, so recovering (process restart) or draining (orphan
+ * adoption) a slot requires re-registering the whole dictionary on the fresh
+ * server before those frames replay. This file is that dictionary. Unlike
+ * {@link AckWatermark} -- a discardable optimization protected by a
+ * {@code max()} clamp -- this file is load-bearing: a surviving frame
+ * that references an id missing from it is unrecoverable. It is therefore held
+ * to a stronger durability contract, and {@link #open} never destroys it (see
+ * "Never recreate over an existing file" below).
+ * 

+ * Layout (little-endian):
+ * 
+ *   offset 0: u32 magic = 'SYD1'
+ *   offset 4: u8  version = 1
+ *   offset 5: 3 bytes reserved (zero)
+ *   offset 8: chunks, each
+ *             [entryCount: varint][entryBytes: varint][entries][crc32c: u32]
+ *             where entries = [len: varint][utf8] repeated entryCount times,
+ *             occupying exactly entryBytes bytes, and the CRC-32C covers the
+ *             two header varints AND the entry region.
+ * 

+ * A recovered dictionary is trusted on its magic, version and per-chunk CRCs.
+ * It cannot outlive the id space it describes: a fresh start truncates it via
+ * {@link #openClean} and refuses the slot outright if that truncation fails,
+ * so a survivor from a prior producer generation is never reachable here.
+ * A chunk is one append -- i.e. exactly the set of symbols one frame
+ * introduces, since the producer persists a frame's new symbols in a single call
+ * before publishing it. Symbol id {@code i} is the {@code i}-th entry across all
+ * chunks (ids are dense and assigned sequentially from 0), so no id is stored.
+ * 

+ * Why the checksum is per chunk, not per entry. The only consumer of the
+ * recovered prefix is the send loop's replay guard, which compares a surviving
+ * frame's {@code deltaStart} against the recovered dictionary size -- and every
+ * {@code deltaStart} is a chunk boundary, because chunks and frame deltas are
+ * written one-for-one. A tear inside a chunk therefore invalidates exactly the
+ * frames a per-entry checksum would have invalidated anyway: per-entry
+ * granularity buys no extra recoverable prefix. It costs a great deal, though.
+ * {@link Crc32c#update} is a native call, so checksumming per entry put one JNI
+ * transition -- plus one sub-cache-line copy and one redundant varint decode --
+ * on the producer thread for every new symbol. On the high-cardinality batch this
+ * feature exists to serve (one new symbol per row), that is a thousand native
+ * calls per flush where the chunk needs one.
+ * 

+ * Durability / write-ahead ordering: the producer appends the symbols a
+ * frame introduces BEFORE that frame is published to the ring, but does NOT
+ * fsync -- matching the rest of store-and-forward, which is page-cache (not
+ * disk) durable. This ordering is sufficient for a process/JVM crash: the
+ * page cache survives, so both the dictionary and the frames survive and the
+ * dictionary is a superset of every recoverable frame's references. It is NOT
+ * sufficient for a host/power crash, where unflushed pages can be lost out
+ * of order and the dictionary may end up torn relative to the frames it serves --
+ * exactly as the segment frames themselves may be lost on a host crash. Two
+ * layers keep a host-crash tear from silently corrupting data:
+ * 

+ * Together these turn every detectable host-crash tear into a fail-clean
+ * "resend required" instead of a silent symbol misattribution -- the same
+ * CRC-32C protection the segment frames carry. A tear that happened to leave a
+ * byte run whose CRC still matches is not distinguished, but that is a 1-in-2^32
+ * collision per corrupted chunk, no weaker than the frames' own checksum.
+ * 

+ * A torn trailing chunk from a crash mid-append is self-healing: {@link #open}
+ * stops parsing at the first incomplete chunk and the next append overwrites it.
+ * 

+ * Never recreate over an existing file. {@link #open} -- the RECOVERY
+ * entry point -- returns {@code null} when an existing file cannot be read or
+ * parsed, and NEVER falls back to recreating it empty. Recreating would mean
+ * {@code O_TRUNC} over the only copy of load-bearing state, so a single transient
+ * read error (an EIO on a flaky disk, a short read) would permanently destroy the
+ * dictionary the surviving delta frames reference -- turning a recoverable outage
+ * into unrecoverable data. A {@code null} instead degrades the sender to full
+ * self-sufficient frames and leaves every byte on disk, so a later attempt, once
+ * the transient clears, can still recover the slot in full. Only
+ * {@link #openClean} -- the FRESH-slot path, where discarding is the whole point
+ * -- truncates.
+ * 

+ * Lifecycle: single-writer (the producer / user thread) for appends. Read
+ * once at {@link #open} to seed in-memory state on recovery or orphan-drain. The
+ * owner (the engine) closes it, and {@code close()} is callable from any thread
+ * (a shutdown hook, test cleanup). {@code close()} and the append methods are
+ * therefore {@code synchronized}: without that, a close racing an in-flight append
+ * could unmap the production append region (or free the fault-test scratch buffer),
+ * close the fd, and let an in-flight write corrupt memory or land on a descriptor
+ * the OS has reused for another file. Not thread-safe for concurrent writers.
+ */
+public final class PersistedSymbolDict implements QuietCloseable {
+
+    /**
+     * Filename within the slot directory. Dot-prefixed so directory
+     * enumerators that filter by the {@code .sfa} suffix (segment recovery,
+     * OrphanScanner, trim) skip it automatically.
+     */
+    public static final String FILE_NAME = ".symbol-dict";
+    static final int CRC_SIZE = 4; // u32 CRC-32C trailing every chunk
+    static final int FILE_MAGIC = 0x31445953; // 'SYD1' little-endian
+    static final int HEADER_SIZE = 8;
+    // One bounded, segment-sized append window avoids the allocate/unmap/mmap
+    // cycle every 64 KiB without geometrically reserving up to 2x a large
+    // dictionary. close() truncates the unused tail back to appendOffset.
+    static final int APPEND_MAP_CAPACITY = 4 * 1024 * 1024;
+    /**
+     * Upper bound on a chunk's two header varints ({@code entryCount} and
+     * {@code entryBytes}): each is at most 5 bytes for a 32-bit value. The
+     * encoders reserve this much in front of the entry region so the header can
+     * be back-filled once the region's exact size is known, keeping header,
+     * entries and CRC one contiguous run.
+     */
+    static final int MAX_CHUNK_HEADER_SIZE = 10;
+    /**
+     * Ceiling for the append scratch buffer, mirroring
+     * {@code CursorWebSocketSendLoop.MAX_SENT_DICT_BYTES}: the capacity math is
+     * int-typed, so a larger buffer cannot be addressed. Exceeding it throws --
+     * {@link #ensureScratch} never silently under-allocates.
+     */
+    static final int MAX_SCRATCH_BYTES = Integer.MAX_VALUE - 8;
+    static final byte VERSION = 1;
+    private static final Logger LOG = LoggerFactory.getLogger(PersistedSymbolDict.class);
+    private final int fd;
+    // Filesystem seam. Production is FilesFacade.INSTANCE (straight to Files);
+    // tests inject a fault facade to exercise recovery I/O failures (a truncate
+    // that cannot drop a torn tail, a short write) without a real broken disk.
+    private final FilesFacade ff;
+    // Slot-qualified path, retained purely so diagnostics can name WHICH dictionary they
+    // are about: one JVM routinely holds many (a foreground sender plus N orphan
+    // drainers), so a warning carrying only the bare filename is unattributable exactly
+    // when it fires.
+    private final String filePath;
+    // Production writes directly into segmented append mappings. Wrapping facades retain the
+    // positioned-write path by default so fault tests can inject short writes through ff.write;
+    // mmap-specific fault facades opt in through FilesFacade.isMmapAllowed().
+    private final boolean mappedAppend;
+    // True only when recovery parsed the file through a temporary read-only
+    // mmap instead of allocating a second native buffer as large as the file.
+    // Test-visible so the peak-memory regression has an observable contract.
+    private final boolean mappedRecoveryInput;
+    // Entry count that corresponds EXACTLY to loadedEntriesAddr/loadedEntriesLen,
+    // fixed at open. Distinct from the live `size`, which appends advance -- including
+    // the recovery-time heal in QwpWebSocketSender.healPersistedDictionary, which runs
+    // BEFORE the send loop is constructed. The loop seeds its mirror from the loaded
+    // BYTES, so it must take its count from here; pairing those bytes with the live
+    // size would let sentDictCount claim symbols the mirror does not hold.
+    private final int recoveredSize;
+    private long appendMapAddr;
+    private long appendMapCapacity;
+    private long appendMapOffset;
+    private int appendMapGrowthCount;
+    private long appendOffset;
+    private long appendWriteCount;
+    private boolean closed;
+    // In-memory copy of the WIRE entry region [len][utf8]... -- chunk headers and
+    // CRCs stripped -- populated only when open() recovered existing chunks
+    // (recovery / orphan-drain). Zero/empty for a freshly created file. READ (not
+    // consumed) to seed the producer's id map and to seed the send loop's catch-up
+    // mirror. Foreground construction transfers ownership to its sole loop after
+    // producer seeding; orphan-drainer loops borrow it because one engine may create
+    // several wire sessions. If ownership was not transferred, close() frees it.
+    private long loadedEntriesAddr;
+    private int loadedEntriesLen;
+    private long scratchAddr;
+    private int scratchCap;
+    private int size;
+
+    private PersistedSymbolDict(
+            FilesFacade ff,
+            String filePath,
+            int fd,
+            long appendOffset,
+            int size,
+            long loadedEntriesAddr,
+            int loadedEntriesLen,
+            boolean mappedRecoveryInput
+    ) {
+        this.ff = ff;
+        this.filePath = filePath;
+        this.fd = fd;
+        this.mappedAppend = ff.isMmapAllowed();
+        this.mappedRecoveryInput = mappedRecoveryInput;
+        this.appendOffset = appendOffset;
+        this.size = size;
+        this.recoveredSize = size;
+        this.loadedEntriesAddr = loadedEntriesAddr;
+        this.loadedEntriesLen = loadedEntriesLen;
+    }
+
+    /**
+     * Opens the dictionary file in {@code slotDir} for RECOVERY, creating it only
+     * when it does not already exist. An existing file is parsed and its complete,
+     * CRC-valid chunks are loaded into memory (see {@link #loadedEntriesAddr()}).
+     * 

+     * Returns {@code null} on any I/O or parse failure -- including an existing file
+     * that cannot be read, carries an unknown version, or fails its checksums. The
+     * caller then falls back to full-dictionary (self-sufficient) frames for this
+     * slot, so a broken side-file degrades gracefully rather than aborting the
+     * sender. Crucially, a {@code null} return NEVER destroys the file: see the
+     * class-level "Never recreate over an existing file" note.
+     */
+    public static PersistedSymbolDict open(String slotDir) {
+        return open(FilesFacade.INSTANCE, slotDir);
+    }
+
+    /**
+     * Facade-aware variant of {@link #open(String)}. Production passes
+     * {@link FilesFacade#INSTANCE}; tests inject a fault facade to drive recovery
+     * I/O failures (e.g. a truncate that cannot drop a torn tail).
+     */
+    public static PersistedSymbolDict open(FilesFacade ff, String slotDir) {
+        String filePath = slotDir + "/" + FILE_NAME;
+        boolean exists = ff.exists(filePath);
+        long existing = exists ? ff.length(filePath) : -1L;
+        if (exists && existing < 0) {
+            // The file is present but its length could not be stat'd (a transient EIO
+            // on a flaky disk). Do NOT fall through to openFresh below, which O_TRUNCs:
+            // truncating the only copy of load-bearing state on a TRANSIENT fault is the
+            // exact destruction the class-level "Never recreate over an existing file"
+            // note forbids -- and unlike the openExisting read path, this routing check
+            // otherwise has no guard. A genuine sub-header stub reports a length in
+            // [0, HEADER_SIZE); only a stat error reports < 0, so the two are
+            // distinguishable. Degrade to full self-sufficient frames and leave every
+            // byte on disk for a later attempt, once the transient clears.
+            LOG.warn("symbol dict {} exists but its length could not be read; "
+                    + "falling back to full-dictionary frames (file left intact)", filePath);
+            return null;
+        }
+        if (existing >= HEADER_SIZE) {
+            // Chunk lengths and the retained contiguous entry region are int-sized,
+            // so a dictionary at or past Integer.MAX_VALUE cannot be represented
+            // safely even though production recovery maps rather than reads it.
+            if (existing >= Integer.MAX_VALUE) {
+                LOG.warn("symbol dict {} too large ({} bytes) to reopen; "
+                        + "falling back to full-dictionary frames (file left intact)", filePath, existing);
+                return null;
+            }
+            // NEVER recreate over an existing file on the recovery path: openFresh
+            // truncates, and these bytes are the only copy of state the surviving
+            // delta frames reference. A null degrades this slot to full
+            // self-sufficient frames and preserves the file for a later attempt.
+            //
+            // The holder catches what openExisting's own catch structurally cannot.
+            // Recovery reads the file through a mapping (Unsafe loads in
+            // scanAndCopyRecoveredChunks), and pre-JDK-21 HotSpot delivers an
+            // unsafe-access fault ASYNCHRONOUSLY -- at the next return or safepoint,
+            // which can be openExisting's own return, in THIS frame. The instance is
+            // fully built by then and owns an fd plus a buffer as large as the file,
+            // but the assignment never happens, so nothing else can release them.
+            // MmapSegment.openExisting takes an inFlight[] for exactly this shape; the
+            // dictionary is at least as exposed, because ensureAppendMap grows the file
+            // with ff.allocate and the reserve truncate is skipped after a crash, so a
+            // sparse tail is routine. Without this, the fault also escapes open()
+            // entirely -- past CursorSendEngine's constructor and out of Sender.build()
+            // -- turning the documented "degrade to full-dictionary frames" into a
+            // sender that cannot be constructed at all.
+            PersistedSymbolDict[] inFlight = new PersistedSymbolDict[1];
+            try {
+                return openExisting(ff, filePath, existing, inFlight);
+            } catch (Throwable t) {
+                if (inFlight[0] != null) {
+                    inFlight[0].close();
+                }
+                LOG.warn("symbol dict {} recovery faulted late; falling back to "
+                        + "full-dictionary frames (file left intact)", filePath, t);
+                return null;
+            }
+        }
+        // Absent, or a sub-header stub left by a crash inside openFresh: no
+        // load-bearing content to lose, so create it. mustTruncate=false: this
+        // is the RECOVERY entry point, and there is nothing here to protect --
+        // an absent/stub file has no id space to preserve, so a create failure
+        // still just degrades to null (full self-sufficient frames), same as
+        // every other recovery I/O failure above.
+        return openFresh(ff, filePath, false);
+    }
+
+    /**
+     * Opens the dictionary in {@code slotDir} as a FRESH, EMPTY file, discarding
+     * any surviving content. This is the fresh-start counterpart to {@link #open}:
+     * a slot with no recovered segments must start with an empty dictionary, so a
+     * dictionary left by a prior lifecycle -- a fully-drained slot whose
+     * best-effort delete failed, or a crash in the close window -- must NOT be
+     * inherited. Unlike {@link #open}, which parses and TRUSTS an existing file for
+     * recovery/orphan-drain replay and never destroys it, this truncates: the
+     * fresh-start producer is not seeded from the dictionary, so trusting a survivor
+     * would leave the producer's ids diverged from the dictionary the send loop
+     * replays and silently misattribute symbols on the next reconnect. Truncating
+     * (rather than relying on an unlink succeeding first) closes the gap even when
+     * the delete is refused -- e.g. a Windows share lock. Returns {@code null} when
+     * the slot has no dictionary at all and a fresh one cannot be created either,
+     * so the caller falls back to full self-sufficient frames exactly as
+     * {@link #open} does. But when a file DOES survive and cannot be truncated,
+     * returning null instead of refusing would proceed anyway: this session would
+     * run full-dict from id 0 while the survivor -- describing a prior
+     * generation's id space -- stays on disk, and the next recovery would trust
+     * it and misattribute symbols with no detectable gap. So that case throws
+     * {@link UnreplayableSlotException} instead of degrading -- see {@link #openFresh}.
+     */
+    public static PersistedSymbolDict openClean(String slotDir) {
+        return openClean(FilesFacade.INSTANCE, slotDir);
+    }
+
+    /**
+     * Facade-aware variant of {@link #openClean(String)}.
+     */
+    public static PersistedSymbolDict openClean(FilesFacade ff, String slotDir) {
+        return openFresh(ff, slotDir + "/" + FILE_NAME, true);
+    }
+
+    /**
+     * Best-effort removal of a stale dictionary file. Used at fully-drained close
+     * (the slot is empty, nothing references the dictionary any more), mirroring
+     * {@link AckWatermark#removeOrphan}. The fresh-start path deliberately does NOT
+     * use this -- it opens a clean dictionary via {@link #openClean} instead, so a
+     * failed delete cannot leave a stale dictionary a new session would trust.
+     */
+    public static void removeOrphan(String slotDir) {
+        removeOrphan(FilesFacade.INSTANCE, slotDir);
+    }
+
+    /**
+     * Facade-aware variant of {@link #removeOrphan(String)}.
+     */
+    public static void removeOrphan(FilesFacade ff, String slotDir) {
+        ff.remove(slotDir + "/" + FILE_NAME);
+    }
+
+    /**
+     * Appends {@code count} wire entries -- {@code [len varint][utf8]...}, the
+     * symbol-dict delta section the frame encoder just wrote -- as ONE chunk.
+     * 

+     * The consistency walk below decodes each entry's length varint, but the bytes
+     * themselves are copied in a SINGLE {@code copyMemory} and checksummed by a
+     * SINGLE {@link Crc32c#update} covering the whole chunk. A per-entry checksum
+     * would put one JNI transition, one sub-cache-line copy and one redundant varint
+     * decode on the producer thread per new symbol; the chunk needs one of each.
+     * 

+     * Advances {@code size} by {@code count}. Same durability/idempotency contract
+     * as {@link #appendSymbols}: no fsync, and a short write throws WITHOUT
+     * advancing {@code size}/{@code appendOffset}, so a retry keyed off
+     * {@link #size()} re-persists the same range at the same offset. No-op when the
+     * range is empty or the dictionary is closed.
+     */
+    public synchronized void appendRawEntries(long addr, int len, int count) {
+        if (closed || count <= 0 || len <= 0) {
+            return;
+        }
+        // Validate the (addr, len, count) triple BEFORE writing anything: an
+        // inconsistent triple would record a chunk whose stored entryCount disagreed
+        // with the entries it holds, shifting the dense id->symbol map on recovery.
+        // The sole caller derives count and len from one beginMessage, so this cannot
+        // fire today -- but the file this guards is the one the "resend required"
+        // contract rests on, so keep the structural guard. Gated behind assert: it
+        // re-walks every entry's length prefix on the common flush path, and the client
+        // library runs without -ea in production (embedded in user apps), so this holds
+        // the guard in the client's own -ea test suite without the per-flush cost in
+        // production.
+        assert validateRawEntries(addr, len, count);
+        int hdrLen = NativeBufferWriter.varintSize(count) + NativeBufferWriter.varintSize(len);
+        if (mappedAppend) {
+            long recLen = (long) hdrLen + len + CRC_SIZE;
+            ensureAppendMap(checkedRequiredOffset(recLen));
+            long recStart = appendMapAddr + appendOffset - appendMapOffset;
+            long p = NativeBufferWriter.writeVarint(recStart, count);
+            NativeBufferWriter.writeVarint(p, len);
+            Unsafe.getUnsafe().copyMemory(addr, recStart + hdrLen, len);
+            commitMappedChunk(recStart, hdrLen, len, count);
+            return;
+        }
+        ensureScratch((long) hdrLen + len + CRC_SIZE);
+        long p = NativeBufferWriter.writeVarint(scratchAddr, count);
+        NativeBufferWriter.writeVarint(p, len);
+        Unsafe.getUnsafe().copyMemory(addr, scratchAddr + hdrLen, len);
+        flushChunk(scratchAddr, hdrLen, len, count);
+    }
+
+    /**
+     * Appends one symbol as a single-entry chunk, extending the on-disk dictionary.
+     * The caller appends a frame's new symbols BEFORE publishing that frame, so the
+     * write ordering (dictionary entry before referencing frame) holds; no fsync is
+     * performed (see the class-level durability note). Assigns the next dense id
+     * implicitly (the entry's position).
+     * 

+     * Test-only: production persists a frame's whole new-symbol range in one chunk
+     * via {@link #appendSymbols} / {@link #appendRawEntries}. Tests use this to
+     * build a dictionary one entry at a time.
+     */
+    @TestOnly
+    public synchronized void appendSymbol(CharSequence symbol) {
+        if (closed) {
+            return;
+        }
+        int utf8Len = Utf8s.utf8Bytes(symbol);
+        int wireLen = NativeBufferWriter.varintSize(utf8Len) + utf8Len; // [len][utf8]
+        if (mappedAppend) {
+            int hdrLen = NativeBufferWriter.varintSize(1) + NativeBufferWriter.varintSize(wireLen);
+            long recLen = (long) hdrLen + wireLen + CRC_SIZE;
+            ensureAppendMap(checkedRequiredOffset(recLen));
+            long recStart = appendMapAddr + appendOffset - appendMapOffset;
+            long p = NativeBufferWriter.writeVarint(recStart, 1);
+            p = NativeBufferWriter.writeVarint(p, wireLen);
+            p = NativeBufferWriter.writeVarint(p, utf8Len);
+            if (utf8Len > 0) {
+                Utf8s.strCpyUtf8(symbol, p, utf8Len);
+            }
+            commitMappedChunk(recStart, hdrLen, wireLen, 1);
+            return;
+        }
+        ensureScratch((long) MAX_CHUNK_HEADER_SIZE + wireLen + CRC_SIZE);
+        long entryStart = scratchAddr + MAX_CHUNK_HEADER_SIZE;
+        long p = NativeBufferWriter.writeVarint(entryStart, utf8Len);
+        if (utf8Len > 0) {
+            Utf8s.strCpyUtf8(symbol, p, utf8Len);
+        }
+        writeChunkFromScratch(wireLen, 1);
+    }
+
+    /**
+     * Appends the dense id range {@code [from .. to]} as one chunk with one
+     * checksum. This is the RE-ENCODE path: the steady-state persist ships a frame's
+     * pre-encoded delta bytes through {@link #appendRawEntries}, and only a retry
+     * after a failed publish rebuilds a range straight from the dictionary. The
+     * mapped path stages the entry region in the scratch buffer with a single UTF-8
+     * walk per symbol, then bulk-copies it into the append window after the header,
+     * and still commits WITHOUT a positioned-write syscall. A direct encode into the
+     * window would walk each symbol's UTF-8 length twice -- the exact entriesLen sizes
+     * both the header varint and the mmap reserve -- and the back-to-back chunk format
+     * leaves no room to reserve-and-back-fill the header in place. The
+     * positioned-write fallback runs only behind an injected filesystem facade so
+     * short-write recovery tests retain their fault seam. Callers pass the dictionary
+     * and the range so the ids resolve to their symbol strings.
+     * 

+     * Same durability and idempotency contract as {@link #appendSymbol}: no fsync,
+     * and a short write throws WITHOUT advancing {@code size}/{@code appendOffset},
+     * so a retry keyed off {@link #size()} re-encodes the same range and overwrites
+     * at the same offset. No-op when the range is empty or the dictionary is closed.
+     */
+    public synchronized void appendSymbols(GlobalSymbolDictionary dict, int from, int to) {
+        if (closed || to < from) {
+            return;
+        }
+        int count = to - from + 1;
+        if (mappedAppend) {
+            // Stage the entry region in scratch with ONE UTF-8 walk per symbol, then
+            // bulk-copy it into the append window after the header (see the method
+            // javadoc for why a direct in-window encode would have to walk each symbol
+            // twice). ensureScratch enforces the same MAX_SCRATCH_BYTES ceiling the old
+            // sizing pass did, throwing before size/appendOffset advance.
+            int entriesLen = 0;
+            for (int id = from; id <= to; id++) {
+                CharSequence symbol = dict.getSymbol(id);
+                int utf8Len = Utf8s.utf8Bytes(symbol);
+                int wireLen = NativeBufferWriter.varintSize(utf8Len) + utf8Len; // [len][utf8]
+                ensureScratch((long) entriesLen + wireLen);
+                long q = NativeBufferWriter.writeVarint(scratchAddr + entriesLen, utf8Len);
+                if (utf8Len > 0) {
+                    Utf8s.strCpyUtf8(symbol, q, utf8Len);
+                }
+                entriesLen += wireLen;
+            }
+            int hdrLen = NativeBufferWriter.varintSize(count)
+                    + NativeBufferWriter.varintSize(entriesLen);
+            long recLen = (long) hdrLen + entriesLen + CRC_SIZE;
+            ensureAppendMap(checkedRequiredOffset(recLen));
+            long recStart = appendMapAddr + appendOffset - appendMapOffset;
+            long p = NativeBufferWriter.writeVarint(recStart, count);
+            NativeBufferWriter.writeVarint(p, entriesLen);
+            if (entriesLen > 0) {
+                Unsafe.getUnsafe().copyMemory(scratchAddr, recStart + hdrLen, entriesLen);
+            }
+            commitMappedChunk(recStart, hdrLen, entriesLen, count);
+            return;
+        }
+        int entriesLen = 0;
+        for (int id = from; id <= to; id++) {
+            CharSequence symbol = dict.getSymbol(id);
+            int utf8Len = Utf8s.utf8Bytes(symbol);
+            int wireLen = NativeBufferWriter.varintSize(utf8Len) + utf8Len; // [len][utf8]
+            ensureScratch((long) MAX_CHUNK_HEADER_SIZE + entriesLen + wireLen + CRC_SIZE);
+            long entryStart = scratchAddr + MAX_CHUNK_HEADER_SIZE + entriesLen;
+            long p = NativeBufferWriter.writeVarint(entryStart, utf8Len);
+            if (utf8Len > 0) {
+                Utf8s.strCpyUtf8(symbol, p, utf8Len);
+            }
+            entriesLen += wireLen;
+        }
+        writeChunkFromScratch(entriesLen, count);
+    }
+
+    @Override
+    public synchronized void close() {
+        if (closed) {
+            return;
+        }
+        closed = true;
+        // Each step in its own try/catch, as CursorSendEngine.close() already does.
+        // `closed` is set first, so a throw from any step short-circuits every retry --
+        // stranding the scratch buffer and the fd for the process's lifetime. Unreachable
+        // with FilesFacade.INSTANCE (munmap and truncate are native calls returning
+        // int/boolean), but the ff seam exists precisely so a test CAN inject a throw,
+        // and a close path must not depend on its own steps never failing.
+        if (loadedEntriesAddr != 0L) {
+            try {
+                Unsafe.free(loadedEntriesAddr, loadedEntriesLen, MemoryTag.NATIVE_DEFAULT);
+            } catch (Throwable ignored) {
+            }
+            // Null after freeing (like scratchAddr below) so a CONSTRUCTION-PHASE reader
+            // that runs after close observes 0 rather than a dangling pointer. This is
+            // NOT a cross-thread guard, and the getters' own javadoc is the accurate
+            // statement: they are unsynchronised reads of non-volatile fields, so a
+            // caller on another thread has no guarantee of seeing this write and can
+            // still dereference freed memory. Ordering, not nulling, is what makes the
+            // borrow safe -- every owner closes its loop before the engine.
+            loadedEntriesAddr = 0L;
+            loadedEntriesLen = 0;
+        }
+        if (appendMapAddr != 0L) {
+            long mapAddr = appendMapAddr;
+            long mapCapacity = appendMapCapacity;
+            appendMapAddr = 0L;
+            appendMapCapacity = 0L;
+            appendMapOffset = 0L;
+            try {
+                ff.munmap(mapAddr, mapCapacity, MemoryTag.MMAP_DEFAULT);
+            } catch (Throwable ignored) {
+            }
+            try {
+                // The active window reserves space past the logical end. Return that tail on
+                // orderly close; after a crash open() treats the zero-filled reserve as
+                // a torn trailing chunk and truncates it to the same appendOffset.
+                if (!ff.truncate(fd, appendOffset)) {
+                    LOG.warn("symbol dict {} could not trim mmap reserve to {}; recovery will "
+                                    + "discard the zero-filled tail on the next open",
+                            filePath, appendOffset);
+                }
+            } catch (Throwable ignored) {
+            }
+        }
+        if (scratchAddr != 0L) {
+            try {
+                Unsafe.free(scratchAddr, scratchCap, MemoryTag.NATIVE_DEFAULT);
+            } catch (Throwable ignored) {
+            }
+            scratchAddr = 0L;
+            scratchCap = 0;
+        }
+        if (fd >= 0) {
+            try {
+                ff.close(fd);
+            } catch (Throwable ignored) {
+            }
+        }
+    }
+
+    @TestOnly
+    public synchronized int appendMapGrowthCount() {
+        return appendMapGrowthCount;
+    }
+
+    @TestOnly
+    public synchronized long appendWriteCount() {
+        return appendWriteCount;
+    }
+
+    /**
+     * Base address of the loaded entry region -- the concatenated
+     * {@code [len][utf8]} bytes of every recovered symbol in id order, exactly as a
+     * delta section carries them (chunk headers and CRCs stripped). Zero when
+     * nothing was recovered.
+     * 

+     * Construction-phase only. This hands out a raw pointer into native
+     * memory that {@link #close()} frees and nulls, with no closed-guard and no
+     * synchronization. It is safe to read only BEFORE the slot's I/O thread and
+     * any producer append start -- i.e. while the send loop is being constructed
+     * or an orphan-drain is seeding its mirror, both of which happen-before those
+     * threads. A caller that reads it from a running thread races {@code close()}
+     * and can dereference freed memory (use-after-free).
+     */
+    public long loadedEntriesAddr() {
+        return loadedEntriesAddr;
+    }
+
+    /**
+     * Byte length of {@link #loadedEntriesAddr()}. Construction-phase only, for
+     * the same reason -- see {@link #loadedEntriesAddr()}.
+     */
+    public int loadedEntriesLen() {
+        return loadedEntriesLen;
+    }
+
+    /**
+     * Number of symbols {@link #open} recovered from disk -- the exact entry count of
+     * {@link #loadedEntriesAddr()} / {@link #loadedEntriesLen()}. Unlike {@link #size()}
+     * this never advances, so a caller seeding from the loaded bytes stays in lockstep
+     * with them even after the producer has appended (the recovery heal does exactly
+     * that, before the send loop is built).
+     */
+    public int recoveredSize() {
+        return recoveredSize;
+    }
+
+    @TestOnly
+    public boolean usedMappedRecoveryInput() {
+        return mappedRecoveryInput;
+    }
+
+    /**
+     * Decodes the recovered entries directly into {@code target} in ascending-id
+     * order. This avoids materialising a cardinality-sized temporary list that
+     * the producer would immediately copy into the global dictionary.
+     * Construction-phase only; see {@link #loadedEntriesAddr()}.
+     */
+    public void addLoadedSymbolsTo(GlobalSymbolDictionary target) {
+        decodeLoadedSymbols(target, null);
+    }
+
+    /**
+     * Materialises the loaded entries as symbol strings in ascending-id order.
+     * Retained for recovery-format tests; production decodes directly through
+     * {@link #addLoadedSymbolsTo(GlobalSymbolDictionary)}.
+     */
+    @TestOnly
+    public ObjList readLoadedSymbols() {
+        ObjList out = new ObjList<>(Math.max(recoveredSize, 1));
+        decodeLoadedSymbols(null, out);
+        return out;
+    }
+
+    private void decodeLoadedSymbols(GlobalSymbolDictionary target, ObjList out) {
+        long p = loadedEntriesAddr;
+        long limit = p + loadedEntriesLen;
+        // open() CRC-validated these bytes and copyRecoveredEntries laid down exactly
+        // `size` entries, so this decode runs on trusted, self-consistent input. Any
+        // mismatch means an internal invariant broke: fail loud like the rest of this
+        // file (copyRecoveredEntries throws too) instead of silently under-populating
+        // the dictionary, which would shift every id above the short point.
+        // recoveredSize, not the live size: this decodes the LOADED region, whose entry
+        // count is fixed at open. Appends (including the recovery heal) advance size
+        // without extending that region, so keying off size would over-read.
+        for (int i = 0; i < recoveredSize; i++) {
+            long len = 0;
+            int shift = 0;
+            boolean terminated = false;
+            while (p < limit) {
+                byte b = Unsafe.getUnsafe().getByte(p++);
+                len |= (long) (b & 0x7F) << shift;
+                if ((b & 0x80) == 0) {
+                    terminated = true;
+                    break;
+                }
+                shift += 7;
+                if (shift > 35) {
+                    break; // over-long run -- a canonical length varint is <= 5 bytes
+                }
+            }
+            if (!terminated || p + len > limit) {
+                // UnreplayableSlotException so Sender.build() can set the slot aside
+                // instead of rethrowing forever -- see CursorSendEngine's twin throw.
+                throw new UnreplayableSlotException("truncated loaded symbol dictionary entry to "
+                        + FILE_NAME + " [entry=" + i + ", size=" + recoveredSize + ']');
+            }
+            String symbol = Utf8s.stringFromUtf8Bytes(p, p + len);
+            if (target != null) {
+                target.addRecoveredSymbol(symbol);
+            } else {
+                out.add(symbol);
+            }
+            p += len;
+        }
+        if (p != limit) {
+            throw new UnreplayableSlotException("loaded symbol dictionary has trailing bytes to "
+                    + FILE_NAME + " [consumed=" + (p - loadedEntriesAddr) + ", length=" + loadedEntriesLen + ']');
+        }
+    }
+
+    /**
+     * Number of symbols the dictionary holds (highest id + 1).
+     */
+    public int size() {
+        return size;
+    }
+
+    /**
+     * @param inFlight single-slot holder the caller must close when this method throws.
+     *                 Set immediately before the return, so a late-delivered mmap access
+     *                 fault -- which pre-JDK-21 HotSpot may raise at this method's RETURN,
+     *                 in the caller's frame, past the catch below -- still leaves the fully
+     *                 built instance (fd plus loaded-entry buffer) reachable by someone.
+     */
+    private static PersistedSymbolDict openExisting(FilesFacade ff, String filePath, long fileLen,
+                                                     PersistedSymbolDict[] inFlight) {
+        int fd = ff.openRW(filePath);
+        if (fd < 0) {
+            LOG.warn("symbol dict {} could not be opened (rc={}); "
+                    + "falling back to full-dictionary frames (file left intact)", filePath, fd);
+            return null;
+        }
+        int len = (int) fileLen; // open() bounds fileLen to [HEADER_SIZE, Integer.MAX_VALUE)
+        boolean mappedInput = ff.isMmapAllowed();
+        long inputAddr = 0L;
+        long entriesAddr = 0L;
+        int entriesCapacity = 0;
+        int entriesLen = 0;
+        try {
+            if (mappedInput) {
+                inputAddr = ff.mmap(fd, fileLen, 0L, Files.MAP_RO, MemoryTag.MMAP_DEFAULT);
+                if (inputAddr == Files.FAILED_MMAP_ADDRESS) {
+                    inputAddr = 0L;
+                    throw new IllegalStateException("could not mmap symbol dictionary for recovery");
+                }
+            } else {
+                // Fault-injection facades retain the positioned-read path so tests
+                // can still force short reads without mapping around the seam.
+                inputAddr = Unsafe.malloc(len, MemoryTag.NATIVE_DEFAULT);
+                long read = ff.read(fd, inputAddr, len, 0);
+                if (read != len) {
+                    throw new IllegalStateException("short read while recovering symbol dictionary "
+                            + "[expected=" + len + ", actual=" + read + ']');
+                }
+            }
+            if (Unsafe.getUnsafe().getInt(inputAddr) != FILE_MAGIC
+                    || Unsafe.getUnsafe().getByte(inputAddr + 4) != VERSION) {
+                throw new IllegalStateException("bad magic or unknown symbol dictionary version");
+            }
+            // ONE pass: validate each chunk's CRC and, once proven good, copy its
+            // entries straight out. The entry region is by construction a subset of
+            // the file, so `len` is a safe upper bound to allocate against -- it
+            // overshoots only by the chunk headers and CRCs -- and the exact size is
+            // reclaimed by the shrink below. The previous shape walked the file
+            // TWICE, re-decoding both header varints of every chunk on the second
+            // pass purely to size one allocation it could have bounded for free.
+            entriesCapacity = len;
+            entriesAddr = Unsafe.malloc(entriesCapacity, MemoryTag.NATIVE_DEFAULT);
+            RecoveryScan scan = scanAndCopyRecoveredChunks(inputAddr, len, entriesAddr);
+            entriesLen = scan.entriesLen;
+            if (entriesLen == 0) {
+                Unsafe.free(entriesAddr, entriesCapacity, MemoryTag.NATIVE_DEFAULT);
+                entriesAddr = 0L;
+                entriesCapacity = 0;
+            } else if (entriesLen < entriesCapacity) {
+                entriesAddr = Unsafe.realloc(
+                        entriesAddr, entriesCapacity, entriesLen, MemoryTag.NATIVE_DEFAULT);
+                entriesCapacity = entriesLen;
+            }
+            if (mappedInput) {
+                ff.munmap(inputAddr, fileLen, MemoryTag.MMAP_DEFAULT);
+            } else {
+                Unsafe.free(inputAddr, len, MemoryTag.NATIVE_DEFAULT);
+            }
+            inputAddr = 0L;
+            // Drop any torn/stale trailing bytes so a LATER, shorter append cannot
+            // leave residue past its own end. The truncate result IS checked: a file
+            // we cannot trim could still expose stale post-end bytes whose
+            // (self-consistent) chunk CRC the parse would accept at a shifted
+            // position, so a failed truncate makes the file untrusted -- return null
+            // (the sender falls back to full self-sufficient frames) and, per the
+            // never-destroy contract, leave every byte on disk.
+            if (scan.validLen < len && !ff.truncate(fd, scan.validLen)) {
+                throw new IllegalStateException("could not drop torn/stale symbol dictionary tail");
+            }
+            PersistedSymbolDict dict = new PersistedSymbolDict(
+                    ff, filePath, fd, scan.validLen, scan.count, entriesAddr, entriesLen, mappedInput);
+            // Publish to the holder BEFORE returning: from here on the caller owns the
+            // cleanup, including when the return itself faults. See the parameter doc.
+            inFlight[0] = dict;
+            return dict;
+        } catch (Throwable t) {
+            if (inputAddr != 0L) {
+                if (mappedInput) {
+                    ff.munmap(inputAddr, fileLen, MemoryTag.MMAP_DEFAULT);
+                } else {
+                    Unsafe.free(inputAddr, len, MemoryTag.NATIVE_DEFAULT);
+                }
+            }
+            if (entriesAddr != 0L) {
+                // capacity, not entriesLen: the shrink may not have happened yet.
+                Unsafe.free(entriesAddr, entriesCapacity, MemoryTag.NATIVE_DEFAULT);
+            }
+            if (fd >= 0) {
+                ff.close(fd);
+            }
+            // Pass the throwable as a trailing argument with no matching placeholder so
+            // slf4j prints the stack trace: this WARN is the only forensic record of why
+            // a load-bearing dictionary was abandoned.
+            LOG.warn("symbol dict {} recovery failed; falling back to full-dictionary frames "
+                    + "(file left intact)", filePath, t);
+            return null;
+        }
+    }
+
+    /**
+     * Validates every chunk and copies the entries of each proven-good one into
+     * {@code dstAddr}, in a single pass. {@code dstAddr} must have room for {@code len}
+     * bytes -- the entry region is a subset of the file, so that always suffices.
+     * Stops at the first chunk that is torn, fails its CRC, or is internally
+     * inconsistent, exactly as the two-pass version did, so the trusted prefix is
+     * unchanged.
+     */
+    private static RecoveryScan scanAndCopyRecoveredChunks(long inputAddr, int len, long dstAddr) {
+        Varint v = new Varint();
+        int count = 0;
+        int diskPos = HEADER_SIZE;
+        long entriesLen = 0L;
+        while (diskPos < len) {
+            if (!v.decode(inputAddr, diskPos, len)) {
+                break;
+            }
+            long entryCount = v.value;
+            if (!v.decode(inputAddr, v.end, len)) {
+                break;
+            }
+            long entryBytes = v.value;
+            int entriesStart = v.end;
+            long chunkEnd = (long) entriesStart + entryBytes;
+            if (chunkEnd + CRC_SIZE > len) {
+                break;
+            }
+            int chunkEndI = (int) chunkEnd;
+            int crcStored = Unsafe.getUnsafe().getInt(inputAddr + chunkEndI);
+            int crcCalc = Crc32c.updateUnsafe(
+                    Crc32c.INIT,
+                    inputAddr + diskPos,
+                    chunkEndI - diskPos);
+            if (crcCalc != crcStored
+                    || entryCount <= 0
+                    // Every entry costs at least its own length varint, so a positive
+                    // entryCount inside a zero-byte region is self-contradictory. The CRC
+                    // proves the bytes are what was WRITTEN, never that the triple is
+                    // consistent, and decodeLoadedSymbols would only discover it later --
+                    // as a throw two layers up rather than a trimmed trusted prefix.
+                    || entryBytes <= 0
+                    || (long) count + entryCount > Integer.MAX_VALUE
+                    || entriesLen + entryBytes > Integer.MAX_VALUE) {
+                break;
+            }
+            if (!isConsistentEntryRegion(inputAddr, entriesStart, entryBytes, entryCount)) {
+                break;
+            }
+            Unsafe.getUnsafe().copyMemory(inputAddr + entriesStart, dstAddr + entriesLen, entryBytes);
+            entriesLen += entryBytes;
+            diskPos = chunkEndI + CRC_SIZE;
+            count += (int) entryCount;
+        }
+        return new RecoveryScan(count, (int) entriesLen, diskPos);
+    }
+
+    /**
+     * Whether {@code [start, start + bytes)} holds exactly {@code count} well-formed
+     * {@code [len varint][utf8]} entries, consuming the region exactly.
+     * 

+     * The chunk CRC proves the bytes are what was WRITTEN; it says nothing about whether
+     * the header triple is self-consistent. The only write-side guard,
+     * {@code validateRawEntries}, sits behind an {@code assert} -- and this library ships
+     * embedded in user applications, which run without {@code -ea}. So a producer bug or
+     * a torn write that happens to re-checksum could record a chunk whose stored
+     * entryCount disagrees with its entries, shifting the dense id->symbol map for
+     * everything above it.
+     * 

+     * Checking here ends the trusted prefix at that chunk -- the same treatment a CRC
+     * failure gets -- instead of letting decodeLoadedSymbols discover it later and throw
+     * two layers up, which quarantines the whole slot rather than salvaging its intact
+     * prefix. It costs one varint decode per entry on a cold path that already walks
+     * every entry immediately afterwards.
+     */
+    private static boolean isConsistentEntryRegion(long addr, int start, long bytes, long count) {
+        long p = start;
+        long limit = start + bytes;
+        for (long i = 0; i < count; i++) {
+            long len = 0;
+            int shift = 0;
+            boolean terminated = false;
+            while (p < limit) {
+                byte b = Unsafe.getUnsafe().getByte(addr + p++);
+                len |= (long) (b & 0x7F) << shift;
+                if ((b & 0x80) == 0) {
+                    terminated = true;
+                    break;
+                }
+                shift += 7;
+                if (shift > 35) {
+                    return false; // over-long run: a canonical length varint is <= 5 bytes
+                }
+            }
+            if (!terminated || len < 0 || p + len > limit) {
+                return false;
+            }
+            p += len;
+        }
+        return p == limit;
+    }
+
+    /**
+     * @param mustTruncate {@code true} for the fresh-slot path ({@link #openClean}),
+     *                      where an existing file MUST be cleared -- a survivor that
+     *                      cannot be truncated describes a prior generation's id
+     *                      space, and proceeding anyway would leave it on disk while
+     *                      this session writes rows against a fresh id space from 0.
+     *                      The next recovery cannot tell the two apart: ids overlap,
+     *                      the CRC is valid, and the catch-up would register the
+     *                      wrong strings under ids this generation's rows reference.
+     *                      So this case throws {@link UnreplayableSlotException} rather
+     *                      than degrading to null. {@code false} for the recovery
+     *                      path ({@link #open}), which only ever reaches this method
+     *                      when the file is absent or a headerless stub -- nothing
+     *                      load-bearing to lose -- so a create failure there still
+     *                      just degrades to null like every other recovery I/O
+     *                      failure.
+     */
+    private static PersistedSymbolDict openFresh(FilesFacade ff, String filePath, boolean mustTruncate) {
+        int fd = ff.openCleanRW(filePath);
+        if (fd < 0) {
+            if (mustTruncate && ff.exists(filePath)) {
+                // A fresh slot MUST start with an empty dictionary. Proceeding without one
+                // leaves the previous generation's entries on disk while this session
+                // writes rows against a fresh id space from 0, and the next recovery
+                // cannot distinguish the two: the ids overlap, the CRC is valid, and the
+                // catch-up registers the wrong strings under ids this generation's rows
+                // reference. Refuse the slot; the bytes stay on disk for an operator.
+                // Typed as UnreplayableSlotException (rather than a plain
+                // LineSenderException) so Sender.build()'s constructor-time catch
+                // quarantines this fresh, dataless slot instead of bricking every
+                // restart under a stable senderId.
+                throw new UnreplayableSlotException("symbol dict ").put(filePath)
+                        .put(" already exists and cannot be truncated (rc=").put(fd)
+                        .put("); refusing to start on a slot whose dictionary describes a")
+                        .put(" different id space -- move or remove it by hand");
+            }
+            LOG.warn("symbol dict {} could not be created (rc={}); proceeding without it", filePath, fd);
+            return null;
+        }
+        long hdr = 0L;
+        try {
+            hdr = Unsafe.malloc(HEADER_SIZE, MemoryTag.NATIVE_DEFAULT);
+            Unsafe.getUnsafe().putInt(hdr, FILE_MAGIC);
+            Unsafe.getUnsafe().putByte(hdr + 4, VERSION);
+            Unsafe.getUnsafe().putByte(hdr + 5, (byte) 0);
+            Unsafe.getUnsafe().putByte(hdr + 6, (byte) 0);
+            Unsafe.getUnsafe().putByte(hdr + 7, (byte) 0);
+            long written = ff.write(fd, hdr, HEADER_SIZE, 0);
+            if (written != HEADER_SIZE) {
+                int fdToClose = fd;
+                fd = -1; // relinquish before close so the catch cannot double-close if close throws
+                ff.close(fdToClose);
+                ff.remove(filePath); // drop the headerless stub rather than litter
+                LOG.warn("symbol dict {} header write failed; proceeding without it", filePath);
+                return null;
+            }
+        } catch (Throwable t) {
+            // Unreachable with FilesFacade.INSTANCE (Files.write is native and returns
+            // -1 rather than throwing; the Unsafe puts target a valid 8-byte buffer and
+            // an 8-byte malloc cannot realistically OOM), but the ff seam exists so
+            // tests CAN inject a throwing facade -- close the fd and drop the stub so
+            // neither leaks.
+            if (fd >= 0) { // the header-write branch relinquished fd to -1 before closing
+                int fdToClose = fd;
+                fd = -1;
+                ff.close(fdToClose);
+                ff.remove(filePath);
+            }
+            LOG.warn("symbol dict {} creation failed; proceeding without it", filePath, t);
+            return null;
+        } finally {
+            if (hdr != 0L) {
+                Unsafe.free(hdr, HEADER_SIZE, MemoryTag.NATIVE_DEFAULT);
+            }
+        }
+        return new PersistedSymbolDict(ff, filePath, fd, HEADER_SIZE, 0, 0L, 0, false);
+    }
+
+    /**
+     * Verifies that {@code count} wire entries ({@code [len varint][utf8]}) occupy
+     * exactly {@code len} bytes from {@code addr}. Returns {@code true} when the triple
+     * is consistent; throws {@link IllegalStateException} (naming the offending entry /
+     * count / consumed bytes) otherwise. Called only from an {@code assert} in
+     * {@link #appendRawEntries}: it guards an internal invariant the sole caller cannot
+     * violate today, so it runs under the test suite's {@code -ea} but is elided from
+     * the per-flush production path (client apps run without {@code -ea}).
+     */
+    private static boolean validateRawEntries(long addr, int len, int count) {
+        long src = addr;
+        long srcLimit = addr + len;
+        for (int i = 0; i < count; i++) {
+            long symLen = 0;
+            int shift = 0;
+            while (src < srcLimit) {
+                byte b = Unsafe.getUnsafe().getByte(src++);
+                symLen |= (long) (b & 0x7F) << shift;
+                if ((b & 0x80) == 0) {
+                    break;
+                }
+                shift += 7;
+                if (shift > 35) {
+                    // A canonical entry-length varint is <= 5 bytes; a longer
+                    // continuation run is corrupt. The bound check below rejects it.
+                    break;
+                }
+            }
+            src += symLen; // src was just past the len varint
+            if (src > srcLimit) {
+                throw new IllegalStateException("malformed raw symbol-dict entries to " + FILE_NAME
+                        + " [entry=" + i + ", count=" + count + ']');
+            }
+        }
+        if (src != srcLimit) {
+            throw new IllegalStateException("raw symbol-dict entries under-filled the buffer to "
+                    + FILE_NAME + " [count=" + count + ", len=" + len
+                    + ", consumed=" + (int) (src - addr) + ']');
+        }
+        return true;
+    }
+
+    private long checkedRequiredOffset(long recordLen) {
+        long required = appendOffset + recordLen;
+        if (recordLen < 0L || required < appendOffset) {
+            throw new IllegalStateException("symbol dict append offset overflow to "
+                    + FILE_NAME + " [offset=" + appendOffset + ", recordLen=" + recordLen + ']');
+        }
+        return required;
+    }
+
+    /**
+     * Commits a chunk already assembled directly in the append mmap. The logical
+     * offset and symbol count advance last, after the CRC and a store fence, so an
+     * interrupted process leaves either a complete chunk or a tail that open()
+     * rejects and truncates.
+     */
+    private void commitMappedChunk(long recStart, int hdrLen, int entriesLen, int count) {
+        long bodyLen = (long) hdrLen + entriesLen;
+        long recLen = bodyLen + CRC_SIZE;
+        // updateUnsafe, NOT the native update: this checksums bytes inside the append
+        // MAPPING. ensureAppendMap grows the file with ff.allocate, whose native fallback
+        // is ftruncate, so the window can cover blocks the filesystem has not committed
+        // (ENOSPC, a quota, a sparse tail left by a crash) -- and touching one raises
+        // SIGBUS. Inside JNI that ABORTS THE JVM with no recovery; at an Unsafe intrinsic
+        // site HotSpot converts it to a catchable InternalError. MmapSegment.scanFrames
+        // already uses updateUnsafe over the same class of mapping for exactly this
+        // reason. Costs nothing measurable and drops a JNI transition per flush.
+        Unsafe.getUnsafe().putInt(
+                recStart + bodyLen,
+                Crc32c.updateUnsafe(Crc32c.INIT, recStart, bodyLen));
+        Unsafe.getUnsafe().storeFence();
+        appendOffset += recLen;
+        size += count;
+    }
+
+    /**
+     * Ensures the production append mmap covers the absolute file offset
+     * {@code required}. The log is mapped in 4 MiB-aligned windows: small flushes
+     * share a segment-sized window, while a large existing dictionary does not force
+     * the process to reserve and remap its whole prefix or geometrically over-allocate it.
+     */
+    private void ensureAppendMap(long required) {
+        if (appendMapAddr != 0L
+                && required >= appendMapOffset
+                && required <= appendMapOffset + appendMapCapacity) {
+            return;
+        }
+        // Page-align, not APPEND_MAP_CAPACITY-align: mmap only requires a page-aligned
+        // file offset. Rounding the START down to a 4 MiB boundary while sizing `needed`
+        // to the record's END meant a chunk straddling that boundary produced a window
+        // spanning BOTH -- 8 MiB mapped and re-allocated, whose lower half covers bytes
+        // already written and never touched again. Steady state then advanced 4 MiB per
+        // remap while always mapping 8.
+        long pageMask = Files.PAGE_SIZE - 1;
+        long newOffset = appendOffset & ~pageMask;
+        long needed = required - newOffset;
+        long newCapacity = Math.max(APPEND_MAP_CAPACITY, needed);
+        long remainder = newCapacity % APPEND_MAP_CAPACITY;
+        if (remainder != 0L) {
+            long padding = APPEND_MAP_CAPACITY - remainder;
+            if (newCapacity > Long.MAX_VALUE - padding) {
+                throw new IllegalStateException("symbol dict mmap capacity overflow to "
+                        + FILE_NAME + " [required=" + required + ']');
+            }
+            newCapacity += padding;
+        }
+        if (newOffset > Long.MAX_VALUE - newCapacity) {
+            throw new IllegalStateException("symbol dict mmap file offset overflow to "
+                    + FILE_NAME + " [offset=" + newOffset + ", capacity=" + newCapacity + ']');
+        }
+        long newFileSize = newOffset + newCapacity;
+        if (!ff.allocate(fd, newFileSize)) {
+            throw new IllegalStateException("could not grow mmap append region for "
+                    + FILE_NAME + " [required=" + required + ", fileSize=" + newFileSize + ']');
+        }
+        if (appendMapAddr != 0L) {
+            long oldAddr = appendMapAddr;
+            long oldCapacity = appendMapCapacity;
+            appendMapAddr = 0L;
+            appendMapCapacity = 0L;
+            appendMapOffset = 0L;
+            ff.munmap(oldAddr, oldCapacity, MemoryTag.MMAP_DEFAULT);
+        }
+        long newAddr = ff.mmap(
+                fd, newCapacity, newOffset, Files.MAP_RW, MemoryTag.MMAP_DEFAULT);
+        if (newAddr == Files.FAILED_MMAP_ADDRESS) {
+            throw new IllegalStateException("could not mmap append region for "
+                    + FILE_NAME + " [offset=" + newOffset + ", capacity=" + newCapacity + ']');
+        }
+        appendMapAddr = newAddr;
+        appendMapCapacity = newCapacity;
+        appendMapOffset = newOffset;
+        appendMapGrowthCount++;
+    }
+
+    /**
+     * Grows the append scratch buffer to hold at least {@code required} bytes.
+     * 

+     * Throws when {@code required} exceeds {@link #MAX_SCRATCH_BYTES} rather than
+     * clamping to it: a clamp would hand back a buffer SMALLER than the caller asked
+     * for and return normally, and every caller then writes {@code required} bytes
+     * into it -- turning a clean out-of-memory into a silent native-heap overflow, on
+     * the very write path the dictionary's integrity rests on. This is the same
+     * loud-failure shape {@code CursorWebSocketSendLoop.ensureSentDictCapacity} uses
+     * on the same bound. Unreachable at any realistic cardinality (it needs a ~2 GiB
+     * dictionary section in a single frame, itself bounded by the server's batch
+     * cap), but a size guard on a data-integrity write path must never
+     * under-allocate.
+     */
+    private void ensureScratch(long required) {
+        if (scratchCap >= required) {
+            return;
+        }
+        if (required > MAX_SCRATCH_BYTES) {
+            throw new IllegalStateException("symbol dict scratch buffer exceeds the maximum size to "
+                    + FILE_NAME + " [required=" + required + ", max=" + MAX_SCRATCH_BYTES + ']');
+        }
+        // Double in long: scratchCap * 2 as an int overflows negative past ~1 GB and
+        // would make the realloc size negative.
+        long newCap = Math.max(required, Math.max(256L, (long) scratchCap * 2));
+        if (newCap > MAX_SCRATCH_BYTES) {
+            newCap = MAX_SCRATCH_BYTES;
+        }
+        scratchAddr = Unsafe.realloc(scratchAddr, scratchCap, (int) newCap, MemoryTag.NATIVE_DEFAULT);
+        scratchCap = (int) newCap;
+    }
+
+    /**
+     * Checksums {@code [recStart, recStart + hdrLen + entriesLen)} in ONE native
+     * call, appends the CRC, and issues ONE positioned write. Advances
+     * {@code size}/{@code appendOffset} only on a complete write, so a short write
+     * throws and a retry keyed off {@link #size()} re-persists the same range at the
+     * same offset.
+     */
+    private void flushChunk(long recStart, int hdrLen, int entriesLen, int count) {
+        // long math, matching commitMappedChunk: hdrLen + entriesLen can reach
+        // Integer.MAX_VALUE (entriesLen is bounded only by MAX_SCRATCH_BYTES), and an
+        // int sum would wrap negative and hand a bogus length to the CRC and the write.
+        long bodyLen = (long) hdrLen + entriesLen;
+        long recLen = bodyLen + CRC_SIZE;
+        Unsafe.getUnsafe().putInt(recStart + bodyLen, Crc32c.update(Crc32c.INIT, recStart, bodyLen));
+        appendWriteCount++;
+        long written = ff.write(fd, recStart, recLen, appendOffset);
+        if (written != recLen) {
+            throw new IllegalStateException("short write to " + FILE_NAME
+                    + " [expected=" + recLen + ", actual=" + written + ']');
+        }
+        appendOffset += recLen;
+        size += count;
+    }
+
+    /**
+     * Writes one chunk whose entry region is ALREADY encoded in scratch at offset
+     * {@link #MAX_CHUNK_HEADER_SIZE}. Back-fills the header immediately in front of
+     * the entries -- the header is at most {@code MAX_CHUNK_HEADER_SIZE} bytes, so it
+     * always fits the reserve -- leaving header, entries and CRC one contiguous run
+     * for a single checksum and a single write.
+     */
+    private void writeChunkFromScratch(int entriesLen, int count) {
+        int hdrLen = NativeBufferWriter.varintSize(count) + NativeBufferWriter.varintSize(entriesLen);
+        long recStart = scratchAddr + MAX_CHUNK_HEADER_SIZE - hdrLen;
+        long p = NativeBufferWriter.writeVarint(recStart, count);
+        NativeBufferWriter.writeVarint(p, entriesLen);
+        flushChunk(recStart, hdrLen, entriesLen, count);
+    }
+
+    private static final class RecoveryScan {
+        private final int count;
+        private final int entriesLen;
+        private final int validLen;
+
+        private RecoveryScan(int count, int entriesLen, int validLen) {
+            this.count = count;
+            this.entriesLen = entriesLen;
+            this.validLen = validLen;
+        }
+    }
+
+    /**
+     * Zero-allocation LEB128 decoder, one instance per recovery pass -- not one
+     * per chunk. The previous {@code long[]}-returning decoder allocated once per
+     * entry, so a million-symbol recovery churned a million throwaway arrays.
+     */
+    private static final class Varint {
+        int end;
+        long value;
+
+        /**
+         * Decodes the varint at {@code buf[pos..limit)}. Returns false -- leaving
+         * {@code value}/{@code end} undefined -- when the varint is truncated (a torn
+         * tail) or runs longer than a canonical 5-byte length.
+         */
+        boolean decode(long buf, int pos, int limit) {
+            long v = 0;
+            int shift = 0;
+            int cur = pos;
+            while (cur < limit) {
+                byte b = Unsafe.getUnsafe().getByte(buf + cur);
+                cur++;
+                v |= (long) (b & 0x7F) << shift;
+                if ((b & 0x80) == 0) {
+                    value = v;
+                    end = cur;
+                    return true;
+                }
+                shift += 7;
+                if (shift > 35) {
+                    return false; // implausible for a chunk header; treat as torn
+                }
+            }
+            return false;
+        }
+    }
+}
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/RecoveredFrameAnalysis.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/RecoveredFrameAnalysis.java
new file mode 100644
index 00000000..790db93b
--- /dev/null
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/RecoveredFrameAnalysis.java
@@ -0,0 +1,388 @@
+/*******************************************************************************
+ *     ___                  _   ____  ____
+ *    / _ \ _   _  ___  ___| |_|  _ \| __ )
+ *   | | | | | | |/ _ \/ __| __| | | |  _ \
+ *   | |_| | |_| |  __/\__ \ |_| |_| | |_) |
+ *    \__\_\\__,_|\___||___/\__|____/|____/
+ *
+ *  Copyright (c) 2014-2019 Appsicle
+ *  Copyright (c) 2019-2026 QuestDB
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ ******************************************************************************/
+
+package io.questdb.client.cutlass.qwp.client.sf.cursor;
+
+import io.questdb.client.cutlass.qwp.client.GlobalSymbolDictionary;
+import io.questdb.client.cutlass.qwp.protocol.QwpConstants;
+import io.questdb.client.std.MemoryTag;
+import io.questdb.client.std.QuietCloseable;
+import io.questdb.client.std.Unsafe;
+import io.questdb.client.std.str.Utf8s;
+
+/**
+ * Engine-owned result of the one ordered frame walk performed during recovery.
+ * It folds commit-boundary detection, delta extrema, gap detection and the raw
+ * symbol suffix into the same pass. Running state is checkpointed only at a
+ * commit-bearing frame, so a deferred orphan tail can be scanned without ever
+ * leaking its metadata or symbols into the committed result.
+ */
+final class RecoveredFrameAnalysis implements QuietCloseable {
+
+    private static final int MAX_RAW_BYTES = Integer.MAX_VALUE - 8;
+    private final long ackedFsn;
+    private final int baseline;
+    private long committedBoundaryFsn = -1L;
+    private long committedCoverage;
+    private boolean committedGap;
+    private long committedMaxDeltaEnd;
+    private long committedMaxDeltaStart;
+    private int committedRawCount;
+    private int committedRawLen;
+    private long framesVisited;
+    // Set when a gap-reset rewinds the raw write cursor, cleared at every commit
+    // checkpoint. Still set at finish() => the committed snapshot's bytes were
+    // overwritten by an uncommitted tail and must not be published. See finish().
+    private boolean hasRewoundSinceCommit;
+    private boolean runningGap;
+    private boolean runningUnackedGap;
+    private long runningMaxDeltaEnd;
+    private long runningMaxDeltaStart;
+    private long symbolEntriesVisited;
+    private long rawAddr;
+    private int rawCapacity;
+    private int runningRawCount;
+    private int runningRawLen;
+    private long runningCoverage;
+
+    RecoveredFrameAnalysis(int baseline, long ackedFsn) {
+        this.baseline = baseline;
+        this.ackedFsn = ackedFsn;
+        this.runningCoverage = baseline;
+        this.committedCoverage = baseline;
+    }
+
+    void accept(long fsn, long payload, int payloadLen) {
+        framesVisited++;
+        boolean isQwp = payloadLen >= QwpConstants.HEADER_SIZE
+                && payloadLen > QwpConstants.HEADER_OFFSET_FLAGS
+                && Unsafe.getUnsafe().getInt(payload) == QwpConstants.MAGIC_MESSAGE;
+        byte flags = isQwp
+                ? Unsafe.getUnsafe().getByte(payload + QwpConstants.HEADER_OFFSET_FLAGS)
+                : 0;
+        if (isQwp && (flags & QwpConstants.FLAG_DELTA_SYMBOL_DICT) != 0) {
+            foldDelta(fsn, payload + QwpConstants.HEADER_SIZE, payload + payloadLen);
+        }
+
+        // Only a positively identified deferred QWP frame can belong to an
+        // uncommitted tail. Short, foreign or otherwise non-QWP payloads remain
+        // retirement barriers.
+        if (!isQwp || (flags & QwpConstants.FLAG_DEFER_COMMIT) == 0) {
+            committedBoundaryFsn = fsn;
+            committedCoverage = runningCoverage;
+            committedGap = runningGap;
+            committedMaxDeltaEnd = runningMaxDeltaEnd;
+            committedMaxDeltaStart = runningMaxDeltaStart;
+            committedRawLen = runningRawLen;
+            committedRawCount = runningRawCount;
+            hasRewoundSinceCommit = false;
+        }
+    }
+
+    void addDecodedSymbolsTo(GlobalSymbolDictionary target) {
+        decodeSymbols(target);
+    }
+
+    private void decodeSymbols(GlobalSymbolDictionary target) {
+        long p = rawAddr;
+        long limit = rawAddr + committedRawLen;
+        for (int i = 0; i < committedRawCount; i++) {
+            long encoded = readVarint(p, limit);
+            if (encoded < 0L) {
+                throw new IllegalStateException("malformed cached symbol dictionary suffix");
+            }
+            int varintLen = (int) (encoded & 7L);
+            long symbolLen = encoded >>> 3;
+            p += varintLen;
+            if (symbolLen > limit - p) {
+                throw new IllegalStateException("truncated cached symbol dictionary suffix");
+            }
+            target.addRecoveredSymbol(Utf8s.stringFromUtf8Bytes(p, p + symbolLen));
+            p += symbolLen;
+        }
+        if (p != limit) {
+            throw new IllegalStateException("overfilled cached symbol dictionary suffix");
+        }
+    }
+
+    int baseline() {
+        return baseline;
+    }
+
+    long commitBoundaryFsn() {
+        return committedBoundaryFsn;
+    }
+
+    long coverage() {
+        return committedGap ? -1L : committedCoverage;
+    }
+
+    long framesVisited() {
+        return framesVisited;
+    }
+
+    /**
+     * Discards native bytes accumulated only while scanning an uncommitted
+     * deferred tail. Call exactly once after the ordered recovery walk.
+     */
+    void finish() {
+        if (hasRewoundSinceCommit) {
+            // A gap-reset rewound the write cursor after the last commit checkpoint, so
+            // the deferred tail has overwritten the bytes committedRawLen/Count still
+            // describe. Publishing them would decode the TAIL's entries under the
+            // committed ids -- exactly the silent misattribution this analysis exists to
+            // prevent. Fail clean instead: no suffix, and a gap so coverage() reports -1
+            // and the producer falls back to the persisted prefix (or, when unacked
+            // frames depend on the missing ids, quarantines).
+            committedRawLen = 0;
+            committedRawCount = 0;
+            committedGap = true;
+        }
+        if (rawCapacity == committedRawLen) {
+            return;
+        }
+        if (committedRawLen == 0) {
+            Unsafe.free(rawAddr, rawCapacity, MemoryTag.NATIVE_DEFAULT);
+            rawAddr = 0L;
+            rawCapacity = 0;
+            return;
+        }
+        rawAddr = Unsafe.realloc(
+                rawAddr,
+                rawCapacity,
+                committedRawLen,
+                MemoryTag.NATIVE_DEFAULT);
+        rawCapacity = committedRawLen;
+    }
+
+    long maxDeltaEnd() {
+        return committedMaxDeltaEnd;
+    }
+
+    long maxDeltaStart() {
+        return committedMaxDeltaStart;
+    }
+
+    long rawAddr() {
+        return rawAddr;
+    }
+
+    int rawCount() {
+        return committedRawCount;
+    }
+
+    int rawLen() {
+        return committedRawLen;
+    }
+
+    int rawCapacity() {
+        return rawCapacity;
+    }
+
+    /**
+     * Releases the cached suffix after a foreground producer and its one send
+     * loop have both consumed it. Recovery metadata remains available, but the
+     * raw entries must not be requested again.
+     */
+    void releaseRawStorage() {
+        if (rawAddr != 0L) {
+            Unsafe.free(rawAddr, rawCapacity, MemoryTag.NATIVE_DEFAULT);
+            rawAddr = 0L;
+            rawCapacity = 0;
+        }
+        runningRawLen = 0;
+        runningRawCount = 0;
+        committedRawLen = 0;
+        committedRawCount = 0;
+    }
+
+    long symbolEntriesVisited() {
+        return symbolEntriesVisited;
+    }
+
+    @Override
+    public void close() {
+        releaseRawStorage();
+    }
+
+    /**
+     * Appends one contiguous run of {@code count} wire entries -- {@code [len][utf8]}
+     * repeated, exactly as a delta section carries them -- in a single copy. Callers
+     * pass a whole frame's new-symbol suffix at once; see {@link #foldDelta} for why
+     * that suffix is always contiguous.
+     */
+    private void appendRaw(long addr, int len, int count) {
+        long required = (long) runningRawLen + len;
+        if (required > MAX_RAW_BYTES) {
+            throw new IllegalStateException("recovered symbol dictionary suffix exceeds maximum size "
+                    + "[required=" + required + ", max=" + MAX_RAW_BYTES + ']');
+        }
+        if (required > rawCapacity) {
+            long newCapacity = Math.max(required, Math.max(4_096L, (long) rawCapacity * 2L));
+            if (newCapacity > MAX_RAW_BYTES) {
+                newCapacity = MAX_RAW_BYTES;
+            }
+            rawAddr = Unsafe.realloc(rawAddr, rawCapacity, (int) newCapacity, MemoryTag.NATIVE_DEFAULT);
+            rawCapacity = (int) newCapacity;
+        }
+        Unsafe.getUnsafe().copyMemory(addr, rawAddr + runningRawLen, len);
+        runningRawLen += len;
+        runningRawCount += count;
+    }
+
+    private void foldDelta(long fsn, long p, long limit) {
+        long encodedStart = readVarint(p, limit);
+        if (encodedStart < 0L) {
+            markGap(fsn);
+            return;
+        }
+        int startLen = (int) (encodedStart & 7L);
+        long deltaStart = encodedStart >>> 3;
+        p += startLen;
+        if (deltaStart > runningMaxDeltaStart) {
+            runningMaxDeltaStart = deltaStart;
+        }
+
+        long encodedCount = readVarint(p, limit);
+        if (encodedCount < 0L) {
+            markGap(fsn);
+            return;
+        }
+        int countLen = (int) (encodedCount & 7L);
+        long deltaCount = encodedCount >>> 3;
+        p += countLen;
+        long deltaEnd = deltaCount > Long.MAX_VALUE - deltaStart
+                ? Long.MAX_VALUE
+                : deltaStart + deltaCount;
+        if (deltaEnd > runningMaxDeltaEnd) {
+            runningMaxDeltaEnd = deltaEnd;
+        }
+        if (runningGap) {
+            // A full dictionary is a new self-sufficient epoch, but it may only
+            // repair a gap that is entirely behind the durable ACK watermark.
+            // If any gapped frame will replay first, accepting this reset would
+            // hide the unsafe wire-order gap and let the server observe missing
+            // ids before it reaches the full frame.
+            if (deltaStart != 0L || runningUnackedGap) {
+                return;
+            }
+            runningGap = false;
+            runningCoverage = baseline;
+            // Rewinding the write cursor to 0 means the NEXT appendRaw overwrites
+            // [0, committedRawLen) in place -- bytes the committed snapshot still
+            // counts. That is harmless when a commit-bearing frame follows (accept()
+            // re-checkpoints and clears this flag), but a reset inside an uncommitted
+            // deferred TAIL never gets that refresh, and finish() would then hand out
+            // the old counts over the tail's bytes: silent symbol misattribution.
+            hasRewoundSinceCommit = true;
+            runningRawLen = 0;
+            runningRawCount = 0;
+        }
+        if (deltaStart > runningCoverage) {
+            markGap(fsn);
+            return;
+        }
+        // The segment scan already CRC-validated this frame. When its entire
+        // range is covered, entry parsing cannot extend recovery state, so skip
+        // the cardinality-proportional payload walk.
+        if (deltaEnd <= runningCoverage) {
+            return;
+        }
+
+        // runningCoverage is loop-invariant here -- it only advances after the walk --
+        // and id ascends from deltaStart, so `id >= runningCoverage` is a step
+        // predicate: the entries this frame contributes are always ONE contiguous run
+        // at the tail of its delta section. Note where that run starts and copy it in a
+        // single memcpy once the walk succeeds, rather than paying a bound check, a
+        // capacity check and a ~12-byte copyMemory per symbol. Recovery walks the whole
+        // backlog, and the workload this feature exists for introduces a new symbol per
+        // ROW, so that is millions of stub-dispatched small copies where one bulk copy
+        // per frame does the job.
+        //
+        // Deferring the copy past the markGap bail-outs also drops the partial prefix
+        // the per-entry version used to leave behind. That residue was already
+        // unreachable -- a gap pins coverage() at -1, and a later self-sufficient frame
+        // resets runningRawLen/runningRawCount -- so not writing it is equivalent, and
+        // leaves less state to reason about.
+        long id = deltaStart;
+        long suffixStart = 0L;
+        int suffixCount = 0;
+        for (long i = 0; i < deltaCount; i++, id++) {
+            symbolEntriesVisited++;
+            long entryStart = p;
+            long encodedLen = readVarint(p, limit);
+            if (encodedLen < 0L) {
+                markGap(fsn);
+                return;
+            }
+            int varintLen = (int) (encodedLen & 7L);
+            long symbolLen = encodedLen >>> 3;
+            p += varintLen;
+            if (symbolLen > limit - p) {
+                markGap(fsn);
+                return;
+            }
+            p += symbolLen;
+            if (id >= runningCoverage) {
+                if (suffixCount == 0) {
+                    suffixStart = entryStart;
+                }
+                suffixCount++;
+            }
+        }
+        if (suffixCount > 0) {
+            appendRaw(suffixStart, (int) (p - suffixStart), suffixCount);
+        }
+        if (deltaEnd > runningCoverage) {
+            runningCoverage = deltaEnd;
+        }
+    }
+
+    private void markGap(long fsn) {
+        runningGap = true;
+        if (fsn > ackedFsn) {
+            runningUnackedGap = true;
+        }
+    }
+
+    /**
+     * Returns {@code (value << 3) | encodedByteCount}, or {@code -1} for an
+     * unterminated/oversized 32-bit protocol varint.
+     */
+    private static long readVarint(long p, long limit) {
+        long value = 0L;
+        int shift = 0;
+        int bytes = 0;
+        while (p < limit && bytes < 5) {
+            byte b = Unsafe.getUnsafe().getByte(p++);
+            value |= (long) (b & 0x7F) << shift;
+            bytes++;
+            if ((b & 0x80) == 0) {
+                return (value << 3) | bytes;
+            }
+            shift += 7;
+        }
+        return -1L;
+    }
+}
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentManager.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentManager.java
index 91deddd3..c4d3e218 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentManager.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentManager.java
@@ -909,51 +909,60 @@ private boolean serviceRing0(RingEntry e) {
                 String path = null;
                 boolean installed = false;
                 try {
-                    // baseSeq is provisional — SegmentRing.appendOrFsn calls
-                    // rebaseSeq() at rotation time to pin the real value. We
-                    // pass the manager's best guess (nextSeqHint at this
-                    // instant), which is fine since it's overwritten anyway.
-                    if (memoryMode) {
-                        spare = MmapSegment.createInMemory(e.ring.nextSeqHint(), segmentSizeBytes);
-                    } else {
-                        path = nextSparePath(e.dir);
-                        // Native path bytes (NUL-terminated) live in pathScratch
-                        // from the call above. Hand them straight to MmapSegment.create
-                        // via its long-ptr overload, bypassing the byte[] + native
-                        // malloc that the String overload would incur on every
-                        // rotation.
-                        spare = MmapSegment.create(filesFacade,
-                                pathScratch.ptr(), path,
-                                e.ring.nextSeqHint(), segmentSizeBytes, true);
-                    }
-                    Runnable installHook = beforeInstallSyncHook;
-                    if (installHook != null) {
-                        installHook.run();
-                    }
-                    if (!memoryMode) {
-                        spare.syncHeader();
-                        if (filesFacade.fsyncDir(e.dir) != 0) {
-                            throw new MmapSegmentException(
-                                    "could not sync hot-spare directory " + e.dir);
+                    // Null if the ring closed between this tick's snapshot and now:
+                    // close() nulls it under the ring's own monitor with no
+                    // coordination with the manager thread. A closed ring rejects
+                    // installHotSpare regardless, so there is nothing to provision
+                    // this tick; skip cleanly rather than provision a spare that will
+                    // be abandoned.
+                    MmapSegment active = e.ring.getActive();
+                    if (active != null) {
+                        // baseSeq is provisional -- SegmentRing.appendOrFsn calls
+                        // rebaseSeq() at rotation time to pin the real value. We
+                        // pass the manager's best guess (nextSeqHint at this
+                        // instant), which is fine since it's overwritten anyway.
+                        if (memoryMode) {
+                            spare = MmapSegment.createInMemory(e.ring.nextSeqHint(), segmentSizeBytes);
+                        } else {
+                            path = nextSparePath(e.dir);
+                            // Native path bytes (NUL-terminated) live in pathScratch
+                            // from the call above. Hand them straight to MmapSegment.create
+                            // via its long-ptr overload, bypassing the byte[] + native
+                            // malloc that the String overload would incur on every
+                            // rotation.
+                            spare = MmapSegment.create(filesFacade,
+                                    pathScratch.ptr(), path,
+                                    e.ring.nextSeqHint(), segmentSizeBytes, true);
                         }
-                    }
-                    // Install + commit atomically under the manager lock.
-                    // If `e.ring` was deregistered between the snapshot
-                    // above and now, abandoning the spare here is the only
-                    // way to keep totalBytes consistent: deregister already
-                    // subtracted ring.totalSegmentBytes() (without the
-                    // spare, since it wasn't installed yet) so a commit at
-                    // this point would inflate totalBytes by one segment
-                    // with no future subtractor. By holding `lock` across
-                    // installHotSpare AND the += commit AND the registration
-                    // check, deregister is forced to either
-                    // observe the spare in the ring (and subtract it) or
-                    // run before installation (so no install happens).
-                    synchronized (lock) {
-                        if (e.isRegistered()) {
-                            e.ring.installHotSpare(spare);
-                            totalBytes += segmentSizeBytes;
-                            installed = true;
+                        Runnable installHook = beforeInstallSyncHook;
+                        if (installHook != null) {
+                            installHook.run();
+                        }
+                        if (!memoryMode) {
+                            spare.syncHeader();
+                            if (filesFacade.fsyncDir(e.dir) != 0) {
+                                throw new MmapSegmentException(
+                                        "could not sync hot-spare directory " + e.dir);
+                            }
+                        }
+                        // Install + commit atomically under the manager lock.
+                        // If `e.ring` was deregistered between the snapshot
+                        // above and now, abandoning the spare here is the only
+                        // way to keep totalBytes consistent: deregister already
+                        // subtracted ring.totalSegmentBytes() (without the
+                        // spare, since it wasn't installed yet) so a commit at
+                        // this point would inflate totalBytes by one segment
+                        // with no future subtractor. By holding `lock` across
+                        // installHotSpare AND the += commit AND the registration
+                        // check, deregister is forced to either
+                        // observe the spare in the ring (and subtract it) or
+                        // run before installation (so no install happens).
+                        synchronized (lock) {
+                            if (e.isRegistered()) {
+                                e.ring.installHotSpare(spare);
+                                totalBytes += segmentSizeBytes;
+                                installed = true;
+                            }
                         }
                     }
                 } catch (Throwable t) {
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java
index d3f43e25..6fd7f5aa 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java
@@ -367,6 +367,49 @@ static Recovery recover(
             long activeBase;
             if (manifest == null) {
                 if (data.size() > 0) {
+                    // Legacy migration is the one branch with no recorded boundary
+                    // to check a set-aside file against: validateContiguous only
+                    // compares SURVIVORS, so a file that sat BELOW data.get(0)
+                    // leaves a contiguous chain, and the manifest this branch is
+                    // about to create would record headBase as if that file's
+                    // frames had been acked. They never were -- the ack seed
+                    // derives from the head.
+                    //
+                    // A chain based at 0 needs no check: nothing can precede it,
+                    // so a set-aside file of any kind was a stray or a spare. And
+                    // a chain based above 0 is the NORMAL steady state for a
+                    // long-lived legacy slot whose head was ack-trimmed, so the
+                    // refusal has to key off positive evidence that a file was
+                    // set aside during THIS recovery, never off the base alone.
+                    long legacyChainStart = data.get(0).baseSeq();
+                    if (legacyChainStart != 0L) {
+                        if (corruptPaths != null) {
+                            // A corrupt file's own baseSeq is unreadable (it is
+                            // inside the bytes that failed), so nothing can place
+                            // it above the chain head. Fail closed.
+                            throw new SfRecoveryException("cannot migrate the legacy SF chain in "
+                                    + sfDir + " based at " + legacyChainStart + ": a corrupt segment"
+                                    + " of unknown identity could be its head, and no manifest"
+                                    + " boundary exists to prove otherwise");
+                        }
+                        for (int i = 0, n = all.size(); i < n; i++) {
+                            MmapSegment segment = all.get(i);
+                            // A frameless segment carrying torn-tail residue is a
+                            // segment whose frame[0] failed: bytes were written to
+                            // it, so it is not an untouched spare. Its header IS
+                            // readable, so its position is provable -- refuse only
+                            // when it really sat below the chain head.
+                            if (segment.frameCount() == 0
+                                    && segment.tornTailBytes() > 0
+                                    && segment.baseSeq() < legacyChainStart) {
+                                throw new SfRecoveryException("cannot migrate the legacy SF chain in "
+                                        + sfDir + " based at " + legacyChainStart + ": the segment at"
+                                        + " base " + segment.baseSeq() + " lost its frames to a torn"
+                                        + " write and sits below that head, so its range cannot be"
+                                        + " shown already-acked");
+                            }
+                        }
+                    }
                     validateContiguous(data);
                     for (int i = 0, n = data.size(); i < n; i++) {
                         chain.add(data.get(i));
@@ -461,9 +504,41 @@ static Recovery recover(
                             long torn = segment.tornTailBytes();
                             segment.close();
                             if (torn > 0) {
+                                // Preserve, not destroy: on success the bytes move to
+                                // .corrupt; if even the rename fails, they stay under
+                                // their original name instead (quarantineFile only
+                                // logs a warning on a failed rename, so a torn,
+                                // data-bearing leftover really can remain a plain
+                                // .sfa file here). Either way the file survives for
+                                // the NEXT recovery to re-examine -- and it is THAT
+                                // recovery's FSN boundary checks, not the name, that
+                                // keep it from being mistaken for this generation's:
+                                // a base above activeBase throws ("segment exists
+                                // beyond committed SF active boundary"), a base
+                                // colliding with the new chain fails
+                                // validateContiguous's expected-next-base check, and
+                                // a base wholly below headBase is excluded from the
+                                // chain and quarantined again as a non-retained extra.
                                 quarantineFile(filesFacade, path);
                             } else if (!filesFacade.remove(path)) {
-                                LOG.warn("could not remove drained SF leftover {}", path);
+                                // Returning EMPTY here makes the caller start fresh at
+                                // baseSeq 0 in a directory that still holds a prior
+                                // generation's file. Their FSN ranges then overlap while
+                                // their symbol ids describe different strings, and no
+                                // downstream guard can see it: contiguity and the
+                                // manifest boundaries both reason about FSNs, and both
+                                // generations number theirs from the same origin. Refuse
+                                // the slot so the state stays representable-by-refusal
+                                // rather than silently mixed.
+                                //
+                                // Operational, not terminal: an unlink can fail for a
+                                // transient reason (a share lock, an antivirus or backup
+                                // handle), exactly as the manifest unlink below can. The
+                                // plain type aborts startup for a retry instead of
+                                // quarantining a slot whose bytes may be perfectly intact.
+                                throw new MmapSegmentException("could not remove drained SF leftover "
+                                        + path + "; refusing to start fresh in a slot that still"
+                                        + " holds a prior generation's segment");
                             }
                         }
                         all.clear();
@@ -1252,32 +1327,28 @@ public synchronized MmapSegment firstTrimmable() {
     }
 
     /**
-     * Walks every published frame in the ring (sealed segments plus the active
-     * segment) and returns the FSN of the LAST frame whose payload does NOT
-     * carry the given flag bit, or {@code -1} when every published frame
-     * carries it (or the ring is empty). All frames above the returned FSN
-     * carry the flag.
+     * Performs the one ordered recovery fold across sealed segments and the
+     * active segment. The returned native suffix remains owned by the caller.
      * 

-     * Recovery-time helper: locates the last commit-bearing QWP frame below a
-     * potentially orphaned FLAG_DEFER_COMMIT tail left behind by a producer
-     * that crashed (or closed) mid-transaction. Call before the I/O loop and
-     * producer start appending; the walk is not synchronized against appends
-     * into the active segment. See
-     * {@link MmapSegment#findLastFrameFsnWithoutPayloadFlag} for the
-     * positive-identification contract: frames that do not parse as protocol
-     * messages count as commit-bearing (retirement barriers), never as
-     * trimmable.
+     * Replaces the former {@code findLastFsnWithoutPayloadFlag} walk: the
+     * deferred-commit tail scan is one of several verdicts the single fold now
+     * produces, so the ring is walked once rather than once per question. Call
+     * before the I/O loop and producer start appending; the walk is not
+     * synchronized against appends into the active segment.
      */
-    public synchronized long findLastFsnWithoutPayloadFlag(int flagsOffset, int flagMask, int headerMagic, int minPayloadLen) {
-        long best = -1L;
-        for (int i = sealedHead, n = sealedSegments.size(); i < n; i++) {
-            long fsn = sealedSegments.get(i).findLastFrameFsnWithoutPayloadFlag(flagsOffset, flagMask, headerMagic, minPayloadLen);
-            if (fsn > best) {
-                best = fsn;
+    synchronized RecoveredFrameAnalysis analyzeRecovery(int symbolBaseline) {
+        RecoveredFrameAnalysis analysis = new RecoveredFrameAnalysis(symbolBaseline, ackedFsn);
+        try {
+            for (int i = sealedHead, n = sealedSegments.size(); i < n; i++) {
+                sealedSegments.get(i).scanRecovery(analysis);
             }
+            active.scanRecovery(analysis);
+            analysis.finish();
+            return analysis;
+        } catch (Throwable t) {
+            analysis.close();
+            throw t;
         }
-        long fsn = active.findLastFrameFsnWithoutPayloadFlag(flagsOffset, flagMask, headerMagic, minPayloadLen);
-        return Math.max(best, fsn);
     }
 
     public MmapSegment getActive() {
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SlotLock.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SlotLock.java
index 17264c5a..e26aa6f3 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SlotLock.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SlotLock.java
@@ -26,20 +26,27 @@
 
 import io.questdb.client.std.Compat;
 import io.questdb.client.std.Files;
+import io.questdb.client.std.FilesFacade;
 import io.questdb.client.std.MemoryTag;
 import io.questdb.client.std.QuietCloseable;
 import io.questdb.client.std.Unsafe;
 import org.jetbrains.annotations.TestOnly;
 
 import java.nio.charset.StandardCharsets;
+import java.nio.file.Path;
+import java.nio.file.Paths;
 
 /**
- * Advisory exclusive lock for a single SF slot directory.
+ * Advisory exclusive locks for a single SF slot.
  * 

- * One {@code .lock} file per slot, held via {@code flock}/{@code LockFileEx}
- * for the entire lifetime of the engine that owns the slot. Normal teardown
- * explicitly unlocks it before closing the fd; hard process exit remains a
- * backstop because the kernel cleans up file locks for terminated processes.
+ * {@link #acquire(String)} locks a {@code .lock} file inside the slot directory
+ * for the entire lifetime of the engine that owns it. {@link #acquireLogical(String)}
+ * locks a sibling file under the parent SF directory for short-lived pathname
+ * transitions and orphan adoption; because it is outside the slot directory,
+ * it remains stable if that directory is renamed. Both use
+ * {@code flock}/{@code LockFileEx}. Normal teardown explicitly unlocks the
+ * descriptor before closing it; hard process exit remains a backstop because
+ * the kernel cleans up file locks for terminated processes.
  * 

  * The holder's PID is written to a sibling {@code .lock.pid} file at
  * acquisition time. A failed acquisition reads it back so the error message
@@ -60,6 +67,7 @@ public final class SlotLock implements QuietCloseable {
     private static final int DEAD_FD_FOR_TESTING = 1_000_000_000;
     private static final String LOCK_FILE_NAME = ".lock";
     private static final String LOCK_PID_FILE_NAME = ".lock.pid";
+    private static final String LOGICAL_LOCK_DIR_NAME = ".slot-locks";
     private static final Object RELEASE_RETRY_LOCK = new Object();
     private static SlotLock releaseRetryHead;
     private final String slotDir;
@@ -89,48 +97,144 @@ public static SlotLock acquire(String slotDir) {
      * parent directory before any segment file is created.
      */
     public static SlotLock acquire(String slotDir, boolean syncParentDirectory) {
-        if (slotDir == null || slotDir.isEmpty()) {
-            throw new IllegalArgumentException("slotDir must not be empty");
-        }
+        validateSlotDir(slotDir);
         // Construction cleanup may have retained locks after explicit unlock
         // failures. Drive every pending owner before opening a new descriptor.
         // Path text cannot identify a physical file portably (symlinks and
         // Windows case aliases are counterexamples), while the pending list is
         // cold, error-only state and normally empty.
         retryPendingReleases();
-        if (!Files.exists(slotDir)) {
-            int rc = Files.mkdir(slotDir, Files.DIR_MODE_DEFAULT);
-            if (rc != 0) {
-                throw new SfOperationalException(
-                        "could not create slot dir: " + slotDir + " rc=" + rc);
-            }
-        }
+        // DIR_MODE_DEFAULT is right here: one process creates its own slot
+        // directory and only that process writes inside it.
+        ensureDirectory(FilesFacade.INSTANCE, slotDir, "slot dir", Files.DIR_MODE_DEFAULT);
         if (syncParentDirectory && Files.fsyncParentDir(slotDir) != 0) {
             throw new SfOperationalException(
                     "could not sync parent directory for SF slot: " + slotDir);
         }
         String lockPath = slotDir + "/" + LOCK_FILE_NAME;
         String pidPath = slotDir + "/" + LOCK_PID_FILE_NAME;
-        int fd = Files.openRW(lockPath);
+        return acquireAt(FilesFacade.INSTANCE, slotDir, lockPath, pidPath);
+    }
+
+    /**
+     * Acquires the stable logical lock for {@code slotDir}, creating its
+     * {@code .slot-locks} parent at {@link Files#DIR_MODE_DEFAULT}. Unlike the
+     * directory-local {@code .lock}, this lock is anchored in the parent SF
+     * directory, so renaming the slot cannot move the lock inode away from the
+     * logical slot name it guards.
+     * 

+     * Callers use this as a short-lived transition/adoption lock, always before
+     * acquiring the directory-local lock. In particular it must cover an
+     * unreplayable slot's close -> rename -> recreate transition, preventing a
+     * queued orphan drainer from adopting the renamed inode and later touching
+     * the fresh directory through the old pathname.
+     */
+    public static SlotLock acquireLogical(String slotDir) {
+        return acquireLogical(FilesFacade.INSTANCE, slotDir);
+    }
+
+    /** Facade-aware variant used to exercise logical-lock I/O failures. */
+    @TestOnly
+    public static SlotLock acquireLogical(FilesFacade ff, String slotDir) {
+        validateSlotDir(slotDir);
+        // Same pre-step as acquire(): a logical lock this process retained after
+        // an unconfirmed unlock would otherwise contend with its own successor.
+        retryPendingReleases();
+        String[] paths = resolveLogicalLock(slotDir);
+        if (paths == null) {
+            throw new IllegalArgumentException(
+                    "slotDir must contain a parent and slot name: " + slotDir);
+        }
+        ensureDirectory(ff, paths[0], "logical slot lock dir", Files.DIR_MODE_DEFAULT);
+        return acquireAt(ff, slotDir, paths[1], paths[2]);
+    }
+
+    /**
+     * Best-effort removal of the parent-anchored logical lock files
+     * ({@code /.slot-locks/.lock} and its {@code .lock.pid}
+     * sidecar) for {@code slotDir}, mirroring {@link AckWatermark#removeOrphan}
+     * and {@link PersistedSymbolDict#removeOrphan} for the slot's in-directory
+     * side-files. The fully-drained close that permanently retires a slot calls
+     * this: the logical lock lives OUTSIDE the slot dir (so it survives a slot
+     * rename), so nothing else reclaims it, and without this
+     * {@code /.slot-locks} accumulates one dead lock+pid pair per
+     * distinct slot name for the lifetime of {@code sf_dir} -- unbounded under
+     * rotating {@code senderId}s.
+     * 

+     * The unlink is best-effort and safe: the retiring engine still holds the
+     * slot's directory-local {@link #acquire} lock (the real multi-writer guard)
+     * across this cleanup, and fully-drained retirement performs no rename, so
+     * the logical lock -- which only guards the close/rename/recreate transition
+     * -- is not in use. An orphan drainer momentarily mid-{@link #acquireLogical}
+     * fails its immediately-following candidacy / directory-lock check (the slot
+     * is gone) and backs off. Unlike {@link #acquireLogical}, an unusable
+     * {@code slotDir} is a silent no-op here rather than a throw.
+     */
+    public static void removeOrphanLogical(String slotDir) {
+        removeOrphanLogical(FilesFacade.INSTANCE, slotDir);
+    }
+
+    /** Facade-aware variant of {@link #removeOrphanLogical(String)}. */
+    public static void removeOrphanLogical(FilesFacade ff, String slotDir) {
+        if (slotDir == null || slotDir.isEmpty()) {
+            return;
+        }
+        String[] paths = resolveLogicalLock(slotDir);
+        if (paths == null) {
+            return;
+        }
+        // Unlink ONLY while holding the lock. Removing a lock file another party holds
+        // does not release that party's flock, but it does free the pathname, so the
+        // next acquirer creates a SECOND inode and locks it successfully -- two owners
+        // of a lock whose only job is mutual exclusion.
+        //
+        // The retiring engine's directory-local lock does not stand in for this. It says
+        // nothing about the LOGICAL lock, which Sender.build() holds across its whole
+        // construct -> connect -> quarantine transition, in a frame ABOVE this one: an
+        // ordinary failed connect closes the engine from inside that scope, reaches here,
+        // and used to unlink the very file build() was holding. Acquiring first narrows
+        // that to a single-syscall window instead of silent double-ownership.
+        SlotLock guard;
+        try {
+            guard = acquireAt(ff, slotDir, paths[1], paths[2]);
+        } catch (Throwable t) {
+            // Contended (someone holds it, possibly a caller above us) or unopenable.
+            // Leaving the pair on disk is the safe outcome -- a live holder's lock must
+            // outlive our cleanup. The next fully-drained retirement reclaims it.
+            return;
+        }
+        try {
+            // Sidecar first: once paths[1] (.lock) is unlinked, a racing acquirer can
+            // create a fresh lock inode and write its own .lock.pid inside our window;
+            // removing the pid file first means no successor's sidecar can be hit.
+            ff.remove(paths[2]);
+            ff.remove(paths[1]);
+        } finally {
+            guard.close();
+        }
+    }
+
+    private static SlotLock acquireAt(FilesFacade ff, String slotDir, String lockPath, String pidPath) {
+        int fd = ff.openRW(lockPath);
         if (fd < 0) {
             throw new SfOperationalException(
                     "could not open slot lock file: " + lockPath);
         }
         boolean ok = false;
         try {
-            int rc = Files.lock(fd);
+            int rc = ff.lock(fd);
             if (rc != 0) {
                 String holder = readHolder(pidPath);
                 throw new SlotLockContentionException(
                         "sf slot already in use by another process [slot="
                                 + slotDir + ", holder=" + holder + "]");
             }
-            writePid(pidPath);
+            writePid(ff, pidPath);
             ok = true;
             return new SlotLock(slotDir, fd);
         } finally {
             if (!ok) {
-                Files.close(fd);
+                ff.close(fd);
             }
         }
     }
@@ -186,6 +290,49 @@ public static String probeHolder(String slotDir) {
         return null;
     }
 
+    private static void ensureDirectory(FilesFacade ff, String path, String description, int mode) {
+        if (!ff.exists(path)) {
+            int rc = ff.mkdir(path, mode);
+            // Multiple senders may create the shared parent lock directory
+            // concurrently. Treat EEXIST as success, just as the builder does
+            // for the SF root itself.
+            if (rc != 0 && !ff.exists(path)) {
+                throw new SfOperationalException(
+                        "could not create " + description + ": " + path + " rc=" + rc);
+            }
+        }
+    }
+
+    /**
+     * Resolves the parent-anchored logical lock layout for {@code slotDir}:
+     * {@code [0]} the {@code .slot-locks} directory, {@code [1]} the
+     * {@code .lock} path, {@code [2]} the {@code .lock.pid} path.
+     * Returns {@code null} when {@code slotDir} has no usable parent or name.
+     * Shared by {@link #acquireLogical} and {@link #removeOrphanLogical} so the
+     * two can never target different files.
+     */
+    private static String[] resolveLogicalLock(String slotDir) {
+        Path slotPath = Paths.get(slotDir);
+        Path parentPath = slotPath.getParent();
+        Path slotNamePath = slotPath.getFileName();
+        if (parentPath == null || slotNamePath == null || slotNamePath.toString().isEmpty()) {
+            return null;
+        }
+        String logicalLockDir = parentPath + "/" + LOGICAL_LOCK_DIR_NAME;
+        String slotName = slotNamePath.toString();
+        return new String[]{
+                logicalLockDir,
+                logicalLockDir + "/" + slotName + LOCK_FILE_NAME,
+                logicalLockDir + "/" + slotName + LOCK_PID_FILE_NAME
+        };
+    }
+
+    private static void validateSlotDir(String slotDir) {
+        if (slotDir == null || slotDir.isEmpty()) {
+            throw new IllegalArgumentException("slotDir must not be empty");
+        }
+    }
+
     /**
      * Replaces the live descriptor with a known-dead value until the returned
      * guard closes. Test-only: exercises release retry paths without exposing
@@ -327,7 +474,7 @@ private static void retryPendingReleases() {
         }
     }
 
-    private static void writePid(String pidPath) {
+    private static void writePid(FilesFacade ff, String pidPath) {
         long pid;
         try {
             pid = Compat.currentPid();
@@ -335,25 +482,25 @@ private static void writePid(String pidPath) {
             // Diagnostic-only — never block lock acquisition on it.
             pid = -1L;
         }
-        int wfd = Files.openRW(pidPath);
+        int wfd = ff.openRW(pidPath);
         if (wfd < 0) {
             // Diagnostic-only — never block lock acquisition on it.
             return;
         }
         try {
-            Files.truncate(wfd, 0L);
+            ff.truncate(wfd, 0L);
             byte[] payload = (pid + "\n").getBytes(StandardCharsets.UTF_8);
             long addr = Unsafe.malloc(payload.length, MemoryTag.NATIVE_DEFAULT);
             try {
                 for (int i = 0; i < payload.length; i++) {
                     Unsafe.getUnsafe().putByte(addr + i, payload[i]);
                 }
-                Files.write(wfd, addr, payload.length, 0L);
+                ff.write(wfd, addr, payload.length, 0L);
             } finally {
                 Unsafe.free(addr, payload.length, MemoryTag.NATIVE_DEFAULT);
             }
         } finally {
-            Files.close(wfd);
+            ff.close(wfd);
         }
     }
 
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/UnreplayableSlotException.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/UnreplayableSlotException.java
new file mode 100644
index 00000000..359b1da8
--- /dev/null
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/UnreplayableSlotException.java
@@ -0,0 +1,65 @@
+/*******************************************************************************
+ *     ___                  _   ____  ____
+ *    / _ \ _   _  ___  ___| |_|  _ \| __ )
+ *   | | | | | | |/ _ \/ __| __| | | |  _ \
+ *   | |_| | |_| |  __/\__ \ |_| |_| | |_) |
+ *    \__\_\\__,_|\___||___/\__|____/|____/
+ *
+ *  Copyright (c) 2014-2019 Appsicle
+ *  Copyright (c) 2019-2026 QuestDB
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ ******************************************************************************/
+
+package io.questdb.client.cutlass.qwp.client.sf.cursor;
+
+import io.questdb.client.cutlass.line.LineSenderException;
+
+/**
+ * A recovered store-and-forward slot that cannot be replayed at all. Three causes, all
+ * rooted in the symbol dictionary:
+ * 

+ * In the first two cases the frames reference ids that nothing still holds, so replaying
+ * them would make the server reject the frame with STATUS_DICTIONARY_GAP.
+ * 

+ * A distinct type, rather than a message match, because the difference between "this slot
+ * is unreplayable" and any other {@link LineSenderException} out of the connect path is the
+ * difference between setting a slot aside and silently discarding one that was fine.
+ * 

+ * It stays a {@code LineSenderException} so that a caller which does NOT handle it -- a test
+ * constructing the sender directly, say -- keeps the old fail-clean behaviour rather than
+ * seeing a new checked type. {@code Sender.build()} and {@code BackgroundDrainer} both treat
+ * it as recoverable: the former sets the slot aside and starts the producer on a fresh one,
+ * the latter drops the {@code .failed} sentinel so the next orphan scan skips the slot.
+ */
+public class UnreplayableSlotException extends LineSenderException {
+
+    public UnreplayableSlotException(CharSequence message) {
+        super(message);
+    }
+}
diff --git a/core/src/main/java/io/questdb/client/impl/ConfigSchema.java b/core/src/main/java/io/questdb/client/impl/ConfigSchema.java
index de8cce43..c9529a13 100644
--- a/core/src/main/java/io/questdb/client/impl/ConfigSchema.java
+++ b/core/src/main/java/io/questdb/client/impl/ConfigSchema.java
@@ -77,6 +77,7 @@ public final class ConfigSchema {
         // (one vocabulary, side-owned application).
         str("max_frame_rejections", Side.INGRESS);
         str("poison_min_escalation_window_millis", Side.INGRESS);
+        str("catch_up_cap_gap_min_escalation_window_millis", Side.INGRESS);
         str("max_name_len", Side.INGRESS);
         str("reconnect_initial_backoff_millis", Side.INGRESS);
         str("reconnect_max_backoff_millis", Side.INGRESS);
diff --git a/core/src/main/java/io/questdb/client/std/Crc32c.java b/core/src/main/java/io/questdb/client/std/Crc32c.java
index d0a2e6a8..738efb1b 100644
--- a/core/src/main/java/io/questdb/client/std/Crc32c.java
+++ b/core/src/main/java/io/questdb/client/std/Crc32c.java
@@ -46,6 +46,7 @@
 public final class Crc32c {
     /** Seed value to start a fresh CRC-32C accumulation. */
     public static final int INIT = 0;
+    private static final int[] CRC32C_TABLE = buildCrc32cTable();
 
     private Crc32c() {
     }
@@ -65,6 +66,73 @@ private Crc32c() {
      */
     public static native int update(int seed, long addr, long len);
 
+    /**
+     * Java/Unsafe slice-by-8 CRC-32C for memory that can fault while it is read,
+     * such as a recovery mmap over a sparse or concurrently truncated file.
+     * Keeping every load at an {@link Unsafe} intrinsic site lets HotSpot turn an
+     * mmap access fault into a catchable {@link InternalError}; the native
+     * {@link #update} path cannot provide that guarantee because a SIGBUS raised
+     * inside JNI aborts the JVM.
+     * 

+     * The hot loop consumes each 8-byte block with two {@code getInt}s rather than
+     * eight {@code getByte}s -- 9 loads per block instead of 16, for the same table
+     * work. {@code getInt} is an {@link Unsafe} intrinsic exactly as {@code getByte}
+     * is, so the fault-catchability above is unaffected. It does make the loop
+     * little-endian, which costs nothing here: the segment and dictionary formats this
+     * checksums are already little-endian throughout (their headers are read back with
+     * {@code getInt}), and the native twin this must agree with asserts the same.
+     *
+     * @param seed previous CRC value, or {@link #INIT} to start fresh
+     * @param addr off-heap address of at least {@code len} readable bytes
+     * @param len  number of bytes to consume
+     * @return the new CRC value
+     */
+    public static int updateUnsafe(int seed, long addr, long len) {
+        assert len >= 0L : "CRC length must be non-negative";
+        int crc = ~seed;
+        int[] table = CRC32C_TABLE;
+        while (len >= Long.BYTES) {
+            // Little-endian: lo holds bytes 0..3 and hi bytes 4..7, ascending from the
+            // low byte, so the per-byte table lookups below stay in wire order.
+            int lo = Unsafe.getUnsafe().getInt(addr) ^ crc;
+            int hi = Unsafe.getUnsafe().getInt(addr + Integer.BYTES);
+            crc = table[7 * 256 + (lo & 0xFF)]
+                    ^ table[6 * 256 + ((lo >>> 8) & 0xFF)]
+                    ^ table[5 * 256 + ((lo >>> 16) & 0xFF)]
+                    ^ table[4 * 256 + (lo >>> 24)]
+                    ^ table[3 * 256 + (hi & 0xFF)]
+                    ^ table[2 * 256 + ((hi >>> 8) & 0xFF)]
+                    ^ table[256 + ((hi >>> 16) & 0xFF)]
+                    ^ table[hi >>> 24];
+            addr += Long.BYTES;
+            len -= Long.BYTES;
+        }
+        while (len-- > 0L) {
+            crc = (crc >>> 8) ^ table[(crc ^ Unsafe.getUnsafe().getByte(addr++)) & 0xFF];
+        }
+        return ~crc;
+    }
+
+    private static int[] buildCrc32cTable() {
+        int[] table = new int[8 * 256];
+        for (int n = 0; n < 256; n++) {
+            int c = n;
+            for (int k = 0; k < 8; k++) {
+                c = (c & 1) != 0 ? (0x82F63B78 ^ (c >>> 1)) : (c >>> 1);
+            }
+            table[n] = c;
+        }
+        for (int slice = 1; slice < 8; slice++) {
+            int previousOffset = (slice - 1) * 256;
+            int offset = slice * 256;
+            for (int n = 0; n < 256; n++) {
+                int previous = table[previousOffset + n];
+                table[offset + n] = (previous >>> 8) ^ table[previous & 0xFF];
+            }
+        }
+        return table;
+    }
+
     static {
         Os.init();
     }
diff --git a/core/src/main/java/io/questdb/client/std/FilesFacade.java b/core/src/main/java/io/questdb/client/std/FilesFacade.java
index 921684a0..fd606c2c 100644
--- a/core/src/main/java/io/questdb/client/std/FilesFacade.java
+++ b/core/src/main/java/io/questdb/client/std/FilesFacade.java
@@ -89,6 +89,16 @@ default int fsyncDir(String dir) {
         return Files.fsyncDir(dir);
     }
 
+    /**
+     * Whether callers should use this facade's mmap path. The production facade
+     * returns {@code true}; wrapping fault facades retain positioned I/O unless
+     * they explicitly opt in, preserving their ability to inject short reads and
+     * writes.
+     */
+    default boolean isMmapAllowed() {
+        return this == INSTANCE;
+    }
+
     /**
      * Returns the current byte length of the file referenced by open descriptor
      * {@code fd}, or a negative value when the descriptor cannot be statted.
@@ -118,6 +128,10 @@ default int mlock(long addr, long len) {
         return Files.mlock(addr, len);
     }
 
+    /**
+     * Maps a file region. Kept on the facade so mmap failures can be injected
+     * without relying on platform-specific filesystem behavior.
+     */
     default long mmap(int fd, long len, long offset, int flags, int memoryTag) {
         return Files.mmap(fd, len, offset, flags, memoryTag);
     }
@@ -134,6 +148,7 @@ default int munlock(long addr, long len) {
         return Files.munlock(addr, len);
     }
 
+    /** Releases a region returned by {@link #mmap(int, long, long, int, int)}. */
     default void munmap(long address, long len, int memoryTag) {
         Files.munmap(address, len, memoryTag);
     }
diff --git a/core/src/test/java/io/questdb/client/test/SenderErrorTest.java b/core/src/test/java/io/questdb/client/test/SenderErrorTest.java
index 486b1679..057a4e60 100644
--- a/core/src/test/java/io/questdb/client/test/SenderErrorTest.java
+++ b/core/src/test/java/io/questdb/client/test/SenderErrorTest.java
@@ -40,13 +40,14 @@ public void testAllCategoriesEnumerable() {
         // Pin the public enum values — adding/removing requires a deliberate spec change
         // (and an update to wire-classification mapping in the I/O loop).
         SenderError.Category[] cats = SenderError.Category.values();
-        Assert.assertEquals(8, cats.length);
+        Assert.assertEquals(9, cats.length);
         Assert.assertEquals(SenderError.Category.SCHEMA_MISMATCH, SenderError.Category.valueOf("SCHEMA_MISMATCH"));
         Assert.assertEquals(SenderError.Category.PARSE_ERROR, SenderError.Category.valueOf("PARSE_ERROR"));
         Assert.assertEquals(SenderError.Category.INTERNAL_ERROR, SenderError.Category.valueOf("INTERNAL_ERROR"));
         Assert.assertEquals(SenderError.Category.SECURITY_ERROR, SenderError.Category.valueOf("SECURITY_ERROR"));
         Assert.assertEquals(SenderError.Category.WRITE_ERROR, SenderError.Category.valueOf("WRITE_ERROR"));
         Assert.assertEquals(SenderError.Category.NOT_WRITABLE, SenderError.Category.valueOf("NOT_WRITABLE"));
+        Assert.assertEquals(SenderError.Category.DICTIONARY_GAP, SenderError.Category.valueOf("DICTIONARY_GAP"));
         Assert.assertEquals(SenderError.Category.PROTOCOL_VIOLATION, SenderError.Category.valueOf("PROTOCOL_VIOLATION"));
         Assert.assertEquals(SenderError.Category.UNKNOWN, SenderError.Category.valueOf("UNKNOWN"));
     }
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/http/client/WebSocketClientTest.java b/core/src/test/java/io/questdb/client/test/cutlass/http/client/WebSocketClientTest.java
index a44c57f2..93a380f8 100644
--- a/core/src/test/java/io/questdb/client/test/cutlass/http/client/WebSocketClientTest.java
+++ b/core/src/test/java/io/questdb/client/test/cutlass/http/client/WebSocketClientTest.java
@@ -32,6 +32,7 @@
 import io.questdb.client.network.PlainSocketFactory;
 import io.questdb.client.network.Socket;
 import io.questdb.client.network.SocketReadinessWaiter;
+import io.questdb.client.std.MemoryTag;
 import io.questdb.client.std.Unsafe;
 import org.junit.Assert;
 import org.junit.Test;
@@ -312,6 +313,41 @@ public void onClose(int code, String reason) {
         });
     }
 
+    @Test
+    public void testSendBinaryAssemblesTwoPayloadSlicesIntoOneMaskedFrame() throws Exception {
+        assertMemoryLeak(() -> {
+            CapturingSocket socket = new CapturingSocket();
+            long first = Unsafe.malloc(3, MemoryTag.NATIVE_DEFAULT);
+            long second = Unsafe.malloc(4, MemoryTag.NATIVE_DEFAULT);
+            try (CapturingWebSocketClient client = new CapturingWebSocketClient(socket)) {
+                setUpgradedTrue(client);
+                for (int i = 0; i < 3; i++) {
+                    Unsafe.getUnsafe().putByte(first + i, (byte) (i + 1));
+                }
+                for (int i = 0; i < 4; i++) {
+                    Unsafe.getUnsafe().putByte(second + i, (byte) (i + 4));
+                }
+
+                client.sendBinary(first, 3, second, 4);
+
+                byte[] frame = socket.sent;
+                Assert.assertNotNull(frame);
+                Assert.assertEquals((byte) 0x82, frame[0]);
+                Assert.assertEquals((byte) (0x80 | 7), frame[1]);
+                byte[] payload = new byte[7];
+                for (int i = 0; i < payload.length; i++) {
+                    payload[i] = (byte) (frame[6 + i] ^ frame[2 + (i & 3)]);
+                }
+                Assert.assertArrayEquals(
+                        new byte[]{1, 2, 3, 4, 5, 6, 7},
+                        payload);
+            } finally {
+                Unsafe.free(first, 3, MemoryTag.NATIVE_DEFAULT);
+                Unsafe.free(second, 4, MemoryTag.NATIVE_DEFAULT);
+            }
+        });
+    }
+
     @Test
     public void testSendCloseFrameDoesNotClobberSendBuffer() throws Exception {
         assertMemoryLeak(() -> {
@@ -463,6 +499,34 @@ public boolean wantsTlsWrite() {
         }
     }
 
+    private static class CapturingSocket extends FakeSocket {
+        private byte[] sent;
+
+        @Override
+        public int send(long bufferPtr, int bufferLen) {
+            sent = new byte[bufferLen];
+            for (int i = 0; i < bufferLen; i++) {
+                sent[i] = Unsafe.getUnsafe().getByte(bufferPtr + i);
+            }
+            return bufferLen;
+        }
+    }
+
+    private static class CapturingWebSocketClient extends WebSocketClient {
+
+        CapturingWebSocketClient(CapturingSocket socket) {
+            super(DefaultHttpClientConfiguration.INSTANCE, (nf, log) -> socket);
+        }
+
+        @Override
+        protected void ioWait(int timeout, int op) {
+        }
+
+        @Override
+        protected void setupIoWait() {
+        }
+    }
+
     /**
      * Socket that serves a fixed byte sequence from recv() and reports
      * every send() as fully written (so close-frame sends succeed).
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/CloseSafetyNetTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/CloseSafetyNetTest.java
index 2a266212..639b7c8d 100644
--- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/CloseSafetyNetTest.java
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/CloseSafetyNetTest.java
@@ -30,6 +30,7 @@
 import io.questdb.client.SenderErrorHandler;
 import io.questdb.client.cutlass.line.LineSenderException;
 import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender;
+import io.questdb.client.cutlass.qwp.client.WebSocketResponse;
 import io.questdb.client.test.cutlass.qwp.websocket.TestWebSocketServer;
 import org.jetbrains.annotations.NotNull;
 import org.junit.Assert;
@@ -37,14 +38,18 @@
 import org.junit.Test;
 import org.junit.rules.TemporaryFolder;
 
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.nio.charset.StandardCharsets;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
 import java.util.concurrent.atomic.AtomicReference;
 
 /**
  * Pins both branches of {@code QwpWebSocketSender.close()}'s safety-net
- * rethrow, strictly — unlike the close assertions in
- * {@link InitialConnectAsyncTest}, which tolerate either outcome:
+ * rethrow using a deterministic server-side parse rejection:
  *