From 5ec096cc512d3714b1d11ba55e19937bd6df4589 Mon Sep 17 00:00:00 2001 From: kirillDevPro <113171057+kirillDevPro@users.noreply.github.com> Date: Thu, 3 Sep 2026 14:41:10 +0200 Subject: [PATCH 1/6] fix(analytics): stop the window flashing when a trade lands With Analytics open, every closed trade made the whole window blink: the numbers vanished behind a failure note and came back a second later. A trade landing is exactly when the report writer is committing, so the writer-driven catch-up read often returns Busy under SQLite contention. Five surfaces (summary, calendar, and the filters, time and coins tuner axes) published that Busy failure over settled data; only the strategy base already guarded against it. The bounded retry then restored the numbers, which is what turned an error into a flash. The open chart popup was also closed on every landed trade, and the measured core-column width cache was dropped on every published base. One rule now covers all twelve call sites: a catch-up result replaces what is on screen only when it carries data, or when it is a non-Busy failure, or when Busy has exhausted its retry allowance. The popup and the width cache survive a landed trade. The debounce gate is untouched. Contract tests pin the preserve rule and the hover asymmetry, and a civil-date rollover is caught explicitly. --- .../src/analytics/calendar/mod.rs | 23 ++- .../src/analytics/calendar/tests.rs | 45 ++++- crates/moon-ui-gpui/src/analytics/mod.rs | 116 +++++++---- crates/moon-ui-gpui/src/analytics/refresh.rs | 34 ++++ .../src/analytics/refresh/tests.rs | 56 +++++- .../src/analytics/tuner/coins/load.rs | 6 +- .../src/analytics/tuner/filter/mod.rs | 15 +- .../src/analytics/tuner/list/mod.rs | 42 +++- .../src/analytics/tuner/list/tests.rs | 65 +++++- .../moon-ui-gpui/src/analytics/tuner/mod.rs | 4 +- .../src/analytics/tuner/time/mod.rs | 16 +- crates/moon-ui-gpui/src/load_state.rs | 17 ++ crates/moon-ui-gpui/src/load_state/tests.rs | 60 +++++- .../tests/theme_contract/analytics.rs | 186 +++++++++++++++++- 14 files changed, 626 insertions(+), 59 deletions(-) diff --git a/crates/moon-ui-gpui/src/analytics/calendar/mod.rs b/crates/moon-ui-gpui/src/analytics/calendar/mod.rs index 522bc687..14db1cac 100644 --- a/crates/moon-ui-gpui/src/analytics/calendar/mod.rs +++ b/crates/moon-ui-gpui/src/analytics/calendar/mod.rs @@ -38,6 +38,10 @@ use moon_core::db::{ProfitScope, ProfitUnit, ReadResult}; /// previous: Previous-month aggregate UI load state. /// period_result: Current cells and optional comparison from one SQLite snapshot. /// has_previous: Whether the active mode requested a comparison period. +/// preserve_on_failure: Whether a same-scope catch-up failure should keep the visible days +/// and previous-period cells instead of publishing the classified error — decided by the +/// caller via `refresh::preserve_on_catch_up_failure` (transient contention with retry +/// allowance left; every other failure is still published). /// /// Returns: /// `true` when the shared period read failed. @@ -46,6 +50,7 @@ fn apply_calendar_results( previous: &mut LoadState>, period_result: ReadResult>, has_previous: bool, + preserve_on_failure: bool, ) -> bool { match period_result { Ok(ProfitScope::Comparable { unit, data: period }) => { @@ -67,8 +72,10 @@ fn apply_calendar_results( false } Err(error) => { - days.apply(Err(error.clone())); - previous.apply(if has_previous { Err(error) } else { Ok(None) }); + if !preserve_on_failure { + days.apply(Err(error.clone())); + previous.apply(if has_previous { Err(error) } else { Ok(None) }); + } true } } @@ -741,7 +748,10 @@ impl AnalyticsView { /// Start the Calendar query with optional post-commit metadata chaining. /// /// Args: - /// after_report: Whether to preserve report-style catch-up and retry semantics. + /// after_report: Whether to preserve report-style catch-up and retry semantics, including + /// keeping the visible cells over a same-scope failure that is transient contention + /// with retry allowance left (`refresh::preserve_on_catch_up_failure`) — every other + /// failure still publishes the classified error. /// show_overlay: Whether this refresh must block interaction with visible progress feedback. /// cx: GPUI context used to execute and publish the reads. fn reload_calendar_inner( @@ -803,18 +813,23 @@ impl AnalyticsView { .cloned(); let metadata_failed = data.cores.as_ref().is_some_and(|cores| cores.is_err()) || data.undated.is_err(); + // Peek at `data.period`'s error before it moves into the call below: only + // transient contention with retry allowance left may keep the visible cells. + let preserve_on_failure = !matches!(this.cal_days, ProfitLoadState::Loading) + && this.keep_on_busy(after_report, data.period.as_ref().err()); let read_failed = apply_calendar_results( &mut this.cal_days, &mut this.cal_prev, data.period, has_previous, + preserve_on_failure, ); if let Some(Ok(cores)) = data.cores { this.cores = cores; this.last_cores_at = Some(std::time::Instant::now()); this.core_refresh_needed = false; } - this.apply_undated_result(data.undated, true); + this.apply_undated_result(data.undated, after_report); this.cal_dirty = super::refresh::report_result_is_stale( report_req, this.current_report_generation(), diff --git a/crates/moon-ui-gpui/src/analytics/calendar/tests.rs b/crates/moon-ui-gpui/src/analytics/calendar/tests.rs index 297fa3c8..4fe6e47f 100644 --- a/crates/moon-ui-gpui/src/analytics/calendar/tests.rs +++ b/crates/moon-ui-gpui/src/analytics/calendar/tests.rs @@ -23,7 +23,7 @@ fn current_and_previous_failures_remain_visible() { let mut days = ProfitLoadState::>::default(); let mut previous = LoadState::>::default(); - let failed = apply_calendar_results(&mut days, &mut previous, Err(failure()), true); + let failed = apply_calendar_results(&mut days, &mut previous, Err(failure()), true, false); assert!(failed); assert!(matches!( @@ -42,6 +42,49 @@ fn current_and_previous_failures_remain_visible() { )); } +/// `analytics/calendar/mod.rs:apply_calendar_results` must leave both settled Calendar snapshots +/// intact for a report-driven failure. Returning `false` or applying that failure clears the cells, +/// so `cal_dirty` can stop the catch-up and the visible calendar stays stale or flashes blank. +#[test] +fn report_catch_up_failure_preserves_current_and_previous_calendar_snapshots() { + let mut days = ProfitLoadState::Ready { + unit: None, + data: Arc::new(vec![DayCell { + start: 123, + ..Default::default() + }]), + }; + let mut previous = LoadState::Ready(Arc::new(Some(Default::default()))); + let original_days = days.data().expect("settled Calendar cells").clone(); + let original_previous = previous + .data() + .expect("settled previous-period total") + .clone(); + let failure = ReadFail::Failed { + kind: FailKind::Busy, + msg: Arc::from("busy calendar"), + }; + + assert!( + apply_calendar_results(&mut days, &mut previous, Err(failure), true, true), + "a failed period read must keep Calendar dirty for the bounded retry" + ); + assert!( + Arc::ptr_eq( + &original_days, + days.data().expect("preserved Calendar cells") + ), + "the catch-up failure must retain the exact current-cell snapshot" + ); + assert!( + Arc::ptr_eq( + &original_previous, + previous.data().expect("preserved previous-period total") + ), + "the catch-up failure must retain the exact comparison snapshot" + ); +} + /// Replacing `calendar::hour_start`'s gap rejection with the shared picker clamp would map both /// Warsaw 02:00 and 03:00 to the same bucket and duplicate that hour's profit in the Day grid. #[test] diff --git a/crates/moon-ui-gpui/src/analytics/mod.rs b/crates/moon-ui-gpui/src/analytics/mod.rs index 1cd7f75a..9ad36108 100644 --- a/crates/moon-ui-gpui/src/analytics/mod.rs +++ b/crates/moon-ui-gpui/src/analytics/mod.rs @@ -9,6 +9,10 @@ //! SQLite query never runs on the UI thread). Direct scope edits reload immediately; stale //! tab/mode entry and committed report generations use the shared quiet-period and maximum-wait //! gate. Hidden surfaces remain marked stale until entry, and automatic scans never overlap. +//! +//! A writer-driven catch-up preserves a settled snapshot only for transient contention that a +//! bounded retry will correct. The final Busy failure still publishes, so stale data never looks +//! current after the retry budget is exhausted. /// The shared spawn+overlay envelope of every background DB read. mod bg; @@ -409,6 +413,9 @@ pub struct AnalyticsView { /// Period currently represented by `data` (summary/strategy list). Entering a tab /// with a different time window triggers a reload. data_period: Period, + /// Resolved `[from, to)` bounds `data` was computed for. Presets survive civil rollovers, so + /// the enum alone cannot tell when their re-resolved bounds make retained hovers stale. + data_range: (i64, i64), /// Whether `data` predates the latest committed report generation. data_dirty: bool, /// Cores from the replica (for the combo box) plus multi-selection (empty = all), using @@ -552,8 +559,8 @@ pub struct AnalyticsView { /// Content-measured preferred width of the strategy list's core column as /// `(font_scale_it_was_measured_under, width_in_base_px)` (`tuner::list::table::core_col_w`). /// Filled lazily on render; measuring lays out a glyph per character for every distinct core - /// name, too much to repay on an idle repaint. Invalidated two ways: cleared to `None` where - /// `strategy_data` is replaced (the names may have changed), and + /// name, too much to repay on an idle repaint. It is cleared only when a published base + /// changes the single-core names (`tuner::core_names_changed`), and /// recomputed when the stored font scale no longer matches the current one, so a Font-slider /// move OR a theme whose mode carries a different base mono size re-measures instead of /// scaling a width that assumed the old base. @@ -967,6 +974,7 @@ impl AnalyticsView { period: saved_period.unwrap_or(Period::CurMonth), strat_period: saved_strat_period.unwrap_or(Period::CurMonth), data_period: saved_period.unwrap_or(Period::CurMonth), + data_range: saved_period.unwrap_or(Period::CurMonth).range(bound_zone), data_dirty: false, cores: Vec::new(), last_cores_at: None, @@ -1218,6 +1226,27 @@ impl AnalyticsView { self.schedule_report_refresh(cx); } + /// Decide whether a writer-driven catch-up failure may keep a visible snapshot instead of + /// publishing it. + /// + /// Centralizing the predicate keeps every load surface aligned as their call sites evolve. + /// + /// Args: + /// after_report: Whether this is a writer-driven catch-up rather than a manual reload. + /// error: The classified failure this read completed with, if any. + /// + /// Returns: + /// `true` only when the failure is transient contention AND a retry will actually follow. + fn keep_on_busy(&self, after_report: bool, error: Option<&ReadFail>) -> bool { + error.is_some_and(|error| { + refresh::preserve_on_catch_up_failure( + after_report, + error, + self.report_busy_retries.has_allowance(), + ) + }) + } + /// Queue a visible catch-up behind any Analytics database work already in flight. /// /// Args: @@ -1621,8 +1650,8 @@ impl AnalyticsView { /// Reload the full Summary without resetting tuner drafts. /// /// Args: - /// after_report: Whether report-style catch-up may preserve the previous undated value - /// while exposing a classified read error. + /// after_report: Whether this writer-driven catch-up may keep a settled snapshot while a + /// transient Busy retry remains. /// show_overlay: Whether this refresh must block interaction with visible progress feedback. /// cx: GPUI context used to run and publish the shared background query. fn reload_summary(&mut self, after_report: bool, show_overlay: bool, cx: &mut Context) { @@ -1639,18 +1668,25 @@ impl AnalyticsView { if !after_report { self.data = ProfitLoadState::default(); } - // Drop every chart hover: the bars/columns under the cursor are about to be - // replaced (and on a single-day period the right card swaps its whole element - // tree), so they never fire `hovered = false` and a stale index would re-open a - // popup with no cursor on it — pointing at another period's bucket. - self.hover_daily_bucket = None; - self.hover_cum_bucket = None; + // Presets keep their enum across civil rollovers while their resolved bounds slide, so + // only the range can tell whether retained bucket hovers still name the same data. + let active_range = self.active_period().range(self.bound_zone()); + let range_moved = active_range != self.data_range; + if !after_report || range_moved { + // Bucket indices are time-ordered and survive a same-scope catch-up, but a moved + // range shifts their meaning without changing the preset enum. + self.hover_daily_bucket = None; + self.hover_cum_bucket = None; + } + // Kinds are profit-sorted on every read, so an existing index can name another kind even + // during a same-scope catch-up; a closed popup is safer than a silently wrong one. self.hover_kind = None; self.seq = self.seq.wrapping_add(1); let req = self.seq; let report_req = self.current_report_generation(); // Record the ACTIVE tab's time window that `data` is being computed for. self.data_period = self.active_period(); + self.data_range = active_range; let q = self.query(); let read_cores = self.core_metadata_due(cx); self.spawn_latest_db( @@ -1691,7 +1727,13 @@ impl AnalyticsView { this.last_cores_at = Some(std::time::Instant::now()); this.core_refresh_needed = false; } - this.data.apply(data); + // A snapshot survives only while a Busy retry can replace it; otherwise stale + // values would look current with no remaining correction path. + let preserve_snapshot = !matches!(this.data, ProfitLoadState::Loading) + && this.keep_on_busy(after_report, data_error.as_ref()); + if !preserve_snapshot { + this.data.apply(data); + } this.data_dirty = refresh::report_result_is_stale( report_req, this.current_report_generation(), @@ -1768,25 +1810,24 @@ impl AnalyticsView { this.last_cores_at = Some(std::time::Instant::now()); this.core_refresh_needed = false; } - // A same-scope automatic failure must leave the last ready snapshot visible while - // the retry gate settles. Manual scope changes have already retired the old data, - // so they publish the classified failure instead of showing stale values. - // - // "Keep what is visible" needs something visible to keep. An automatic catch-up - // can reach here over a NEVER-SETTLED state, and there the rule would leave the - // tuner reading "Loading" forever with nothing in flight — a pending state that - // was really a classified failure, which is the one thing this window must never - // show. - // - // The test is `Loading`, deliberately NOT "has scalar data": `Split` carries a - // real per-quote breakdown the toolbar renders, and `NotReady` / `Failed` are - // settled answers too. All three stay preserved exactly as before, so no path - // that existed before this change behaves differently. - let preserve_snapshot = - after_report && !matches!(this.strategy_data, ProfitLoadState::Loading); - if !preserve_snapshot || data_error.is_none() { + // Read `data` before `apply` moves it: the comparison needs both the currently + // shown group set and the one about to be published. + let core_names_changed = tuner::core_names_changed( + this.strategy_data.data().map(|d| d.strategies.as_slice()), + tuner::published_groups(&data), + ); + // Keep a settled same-scope snapshot only until a Busy retry replaces it. An + // initial `Loading` state must publish its failure rather than remain pending + // forever; `Split`, `NotReady`, and `Failed` are already settled snapshots. + let preserve_snapshot = !matches!(this.strategy_data, ProfitLoadState::Loading) + && this.keep_on_busy(after_report, data_error.as_ref()); + if !preserve_snapshot { this.strategy_data.apply(data); - this.strat_core_w = None; + // Measuring every core name is expensive, so invalidate only when its input + // text changed. + if core_names_changed { + this.strat_core_w = None; + } // Both caches describe the group set that was just replaced. The memo's key // also carries that set's address, but an address is only unique among LIVE // allocations: a failed load drops the old buffer and a later successful one @@ -1818,11 +1859,19 @@ impl AnalyticsView { ); } - /// Apply an undated-close result without erasing a same-scope value on automatic failure. + /// Apply an undated-close result without replacing a settled strip with a transient Busy + /// alert that the bounded retry will immediately remove. + /// + /// Every other failure still publishes: otherwise stale counts would look current without a + /// correction path. + /// + /// Args: + /// result: Current undated-close read outcome. + /// after_report: Whether this is a writer-driven catch-up rather than a manual reload. fn apply_undated_result( &mut self, result: moon_core::db::ReadResult, - preserve_previous: bool, + after_report: bool, ) { match result { Ok(undated) => { @@ -1830,9 +1879,10 @@ impl AnalyticsView { self.undated_error = None; } Err(error) => { - if !preserve_previous { - self.undated = None; + if self.keep_on_busy(after_report, Some(&error)) { + return; } + self.undated = None; self.undated_error = Some(error); } } diff --git a/crates/moon-ui-gpui/src/analytics/refresh.rs b/crates/moon-ui-gpui/src/analytics/refresh.rs index 06225cfc..52897aad 100644 --- a/crates/moon-ui-gpui/src/analytics/refresh.rs +++ b/crates/moon-ui-gpui/src/analytics/refresh.rs @@ -9,9 +9,16 @@ //! runs immediately: [`RefreshUrgency`] is what separates the two, and it is the difference //! between a tuner tab that opens now and one that opens a second from now. The `db_active` //! interlock is unaffected: nothing here ever overlaps work already in flight. +//! +//! [`preserve_on_catch_up_failure`] is the one rule every writer-driven completion handler +//! consults before publishing a failed read: only transient contention with retry allowance +//! left may keep a settled surface on screen instead of the classified error, because that is +//! the only kind of failure the bounded [`BusyRetryBudget`] retry will actually clear. use std::time::{Duration, Instant}; +use moon_core::db::{FailKind, ReadFail}; + use super::Tab; /// Quiet time that coalesces adjacent report commits into one analytical scan. @@ -58,6 +65,13 @@ impl BusyRetryBudget { true } + /// Whether a Busy failure can still be corrected automatically. + /// + /// This is read-only because only `claim` may spend an attempt. + pub(super) fn has_allowance(&self) -> bool { + self.attempts < MAX_BUSY_RETRIES + } + /// Close a retry episode after a read completes without SQLite contention. pub(super) fn resolve(&mut self) { self.attempts = 0; @@ -133,6 +147,26 @@ pub(super) fn report_result_is_stale( read_failed || started_generation != current_generation } +/// Decide whether a writer-driven catch-up may keep a settled snapshot for a failed read. +/// +/// Only a Busy failure with another retry may be hidden: any other result, including an exhausted +/// Busy budget, must publish so stale values never look current without a correction path. +/// +/// Args: +/// after_report: Whether this is a writer-driven catch-up rather than a manual reload. +/// error: The classified failure this read completed with. +/// retry_allowance: Whether another automatic Busy retry remains for this episode. +/// +/// Returns: +/// `true` only when the failure is transient contention AND a retry will actually follow. +pub(super) fn preserve_on_catch_up_failure( + after_report: bool, + error: &ReadFail, + retry_allowance: bool, +) -> bool { + after_report && error.kind() == Some(FailKind::Busy) && retry_allowance +} + /// Decide whether the compact Strategies base may continue into its visible axis. /// /// The axis is a child of the compound base refresh. Starting it after a partial base failure diff --git a/crates/moon-ui-gpui/src/analytics/refresh/tests.rs b/crates/moon-ui-gpui/src/analytics/refresh/tests.rs index 6067162f..3029e868 100644 --- a/crates/moon-ui-gpui/src/analytics/refresh/tests.rs +++ b/crates/moon-ui-gpui/src/analytics/refresh/tests.rs @@ -4,9 +4,63 @@ use std::time::{Duration, Instant}; use super::{ BusyRetryBudget, RefreshGate, RefreshPlan, RefreshUrgency, VisibleRefresh, core_metadata_wait, - report_result_is_stale, strategy_base_allows_axis, visible_refresh, + preserve_on_catch_up_failure, report_result_is_stale, strategy_base_allows_axis, + visible_refresh, }; use crate::analytics::Tab; +use moon_core::db::{FailKind, ReadFail}; +use std::sync::Arc; + +/// Construct a classified database failure without depending on display text. +fn failure(kind: FailKind) -> ReadFail { + ReadFail::Failed { + kind, + msg: Arc::from("test failure"), + } +} + +/// `analytics/refresh.rs:preserve_on_catch_up_failure` must preserve only a Busy failure that an +/// automatic retry can still erase. Broadening the predicate lets a permanent failure, NotReady, +/// or the final exhausted Busy failure leave stale Analytics numbers looking current forever. +#[test] +fn catch_up_preservation_requires_busy_and_an_available_retry() { + for (label, error, is_busy) in [ + ("Busy", failure(FailKind::Busy), true), + ("Corrupt", failure(FailKind::Corrupt), false), + ("Other", failure(FailKind::Other), false), + ("NotReady", ReadFail::NotReady, false), + ("IncomparableQuote", ReadFail::IncomparableQuote, false), + ("PeriodOutOfRange", ReadFail::PeriodOutOfRange, false), + ] { + for after_report in [false, true] { + for retry_allowance in [false, true] { + assert_eq!( + preserve_on_catch_up_failure(after_report, &error, retry_allowance), + after_report && is_busy && retry_allowance, + "{label}: after_report={after_report}, retry_allowance={retry_allowance}" + ); + } + } + } +} + +/// `analytics/refresh.rs:BusyRetryBudget::has_allowance` must read without spending a retry. +/// Letting this publication query increment attempts exhausts a transient Busy recovery budget +/// before `claim` schedules the retry, leaving stale data or a false error. +#[test] +fn checking_the_busy_retry_allowance_does_not_consume_an_attempt() { + let mut budget = BusyRetryBudget::default(); + + assert!(budget.has_allowance()); + assert!(budget.has_allowance()); + assert!(budget.claim()); + assert!(budget.has_allowance()); + assert!(budget.claim()); + assert!(budget.has_allowance()); + assert!(budget.claim()); + assert!(!budget.has_allowance()); + assert!(!budget.claim()); +} /// `analytics/refresh.rs:report_result_is_stale` must compare the start and completion /// generations; removing that comparison lets a scan started before a new trade clear the diff --git a/crates/moon-ui-gpui/src/analytics/tuner/coins/load.rs b/crates/moon-ui-gpui/src/analytics/tuner/coins/load.rs index e9a0fda6..c654b9e2 100644 --- a/crates/moon-ui-gpui/src/analytics/tuner/coins/load.rs +++ b/crates/moon-ui-gpui/src/analytics/tuner/coins/load.rs @@ -207,7 +207,8 @@ impl AnalyticsView { this.current_report_generation(), read_failed, ); - this.coins.stats.apply(stats); + let keep_stats = this.keep_on_busy(after_report, stats.as_ref().err()); + this.coins.stats.apply_or_keep(stats, keep_stats); // Only a confirmed strategies snapshot may replace the baseline. On a failed // read, preserving both sets is safer than replaying the draft against a // fabricated empty list and silently changing the next Save. @@ -216,7 +217,8 @@ impl AnalyticsView { // left to show; fall back before the table renders "no matches". this.coins.settle_filter(); if !edited { - this.coins.kpi.apply(kpi); + let keep_kpi = this.keep_on_busy(after_report, kpi.as_ref().err()); + this.coins.kpi.apply_or_keep(kpi, keep_kpi); this.coins.kpi_bl = bl_n; this.coins.kpi_wl = wl_n; } else { diff --git a/crates/moon-ui-gpui/src/analytics/tuner/filter/mod.rs b/crates/moon-ui-gpui/src/analytics/tuner/filter/mod.rs index 0a21a502..8f799219 100644 --- a/crates/moon-ui-gpui/src/analytics/tuner/filter/mod.rs +++ b/crates/moon-ui-gpui/src/analytics/tuner/filter/mod.rs @@ -219,7 +219,8 @@ impl AnalyticsView { if this.tuner.hist_seq == hist_req { this.tuner.hist_loading = false; hist_error = histogram.as_ref().err().cloned(); - this.tuner.hist.apply(histogram); + let keep_hist = this.keep_on_busy(after_report, hist_error.as_ref()); + this.tuner.hist.apply_or_keep(histogram, keep_hist); this.tuner.hist_dirty = super::super::refresh::report_result_is_stale( report_req, this.current_report_generation(), @@ -234,9 +235,12 @@ impl AnalyticsView { return; } let error = stats.as_ref().err().cloned(); - // A completed non-data result clears stale numbers because - // values under a changed period label must belong to it. - this.tuner.stats.apply(stats); + // A completed non-data result clears stale numbers, unless it is transient + // contention with retry allowance left (`keep_on_busy`) — values + // under a changed period label must belong to it, but a settled snapshot the + // bounded retry is about to replace anyway must not blink through it. + let keep_stats = this.keep_on_busy(after_report, error.as_ref()); + this.tuner.stats.apply_or_keep(stats, keep_stats); // `strategy_filters` is intentionally lossy and reports an unreadable row as // `found=false`. An automatic report refresh must not turn that ambiguity into // an empty Save baseline; explicit scope changes may clear the old strategy. @@ -312,7 +316,8 @@ impl AnalyticsView { return; } let error = stats.as_ref().err().cloned(); - this.tuner.stats.apply(stats); + let keep_stats = this.keep_on_busy(after_report, error.as_ref()); + this.tuner.stats.apply_or_keep(stats, keep_stats); this.tuner.apply_strategy_read(filters, after_report); this.tuner.dirty = super::super::refresh::report_result_is_stale( report_req, diff --git a/crates/moon-ui-gpui/src/analytics/tuner/list/mod.rs b/crates/moon-ui-gpui/src/analytics/tuner/list/mod.rs index 5f7f06f1..f014400e 100644 --- a/crates/moon-ui-gpui/src/analytics/tuner/list/mod.rs +++ b/crates/moon-ui-gpui/src/analytics/tuner/list/mod.rs @@ -33,7 +33,8 @@ use super::{ }; use crate::design; use crate::design::moon; -use moon_core::db::analytics::GroupStat; +use moon_core::db::ProfitScope; +use moon_core::db::analytics::{GroupStat, StrategyBase}; /// "Which strategies name coins in a list?" — the list filter of the strategy table. /// @@ -244,6 +245,45 @@ fn memo_is_fresh(cached: Option<&VisibleRows>, key: &VisibleKey) -> bool { cached.is_some_and(|c| &c.key == key) } +/// Whether a published group set changes the names measured for the core-column width. +/// +/// This matches `core_col_w`'s `cores_n <= 1` filter so the cache invalidates only when its +/// measured text changes; multi-core rows render an aggregate instead of a name. No new group +/// set means no replacement was published, while a missing old set must measure the first one. +pub(in crate::analytics) fn core_names_changed( + old: Option<&[GroupStat]>, + new: Option<&[GroupStat]>, +) -> bool { + let Some(new) = new else { + return false; + }; + let Some(old) = old else { + return true; + }; + fn names(rows: &[GroupStat]) -> std::collections::HashSet<&str> { + rows.iter() + .filter(|g| g.cores_n <= 1) + .map(|g| g.core.as_str()) + .collect() + } + names(old) != names(new) +} + +/// Borrow the strategy rows a just-completed base read published, when it published any. +/// +/// `Split` carries no comparable group set (only raw per-quote totals), and a failed read +/// carries none at all — both read as "nothing new to measure", not as "the set became empty". +pub(in crate::analytics) fn published_groups( + result: &moon_core::db::ReadResult>, +) -> Option<&[GroupStat]> { + match result { + Ok(ProfitScope::Comparable { data, .. }) | Ok(ProfitScope::Empty(data)) => { + Some(&data.strategies) + } + Ok(ProfitScope::Split(_)) | Err(_) => None, + } +} + impl AnalyticsView { /// Rebuild the row order unless the cache already holds the same inputs. /// diff --git a/crates/moon-ui-gpui/src/analytics/tuner/list/tests.rs b/crates/moon-ui-gpui/src/analytics/tuner/list/tests.rs index 0a8e34cb..3259b76f 100644 --- a/crates/moon-ui-gpui/src/analytics/tuner/list/tests.rs +++ b/crates/moon-ui-gpui/src/analytics/tuner/list/tests.rs @@ -6,8 +6,8 @@ use moon_core::db::analytics::GroupStat; use super::{ - SORT_NAME, StratListFilter, VisibleKey, VisibleRows, filter_sort_indices, memo_is_fresh, - restore_strat_sort, + SORT_NAME, StratListFilter, VisibleKey, VisibleRows, core_names_changed, filter_sort_indices, + memo_is_fresh, restore_strat_sort, }; /// A group with just the fields these tests filter and sort on. @@ -23,6 +23,67 @@ fn g(name: &str, kind: &str, alive: Option, bl: i64, profit: f64) -> GroupS } } +/// A group whose Core column either names one core or summarizes several. +fn core_group(core: &str, cores_n: i64) -> GroupStat { + let mut group = g(core, "K", Some(2), 0, 0.0); + group.core = core.to_string(); + group.cores_n = cores_n; + group +} + +/// `analytics/tuner/list/mod.rs:core_names_changed` must compare the set of names measured by +/// the Core column. Replacing its set comparison with row order would remeasure every glyph after +/// an otherwise identical report refresh, restoring the writer-path hitch this cache avoids. +#[test] +fn core_name_changes_ignore_row_order_and_multi_core_summaries() { + let old = vec![core_group("Alpha", 1), core_group("Beta", 1)]; + let reordered = vec![core_group("Beta", 1), core_group("Alpha", 1)]; + let with_summary_only = vec![ + core_group("Beta", 1), + core_group("Alpha", 1), + core_group("Many cores", 2), + ]; + let with_new_single_core = vec![ + core_group("Alpha", 1), + core_group("Beta", 1), + core_group("Gamma", 1), + ]; + + assert!( + !core_names_changed(Some(&old), Some(&reordered)), + "the same measured names in another row order retain the width cache" + ); + assert!( + !core_names_changed(Some(&old), Some(&with_summary_only)), + "a multi-core summary is not a separately measured Core-column name" + ); + assert!( + core_names_changed(Some(&old), Some(&with_new_single_core)), + "a newly measured single-core name needs a fresh width" + ); + assert!( + core_names_changed(Some(&old), Some(&[core_group("Alpha", 1)])), + "a disappearing measured name also changes the rendered column" + ); +} + +/// `analytics/tuner/list/mod.rs:core_names_changed` must treat an unpublished replacement as no +/// change. Treating `new == None` as changed clears the settled width cache for every failed or +/// split read, bringing the visible writer-path hitch back without new names to measure. +#[test] +fn unpublished_core_names_do_not_invalidate_a_settled_width() { + let old = vec![core_group("Alpha", 1)]; + + assert!( + core_names_changed(None, Some(&old)), + "the first published Core-column set needs an initial measurement" + ); + assert!( + !core_names_changed(Some(&old), None), + "a failed or split replacement publishes no names and keeps the old width" + ); +} + /// The default key over group set `base`: no search, no kind, no list filter, active-only off, /// unsorted. fn key_over(base: usize) -> VisibleKey { diff --git a/crates/moon-ui-gpui/src/analytics/tuner/mod.rs b/crates/moon-ui-gpui/src/analytics/tuner/mod.rs index ab8eeb47..c7b06a21 100644 --- a/crates/moon-ui-gpui/src/analytics/tuner/mod.rs +++ b/crates/moon-ui-gpui/src/analytics/tuner/mod.rs @@ -47,7 +47,9 @@ mod time; pub(super) use coins::picker::CoinListsState; pub(super) use coins::state::CoinsState; pub(super) use filter::state::TunerState; -pub(super) use list::{StratListFilter, VisibleRows, restore_strat_sort}; +pub(super) use list::{ + StratListFilter, VisibleRows, core_names_changed, published_groups, restore_strat_sort, +}; pub(super) use time::state::TimeTunerState; // Column descriptors of the comparison tables — re-exported so the submodules (`list`) take diff --git a/crates/moon-ui-gpui/src/analytics/tuner/time/mod.rs b/crates/moon-ui-gpui/src/analytics/tuner/time/mod.rs index 1ce0a742..a42a01a9 100644 --- a/crates/moon-ui-gpui/src/analytics/tuner/time/mod.rs +++ b/crates/moon-ui-gpui/src/analytics/tuner/time/mod.rs @@ -150,10 +150,18 @@ impl AnalyticsView { profiles.is_err() || stats.is_err() || slider.is_err() || current.is_none(), ); // The profile, slider colors, and KPI are independent load surfaces, so each - // retains its own classified error instead of collapsing it to "no data". - this.time_tuner.profiles.apply(profiles); - this.time_tuner.slider.apply(slider); - this.time_tuner.stats.apply(stats); + // retains its own classified error instead of collapsing it to "no data" — unless + // that error is transient contention with retry allowance left (`keep_on_busy`), + // in which case its own settled snapshot survives the catch-up instead of + // blinking through the failure. + let keep_profiles = this.keep_on_busy(after_report, profiles.as_ref().err()); + let keep_slider = this.keep_on_busy(after_report, slider.as_ref().err()); + let keep_stats = this.keep_on_busy(after_report, stats.as_ref().err()); + this.time_tuner + .profiles + .apply_or_keep(profiles, keep_profiles); + this.time_tuner.slider.apply_or_keep(slider, keep_slider); + this.time_tuner.stats.apply_or_keep(stats, keep_stats); this.time_tuner.apply_current_read(current, after_report); if after_report { this.settle_report_refresh_retry(retry.as_ref(), cx); diff --git a/crates/moon-ui-gpui/src/load_state.rs b/crates/moon-ui-gpui/src/load_state.rs index 185ec6d8..670644fa 100644 --- a/crates/moon-ui-gpui/src/load_state.rs +++ b/crates/moon-ui-gpui/src/load_state.rs @@ -74,6 +74,23 @@ impl LoadState { }; } + /// Keep revalidation stale data only for a caller-approved transient failure. + /// + /// `NotReady` never qualifies: it is a completed absence, so preserving data would promise a + /// recovery that no retry can provide. + pub(crate) fn apply_or_keep(&mut self, r: Result, keep_on_failure: bool) { + let keeps = keep_on_failure + && matches!(r, Err(ref e) if !matches!(e, ReadFail::NotReady)) + && matches!(self, LoadState::Loading { stale: Some(_) }); + if keeps { + if let LoadState::Loading { stale: Some(v) } = self { + *self = LoadState::Ready(Arc::clone(v)); + } + return; + } + self.apply(r); + } + /// What this surface should render: either the data, or the placeholder /// standing in for it. `empty` decides whether loaded data counts as empty. /// diff --git a/crates/moon-ui-gpui/src/load_state/tests.rs b/crates/moon-ui-gpui/src/load_state/tests.rs index b67272ca..6ace1298 100644 --- a/crates/moon-ui-gpui/src/load_state/tests.rs +++ b/crates/moon-ui-gpui/src/load_state/tests.rs @@ -1,7 +1,65 @@ //! Classified load-state presentation regression tests. use super::{LoadState, Note}; -use moon_core::db::ReadFail; +use moon_core::db::{FailKind, ReadFail}; +use std::sync::Arc; + +/// Construct a classified read failure without coupling tests to a rendered error message. +fn failure(kind: FailKind) -> ReadFail { + ReadFail::Failed { + kind, + msg: Arc::from("test failure"), + } +} + +/// `load_state.rs:LoadState::apply_or_keep` must retain only a settled stale snapshot when a +/// report catch-up gets Busy. Keeping a NotReady result would show numbers from an unavailable +/// replica, while dropping a valid stale snapshot makes the Analytics surface flash to an error. +#[test] +fn catch_up_failure_keeps_only_a_real_stale_snapshot() { + let mut stale = LoadState::Ready(Arc::new(vec![42])); + stale.begin(); + stale.apply_or_keep(Err(failure(FailKind::Busy)), true); + assert!( + matches!(&stale, LoadState::Ready(_)), + "a Busy revalidation restores the settled stale snapshot" + ); + + let mut no_stale = LoadState::>::default(); + no_stale.apply_or_keep(Err(failure(FailKind::Busy)), true); + assert!( + matches!(&no_stale, LoadState::Failed(_)), + "an initial read failure has no snapshot to preserve" + ); + + let mut unavailable = LoadState::Ready(Arc::new(vec![7])); + unavailable.begin(); + unavailable.apply_or_keep(Err(ReadFail::NotReady), true); + assert!( + matches!(&unavailable, LoadState::NotReady), + "NotReady is a completed unavailable-replica answer, never stale data" + ); +} + +/// `load_state.rs:LoadState::apply_or_keep` must publish a failure when preservation is disabled. +/// Keeping the old snapshot after a user scope change would put the prior scope's numbers under +/// the new label, which is worse than a visible classified read failure. +#[test] +fn scope_changes_do_not_keep_the_previous_snapshot_after_a_failure() { + let mut state = LoadState::Ready(Arc::new(vec![42])); + state.begin(); + state.apply_or_keep(Err(failure(FailKind::Busy)), false); + assert!( + matches!(&state, LoadState::Failed(_)), + "a non-preserving scope change must surface the completed failure" + ); + + state.apply_or_keep(Ok(vec![9]), true); + assert!( + matches!(&state, LoadState::Ready(_)), + "a successful replacement still settles normally" + ); +} /// Incomparable quote scope is guidance, not a reports-database failure. /// diff --git a/crates/moon-ui-gpui/tests/theme_contract/analytics.rs b/crates/moon-ui-gpui/tests/theme_contract/analytics.rs index 6764eab1..3638134b 100644 --- a/crates/moon-ui-gpui/tests/theme_contract/analytics.rs +++ b/crates/moon-ui-gpui/tests/theme_contract/analytics.rs @@ -1425,22 +1425,200 @@ fn automatic_strategy_refresh_keeps_the_visible_snapshot() { manual.contains("self.reload_strategy_base(false, true, true, cx)"), "manual scope refresh must retire values from the previous scope" ); - let preserve_snapshot = "let preserve_snapshot =\n after_report && !matches!(this.strategy_data, ProfitLoadState::Loading);"; + let preserve_snapshot = "let preserve_snapshot = !matches!(this.strategy_data, ProfitLoadState::Loading)\n && this.keep_on_busy(after_report, data_error.as_ref());"; assert!( reload.contains(preserve_snapshot), "only a settled strategy snapshot may survive an automatic report refresh" ); let automatic_result = chain_between( reload, - "if !preserve_snapshot || data_error.is_none() {", + "let preserve_snapshot =", "this.strategy_dirty = refresh::report_result_is_stale(", "automatic strategy result publication", ); assert!( - automatic_result.contains("this.strategy_data.apply(data);") + automatic_result.contains("this.keep_on_busy(after_report, data_error.as_ref())") + && !automatic_result.contains("if !preserve_snapshot || data_error.is_none() {") + && automatic_result.contains("this.strategy_data.apply(data)") && automatic_result.contains("this.strat_core_w = None;") && automatic_result.contains("this.strat_visible = None;"), - "an automatic read failure must preserve the complete visible strategy snapshot" + "only a retryable Busy failure may preserve the complete visible strategy snapshot" + ); +} + +/// `analytics/mod.rs:reload_summary` must retain only time-ordered chart hovers across a catch-up. +/// Moving a time-bucket clear back outside the scope-reset branch closes a valid popup on every +/// landed trade, while retaining profit-ordered `hover_kind` can show another kind's data. +#[test] +fn report_catch_up_keeps_time_bucket_hovers_but_clears_profit_ordered_kind_hover() { + let analytics = read_src("analytics/mod.rs"); + let reload = braced_body(&analytics, "fn reload_summary("); + let before_request = chain_between( + reload, + "fn reload_summary(", + "self.seq = self.seq.wrapping_add(1);", + "Summary reset path", + ); + assert!( + before_request.contains("if !after_report || range_moved {"), + "a resolved period rollover must clear time-bucket hovers even for an automatic catch-up" + ); + let scope_reset = code_only(braced_body( + before_request, + "if !after_report || range_moved", + )); + + for assignment in [ + "self.hover_daily_bucket = None;", + "self.hover_cum_bucket = None;", + ] { + assert!( + scope_reset.contains(assignment), + "{assignment} must belong only to the manual scope-reset branch" + ); + assert_eq!( + before_request.matches(assignment).count(), + 1, + "{assignment} must not also clear a report-driven catch-up" + ); + } + assert!( + !scope_reset.contains("self.hover_kind = None;"), + "kind hovers are ordered by profit and must clear during a same-scope catch-up" + ); + assert_eq!( + before_request.matches("self.hover_kind = None;").count(), + 1, + "kind hover must have one unconditional clear before the replacement request" + ); +} + +/// `analytics/mod.rs:reload_strategy_base` must invalidate the Core-column width only when the +/// measured name set changed. Unwrapping that assignment makes every published writer result +/// remeasure glyphs and visibly hitches the Strategies list despite unchanged core names. +#[test] +fn strategy_base_remeasures_core_width_only_for_changed_names() { + let analytics = read_src("analytics/mod.rs"); + let reload = braced_body(&analytics, "fn reload_strategy_base("); + let publication = chain_between( + reload, + "let preserve_snapshot =", + "this.strategy_dirty = refresh::report_result_is_stale(", + "strategy base publication", + ); + let name_change = code_only(braced_body(publication, "if core_names_changed")); + + assert_eq!( + publication.matches("this.strat_core_w = None;").count(), + 1, + "the width cache must have one invalidation site in result publication" + ); + assert!( + name_change.contains("this.strat_core_w = None;"), + "the sole width-cache invalidation must be guarded by changed measured names" + ); +} + +/// `analytics/mod.rs:reload_summary` must preserve a settled snapshot only across a retryable Busy +/// catch-up failure. Broadening that publication leaves stale Summary numbers current-looking after +/// NotReady, corruption, or an exhausted Busy budget with no retry left to correct them. +#[test] +fn summary_catch_up_preserves_only_a_retryable_busy_snapshot() { + let analytics = read_src("analytics/mod.rs"); + let reload = braced_body(&analytics, "fn reload_summary("); + let decision = code_only(chain_between( + reload, + "let preserve_snapshot =", + "this.data_dirty = refresh::report_result_is_stale(", + "Summary result publication", + )); + + assert!( + reload.contains("let preserve_snapshot ="), + "Summary completion must distinguish automatic settled snapshots from initial loading" + ); + assert!( + decision.contains("this.keep_on_busy(after_report, data_error.as_ref())") + && !decision.contains("if !preserve_snapshot || data_error.is_none() {") + && decision.contains("this.data.apply(data);"), + "only a retryable Busy failure may retain the Summary snapshot" + ); +} + +/// The report-generation observer chain must not repaint Analytics before a result lands. +/// Adding `cx.notify()` to bookkeeping or scheduling turns each report generation into a whole +/// window rebuild rather than the one repaint owed by each completed visible result. +#[test] +fn report_generation_bookkeeping_has_no_repaint_and_each_completion_has_one() { + let analytics = read_src("analytics/mod.rs"); + for signature in [ + "fn observe_report_generation(", + "fn mark_report_data_stale(", + "fn schedule_report_refresh(", + "fn refresh_visible_report_data(", + ] { + let body = code_only(braced_body(&analytics, signature)); + assert!( + !body.contains("cx.notify()"), + "{signature} must only mark, schedule, or start work; completion owns repaint" + ); + } + + for (rel, signature, completion) in [ + ( + "analytics/mod.rs", + "fn reload_summary(", + "move |this, result, cx|", + ), + ( + "analytics/mod.rs", + "fn reload_strategy_base(", + "move |this, result, cx|", + ), + ( + "analytics/calendar/mod.rs", + "fn reload_calendar_inner(", + "move |this, data, cx|", + ), + ] { + let source = if rel == "analytics/mod.rs" { + analytics.clone() + } else { + read_src(rel) + }; + let reload = braced_body(&source, signature); + let callback = code_only(braced_body(reload, completion)); + assert_eq!( + reload.matches("cx.notify();").count(), + 1, + "{rel}: {signature} must repaint exactly once per landed result" + ); + assert_eq!( + callback.matches("cx.notify();").count(), + 1, + "{rel}: {signature} must put its one repaint in the database completion" + ); + } +} + +/// `analytics/mod.rs:apply_undated_result` must call `keep_on_busy` before it publishes an error. +/// Dropping that gate flashes a retryable Busy catch-up, while broadening it hides an exhausted +/// Busy failure and leaves the retained undated count falsely current forever. +#[test] +fn undated_busy_gate_runs_before_the_error_is_published() { + let analytics = read_src("analytics/mod.rs"); + let apply = braced_body(&analytics, "fn apply_undated_result("); + let failure_arm = code_only(braced_body(apply, "Err(error) =>")); + let before_error = chain_between( + &failure_arm, + "Err(error) =>", + "self.undated_error = Some(error);", + "undated failure publication", + ); + + assert!( + before_error.contains("self.keep_on_busy(after_report, Some(&error))"), + "the retryable-Busy gate must run before the undated error can replace a retained count" ); } From 0912aaa97513ba68ddcab31fa04615ade4737482 Mon Sep 17 00:00:00 2001 From: kirillDevPro <113171057+kirillDevPro@users.noreply.github.com> Date: Thu, 3 Sep 2026 18:23:07 +0200 Subject: [PATCH 2/6] feat(trade-replay): draw a tick chart in the trade window where the venue publishes trades The trade-history window opened from the Report always replayed one-minute candles, even on venues whose public REST publishes time-ranged trades, so the detail view was coarser than the main chart's tick view. Add a trade-replay route beside the kline route: fetch public trades (Binance spot/USD-M/COIN-M, Gate spot/futures, Bitget spot/mix, OKX) in a two-stage worker, sort the pages globally before aggregation, claim the per-host send permit for the tick stage and release it on every exit path, abandon a partial page rather than shipping a hole as a whole window, and let the window accept a tick upgrade over an already-drawn candle replay. Bybit and Hyperliquid publish no such endpoint, so they keep candles. --- .../moon-core/src/market/trade_replay/mod.rs | 85 ++- .../src/market/trade_replay/rest/binance.rs | 153 ++++- .../src/market/trade_replay/rest/bitget.rs | 167 ++++- .../market/trade_replay/rest/bitget/tests.rs | 36 + .../src/market/trade_replay/rest/gateio.rs | 245 ++++++- .../src/market/trade_replay/rest/mod.rs | 97 ++- .../src/market/trade_replay/rest/okx.rs | 155 ++++- .../src/market/trade_replay/tests.rs | 84 +++ .../src/market/trade_replay/venue_caps.rs | 206 ++++++ .../src/market/trade_replay/worker.rs | 641 +++++++++++++++++- crates/moon-ui-gpui/src/trade_window/mod.rs | 170 ++++- .../moon-ui-gpui/src/trade_window/window.rs | 26 +- 12 files changed, 1998 insertions(+), 67 deletions(-) diff --git a/crates/moon-core/src/market/trade_replay/mod.rs b/crates/moon-core/src/market/trade_replay/mod.rs index a2bebb2a..e2ef2057 100644 --- a/crates/moon-core/src/market/trade_replay/mod.rs +++ b/crates/moon-core/src/market/trade_replay/mod.rs @@ -291,6 +291,48 @@ pub fn pages(window: ReplayWindow, bar_ms: i64, max_rows: usize) -> Vec<(i64, i6 out } +/// Split one window into requests no larger than a trade route's documented query span. +/// +/// The frozen shape [`super::worker`] and every `rest::::fetch_trades` build against: +/// `None` answers one slice covering the whole window; `Some(span)` tiles the window into +/// requests of at most `span` ms with no gap and no overlap, the same tiling discipline [`pages`] +/// uses for the kline pager, just keyed on a request's time SPAN rather than its row count — +/// several trade routes ([`super::venue_caps::TradeRoute::max_query_ms`]) cap a request's window +/// rather than its row count. +/// +/// A non-positive span returns no slices: unlike [`pages`], which derives its step from a +/// `bar_ms * max_rows` product that route constants keep positive, a caller here supplies the +/// span directly. [`super::venue_caps::TradeRoute::max_query_ms`] represents an unbounded route +/// as `None`, so every `Some` value remains a finite vendor-imposed request window. +/// +/// Args: +/// window: The window to cover. +/// max_span_ms: Widest span one request may cover, or `None` when the route documents no +/// cap. +/// +/// Returns: +/// Inclusive `(from_ms, to_ms)` pairs in ascending order; empty when `max_span_ms` is +/// `Some(n)` with `n <= 0`, which no route reports. +pub fn time_slices(window: ReplayWindow, max_span_ms: Option) -> Vec<(i64, i64)> { + let Some(span) = max_span_ms else { + return vec![(window.from_ms, window.to_ms)]; + }; + if span <= 0 { + return Vec::new(); + } + let mut out = Vec::new(); + let mut cursor = window.from_ms; + while cursor <= window.to_ms { + let end = cursor.saturating_add(span - 1).min(window.to_ms); + out.push((cursor, end)); + if end == window.to_ms { + break; + } + cursor = end + 1; + } + out +} + /// Whether cached rows already cover a window densely enough to skip the network. /// /// COVERAGE, not presence, is the question. A partial prefix is exactly what a previously @@ -370,6 +412,30 @@ pub fn replay_revision(identity: u64, tf_ms: i64, from_bucket: i64, to_bucket: i hash.max(1) } +/// Per-source salt for [`replay_revision`], so a tick series and its sibling candle series of the +/// same identity, timeframe and window never share a revision. +/// +/// The bug this exists to prevent: [`TradeReplaySeries::read_into`] derives `revision` from +/// `(identity, tf_ms, from_bucket, to_bucket)`, and a tick upgrade shares every one of those four +/// with the kline series it replaces — same `identity` ([`super::worker`] never changes it +/// between the two outcomes), same window, same `tf_ms == 60_000`. Unsalted, the upgrade's +/// revision would equal the one the pane already shipped, `read_into`'s `candles_changed` would +/// stay `false`, and the pane would keep drawing exchange klines forever under the new tick +/// points. +/// +/// Args: +/// source: Which kind of series is being read. +/// +/// Returns: +/// `0` for [`TradeReplaySource::Klines1m`], so every existing revision stays bit-identical to +/// today's; a fixed non-zero constant for [`TradeReplaySource::Ticks`]. +pub fn tick_identity_salt(source: TradeReplaySource) -> u64 { + match source { + TradeReplaySource::Klines1m => 0, + TradeReplaySource::Ticks => 0x9E37_79B9_7F4A_7C15, + } +} + /// One trade's frozen market history, ready to be drawn. #[derive(Clone, Debug)] pub struct TradeReplaySeries { @@ -381,9 +447,14 @@ pub struct TradeReplaySeries { pub window: ReplayWindow, /// Timeframe of [`Self::candles`] in milliseconds; one minute for every current route. pub tf_ms: i64, - /// Bars in ascending open time; empty when [`Self::source`] is [`TradeReplaySource::Ticks`]. + /// Bars in ascending open time. A [`TradeReplaySource::Klines1m`] series carries these alone; + /// a [`TradeReplaySource::Ticks`] series carries these TOO — its own aggregated candles, + /// computed from [`Self::ticks`] via [`crate::market::candles::aggregate_trades`] — alongside + /// the points, never instead of them. pub candles: Vec, - /// Trade points in ascending time; empty when the source is bars. + /// Trade points in ascending time. Empty when [`Self::source`] is + /// [`TradeReplaySource::Klines1m`]; carried alongside [`Self::candles`] for + /// [`TradeReplaySource::Ticks`], never in place of them. pub ticks: Vec, /// Stable discriminator feeding [`replay_revision`], so two open windows never collide. pub identity: u64, @@ -459,8 +530,13 @@ impl TradeReplaySeries { let to_ms = ((epoch_ms + f64::from(to_rel_ms.max(from_rel_ms))).round() as i64) .max(from_ms.saturating_add(1)); let tf_ms = candle_params.map_or(self.tf_ms, |p| p.tf_ms).max(1); + // Salted by source: a tick series and its sibling candle series otherwise share + // `(identity, tf_ms, from_bucket, to_bucket)` bit-for-bit (§4 of the tick-replay plan), + // so the tick upgrade's revision would equal the one the pane already shipped and + // `candles_changed` below would stay false forever. See `tick_identity_salt`. + let salted_identity = self.identity ^ tick_identity_salt(self.source); let revision = replay_revision( - self.identity, + salted_identity, tf_ms, from_ms.div_euclid(tf_ms), to_ms.div_euclid(tf_ms), @@ -469,7 +545,8 @@ impl TradeReplaySeries { read.candles_revision = revision; // Points first: they are re-emitted whole on every read, because a frozen series has no - // live edge to drain incrementally and the whole window is a few hundred rows at most. + // live edge to drain incrementally and the whole window is bounded — a few hundred rows + // for a candle-only series, or up to `worker::TICK_BUDGET` for a tick one. out.ticks.extend( self.ticks .iter() diff --git a/crates/moon-core/src/market/trade_replay/rest/binance.rs b/crates/moon-core/src/market/trade_replay/rest/binance.rs index eecaf75e..3d053114 100644 --- a/crates/moon-core/src/market/trade_replay/rest/binance.rs +++ b/crates/moon-core/src/market/trade_replay/rest/binance.rs @@ -5,9 +5,10 @@ use serde_json::Value; -use super::{FetchError, cell_f32}; +use super::{FetchError, TradeCursor, TradePage, cell_f32}; +use crate::feed::types::{Side, Tick}; use crate::market::candles::{ChartCandle, estimate_quote_volume}; -use crate::market::trade_replay::venue_caps::KlineRoute; +use crate::market::trade_replay::venue_caps::{KlineRoute, TradeRoute}; /// Positional index of the cell holding QUOTE-asset turnover on a spot or USD-M row. /// @@ -149,3 +150,151 @@ fn parse_row(row: &Value, quote_source: QuoteSource) -> Option { quote_volume, }) } + +/// Fetch one page of public aggregate trades and return the decoded body. +/// +/// # Continuation is by id, and the FIRST page is the only one carrying a time range +/// +/// The first page of a slice is asked with `startTime`/`endTime`. Every later page instead +/// carries `fromId` ALONE, Binance's own forward cursor — an aggregate-trade id is unambiguous +/// where a millisecond timestamp is not, since more than one trade can share a millisecond — and +/// `startTime`/`endTime` are dropped rather than kept alongside it. That is the vendor's own +/// documented contract, not a preference: Binance's `aggTrades` page states "Sending both +/// startTime/endTime and fromId might cause response timeout, please send either fromId or +/// startTime/endTime." A dense slice needing a second page would otherwise take a documented +/// timeout-prone path on exactly the busy markets where ticks matter most. The client-side clip +/// against `to_ms` in [`parse_agg_trades`] is what bounds a continuation page instead. +/// +/// **Resolves the earlier open assumption** ("nothing says `endTime` is rejected alongside +/// `fromId`"): the vendor's own page does warn against the combination, so the assumption is +/// refuted by the docs, not merely tidied away. +/// +/// Args: +/// agent: Shared client. +/// route: One of the three Binance trade routes. +/// market: Exchange-native market name. +/// from_ms: Left edge of this slice, inclusive; used only on the first page. +/// to_ms: Right edge of this slice, inclusive; used only on the first page. +/// max_rows: Row cap for this request. +/// cursor: Continuation from a previous page, or `None` for the first page. +/// +/// Returns: +/// The decoded response, or a classified failure. +pub(super) fn fetch_trades( + agent: &ureq::Agent, + route: TradeRoute, + market: &str, + from_ms: i64, + to_ms: i64, + max_rows: usize, + cursor: Option, +) -> Result { + let request = agent + .get(route.url()) + .query("symbol", market) + .query("limit", max_rows.to_string()); + let request = match cursor { + Some(TradeCursor::FromId(id)) => request.query("fromId", id.to_string()), + None => request + .query("startTime", from_ms.to_string()) + .query("endTime", to_ms.to_string()), + Some(_) => { + debug_assert!(false, "binance trade routes hand back only FromId"); + request + .query("startTime", from_ms.to_string()) + .query("endTime", to_ms.to_string()) + } + }; + let response = request + .call() + .map_err(|error| FetchError::Transient(error.to_string()))?; + super::decode_and_classify(response, "binance", classify) +} + +/// Parse a Binance aggregate-trade array into a page of ticks. +/// +/// Args: +/// body: Decoded response. +/// to_ms: Right edge of the slice this page belongs to, so completeness is judged from the +/// last row's own timestamp rather than assumed from the row count alone. +/// max_rows: Row cap that was sent, so a FULL page can be told from a short, final one. +/// +/// Returns: +/// The page, or a failure when the envelope is not an array. +pub(super) fn parse_agg_trades( + body: &Value, + to_ms: i64, + max_rows: usize, +) -> Result { + let rows = body + .as_array() + .ok_or_else(|| FetchError::Transient("binance: response is not an array".to_string()))?; + let ticks: Vec = rows.iter().filter_map(parse_trade_row).collect(); + // A page holding a malformed row alongside valid ones can still finish pagination with a + // non-empty tick vector that is silently missing rows — dropping one bad row in a thousand is + // fine for candles, where a missing bar is visible, but not here, where a hole must send the + // window to candles instead of drawing a partial tape as if it were whole. + if ticks.len() < rows.len() { + return Err(FetchError::Transient(format!( + "binance: page held {} unparseable row(s) of {} (parsed {})", + rows.len() - ticks.len(), + rows.len(), + ticks.len() + ))); + } + let last_id = rows.last().and_then(|r| r.get("a")).and_then(Value::as_u64); + let last_time_ms = rows.last().and_then(|r| r.get("T")).and_then(Value::as_i64); + let full = rows.len() >= max_rows; + let covered = last_time_ms.is_some_and(|t| t >= to_ms); + let next = match (full, covered, last_id) { + (true, false, Some(id)) => Some(TradeCursor::FromId(id + 1)), + // The page is full and the window is not yet covered, but the last row's own `a` (trade + // id) did not parse: the cursor to continue from is unknowable. Treating this as + // completion would silently ship a truncated window as a whole one — every field name + // here is inferred with no fixture behind it, so a wrong name yields exactly this shape + // on every page. + (true, false, None) => { + return Err(FetchError::Transient( + "binance: full page, window not covered, but the last row's `a` did not parse" + .to_string(), + )); + } + _ => None, + }; + Ok(TradePage { ticks, next }) +} + +/// Parse one Binance aggregate-trade row. +/// +/// **COIN-M unit note**: on a `BinanceCoinMAggTrades` row, `q` is a CONTRACT count, not a +/// base-currency amount — `aggTrades` exposes no `baseQty` alternative for COIN-M, the same fact +/// [`COIN_M_BASE_VOLUME_CELL`] already states for klines. `Tick::qty` therefore holds contracts +/// rather than base currency for that one route; no base-asset amount is available from this +/// endpoint. The chart's volume bars stay shape-correct regardless: the chart normalises the +/// visible window against its own maximum `qty`, and a per-instrument contract multiplier is a +/// constant that cancels out of that normalisation — only an ABSOLUTE volume figure would be +/// wrong, and a tick series' aggregated candles never reach the shared SQLite kline cache where +/// one could be read. See `venue_caps.rs`'s trade-route table for the same note. +/// +/// Args: +/// row: One element of the response array. +/// +/// Returns: +/// The tick, or `None` when the row is malformed. +fn parse_trade_row(row: &Value) -> Option { + let price = cell_f32(row.get("p")?)?; + let qty = cell_f32(row.get("q")?)?; + let time_ms = row.get("T")?.as_i64()? as f64; + // `m == true`: the BUYER was the maker, meaning the TAKER — the side this tape reports — SOLD. + // Time is read from `T`, never from the aggregate id `a`. + let side = match row.get("m")?.as_bool()? { + true => Side::Sell, + false => Side::Buy, + }; + Some(Tick { + time_ms, + price, + qty, + side, + }) +} diff --git a/crates/moon-core/src/market/trade_replay/rest/bitget.rs b/crates/moon-core/src/market/trade_replay/rest/bitget.rs index 43ae30b5..94feefe2 100644 --- a/crates/moon-core/src/market/trade_replay/rest/bitget.rs +++ b/crates/moon-core/src/market/trade_replay/rest/bitget.rs @@ -17,9 +17,10 @@ use serde_json::Value; -use super::{FetchError, cell_f32, cell_i64}; +use super::{FetchError, TradeCursor, TradePage, cell_f32, cell_i64}; +use crate::feed::types::{Side, Tick}; use crate::market::candles::ChartCandle; -use crate::market::trade_replay::venue_caps::KlineRoute; +use crate::market::trade_replay::venue_caps::{KlineRoute, TradeRoute}; /// Fetch one page and return the decoded body. /// @@ -147,5 +148,167 @@ fn parse_row(row: &Value) -> Option { }) } +/// Fetch one page of public fills and return the decoded body. +/// +/// Args: +/// agent: Shared client. +/// route: [`TradeRoute::BitgetSpotFills`] or [`TradeRoute::BitgetMixFills`]. +/// market: Exchange-native market name, e.g. `BTCUSDT`. +/// from_ms: First millisecond of this slice, inclusive. +/// to_ms: Last millisecond of this slice, inclusive; the vendor caps one request's span at +/// 7 days, see [`super::super::venue_caps::TradeRoute::max_query_ms`]. +/// cursor: Continuation from a previous page, or `None` for the first page. +/// +/// Returns: +/// The decoded response, or a classified failure. +pub(super) fn fetch_trades( + agent: &ureq::Agent, + route: TradeRoute, + market: &str, + from_ms: i64, + to_ms: i64, + cursor: Option, +) -> Result { + let futures = matches!(route, TradeRoute::BitgetMixFills); + let mut request = agent + .get(route.url()) + .query("symbol", market) + .query("startTime", from_ms.to_string()) + .query("endTime", to_ms.to_string()) + .query("limit", route.max_rows().to_string()); + if futures { + request = request.query("productType", "USDT-FUTURES"); + } + if let Some(TradeCursor::LessThanId(id)) = cursor { + request = request.query("idLessThan", id.to_string()); + } + let response = request + .call() + .map_err(|error| FetchError::Transient(error.to_string()))?; + super::decode_and_classify(response, "bitget", classify_fills) +} + +/// Classify a BitGet fills-history response by status and envelope `code`. +/// +/// **Assumption, not settled by vendor docs read for this task**: `fills-history` shares its +/// unknown-symbol codes (`40034` mix, `400172` spot) with `history-candles`. No vendor page for +/// this specific endpoint documents its own unknown-symbol code the way the candle endpoint's +/// pages do — see [`classify`] for the pair this borrows. If a later reader finds this endpoint +/// answers a different code for an unknown symbol, that market falls through to `Transient` +/// below rather than being misclassified as permanent, so the failure mode of a wrong guess here +/// is a retry loop, not a silently wrong "this market does not exist". +/// +/// Args: +/// status: HTTP status. +/// body: Decoded response. +/// +/// Returns: +/// `Ok(())` on success, or the classified failure. +pub(super) fn classify_fills(status: u16, body: &Value) -> Result<(), FetchError> { + if !(200..300).contains(&status) { + return Err(FetchError::Transient(format!("bitget HTTP {status}"))); + } + let code = body.get("code").and_then(Value::as_str).unwrap_or_default(); + match code { + "00000" => Ok(()), + "40034" | "400172" => Err(FetchError::UnknownSymbol), + "" => Err(FetchError::Transient(format!( + "bitget HTTP {status}: empty code" + ))), + other => { + let message = body.get("msg").and_then(Value::as_str).unwrap_or("unknown"); + Err(FetchError::Transient(format!("bitget {other}: {message}"))) + } + } +} + +/// Parse a BitGet fills envelope into a page of ticks. +/// +/// Rows arrive DESCENDING (newest first, per the vendor's own doc), so the oldest row in this +/// page is the LAST one — that is what both the next cursor and the window-covered check key on. +/// +/// Args: +/// body: Decoded response. +/// max_rows: Row cap that was sent, so a FULL page can be told from a short, final one. +/// from_ms: Left edge of the slice, so completeness is judged from the oldest row's own +/// timestamp rather than assumed from the row count alone. +/// +/// Returns: +/// The page, or a failure when the envelope is missing. +pub(super) fn parse_fills( + body: &Value, + max_rows: usize, + from_ms: i64, +) -> Result { + let rows = body + .get("data") + .and_then(Value::as_array) + .ok_or_else(|| FetchError::Transient("bitget: missing data".to_string()))?; + let ticks: Vec = rows.iter().filter_map(parse_fill_row).collect(); + // A page holding a malformed row alongside valid ones can still finish pagination with a + // non-empty tick vector that is silently missing rows — a hole must send the window to + // candles instead of drawing a partial tape as if it were whole. + if ticks.len() < rows.len() { + return Err(FetchError::Transient(format!( + "bitget: page held {} unparseable row(s) of {} (parsed {})", + rows.len() - ticks.len(), + rows.len(), + ticks.len() + ))); + } + let oldest_id = rows + .last() + .and_then(|r| r.get("tradeId")) + .and_then(cell_i64) + .map(|v| v as u64); + let oldest_time_ms = rows.last().and_then(|r| r.get("ts")).and_then(cell_i64); + let full = rows.len() >= max_rows; + let covered = oldest_time_ms.is_some_and(|t| t <= from_ms); + let next = match (full, covered, oldest_id) { + (true, false, Some(id)) => Some(TradeCursor::LessThanId(id)), + // Full page, window not covered, but the oldest row's own `tradeId` did not parse: the + // cursor to continue from is unknowable. Treating this as completion would silently ship + // a truncated window as a whole one. + (true, false, None) => { + return Err(FetchError::Transient( + "bitget: full page, window not covered, but the oldest row's tradeId did not parse" + .to_string(), + )); + } + _ => None, + }; + Ok(TradePage { ticks, next }) +} + +/// Parse one BitGet fills row. +/// +/// The vendor's own `side` casing contradicts itself between its field table and its worked +/// example, so it is read case-insensitively here rather than trusted literally. +/// +/// Args: +/// row: One element of `data`. +/// +/// Returns: +/// The tick, or `None` when the row is malformed. +fn parse_fill_row(row: &Value) -> Option { + let price = cell_f32(row.get("price")?)?; + let qty = cell_f32(row.get("size")?)?; + let time_ms = cell_i64(row.get("ts")?)? as f64; + let side = match row + .get("side") + .and_then(Value::as_str)? + .eq_ignore_ascii_case("sell") + { + true => Side::Sell, + false => Side::Buy, + }; + Some(Tick { + time_ms, + price, + qty, + side, + }) +} + #[cfg(test)] mod tests; diff --git a/crates/moon-core/src/market/trade_replay/rest/bitget/tests.rs b/crates/moon-core/src/market/trade_replay/rest/bitget/tests.rs index b707b011..15b19218 100644 --- a/crates/moon-core/src/market/trade_replay/rest/bitget/tests.rs +++ b/crates/moon-core/src/market/trade_replay/rest/bitget/tests.rs @@ -1,5 +1,7 @@ use super::*; +use serde_json::json; + fn fixture(name: &str) -> Value { let text = match name { "spot" => include_str!("fixtures/spot_klines.json"), @@ -62,3 +64,37 @@ fn bitget_unknown_symbol_codes_are_permanent() { Err(FetchError::UnknownSymbol) ); } + +/// `rest/bitget.rs:parse_fills` treating a full, uncovered page with an unparseable oldest +/// `tradeId` as complete silently ships a truncated tick window as a complete chart. +#[test] +fn bitget_rejects_a_full_uncovered_page_without_a_cursor() { + let body = json!({ + "data": [ + {"price": "100", "size": "1", "ts": "2000", "side": "buy", "tradeId": "101"}, + {"price": "99", "size": "1", "ts": "1000", "side": "sell", "tradeId": "not-an-id"} + ] + }); + + assert!( + parse_fills(&body, 2, 0).is_err(), + "an uncovered full page cannot be complete when its continuation cursor is unknowable" + ); +} + +/// `rest/bitget.rs:parse_fills` dropping its raw-row guard accepts a partly malformed page and +/// draws a gap-ridden tick series as complete market history. +#[test] +fn bitget_rejects_a_page_with_any_unparseable_row() { + let body = json!({ + "data": [ + {"price": "100", "size": "1", "ts": "2000", "side": "buy", "tradeId": "101"}, + {"size": "1", "ts": "1000", "side": "sell", "tradeId": "100"} + ] + }); + + assert!( + parse_fills(&body, 2, 0).is_err(), + "a response with a malformed row is incomplete even when another row parsed" + ); +} diff --git a/crates/moon-core/src/market/trade_replay/rest/gateio.rs b/crates/moon-core/src/market/trade_replay/rest/gateio.rs index b19182ec..37ae1d89 100644 --- a/crates/moon-core/src/market/trade_replay/rest/gateio.rs +++ b/crates/moon-core/src/market/trade_replay/rest/gateio.rs @@ -10,9 +10,10 @@ use serde_json::Value; -use super::{FetchError, cell_f32, cell_i64}; +use super::{FetchError, TradeCursor, TradePage, cell_f32, cell_i64}; +use crate::feed::types::{Side, Tick}; use crate::market::candles::ChartCandle; -use crate::market::trade_replay::venue_caps::KlineRoute; +use crate::market::trade_replay::venue_caps::{KlineRoute, TradeRoute}; /// Milliseconds per second, for Gate's second-resolution window and timestamps. const MS_PER_S: i64 = 1_000; @@ -203,5 +204,245 @@ fn parse_futures_row(row: &Value) -> Option { }) } +/// Largest page number the SPOT trades endpoint's own cap allows. +/// +/// Gate documents `limit*(page-1) <= 100000` for `/spot/trades`; a page beyond that is refused. +/// Futures pagination is by row `offset` and carries no such cap. +const SPOT_PAGE_ROW_CAP: usize = 100_000; + +/// Fetch one page of public trades and return the decoded body. +/// +/// # Order is UNDOCUMENTED here, unlike the candle endpoint +/// +/// Both routes' pagination below therefore keys on the ROW COUNT alone, never on a row's own +/// timestamp: nothing here assumes ascending or descending order, and +/// [`super::super::worker::serve_ticks`] is what sorts the complete tick vector once every page +/// of a stage is in. +/// +/// Args: +/// agent: Shared client. +/// route: [`TradeRoute::GateSpotTrades`] or [`TradeRoute::GateFuturesTrades`]. +/// market: Exchange-native market name, e.g. `BTC_USDT`. +/// from_ms: First millisecond of this slice, inclusive. +/// to_ms: Last millisecond of this slice, inclusive. +/// cursor: Continuation from a previous page, or `None` for the first page. +/// +/// Returns: +/// The decoded response, or a classified failure. +pub(super) fn fetch_trades( + agent: &ureq::Agent, + route: TradeRoute, + market: &str, + from_ms: i64, + to_ms: i64, + cursor: Option, +) -> Result { + let futures = matches!(route, TradeRoute::GateFuturesTrades); + let market_param = match futures { + true => "contract", + false => "currency_pair", + }; + let mut request = agent + .get(route.url()) + .query(market_param, market) + .query("limit", route.max_rows().to_string()) + .query("from", (from_ms / MS_PER_S).to_string()) + .query("to", (to_ms / MS_PER_S).to_string()); + request = match (futures, cursor) { + (false, Some(TradeCursor::Page(page))) => request.query("page", page.to_string()), + (false, _) => request.query("page", "1"), + (true, Some(TradeCursor::Offset(offset))) => request.query("offset", offset.to_string()), + (true, _) => request.query("offset", "0"), + }; + let response = request + .call() + .map_err(|error| FetchError::Transient(error.to_string()))?; + super::decode_and_classify(response, "gate", classify) +} + +/// Parse a Gate SPOT trades array into a page of ticks. +/// +/// Args: +/// body: Decoded response. +/// max_rows: Row cap that was sent, so a FULL page can be told from a short, final one. +/// cursor: The cursor this request was sent with, so the NEXT page number is derived from it +/// rather than reconstructed. +/// +/// Returns: +/// The page, or a failure when the envelope is not an array. +pub(super) fn parse_spot_trades( + body: &Value, + max_rows: usize, + cursor: Option, +) -> Result { + let rows = body + .as_array() + .ok_or_else(|| FetchError::Transient("gate: spot response is not an array".to_string()))?; + let ticks: Vec = rows.iter().filter_map(parse_spot_trade_row).collect(); + // A page holding a malformed row alongside valid ones can still finish pagination with a + // non-empty tick vector that is silently missing rows — a hole must send the window to + // candles instead of drawing a partial tape as if it were whole. + if ticks.len() < rows.len() { + return Err(FetchError::Transient(format!( + "gate: spot page held {} unparseable row(s) of {} (parsed {})", + rows.len() - ticks.len(), + rows.len(), + ticks.len() + ))); + } + let page = match cursor { + Some(TradeCursor::Page(page)) => page, + _ => 1, + }; + let next_page = page.saturating_add(1); + let within_cap = + (next_page.saturating_sub(1) as usize).saturating_mul(max_rows) <= SPOT_PAGE_ROW_CAP; + let next = match rows.len() >= max_rows && within_cap { + true => Some(TradeCursor::Page(next_page)), + false => None, + }; + Ok(TradePage { ticks, next }) +} + +/// Parse one Gate spot trade row. +/// +/// Args: +/// row: One element of the response array. +/// +/// Returns: +/// The tick, or `None` when the row is malformed. +fn parse_spot_trade_row(row: &Value) -> Option { + let price = cell_f32(row.get("price")?)?; + let qty = cell_f32(row.get("amount")?)?; + // Prefer the sub-second `create_time_ms`; fall back to `create_time`, which is SECONDS. + let time_ms = row + .get("create_time_ms") + .and_then(Value::as_str) + .and_then(|s| s.parse::().ok()) + .or_else(|| { + row.get("create_time") + .and_then(cell_i64) + .map(|s| (s * MS_PER_S) as f64) + })?; + let side = match row.get("side").and_then(Value::as_str)? { + "sell" => Side::Sell, + _ => Side::Buy, + }; + Some(Tick { + time_ms, + price, + qty, + side, + }) +} + +/// Parse a Gate USDT-perpetual trades array into a page of ticks. +/// +/// # No `side` field — the SIGN of `size` is the side +/// +/// Positive `size` is a buyer-taker fill and negative a seller-taker one; the magnitude is the +/// contract quantity. **Assumption, not settled by vendor docs read for this task**: which sign +/// means which side. Every other Gate/Binance/Bitget/OKX parser in this module reads an explicit +/// `side`/`m` field; this is the one route with no such field at all, so the mapping below is +/// inferred from the vendor's own field name (`size`, signed) rather than confirmed against a +/// recorded response, and it is the first place to look if a Gate futures tick chart shows every +/// trade on the wrong side. +/// +/// A FULL page is ALWAYS treated as incomplete regardless of any other signal: this endpoint +/// truncates SILENTLY at `limit` with no error, so a full page means "ask again", never "that was +/// all" — see [`super::super::rest::TradePage::next`]'s own doc for why that rule is frozen. +/// +/// Args: +/// body: Decoded response. +/// max_rows: Row cap that was sent. +/// cursor: The cursor this request was sent with, so the next `offset` is derived from it. +/// +/// Returns: +/// The page, or a failure when the envelope is not an array. +pub(super) fn parse_futures_trades( + body: &Value, + max_rows: usize, + cursor: Option, +) -> Result { + let rows = body.as_array().ok_or_else(|| { + FetchError::Transient("gate: futures response is not an array".to_string()) + })?; + let ticks: Vec = rows.iter().filter_map(parse_futures_trade_row).collect(); + // A page holding a malformed row alongside valid ones can still finish pagination with a + // non-empty tick vector that is silently missing rows — a hole must send the window to + // candles instead of drawing a partial tape as if it were whole. + if ticks.len() < rows.len() { + return Err(FetchError::Transient(format!( + "gate: futures page held {} unparseable row(s) of {} (parsed {})", + rows.len() - ticks.len(), + rows.len(), + ticks.len() + ))); + } + let offset = match cursor { + Some(TradeCursor::Offset(offset)) => offset, + _ => 0, + }; + let next = match rows.len() >= max_rows { + true => Some(TradeCursor::Offset( + offset.saturating_add(rows.len() as u32), + )), + false => None, + }; + Ok(TradePage { ticks, next }) +} + +/// Parse one Gate futures trade row. +/// +/// **Unit note**: `size` is a signed CONTRACT count, not a base-currency amount — its base +/// amount depends on the contract's `quanto_multiplier`, served by a different endpoint, so no +/// base-asset amount is available here and `Tick::qty` holds contracts for this route. The +/// chart's volume bars stay shape-correct regardless: the chart normalises the visible window +/// against its own maximum `qty`, and a per-instrument contract multiplier is a constant that +/// cancels out of that normalisation — only an ABSOLUTE volume figure would be wrong, and a tick +/// series' aggregated candles never reach the shared SQLite kline cache where one could be read. +/// See `venue_caps.rs`'s trade-route table for the same note. +/// +/// `size` is checked for finiteness and sign on the NARROWED `f32`, not the `f64` behind it, +/// exactly as [`cell_f32`] does and for the same reason: narrowing is itself a way to become +/// non-finite, so a check applied before the cast can pass a value that becomes infinite after +/// it. +/// +/// Args: +/// row: One element of the response array. +/// +/// Returns: +/// The tick, or `None` when the row is malformed. +fn parse_futures_trade_row(row: &Value) -> Option { + let price = cell_f32(row.get("price")?)?; + let size_raw = match row.get("size")? { + Value::String(text) => text.parse::().ok()?, + other => other.as_f64()?, + }; + let qty = size_raw.abs() as f32; + if !(qty.is_finite() && qty > 0.0) { + return None; + } + let side = match size_raw.is_sign_negative() { + true => Side::Sell, + false => Side::Buy, + }; + let time_ms = row + .get("create_time_ms") + .and_then(cell_i64) + .map(|v| v as f64) + .or_else(|| { + row.get("create_time") + .and_then(cell_i64) + .map(|s| (s * MS_PER_S) as f64) + })?; + Some(Tick { + time_ms, + price, + qty, + side, + }) +} + #[cfg(test)] mod tests; diff --git a/crates/moon-core/src/market/trade_replay/rest/mod.rs b/crates/moon-core/src/market/trade_replay/rest/mod.rs index 85b51f33..75dfff42 100644 --- a/crates/moon-core/src/market/trade_replay/rest/mod.rs +++ b/crates/moon-core/src/market/trade_replay/rest/mod.rs @@ -38,7 +38,8 @@ use std::time::Duration; use serde_json::Value; -use super::venue_caps::KlineRoute; +use super::venue_caps::{KlineRoute, TradeRoute}; +use crate::feed::types::Tick; use crate::market::candles::ChartCandle; /// Bounded lifetime of one HTTP request. @@ -149,6 +150,100 @@ pub fn fetch_klines( Ok(rows) } +/// Continuation token for one public-trade page. +/// +/// FIVE variants, because the venues genuinely paginate five ways and abusing one venue's +/// semantics for another silently truncates a window: Binance walks forward by aggregate-trade +/// id; OKX and Bitget walk BACKWARD by trade id; Gate spot +/// walks by 1-based `page`; Gate futures walks by row `offset`. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum TradeCursor { + /// Binance: next aggregate-trade id to ask for, walking forward. + FromId(u64), + /// Timestamp-bound `after` cursor. No current trade route emits it; OKX switches to + /// [`Self::LessThanId`] after its initial timestamp-bound request. + AfterMs(i64), + /// Bitget: next `idLessThan` bound, walking backward. + LessThanId(u64), + /// Gate spot: next 1-based page number. + Page(u32), + /// Gate futures: next row offset. + Offset(u32), +} + +/// One fetched page of public trades. +#[derive(Clone, Debug)] +pub struct TradePage { + /// Rows in the vendor's OWN order — never sorted here. The global ascending sort and window + /// clip belong to [`super::worker::serve_ticks`], after every page of a stage is in. + pub ticks: Vec, + /// Continuation, Some ONLY when this page was FULL and the window is not yet covered. + /// + /// A full Gate futures page is NEVER accepted as complete: that endpoint truncates silently + /// at `limit` with no error, so a full page means "ask again", never "that was all". + pub next: Option, +} + +/// Fetch one page of public trades. +/// +/// The match below is deliberately exhaustive and carries no `_` arm, matching +/// [`fetch_klines`]'s own discipline. No `category` parameter, deliberately: [`fetch_klines`] +/// carries one only for [`KlineRoute::Bybit`], Bybit has NO trade route, and Bitget derives its +/// required `productType` from the route itself. A parameter with no valid consumer is a +/// boundary leak, not future-proofing. +/// +/// Args: +/// agent: Shared client. +/// route: Which endpoint family to ask. +/// market: Exchange-native market name, as the core reports it. +/// from_ms: First millisecond of this request's slice, inclusive. +/// to_ms: Last millisecond of this request's slice, inclusive. +/// cursor: Continuation from a previous page of this same slice, or `None` for the first. +/// +/// Returns: +/// One page of ticks in the vendor's own order, or a classified failure. +pub fn fetch_trades( + agent: &ureq::Agent, + route: TradeRoute, + market: &str, + from_ms: i64, + to_ms: i64, + cursor: Option, +) -> Result { + match route { + TradeRoute::BinanceSpotAggTrades + | TradeRoute::BinanceUsdMAggTrades + | TradeRoute::BinanceCoinMAggTrades => { + let value = binance::fetch_trades( + agent, + route, + market, + from_ms, + to_ms, + route.max_rows(), + cursor, + )?; + binance::parse_agg_trades(&value, to_ms, route.max_rows()) + } + TradeRoute::GateSpotTrades => { + let value = gateio::fetch_trades(agent, route, market, from_ms, to_ms, cursor)?; + gateio::parse_spot_trades(&value, route.max_rows(), cursor) + } + TradeRoute::GateFuturesTrades => { + let value = gateio::fetch_trades(agent, route, market, from_ms, to_ms, cursor)?; + gateio::parse_futures_trades(&value, route.max_rows(), cursor) + } + TradeRoute::BitgetSpotFills | TradeRoute::BitgetMixFills => { + let value = bitget::fetch_trades(agent, route, market, from_ms, to_ms, cursor)?; + bitget::parse_fills(&value, route.max_rows(), from_ms) + } + TradeRoute::OkxHistoryTrades => { + let value = okx::fetch_trades(agent, route, market, to_ms, route.max_rows(), cursor)?; + okx::parse_history_trades(&value, route.max_rows(), from_ms) + } + } +} + /// Decode one response body and put it through that venue's classifier. /// /// The five GET venues repeat this exact sequence, differing only in which classifier runs and in diff --git a/crates/moon-core/src/market/trade_replay/rest/okx.rs b/crates/moon-core/src/market/trade_replay/rest/okx.rs index fa683384..3f3609b8 100644 --- a/crates/moon-core/src/market/trade_replay/rest/okx.rs +++ b/crates/moon-core/src/market/trade_replay/rest/okx.rs @@ -5,9 +5,10 @@ use serde_json::Value; -use super::{FetchError, cell_f32, cell_i64}; +use super::{FetchError, TradeCursor, TradePage, cell_f32, cell_i64}; +use crate::feed::types::{Side, Tick}; use crate::market::candles::ChartCandle; -use crate::market::trade_replay::venue_caps::KlineRoute; +use crate::market::trade_replay::venue_caps::{KlineRoute, TradeRoute}; /// Positional index of the cell holding BASE-asset volume on a SPOT row. pub(super) const SPOT_VOLUME_CELL: usize = 5; @@ -168,5 +169,155 @@ fn parse_row(row: &Value, volume_cell: usize) -> Option { }) } +/// Fetch one page of public trades and return the decoded body. +/// +/// # No time range at all — the FIRST page anchors on a timestamp, every later one on a trade id +/// +/// `history-trades` has no `startTime`/`endTime` pair whatsoever, unlike the candle endpoint. The +/// FIRST page of a slice has nothing else to anchor to, so it uses `type=2&after=`, walking strictly backward from the slice's own right edge. +/// +/// Every LATER page switches to `type=1&after=` instead of continuing with +/// `type=2` at the previous page's oldest timestamp. `type=2`'s `after` is EXCLUSIVE of the given +/// millisecond, and a full 100-row page routinely ends in the middle of several trades sharing +/// one millisecond on a liquid pair; continuing from that same timestamp would silently drop +/// every sibling trade at the boundary, a hole `full`/`covered` never notice and that ships as a +/// complete window. A trade id has no such collision. +/// +/// Args: +/// agent: Shared client. +/// route: [`TradeRoute::OkxHistoryTrades`]. +/// market: Exchange-native instrument id, e.g. `BTC-USDT` or `BTC-USDT-SWAP`. +/// to_ms: Right edge of this slice, inclusive; the first page's `after` bound. +/// max_rows: Row cap for this request. +/// cursor: Continuation from a previous page, or `None` for the first page. +/// +/// Returns: +/// The decoded response, or a classified failure. +pub(super) fn fetch_trades( + agent: &ureq::Agent, + route: TradeRoute, + market: &str, + to_ms: i64, + max_rows: usize, + cursor: Option, +) -> Result { + let request = agent + .get(route.url()) + .query("instId", market) + .query("limit", max_rows.to_string()); + let request = match cursor { + Some(TradeCursor::LessThanId(id)) => { + request.query("type", "1").query("after", id.to_string()) + } + None => request.query("type", "2").query("after", to_ms.to_string()), + Some(_) => { + debug_assert!( + false, + "okx trade route hands back only LessThanId after the first page" + ); + request.query("type", "2").query("after", to_ms.to_string()) + } + }; + let response = request + .call() + .map_err(|error| FetchError::Transient(error.to_string()))?; + super::decode_and_classify(response, "okx", classify) +} + +/// Parse an OKX `history-trades` envelope into a page of ticks. +/// +/// Rows arrive DESCENDING (newest first, walking backward), so the oldest row in this page is +/// the LAST one — that is what both the next cursor and the window-covered check key on. +/// +/// Args: +/// body: Decoded response. +/// max_rows: Row cap that was sent, so a FULL page can be told from a short, final one. +/// from_ms: Left edge of the slice, so completeness is judged from the oldest row's own +/// timestamp rather than assumed from the row count alone. +/// +/// Returns: +/// The page, or a failure when the envelope is missing. +pub(super) fn parse_history_trades( + body: &Value, + max_rows: usize, + from_ms: i64, +) -> Result { + let rows = body + .get("data") + .and_then(Value::as_array) + .ok_or_else(|| FetchError::Transient("okx: missing data".to_string()))?; + let ticks: Vec = rows.iter().filter_map(parse_trade_row).collect(); + // A page holding a malformed row alongside valid ones can still finish pagination with a + // non-empty tick vector that is silently missing rows — a hole must send the window to + // candles instead of drawing a partial tape as if it were whole. + if ticks.len() < rows.len() { + return Err(FetchError::Transient(format!( + "okx: page held {} unparseable row(s) of {} (parsed {})", + rows.len() - ticks.len(), + rows.len(), + ticks.len() + ))); + } + let oldest_ms = rows.last().and_then(|r| r.get("ts")).and_then(cell_i64); + let oldest_id = rows + .last() + .and_then(|r| r.get("tradeId")) + .and_then(cell_i64) + .map(|v| v as u64); + let full = rows.len() >= max_rows; + let covered = oldest_ms.is_some_and(|t| t < from_ms); + let next = match (full, covered, oldest_id) { + // Continuations paginate by TRADE ID, never by timestamp — see `fetch_trades`'s doc for + // why the boundary millisecond is unsafe to resume from. + (true, false, Some(id)) => Some(TradeCursor::LessThanId(id)), + // Full page, window not covered, but the oldest row's own `tradeId` did not parse: the + // cursor to continue from is unknowable. Treating this as completion would silently ship + // a truncated window as a whole one. + (true, false, None) => { + return Err(FetchError::Transient( + "okx: full page, window not covered, but the oldest row's tradeId did not parse" + .to_string(), + )); + } + _ => None, + }; + Ok(TradePage { ticks, next }) +} + +/// Parse one OKX `history-trades` row. +/// +/// **Unit note**: `sz` is a base-asset amount for SPOT but a CONTRACT COUNT for swap/futures — +/// [`TradeRoute::OkxHistoryTrades`] serves both markets through this one parser, unlike the +/// kline sibling, which is deliberately split into `OkxSpot`/`OkxSwap` for exactly this reason +/// (see `venue_caps.rs`'s doc on `OkxSwap`). No base-asset amount is available from this endpoint +/// for a swap instrument, so `Tick::qty` holds contracts there. The chart's volume bars stay +/// shape-correct regardless: the chart normalises the visible window against its own maximum +/// `qty`, and a per-instrument contract multiplier is a constant that cancels out of that +/// normalisation — only an ABSOLUTE volume figure would be wrong, and a tick series' aggregated +/// candles never reach the shared SQLite kline cache where one could be read. See +/// `venue_caps.rs`'s trade-route table for the same note. +/// +/// Args: +/// row: One element of `data`. +/// +/// Returns: +/// The tick, or `None` when the row is malformed. +fn parse_trade_row(row: &Value) -> Option { + let price = cell_f32(row.get("px")?)?; + let qty = cell_f32(row.get("sz")?)?; + let time_ms = cell_i64(row.get("ts")?)? as f64; + let side = match row.get("side").and_then(Value::as_str)? { + "sell" => Side::Sell, + _ => Side::Buy, + }; + Some(Tick { + time_ms, + price, + qty, + side, + }) +} + #[cfg(test)] mod tests; diff --git a/crates/moon-core/src/market/trade_replay/tests.rs b/crates/moon-core/src/market/trade_replay/tests.rs index 0dd6033c..df0e6e41 100644 --- a/crates/moon-core/src/market/trade_replay/tests.rs +++ b/crates/moon-core/src/market/trade_replay/tests.rs @@ -348,3 +348,87 @@ fn replay_window_keeps_trade_and_floors_when_trimming_the_budget() { "replay_window discarding floors above MAX_SPAN_MS would hide that the request exceeds its budget" ); } + +/// `market/trade_replay/mod.rs:TradeReplaySeries::read_into` dropping the source identity salt, +/// or salting `Klines1m`, makes a tick upgrade leave exchange candles on screen or makes every +/// existing replay look changed to the chart. +#[test] +fn tick_and_kline_replays_keep_distinct_revisions_without_changing_kline_revision() { + let kline = bars_only_series(); + let mut ticks = kline.clone(); + ticks.source = TradeReplaySource::Ticks; + ticks.ticks = vec![crate::feed::types::Tick { + time_ms: MINUTE_MS as f64, + price: 101.0, + qty: 2.0, + side: crate::feed::types::Side::Buy, + }]; + + let mut kline_out = ChartHistoryBuffers::default(); + let kline_read = kline.read_into( + 0.0, + 0.0, + (2 * MINUTE_MS) as f32, + Some(&candle_params(0)), + &mut kline_out, + ); + let mut tick_out = ChartHistoryBuffers::default(); + let tick_read = ticks.read_into( + 0.0, + 0.0, + (2 * MINUTE_MS) as f32, + Some(&candle_params(0)), + &mut tick_out, + ); + + let unchanged_kline_revision = replay_revision(kline.identity, MINUTE_MS, 0, 2); + assert_eq!( + kline_read.revision, unchanged_kline_revision, + "Klines1m keeps the established replay revision for the same identity and window" + ); + assert_ne!( + tick_read.revision, kline_read.revision, + "a tick upgrade must force its aggregated candles to replace already shipped klines" + ); +} + +/// `market/trade_replay/mod.rs:time_slices` representing an unlimited query span as a saturating +/// integer can make pagination step backwards forever, hanging the sole replay worker and every +/// later trade-detail window. +#[test] +fn time_slices_keeps_unbounded_windows_whole_and_bounded_windows_gap_free() { + let window = ReplayWindow { + from_ms: 1_000, + to_ms: 7_200_999, + over_budget: false, + }; + + assert_eq!( + time_slices(window, None), + vec![(window.from_ms, window.to_ms)], + "an unlimited route issues one request for precisely its requested window" + ); + + let span_ms = 3_600_000; + let slices = time_slices(window, Some(span_ms)); + assert_eq!( + slices.first().copied(), + Some((window.from_ms, window.from_ms + span_ms - 1)), + "the first bounded request starts at the requested left edge and consumes one legal span" + ); + assert_eq!( + slices.last().copied().map(|(_, end)| end), + Some(window.to_ms), + "the final bounded request reaches the requested right edge" + ); + assert!( + slices + .iter() + .all(|(start, end)| end >= start && end - start < span_ms), + "each slice stays strictly within the documented exclusive maximum span" + ); + assert!( + slices.windows(2).all(|pair| pair[0].1 + 1 == pair[1].0), + "adjacent requests neither leave a market-data gap nor re-fetch a boundary millisecond" + ); +} diff --git a/crates/moon-core/src/market/trade_replay/venue_caps.rs b/crates/moon-core/src/market/trade_replay/venue_caps.rs index 4c174aa1..3bac6332 100644 --- a/crates/moon-core/src/market/trade_replay/venue_caps.rs +++ b/crates/moon-core/src/market/trade_replay/venue_caps.rs @@ -64,6 +64,40 @@ //! distinguish an unknown coin from an outage**: both answer HTTP 500 with a body of literally //! `null`, so its classifier calls every failure transient, which is the direction that caches //! nothing. +//! +//! # The trade-route directory, in one table +//! +//! [`TradeRoute`] is the same idea one level down: which venues can answer a public "trades +//! between T1 and T2" question, for the trade-detail window's tick replay. Every row was read off +//! the vendor's own documentation, same discipline as the kline table above. Bybit, Hyperliquid +//! and HTX document no such endpoint reachable from this build, so [`trade_route`] answers +//! [`None`] for all three — the same honest degradation [`kline_route`] already gives HTX. +//! +//! | route | endpoint | gate key | rows/page | max query window | retention | cursor | order | unknown-symbol | doc | +//! |---|---|---|---|---|---|---|---|---|---| +//! | `BinanceSpotAggTrades` | `/api/v3/aggTrades` | `data-api.binance.vision` | 1000 | none documented | none documented | `fromId` | undocumented | HTTP 4xx `-1121` | developers.binance.com/docs/binance-spot-api-docs/rest-api/market-data-endpoints | +//! | `BinanceUsdMAggTrades` | `/fapi/v1/aggTrades` | `fapi.binance.com` | 1000 | **< 1 h** | **48 h** | `fromId` | undocumented | HTTP 4xx `-1121` | developers.binance.com/.../derivatives/usds-margined-futures/market-data/rest-api/Compressed-Aggregate-Trades-List | +//! | `BinanceCoinMAggTrades` | `/dapi/v1/aggTrades` | `dapi.binance.com` | 1000 | **< 1 h** | **48 h** | `fromId` | undocumented | HTTP 4xx `-1121` | developers.binance.com/.../derivatives/coin-margined-futures/market-data/rest-api/Compressed-Aggregate-Trades-List | +//! | `GateSpotTrades` | `/api/v4/spot/trades` | `api.gateio.ws` | 1000 | none (page cap `limit*(page-1) <= 100000`) | ~30 d | `from`/`to` in **SECONDS** + `page` | undocumented | label `INVALID_CURRENCY_PAIR` | gateio/gateapi-python docs/SpotApi.md | +//! | `GateFuturesTrades` | `/api/v4/futures/usdt/trades` | `api.gateio.ws` | **undocumented, default 100** | none | none documented | `from`/`to` in **SECONDS** + `offset` | undocumented | label `CONTRACT_NOT_FOUND` | gateio/gateapi-python docs/FuturesApi.md | +//! | `BitgetSpotFills` | `/api/v2/spot/market/fills-history` | `api.bitget.com` | 1000 | **7 d** | **90 d** | `idLessThan` | **desc** | envelope `code != "00000"` | bitget.com/api-doc/classic/spot/market/Get-Market-Trades | +//! | `BitgetMixFills` | `/api/v2/mix/market/fills-history` | `api.bitget.com` | 1000 | **7 d** | **90 d** | `idLessThan` (+`productType` required) | **desc** | envelope `code != "00000"` | bitget.com/api-doc/classic/contract/market/Get-Fills-History | +//! | `OkxHistoryTrades` | `/api/v5/market/history-trades` | `www.okx.com` | **100** | none (no `startTime`/`endTime` at all) | **3 months** | `type=2&after=` for the FIRST page, `type=1&after=` for every later one | **desc** | HTTP 200 + `code 51001` | okx.com/docs-v5/en/#order-book-trading-market-data-get-trades-history | +//! +//! Two venue facts that are silently wrong when mistaken: **Binance futures (both arms) retain +//! only 48 hours of aggTrades**, so the retention check must run before any request is spent; and +//! **Gate's `from`/`to` are in SECONDS**, while every other timestamp in this module is +//! milliseconds. +//! +//! **Three trade routes report a CONTRACT count where `Tick::qty` is documented as base-currency +//! quantity**, and this is deliberate rather than an oversight: `OkxHistoryTrades` +//! (SWAP instruments only — `sz` is base currency for SPOT), `BinanceCoinMAggTrades` (`q` on a +//! dapi row, with no `baseQty` alternative), and `GateFuturesTrades` (`size`, whose base amount +//! depends on the contract's `quanto_multiplier`). No code compensates for this: the chart's +//! volume bars stay shape-correct because the drawn scale is window-relative and a per-instrument +//! multiplier is a constant that cancels out of it, and a tick series' aggregated candles never +//! reach the shared kline cache where an absolute figure could be read as genuine history. See +//! each route's own parser for the fact restated in the vendor's own terms. use crate::venue::{Brand, MarketKind, Venue}; @@ -215,6 +249,178 @@ impl KlineRoute { } } +/// Route serving PUBLIC individual trades over a bounded past window. +/// +/// A variant is a REQUEST SHAPE, exactly as [`KlineRoute`] is one level up: the host, the query +/// grammar, the pagination cursor and the response envelope differ per family, and [`super::rest`] +/// matches on this to build and parse the call. Coverage is narrower than [`KlineRoute`]'s on +/// purpose — Bybit and Hyperliquid document no public trade-history endpoint this build can use, +/// and HTX inherits [`kline_route`]'s own reason for answering [`None`]. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub enum TradeRoute { + /// `GET https://data-api.binance.vision/api/v3/aggTrades`. + BinanceSpotAggTrades, + /// `GET https://fapi.binance.com/fapi/v1/aggTrades` — retains only 48h, `< 1h` per request. + BinanceUsdMAggTrades, + /// `GET https://dapi.binance.com/dapi/v1/aggTrades` — retains only 48h, `< 1h` per request. + BinanceCoinMAggTrades, + /// `GET https://api.gateio.ws/api/v4/spot/trades` — `from`/`to` in SECONDS. + GateSpotTrades, + /// `GET https://api.gateio.ws/api/v4/futures/usdt/trades` — `from`/`to` in SECONDS, truncates + /// silently at its page size with no error. + GateFuturesTrades, + /// `GET https://api.bitget.com/api/v2/spot/market/fills-history` — answers descending. + BitgetSpotFills, + /// `GET https://api.bitget.com/api/v2/mix/market/fills-history` — answers descending, needs + /// `productType`. + BitgetMixFills, + /// `GET https://www.okx.com/api/v5/market/history-trades` — spot and swap alike, no + /// `startTime`/`endTime` at all; paginates backward with `after` only. + OkxHistoryTrades, +} + +impl TradeRoute { + /// Return the fully qualified request URL for this route. + /// + /// Returns: + /// Absolute HTTPS endpoint, without any query string. + pub const fn url(self) -> &'static str { + match self { + Self::BinanceSpotAggTrades => "https://data-api.binance.vision/api/v3/aggTrades", + Self::BinanceUsdMAggTrades => "https://fapi.binance.com/fapi/v1/aggTrades", + Self::BinanceCoinMAggTrades => "https://dapi.binance.com/dapi/v1/aggTrades", + Self::GateSpotTrades => "https://api.gateio.ws/api/v4/spot/trades", + Self::GateFuturesTrades => "https://api.gateio.ws/api/v4/futures/usdt/trades", + Self::BitgetSpotFills => "https://api.bitget.com/api/v2/spot/market/fills-history", + Self::BitgetMixFills => "https://api.bitget.com/api/v2/mix/market/fills-history", + Self::OkxHistoryTrades => "https://www.okx.com/api/v5/market/history-trades", + } + } + + /// Rate-limit key, and it is DERIVED, never re-typed: every arm returns the corresponding + /// [`KlineRoute`]'s own [`KlineRoute::host`]. + /// + /// [`super::gate::ReplayGate`] keys pacing and refusal history by this literal, so a second + /// hand-typed copy of the same string is two production authorities that can drift on a later + /// endpoint change and split ONE real IP budget into two independent permits. Delegation makes + /// that drift unrepresentable rather than merely tested. + /// + /// Returns: + /// Bare host name, without scheme or path. + pub const fn host(self) -> &'static str { + match self { + Self::BinanceSpotAggTrades => KlineRoute::BinanceSpot.host(), + Self::BinanceUsdMAggTrades => KlineRoute::BinanceUsdM.host(), + Self::BinanceCoinMAggTrades => KlineRoute::BinanceCoinM.host(), + Self::GateSpotTrades => KlineRoute::GateSpot.host(), + Self::GateFuturesTrades => KlineRoute::GateFutures.host(), + Self::BitgetSpotFills => KlineRoute::BitgetSpot.host(), + Self::BitgetMixFills => KlineRoute::BitgetFutures.host(), + Self::OkxHistoryTrades => KlineRoute::OkxSpot.host(), + } + } + + /// Largest number of rows one request may ask for. + /// + /// Returns: + /// Maximum rows per request. + pub const fn max_rows(self) -> usize { + match self { + Self::BinanceSpotAggTrades + | Self::BinanceUsdMAggTrades + | Self::BinanceCoinMAggTrades => 1_000, + Self::GateSpotTrades => 1_000, + // UNDOCUMENTED: this is the MEASURED default page size, not a vendor-stated cap. + Self::GateFuturesTrades => 100, + Self::BitgetSpotFills | Self::BitgetMixFills => 1_000, + Self::OkxHistoryTrades => 100, + } + } + + /// Largest span one REQUEST may cover, or [`None`] when the vendor documents no cap. + /// + /// NEVER expressed as `i64::MAX`: [`None`] preserves the distinction between no documented + /// cap and a finite maximum span. + /// + /// Returns: + /// Widest request window in milliseconds, or `None` when unbounded. + pub const fn max_query_ms(self) -> Option { + const HOUR_MS: i64 = 3_600_000; + const DAY_MS: i64 = 24 * HOUR_MS; + match self { + Self::BinanceUsdMAggTrades | Self::BinanceCoinMAggTrades => Some(HOUR_MS), + Self::BitgetSpotFills | Self::BitgetMixFills => Some(7 * DAY_MS), + Self::BinanceSpotAggTrades + | Self::GateSpotTrades + | Self::GateFuturesTrades + | Self::OkxHistoryTrades => None, + } + } + + /// How far back the vendor documents that this endpoint answers, or [`None`] when it + /// documents no limit. + /// + /// Evaluated by [`super::worker::inside_retention`] BEFORE any request is spent, so an + /// out-of-retention window costs zero. + /// + /// Returns: + /// Retention window in milliseconds, or `None` when unbounded. + pub const fn retention_ms(self) -> Option { + const HOUR_MS: i64 = 3_600_000; + const DAY_MS: i64 = 24 * HOUR_MS; + match self { + Self::BinanceUsdMAggTrades | Self::BinanceCoinMAggTrades => Some(48 * HOUR_MS), + Self::GateSpotTrades => Some(30 * DAY_MS), + Self::BitgetSpotFills | Self::BitgetMixFills => Some(90 * DAY_MS), + Self::OkxHistoryTrades => Some(90 * DAY_MS), + Self::BinanceSpotAggTrades | Self::GateFuturesTrades => None, + } + } +} + +/// Return the public-trade route this venue is served by, if this build knows one. +/// +/// EVERY arm spelled out; no `_` catch-all, same discipline as [`kline_route`]. +/// +/// `(Binance, Quarterly)` maps to [`TradeRoute::BinanceCoinMAggTrades`], NOT to `None`: +/// `venue.rs:260-265` defines [`MarketKind::Quarterly`] as Binance COIN-M and `venue.rs:323-326` +/// maps the reachable `QBinance` code to it, so a blanket "Quarterly -> None" rule would make the +/// COIN-M route unreachable here exactly as it would in [`kline_route`]. +/// +/// [`None`] for Bybit, Hyperliquid, HTX — none document a public trade-history endpoint this +/// build can reach — and for the Gate / BitGet / OKX `Quarterly` arms, which name no product +/// those brands actually have, the same reason [`kline_route`] already gives them. +/// +/// Args: +/// venue: Venue resolved from the core's reported platform ordinal. +/// +/// Returns: +/// The route, or `None` when no verified public trade endpoint exists for it in this build. +pub const fn trade_route(venue: Venue) -> Option { + match (venue.brand, venue.kind) { + (Brand::Binance, MarketKind::Spot) => Some(TradeRoute::BinanceSpotAggTrades), + (Brand::Binance, MarketKind::Futures) => Some(TradeRoute::BinanceUsdMAggTrades), + (Brand::Binance, MarketKind::Quarterly) => Some(TradeRoute::BinanceCoinMAggTrades), + (Brand::Gate, MarketKind::Spot) => Some(TradeRoute::GateSpotTrades), + (Brand::Gate, MarketKind::Futures) => Some(TradeRoute::GateFuturesTrades), + (Brand::BitGet, MarketKind::Spot) => Some(TradeRoute::BitgetSpotFills), + (Brand::BitGet, MarketKind::Futures) => Some(TradeRoute::BitgetMixFills), + (Brand::Okx, MarketKind::Spot) | (Brand::Okx, MarketKind::Futures) => { + Some(TradeRoute::OkxHistoryTrades) + } + // Neither venue documents a public trade-history endpoint reachable from this build. + (Brand::Bybit, _) => None, + (Brand::Hyperliquid, _) => None, + // HTX has no trade route either, for the same reason `kline_route` gives it none. + (Brand::Htx, _) => None, + // `venue` yields no quarterly market for these three brands today; spelled out rather + // than folded into a catch-all for the same reason `kline_route` spells them out. + (Brand::Gate, MarketKind::Quarterly) + | (Brand::BitGet, MarketKind::Quarterly) + | (Brand::Okx, MarketKind::Quarterly) => None, + } +} + /// Return the one-minute-bar route this venue is served by, if this build knows one. /// /// Args: diff --git a/crates/moon-core/src/market/trade_replay/worker.rs b/crates/moon-core/src/market/trade_replay/worker.rs index 10ca68a7..bfee5898 100644 --- a/crates/moon-core/src/market/trade_replay/worker.rs +++ b/crates/moon-core/src/market/trade_replay/worker.rs @@ -25,11 +25,12 @@ use std::sync::{Arc, Mutex, OnceLock}; use std::time::{Duration, Instant}; use super::gate::ReplayGate; -use super::venue_caps::{bybit_category, kline_route}; +use super::venue_caps::{TradeRoute, bybit_category, kline_route, trade_route}; use super::{ ReplayWindow, TradeReplayEmpty, TradeReplayFailure, TradeReplayOutcome, TradeReplaySeries, - TradeReplaySource, pages, rest, + TradeReplaySource, pages, rest, time_slices, }; +use crate::feed::types::Tick; use crate::market::candles::ChartCandle; use crate::market::kline_cache::{KlineCache, MergeItem}; use crate::market::source::ReplayAddress; @@ -58,6 +59,27 @@ const JOB_DEADLINE: Duration = Duration::from_secs(45); /// holds one bounded window's rows. const OUTCOME_CACHE_LEN: usize = 8; +/// Ceiling on the total number of ticks held across every remembered entry. +/// +/// A single entry can carry up to [`TICK_BUDGET`] ticks, and [`OUTCOME_CACHE_LEN`] entries of +/// that size would let the ring's own memory dwarf the point ring it feeds. This bounds the ring +/// independently of its entry count: eviction runs oldest-first, exactly as the entry-count +/// eviction does, and never touches the entry that was just inserted, so one huge series is held +/// rather than immediately discarded and re-fetched. +const OUTCOME_CACHE_MAX_TICKS: usize = 2 * TICK_BUDGET; + +/// Bounds MEMORY and the GPU point ring for one tick series. +/// +/// Sits under the live chart's default `trades_limit` of 50 000 (`candles.rs:93`), so a tick +/// replay never asks the point ring for more than the main chart already draws. +pub(crate) const TICK_BUDGET: usize = 40_000; + +/// Bounds WALL TIME on the single worker thread for one tick stage. +/// +/// 60 pages at [`super::gate::ReplayGate::pace`]'s 100 ms floor plus a ~250 ms round trip is +/// about 21 s, inside [`JOB_DEADLINE`] with room for a slow venue. +const TICK_PAGE_BUDGET: usize = 60; + /// What one answered question is remembered as. /// /// An authoritative EMPTY is an answer too, and a valuable one: a delisted or halted market @@ -66,7 +88,13 @@ const OUTCOME_CACHE_LEN: usize = 8; #[derive(Clone, Debug)] enum Remembered { /// Rows to draw. - Ready(TradeReplaySeries), + Ready { + series: TradeReplaySeries, + /// Whether these rows are already a SETTLED tick series, so a reopen never re-asks for + /// ticks it already has, and a fresh entry (candles only, no tick attempt made yet) still + /// earns one. + ticks_settled: bool, + }, /// The venue answered and its answer held nothing in this window. Empty, } @@ -93,6 +121,109 @@ struct OutcomeKey { to_ms: i64, } +/// Which route and cache key a queued tick stage answers. +/// +/// Carried on [`Job::Ticks`] alongside the original [`TradeReplayRequest`] rather than +/// re-derived when the stage finally runs, so a stage queued behind a long line of candle jobs +/// answers the same question `serve` decided it should — never a re-lookup that could disagree. +#[derive(Clone, Debug)] +pub(crate) struct TickStage { + /// Which venue endpoint to ask. + route: TradeRoute, + /// The ring key this stage's answer replaces on success. + key: OutcomeKey, +} + +/// One unit of the worker's internal priority queue. +/// +/// A candle job and its own tick upgrade are two separate units on purpose: queuing the tick +/// stage inline would make a second report-row double-click wait behind it for its OWN candles — +/// see [`next_job`], which is what keeps candle jobs strictly ahead. +pub(crate) enum Job { + Candles(TradeReplayRequest), + Ticks(TradeReplayRequest, TickStage), +} + +/// Pop the next unit of work: any pending [`Job::Candles`] strictly ahead of every +/// [`Job::Ticks`], oldest first within each kind. +/// +/// Args: +/// queue: The worker's own pending-work deque. +/// +/// Returns: +/// The next job to run, or `None` when the queue is empty. +fn next_job(queue: &mut VecDeque) -> Option { + match queue.iter().position(|job| matches!(job, Job::Candles(_))) { + Some(index) => queue.remove(index), + None => queue.pop_front(), + } +} + +/// Why a tick stage produced nothing, so a caller and a test can each name the reason precisely. +/// +/// Each arm is a DIFFERENT log line and a different test; see [`paginate_ticks`]. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum TickAbandon { + Cancelled, + Deadline, + Transient, + Empty, + UnknownSymbol, + OverTickBudget, + OverPageBudget, + /// The stage's own [`TickObserver::claim`] was refused: an active refusal is already + /// recorded for this host by some other request, and the tick stage must respect it rather + /// than send anyway on the strength of a candle stage's claim that already cleared. + RateLimited, +} + +/// What one tick stage produced. +/// +/// `Ready` carries the ticks EXACTLY as fetched, page order untouched: the global sort and the +/// window clip happen in [`serve_ticks`] after this returns, so a test can hand in DESCENDING +/// pages and observe that the SORT, not the pagination, is what fixes them. +#[derive(Debug)] +pub(crate) enum TickVerdict { + Ready(Vec), + Abandoned(TickAbandon), +} + +/// Records the gate calls one tick stage makes, so a test can assert exactly one claim per stage +/// and exactly one pace per fetched page, without a network or a real gate. +/// +/// `claim` takes a REAL send permit rather than merely observing one: the candle stage that ran +/// immediately before this one claimed and cleared its OWN permit already, and neither a +/// cache-answered candle stage nor a memory-ring reopen ever reaches `gate.claim` at all, so a +/// tick stage that trusted that prior claim would send its up-to-60 requests blind to an active +/// refusal recorded for this host by any other request — the exact escalation-to-ban path +/// [`ReplayGate`] exists to prevent. +pub(crate) trait TickObserver { + fn claim(&mut self, host: &str) -> Result<(), u32>; + fn pace(&mut self, host: &str); +} + +/// Bridges the pure [`TickObserver`] seam to the real [`ReplayGate`] for production use. +/// +/// Holds its own `host` rather than trusting the one handed to each call: [`TickObserver`]'s +/// methods take `&str` so the pure seam stays free of a lifetime a test double has no reason to +/// carry, while [`ReplayGate::claim`] and [`ReplayGate::pace`] need the `'static` the route +/// itself already guarantees. Pinning it at construction resolves that without widening the +/// trait's parameter type. +struct GateObserver<'a> { + gate: &'a ReplayGate, + host: &'static str, +} + +impl TickObserver for GateObserver<'_> { + fn claim(&mut self, _host: &str) -> Result<(), u32> { + self.gate.claim(self.host, Instant::now()) + } + + fn pace(&mut self, _host: &str) { + self.gate.pace(self.host); + } +} + /// One replay request. pub struct TradeReplayRequest { /// Exchange addressing resolved from the live source before the request was queued. @@ -143,7 +274,13 @@ pub fn request(request: TradeReplayRequest) { } } -/// Worker loop: one request at a time, forever. +/// Worker loop: an internal priority queue, forever. +/// +/// One request produces up to two jobs, run at different priorities rather than back to back — +/// see [`next_job`] for why an inline tick stage would break the first outcome's own promise. +/// Every iteration blocks on [`Receiver::recv`] only when the queue is empty; otherwise every +/// already-queued request is drained non-blockingly first, so a burst of report-row clicks is +/// batched into the queue before priority is applied rather than served one at a time. /// /// Args: /// rx: Queue of pending requests. @@ -151,18 +288,70 @@ fn run(rx: &Receiver) { let agent = rest::agent(); let gate = ReplayGate::new(); let cache: Mutex> = Mutex::new(VecDeque::new()); - while let Ok(request) = rx.recv() { - // A window that closed while its request sat in the queue costs nothing at all: this is - // the cheapest of the three cancellation guards and the only one that prevents the work. - if request.cancel.load(Ordering::Relaxed) { + let mut queue: VecDeque = VecDeque::new(); + loop { + if queue.is_empty() { + match rx.recv() { + Ok(request) => queue.push_back(Job::Candles(request)), + // Every sender lives inside `WORKER`, which is never dropped, so this is + // unreachable in practice; exiting is the honest answer if it ever happens. + Err(_) => return, + } + } + while let Ok(request) = rx.try_recv() { + queue.push_back(Job::Candles(request)); + } + let Some(job) = next_job(&mut queue) else { continue; + }; + match job { + Job::Candles(request) => { + // A window that closed while its request sat in the queue costs nothing at all: + // this is the cheapest of the three cancellation guards and the only one that + // prevents the work. + if request.cancel.load(Ordering::Relaxed) { + continue; + } + let served = serve(&agent, &gate, &cache, &request); + // The receiver is gone whenever the window closed mid-fetch. Normal, not an + // error — and exactly the signal that a queued tick stage would now answer no + // one, so it is never queued on a failed send. + let sent = request.reply.send(served.outcome).is_ok(); + if sent { + if let Some(stage) = served.tick_stage { + queue.push_back(Job::Ticks(request, stage)); + } + } + } + Job::Ticks(request, stage) => { + if request.cancel.load(Ordering::Relaxed) { + continue; + } + if let Some(series) = serve_ticks(&agent, &gate, &request, &stage) { + remember_store( + &cache, + stage.key, + Remembered::Ready { + series: series.clone(), + ticks_settled: true, + }, + ); + // Normal, not an error, for the same reason as the candle send above. + let _ = request.reply.send(TradeReplayOutcome::Ready(series)); + } + } } - let outcome = serve(&agent, &gate, &cache, &request); - // The receiver is gone whenever the window closed mid-fetch. Normal, not an error. - let _ = request.reply.send(outcome); } } +/// What one candle job resolves to: the outcome to send, and whether it earned a tick upgrade. +struct Served { + outcome: TradeReplayOutcome, + /// `Some` only when the CANDLE outcome above was `Ready`, so `run` may queue it onto the + /// BACK of the deque; see [`tick_stage_for`] for the four conditions that gate it. + tick_stage: Option, +} + /// Answer one request: memory cache, then SQLite cache, then the network. /// /// The order is fixed and each step earns its place. The memory cache answers a reopen with no @@ -177,16 +366,19 @@ fn run(rx: &Receiver) { /// request: The request being served. /// /// Returns: -/// The outcome to send back. +/// The outcome to send back, and the tick stage to queue behind it, if any. fn serve( agent: &ureq::Agent, gate: &ReplayGate, cache: &Mutex>, request: &TradeReplayRequest, -) -> TradeReplayOutcome { +) -> Served { let venue = request.address.venue; let Some(route) = kline_route(venue) else { - return TradeReplayOutcome::Empty(TradeReplayEmpty::NoEndpoint { brand: venue.brand }); + return Served { + outcome: TradeReplayOutcome::Empty(TradeReplayEmpty::NoEndpoint { brand: venue.brand }), + tick_stage: None, + }; }; let key = OutcomeKey { venue, @@ -196,9 +388,21 @@ fn serve( to_ms: request.window.to_ms, }; match remember_lookup(cache, &key, request.identity) { - Some(Remembered::Ready(series)) => return TradeReplayOutcome::Ready(series), + Some(Remembered::Ready { + series, + ticks_settled, + }) => { + let tick_stage = tick_stage_for(venue, request.window, &key, ticks_settled); + return Served { + outcome: TradeReplayOutcome::Ready(series), + tick_stage, + }; + } Some(Remembered::Empty) => { - return TradeReplayOutcome::Empty(TradeReplayEmpty::NoDataInWindow); + return Served { + outcome: TradeReplayOutcome::Empty(TradeReplayEmpty::NoDataInWindow), + tick_stage: None, + }; } None => {} } @@ -206,12 +410,26 @@ fn serve( // The SQLite cache is read first and unconditionally: it costs no request and is not gated. if let Some(rows) = read_cached_bars(request.address.cache.as_ref(), request) { let series = compose(request, venue, rows); - remember_store(cache, key, Remembered::Ready(series.clone())); - return TradeReplayOutcome::Ready(series); + remember_store( + cache, + key.clone(), + Remembered::Ready { + series: series.clone(), + ticks_settled: false, + }, + ); + let tick_stage = tick_stage_for(venue, request.window, &key, false); + return Served { + outcome: TradeReplayOutcome::Ready(series), + tick_stage, + }; } if let Err(retry_in_s) = gate.claim(route.host(), Instant::now()) { - return TradeReplayOutcome::Failed(TradeReplayFailure::RateLimited { retry_in_s }); + return Served { + outcome: TradeReplayOutcome::Failed(TradeReplayFailure::RateLimited { retry_in_s }), + tick_stage: None, + }; } let category = bybit_category(venue, &request.market); let deadline = Instant::now() + JOB_DEADLINE; @@ -228,9 +446,12 @@ fn serve( break; } if Instant::now() >= deadline { - return TradeReplayOutcome::Failed(TradeReplayFailure::Transient { - diagnostic: format!("trade replay exceeded {}s", JOB_DEADLINE.as_secs()), - }); + return Served { + outcome: TradeReplayOutcome::Failed(TradeReplayFailure::Transient { + diagnostic: format!("trade replay exceeded {}s", JOB_DEADLINE.as_secs()), + }), + tick_stage: None, + }; } gate.pace(route.host()); match rest::fetch_klines( @@ -248,10 +469,18 @@ fn serve( // claim here would make one bad market throttle every other market on that host, // and five of them would push it to the backoff ceiling for nothing. gate.clear(route.host()); - return TradeReplayOutcome::Failed(TradeReplayFailure::UnknownSymbol); + return Served { + outcome: TradeReplayOutcome::Failed(TradeReplayFailure::UnknownSymbol), + tick_stage: None, + }; } Err(rest::FetchError::Transient(diagnostic)) => { - return TradeReplayOutcome::Failed(TradeReplayFailure::Transient { diagnostic }); + return Served { + outcome: TradeReplayOutcome::Failed(TradeReplayFailure::Transient { + diagnostic, + }), + tick_stage: None, + }; } } } @@ -284,14 +513,21 @@ fn serve( // window has not been fully answered yet. Without this, a window whose only bar is the forming // one empties out and is remembered as an authoritative "this market did not trade". complete = complete && rows.len() == before_drop; - write_cached_bars(request.address.cache.as_ref(), request, &rows); + write_cached_bars( + request.address.cache.as_ref(), + request, + rows_for_cache(TradeReplaySource::Klines1m, &rows), + ); if rows.is_empty() { // Only a COMPLETE run may be remembered, empty or not: a cancelled one proves nothing // about the window it never finished reading. if complete { remember_store(cache, key, Remembered::Empty); } - return TradeReplayOutcome::Empty(TradeReplayEmpty::NoDataInWindow); + return Served { + outcome: TradeReplayOutcome::Empty(TradeReplayEmpty::NoDataInWindow), + tick_stage: None, + }; } let series = compose(request, venue, rows); // Only a COMPLETE run may be remembered. Pages are issued left to right, so a cancelled run @@ -299,10 +535,324 @@ fn serve( // and the in-memory ring, unlike the SQLite path, has no coverage re-check to catch that on // read. Storing it would serve a silently truncated chart as `Ready` for the life of the // entry. The SQLite merge above is unaffected: `cache_covers` re-checks it on every read. + let tick_stage = match complete { + true => tick_stage_for(venue, request.window, &key, false), + false => None, + }; if complete { - remember_store(cache, key, Remembered::Ready(series.clone())); + remember_store( + cache, + key, + Remembered::Ready { + series: series.clone(), + ticks_settled: false, + }, + ); + } + Served { + outcome: TradeReplayOutcome::Ready(series), + tick_stage, + } +} + +/// Decide whether a just-served CANDLE outcome earns a queued tick upgrade. +/// +/// `Some` only when every one of four conditions holds: a trade route exists for this venue +/// ([`trade_route`]), the ring entry backing this key does not already carry a SETTLED tick +/// series, and the window is inside the route's own documented retention +/// ([`inside_retention`]) — evaluated here, BEFORE any request is spent, so an out-of-retention +/// window costs zero. The fourth condition, "the outcome was `Ready{Klines1m}`", is enforced by +/// every call site: this is reached only from a branch that just composed or looked up a candle +/// `Ready` outcome. +/// +/// A clock that cannot be read (`now_unix_ms_i64` answering `0`) is treated as INSIDE retention +/// rather than refused, the same permissive default [`serve`] already applies to the closed-bar +/// drop above: nothing here can prove the window is too old, so nothing here refuses it. +/// +/// Args: +/// venue: Venue the candles came from. +/// window: The window the candles cover. +/// key: The ring key this stage would replace on success. +/// ticks_settled: Whether the ring entry for this key already carries a settled tick series. +/// +/// Returns: +/// The stage to queue, or `None`. +fn tick_stage_for( + venue: crate::venue::Venue, + window: ReplayWindow, + key: &OutcomeKey, + ticks_settled: bool, +) -> Option { + if ticks_settled { + return None; + } + let route = trade_route(venue)?; + let now_ms = crate::util::time::now_unix_ms_i64(); + if now_ms > 0 && !inside_retention(route, window, now_ms) { + return None; + } + Some(TickStage { + route, + key: key.clone(), + }) +} + +/// Whether a window is within a trade route's own documented retention. +/// +/// Free, and evaluated BEFORE any request is spent — see [`tick_stage_for`], the only caller. +/// +/// Args: +/// route: The trade route in question. +/// window: The window to check. +/// now_ms: Current Unix time in milliseconds. +/// +/// Returns: +/// `true` when the route documents no retention limit, or when the window's left edge falls +/// inside the one it does document. +pub(crate) fn inside_retention(route: TradeRoute, window: ReplayWindow, now_ms: i64) -> bool { + route + .retention_ms() + .is_none_or(|r| window.from_ms >= now_ms - r) +} + +/// Run one queued tick stage to completion. +/// +/// Args: +/// agent: Shared HTTP client. +/// gate: Per-host pacing. +/// request: The original request this stage upgrades. +/// stage: Which route and cache key this stage answers. +/// +/// Returns: +/// The tick series to send as the SECOND outcome, or `None` when it could not be produced — +/// in which case the candle picture already sent stands and nothing more is sent. +fn serve_ticks( + agent: &ureq::Agent, + gate: &ReplayGate, + request: &TradeReplayRequest, + stage: &TickStage, +) -> Option { + let route = stage.route; + let deadline = Instant::now() + JOB_DEADLINE; + let slices = time_slices(request.window, route.max_query_ms()); + let mut observer = GateObserver { + gate, + host: route.host(), + }; + let verdict = paginate_ticks( + route, + &slices, + TICK_BUDGET, + TICK_PAGE_BUDGET, + || request.cancel.load(Ordering::Relaxed), + || Instant::now() >= deadline, + &mut observer, + |from_ms, to_ms, cursor| { + rest::fetch_trades(agent, route, &request.market, from_ms, to_ms, cursor) + }, + ); + let mut ticks = match verdict { + TickVerdict::Ready(ticks) => { + // The venue answered every slice, so its refusal history is stale whatever the rows + // say — exactly the candle stage's own `gate.clear` above. A `Transient` page never + // reaches here: it abandons through the branch below, and the refusal it recorded + // must stay. + gate.clear(route.host()); + ticks + } + TickVerdict::Abandoned(reason) => { + // RELEASE THE PERMIT THIS STAGE TOOK, unless the venue is the reason we stopped. + // + // `TickObserver::claim` records a real attempt on the host's shared claim map, and + // only an explicit `clear` erases it. So an abandonment that is OUR OWN doing — the + // user closed the window, the job deadline expired, either budget was crossed, the + // market was simply quiet, or the venue answered that it does not list this symbol — + // would otherwise leave that attempt standing and put the host into 30-600 s of + // backoff. The next request to the SAME host is then refused, and because one host + // serves several venues that request belongs to an unrelated trade, and is usually a + // CANDLE stage that would have worked. The candle path one function up makes exactly + // this distinction already: it clears unconditionally after its own loop, its own + // cancellation break included, and clears on `UnknownSymbol` for the stated reason + // that "one bad market would throttle every other market on that host". + // + // TWO reasons keep the record, and both are the venue's own word rather than ours: + // `Transient` is a refusal or failure it just gave us, and `RateLimited` means our + // claim was REFUSED — we recorded nothing, so clearing would erase somebody else's + // legitimate backoff. + match reason { + TickAbandon::Transient | TickAbandon::RateLimited => {} + TickAbandon::Cancelled + | TickAbandon::Deadline + | TickAbandon::Empty + | TickAbandon::UnknownSymbol + | TickAbandon::OverTickBudget + | TickAbandon::OverPageBudget => gate.clear(route.host()), + } + log::info!( + "[x] trade-replay tick stage abandoned on {}: {reason:?}", + route.host() + ); + return None; + } + }; + // The complete tick vector, stably sorted ascending and clipped to the window, BEFORE + // anything else: per-page sorting is NOT enough for the two BACKWARD-paginating venues + // (Bitget, OKX), whose concatenated pages walk backwards across every page boundary, and + // `aggregate_trades`'s own trailing arm drops a trade older than the whole series rather than + // reordering it — a tick chart missing whole minutes, with no error anywhere. + ticks.sort_by(|a, b| { + a.time_ms + .partial_cmp(&b.time_ms) + .unwrap_or(std::cmp::Ordering::Equal) + }); + ticks.retain(|t| { + t.time_ms.is_finite() + && (t.time_ms as i64) >= request.window.from_ms + && (t.time_ms as i64) <= request.window.to_ms + }); + if ticks.is_empty() { + log::info!( + "[x] trade-replay tick stage abandoned on {}: no ticks left inside the window after clipping", + route.host() + ); + return None; + } + Some(compose_ticks(request, request.address.venue, ticks)) +} + +/// Walk every slice of one tick stage, paginating each with the venue's own cursor, until the +/// stage completes, is abandoned, or either budget is spent. +/// +/// `fetch` is the injected seam — no network, no clock, no gate inside this function, which is +/// what makes it testable with a fake fetcher and a fake [`TickObserver`]. Cancellation and the +/// deadline are checked before every page, ahead of the page budget, ahead of the tick budget; +/// an empty result is recognised only once every slice has been walked with nothing collected. +/// +/// Args: +/// route: Which venue endpoint this stage answers. +/// slices: The window's own [`time_slices`] output; one venue-side pagination loop runs per +/// slice. +/// tick_budget: Ceiling on the total ticks collected across every slice. +/// page_budget: Ceiling on the total pages fetched across every slice. +/// cancelled: Answers whether the requester's window has closed. +/// expired: Answers whether this stage's own deadline has passed. +/// observer: Records the `claim`/`pace` calls this stage makes. +/// fetch: Fetches one page for a given slice and cursor. +/// +/// Returns: +/// The collected ticks in the vendor's own per-page order (unsorted across pages), or the +/// reason nothing was collected. +pub(crate) fn paginate_ticks( + route: TradeRoute, + slices: &[(i64, i64)], + tick_budget: usize, + page_budget: usize, + cancelled: impl Fn() -> bool, + expired: impl Fn() -> bool, + observer: &mut O, + mut fetch: F, +) -> TickVerdict +where + F: FnMut(i64, i64, Option) -> Result, + O: TickObserver, +{ + if slices.is_empty() { + return TickVerdict::Abandoned(TickAbandon::Empty); + } + if observer.claim(route.host()).is_err() { + return TickVerdict::Abandoned(TickAbandon::RateLimited); + } + let mut ticks: Vec = Vec::new(); + let mut pages_fetched = 0usize; + for &(slice_from, slice_to) in slices { + let mut cursor: Option = None; + loop { + if cancelled() { + return TickVerdict::Abandoned(TickAbandon::Cancelled); + } + if expired() { + return TickVerdict::Abandoned(TickAbandon::Deadline); + } + if pages_fetched >= page_budget { + return TickVerdict::Abandoned(TickAbandon::OverPageBudget); + } + observer.pace(route.host()); + let page = match fetch(slice_from, slice_to, cursor) { + Ok(page) => page, + Err(rest::FetchError::UnknownSymbol) => { + return TickVerdict::Abandoned(TickAbandon::UnknownSymbol); + } + Err(rest::FetchError::Transient(_)) => { + return TickVerdict::Abandoned(TickAbandon::Transient); + } + }; + pages_fetched += 1; + ticks.extend(page.ticks); + if ticks.len() > tick_budget { + return TickVerdict::Abandoned(TickAbandon::OverTickBudget); + } + match page.next { + Some(next_cursor) => cursor = Some(next_cursor), + None => break, + } + } + } + match ticks.is_empty() { + true => TickVerdict::Abandoned(TickAbandon::Empty), + false => TickVerdict::Ready(ticks), + } +} + +/// Build the frozen TICK series one tick stage answers with. +/// +/// `ticks` must already be globally sorted ascending and clipped to `request.window` — this +/// function does neither; [`serve_ticks`] does both before calling it. +/// +/// Args: +/// request: The request being served. +/// venue: Venue the ticks came from. +/// ticks: Trade points, ascending, already clipped to the window. +/// +/// Returns: +/// The series to hand the chart. +fn compose_ticks( + request: &TradeReplayRequest, + venue: crate::venue::Venue, + ticks: Vec, +) -> TradeReplaySeries { + let mut candles = Vec::new(); + crate::market::candles::aggregate_trades(&ticks, BAR_MS, &mut candles); + TradeReplaySeries { + source: TradeReplaySource::Ticks, + venue, + window: request.window, + tf_ms: BAR_MS, + candles, + ticks, + identity: request.identity, + } +} + +/// Answer the rows one cache write may actually carry, keyed on the series it came from. +/// +/// The SQLite isolation seam (acceptance criterion 7): a tick series' own aggregated candles must +/// NEVER reach [`write_cached_bars`], because that table is the SHARED kline cache the live +/// recorder writes too, and a candle aggregated from ticks is not a genuine exchange bar. [`serve`] +/// routes its cache write through this and nothing else, so the invariant is enforced at the one +/// call site rather than merely honoured by omission. +/// +/// Args: +/// source: Which kind of series `rows` was built for. +/// rows: The candidate rows. +/// +/// Returns: +/// `rows` unchanged for [`TradeReplaySource::Klines1m`]; an empty slice for +/// [`TradeReplaySource::Ticks`]. +pub(crate) fn rows_for_cache(source: TradeReplaySource, rows: &[ChartCandle]) -> &[ChartCandle] { + match source { + TradeReplaySource::Klines1m => rows, + TradeReplaySource::Ticks => &[], } - TradeReplayOutcome::Ready(series) } /// Build the frozen series one request answers with. @@ -349,12 +899,18 @@ fn remember_lookup( .unwrap_or_else(std::sync::PoisonError::into_inner); let hit = cache.iter().find(|(k, _)| k == key)?; Some(match hit.1.clone() { - Remembered::Ready(mut series) => { + Remembered::Ready { + mut series, + ticks_settled, + } => { // The identity belongs to the WINDOW that asked, not to the cached rows: two windows // on the same trade must not share a chart revision, or the second would be told // nothing changed and would draw nothing. series.identity = identity; - Remembered::Ready(series) + Remembered::Ready { + series, + ticks_settled, + } } Remembered::Empty => Remembered::Empty, }) @@ -362,6 +918,11 @@ fn remember_lookup( /// Remember one answered window, evicting the oldest when full. /// +/// Two independent ceilings, both enforced oldest-first: [`OUTCOME_CACHE_LEN`] bounds the number +/// of entries, [`OUTCOME_CACHE_MAX_TICKS`] bounds their combined tick count. Neither ever evicts +/// the entry this call just inserted, so a single series alone can outrun the tick ceiling +/// without being immediately discarded. +/// /// Args: /// cache: The ring. /// key: The question that was answered. @@ -379,6 +940,26 @@ fn remember_store( while cache.len() > OUTCOME_CACHE_LEN { cache.pop_front(); } + while cache.len() > 1 && total_ticks(&cache) > OUTCOME_CACHE_MAX_TICKS { + cache.pop_front(); + } +} + +/// Sum the ticks carried by every remembered entry. +/// +/// Args: +/// cache: The ring. +/// +/// Returns: +/// Combined tick count across every entry. +fn total_ticks(cache: &VecDeque<(OutcomeKey, Remembered)>) -> usize { + cache + .iter() + .map(|(_, answer)| match answer { + Remembered::Ready { series, .. } => series.ticks.len(), + Remembered::Empty => 0, + }) + .sum() } /// Read the window's bars from the shared kline cache, when it covers the window. diff --git a/crates/moon-ui-gpui/src/trade_window/mod.rs b/crates/moon-ui-gpui/src/trade_window/mod.rs index faa7ba8e..b8b2084a 100644 --- a/crates/moon-ui-gpui/src/trade_window/mod.rs +++ b/crates/moon-ui-gpui/src/trade_window/mod.rs @@ -227,6 +227,66 @@ pub(super) fn trade_labels( ) } +/// What one worker answer does to an already-drawn (or still-loading) window. +/// +/// One function rather than three predicates, so the whole state transition is pinnable by a +/// test without a GPUI harness — nothing in this repo renders GPUI in tests. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) struct Fold { + /// Whether `outcome` replaces `state` at all. A drawn window regressing to an error or an + /// empty message is never wanted, so once `state` is [`TradeWindowState::Ready`] only a + /// strictly better answer — a tick series for the exact request already on screen — is let + /// through. + pub accept: bool, + /// Whether the user's own candle mode should be restored before this outcome is drawn. + pub restore_candle_mode: bool, + /// Whether the trade's arrows and viewport should be (re)framed for this outcome. + pub frame: bool, +} + +/// Decide what one worker answer does to the window's state. +/// +/// Args: +/// state: What the window currently shows, i.e. what the LAST accepted outcome produced. +/// published_this_sequence: Whether a picture has already been framed for the fetch +/// `outcome` belongs to — `false` for a request's first outcome, `true` for its second. +/// outcome: The answer being folded in. +/// +/// Returns: +/// The decision. `accept == false` means `outcome` is dropped and `state` is left exactly as +/// it is. +pub(super) fn fold_outcome( + state: &TradeWindowState, + published_this_sequence: bool, + outcome: &TradeReplayOutcome, +) -> Fold { + let accept = match state { + // Protocol item 3 guarantees the worker never sends a second `Empty` or `Failed`, so this + // arm only ever meets a tick `Ready`, but the rule is stated for what it means rather than + // for what the worker happens to send: a chart already showing something never regresses. + TradeWindowState::Ready { .. } => matches!( + outcome, + TradeReplayOutcome::Ready(series) + if series.source == TradeReplaySource::Ticks && !series.is_empty() + ), + // Loading, Empty or Failed has nothing on screen to protect. + TradeWindowState::Loading | TradeWindowState::Empty(_) | TradeWindowState::Failed(_) => { + true + } + }; + Fold { + accept, + // Keyed on the outcome's SOURCE, never on its position in the sequence: protocol item 2's + // reopen case delivers a settled `Ticks` series as the FIRST outcome, and a position-keyed + // rule would leave a reopened window stuck in the candle stage's forced mode. + restore_candle_mode: accept + && matches!(outcome, TradeReplayOutcome::Ready(series) if series.source == TradeReplaySource::Ticks), + // The upgrade re-uses the picture already framed rather than re-framing it, so a user who + // panned while the ticks loaded is not yanked back to the trade. + frame: accept && !published_this_sequence, + } +} + /// Name one strategy through the session's own store. /// /// `as u64` because the store keys strategies by the same bits the wire carries, which is the @@ -260,6 +320,15 @@ pub(crate) struct TradeWindowView { /// user clicked. stamps: (String, String), state: TradeWindowState, + /// Whether the current sequence's picture has already been framed (arrows + viewport), so a + /// tick upgrade landing on top of it does not yank back a user who panned while it loaded. + /// Reset to `false` every [`Self::fetch`], set once `fold_outcome`'s `frame` fires. + framed_this_sequence: bool, + /// The mode the user's own chart is actually on, captured in `window.rs` before the candle + /// stage's `CANDLE_MODE_OFF` force is applied. A tick outcome restores it — see + /// [`Self::restore_candle_mode`] — because a tick series has no reason to hide behind that + /// force: it draws points, not candles. + user_candle_mode: u8, /// Monotonic dispatch counter, so a Retry supersedes an in-flight fetch instead of racing it. sequence: u64, /// What the replica said this trade carried, kept so the strategy can be named LATER. @@ -473,6 +542,7 @@ impl TradeWindowView { self.sequence = self.sequence.wrapping_add(1); let sequence = self.sequence; self.state = TradeWindowState::Loading; + self.framed_this_sequence = false; cx.notify(); let (tx, rx) = mpsc::channel(); @@ -486,22 +556,47 @@ impl TradeWindowView { }); cx.spawn(async move |this, cx| { let executor = cx.update(|cx| cx.background_executor().clone()); - // The blocking receive sits on the background executor; the worker's own job deadline - // is what bounds it, so this task cannot outlive a stalled fetch indefinitely. - let Ok(outcome) = executor.spawn(async move { rx.recv() }).await else { - return; - }; - cx.update(|cx| { - let _ = this.update(cx, |this, cx| { - this.apply(sequence, outcome, buy_utc, close_utc, cx) + // One `TradeReplayRequest` now answers with ONE or TWO outcomes before the worker + // drops `reply` — the candle stage, then an optional tick upgrade — so this task + // receives in a loop rather than once, and the drop (a `RecvError`) is its exit + // signal, exactly like the view going away is (`this.update` failing below). The + // receiver travels into the background task and back out each iteration instead of + // living behind a lock across the await, so the blocking `recv` never holds anything + // this foreground task still needs between messages. + let mut rx = rx; + loop { + let (returned_rx, received) = executor + .spawn(async move { + let received = rx.recv(); + (rx, received) + }) + .await; + rx = returned_rx; + let Ok(outcome) = received else { + return; + }; + let applied = cx.update(|cx| { + this.update(cx, |this, cx| { + this.apply(sequence, outcome, buy_utc, close_utc, cx) + }) }); - }); + // The window closed while this outcome was in flight; nothing left to fold it + // into, and no later outcome for this sequence has anywhere to land either. + if applied.is_err() { + return; + } + } }) .detach(); } /// Fold one fetch answer into the window. /// + /// A `fetch` may now deliver up to two answers on the same `sequence` — the candle stage, + /// then an optional tick upgrade — so this runs once per outcome rather than once per fetch. + /// [`fold_outcome`] carries every decision that depends on what is already on screen; this + /// method only carries it out. + /// /// Args: /// sequence: Dispatch counter the answer belongs to. /// outcome: What the worker produced. @@ -522,6 +617,10 @@ impl TradeWindowView { if sequence != self.sequence { return; } + let fold = fold_outcome(&self.state, self.framed_this_sequence, &outcome); + if !fold.accept { + return; + } self.state = match outcome { TradeReplayOutcome::Ready(series) if series.is_empty() => { TradeWindowState::Empty(TradeReplayEmpty::NoDataInWindow) @@ -536,7 +635,13 @@ impl TradeWindowView { // range: a floor of one minute for anything finer, and no `as` wrap for anything // absurdly coarse. let tf_min = (series.tf_ms / 60_000).clamp(1, i64::from(u16::MAX)) as u16; - self.publish(series, buy_utc, close_utc, cx); + if fold.restore_candle_mode { + self.restore_candle_mode(cx); + } + self.publish(series, buy_utc, close_utc, fold.frame, cx); + if fold.frame { + self.framed_this_sequence = true; + } TradeWindowState::Ready { source, tf_min } } TradeReplayOutcome::Empty(empty) => TradeWindowState::Empty(empty), @@ -552,6 +657,23 @@ impl TradeWindowView { cx.notify(); } + /// Restore the user's own candle mode once a tick series is about to be drawn. + /// + /// `window.rs` forces candles on while no tick series has arrived because a candle replay has + /// no ticks; a tick series does, so that force's reason is gone and the user's real choice — + /// including Off, a pure tick chart — comes back. + /// + /// Args: + /// cx: View context. + fn restore_candle_mode(&mut self, cx: &mut Context) { + let mode = self.user_candle_mode; + self.panel.update(cx, |panel, pcx| { + let mut view = panel.effective_candle_view(pcx); + view.mode = mode; + panel.set_candle_view(Some(view), pcx); + }); + } + /// The price scale this window's chart is set to, for its own control to state. /// /// Read off the PANEL rather than back out of the layout, so the trigger reports what this @@ -589,7 +711,8 @@ impl TradeWindowView { cx.notify(); } - /// Hand a fetched series to this window's chart and focus it on the trade. + /// Hand a fetched series to this window's chart, and focus it on the trade on the FIRST + /// publish of a sequence. /// /// Args: /// series: The frozen rows. @@ -599,27 +722,40 @@ impl TradeWindowView { /// necessarily frames the exact pair `fetch` used to build the REST request whose /// `series` this now is. /// close_utc: True-UTC exit stamp, same provenance as `buy_utc`. + /// first_publish: Whether this is the first publish of the fetch's sequence. `false` is a + /// tick upgrade landing on a picture the candle stage already framed — re-running the + /// arrows and the viewport would yank back a user who panned while it loaded, so both + /// are skipped and only the chart's own rows are replaced. /// cx: View context. fn publish( &mut self, series: TradeReplaySeries, buy_utc: i64, close_utc: i64, + first_publish: bool, cx: &mut Context, ) { // Read off the series BEFORE it is moved into the panel, exactly as `source` and `tf_min` - // are in `apply`. - let frame = frame::trade_frame( - buy_utc.saturating_mul(1_000), - close_utc.saturating_mul(1_000), - series.tf_ms, - ); + // are in `apply`. Skipped entirely on an upgrade: the frame this trade opened on is not + // recomputed, only reused. + let frame = first_publish + .then(|| { + frame::trade_frame( + buy_utc.saturating_mul(1_000), + close_utc.saturating_mul(1_000), + series.tf_ms, + ) + }) + .flatten(); // The RAW record, deliberately: correcting it here would double-correct, since B.1 // (`chartdx/trade_history_sync.rs`) already applies the axis inside // `append_trade_history_geometry`. let record = self.record.clone(); self.panel.update(cx, |panel, pcx| { panel.attach_trade_replay(Some(std::rc::Rc::new(series)), pcx); + if !first_publish { + return; + } // THE ENTRY AND EXIT ARROWS. Owning a `ChartPanel` is not enough on its own: the // marker geometry is built during the userdata pass, and that pass only ever sees the // trades handed to this layer. The live Report path fills it from the open-request diff --git a/crates/moon-ui-gpui/src/trade_window/window.rs b/crates/moon-ui-gpui/src/trade_window/window.rs index 14685f2f..1724255a 100644 --- a/crates/moon-ui-gpui/src/trade_window/window.rs +++ b/crates/moon-ui-gpui/src/trade_window/window.rs @@ -195,20 +195,30 @@ pub(crate) fn open_trade_window( // minute rows into that coarser bucket // (`moon-core/src/market/trade_replay/mod.rs`: the caller's `tf_ms` wins, and a wider one // resamples), while the caption underneath claims minutes. That is the whole of the user's - // "нужны минутные свечи, а не 5минутные". + // "I need one-minute candles, not five-minute candles." // // Only `tf_min` is forced. Starting from the EFFECTIVE settings rather than // `CandleViewCfg::default()` keeps the user's candle mode, outline width, in-zone colours // and MoonShot corridor exactly as they are on their own charts. + // Captured immediately BEFORE the candle-stage force below, so `TradeWindowView::apply` + // can hand it back to `restore_candle_mode` once a tick outcome lands and that force's + // reason is gone. A window that never gets past the candle stage never reaches the + // restore path either, and keeps the forced mode for its whole life by construction — + // which is exactly right, since a candle replay genuinely has no ticks to fall back to. + let mut user_candle_mode = 0; panel.update(cx, |panel, pcx| { let mut view = panel.effective_candle_view(pcx); view.tf_min = 1; - // The SECOND thing the effective settings can carry that makes this window useless: - // candle mode Off is a pure TICK chart, and a candle replay has no ticks. Inherited - // unchanged it would draw an empty pane under a caption naming candles — the same - // dishonesty the timeframe pin exists to remove, one step further along. A user who - // turned candles off on their live chart still asked to SEE this trade, so the window - // falls back to the shipped drawing mode rather than to nothing. + user_candle_mode = view.mode; + // THE INITIAL CANDLE-MODE FORCE. It holds until a tick outcome arrives: a fresh + // request yields candles before its optional tick upgrade, while a settled tick-cache + // hit can yield ticks as its FIRST outcome. Candle mode Off is a pure TICK chart, and + // a candle replay has no ticks. Inherited unchanged it would draw an empty pane under + // a caption naming candles — the same dishonesty the timeframe pin exists to remove, + // one step further along. A user who turned candles off on their live chart still + // asked to SEE this trade, so the window falls back to the shipped drawing mode rather + // than to nothing, until `apply` restores the real choice once a tick series is on + // offer. if view.mode == moon_core::market::candles::CANDLE_MODE_OFF { view.mode = moon_core::market::CandleViewCfg::default().mode; } @@ -238,6 +248,8 @@ pub(crate) fn open_trade_window( market: market.clone(), stamps: stamps.clone(), state: TradeWindowState::Loading, + framed_this_sequence: false, + user_candle_mode, meta, strategy_pending: !named, // Nothing searched yet; the first notification does the walk. From 00914be4ee719e676d085d3f9f66f0c8ee0e8768 Mon Sep 17 00:00:00 2001 From: kirillDevPro <113171057+kirillDevPro@users.noreply.github.com> Date: Thu, 3 Sep 2026 21:41:27 +0200 Subject: [PATCH 3/6] fix(connections): give all three text columns one width policy In Settings > Connections the Name and Group columns both grew with the window while the Key column was pinned at 200px. Flexbox splits free space evenly between equal grow factors, so on a wide window the one-word group name took as much room as the core name, which truncated beside it, and the masked key stayed cramped behind its own two icons. All three text columns now grow under one TextScaled width policy (a design value scaled on the Font delta, no floor): Name is uncapped, Key grows to a 260px cap and Group to a 140px cap. A cap rather than a rigid width, because the default 860px window is already a shrink regime where anything flex_shrink_0 is width the text columns never get back. The cap is a new ConnCol.max resolved by the same policy as the basis and applied by the table's cell as max_w. --- .../src/settings/connections/columns.rs | 170 +++++++++++++++--- .../src/settings/connections/columns/tests.rs | 99 +++++++++- .../src/settings/connections/table.rs | 7 + .../tests/theme_contract/theme.rs | 18 ++ 4 files changed, 265 insertions(+), 29 deletions(-) diff --git a/crates/moon-ui-gpui/src/settings/connections/columns.rs b/crates/moon-ui-gpui/src/settings/connections/columns.rs index 211327ea..432b0fe2 100644 --- a/crates/moon-ui-gpui/src/settings/connections/columns.rs +++ b/crates/moon-ui-gpui/src/settings/connections/columns.rs @@ -16,6 +16,18 @@ //! the two kinds genuinely coexist in one row, and scaling all of them would be as wrong as //! scaling none. //! +//! The three columns holding user-typed text -- name, key, group -- all GROW, and two of them +//! carry a [`ConnCol::max`]. That is the second thing stated exactly once here, and it is a +//! response to the row having TWO regimes rather than one. `table.rs::cell` gives a non-growing +//! column `flex_shrink_0` as well, so `grow: false` means rigid in both directions; the rigid +//! columns plus the inset, the scrollbar gutter and the twelve gaps come to about 661px of the +//! 824px body the DEFAULT 860px Settings window leaves, against about 487px of text-column bases. +//! So the default window is a SHRINK regime -- the caps are inert there and the BASES decide who +//! keeps what -- while a wide window is a GROW regime, where the caps are what stop the key and +//! the group from spending width they have nothing readable to put in it. Both dials are needed, +//! and neither is the other's fallback. All three carry ONE width policy, which is what makes +//! their 150 > 140 > 85 ordering a property of the literals rather than of the Font slider. +//! //! Pure and GPUI-free on purpose -- its sibling test file can assert the header and the rows agree //! without a window. @@ -60,6 +72,24 @@ pub(super) enum ConnColWidth { Raw, /// A design reference the Micro dropdown trigger scales by `tokens.font(10) / 10`. MicroTrigger, + /// A design reference stated in CHARACTERS of a `MoonInput::small()`, scaled by the same + /// `tokens.font(10) / 10` ratio. Carried by all three text columns -- `h-name`, `h-key` and + /// `h-group`. + /// + /// The small input renders its text at `tokens.font(10)` (MoonUI + /// `MoonInputMetrics::base_for_size(Size::Small)`), so a width that means "this many readable + /// characters" is only true at the default Font setting unless it follows that ratio -- at the + /// shipped +3 delta a raw 140px cell holds about 18 characters where the design reference + /// promised 24. It shares [`MicroTriggerMetrics::scale`] because that IS the Font ratio, and + /// deliberately NOT the floor beside it: `min_width` is the dropdown TRIGGER's own minimum and + /// means nothing to an input. + /// + /// The three text columns share it for a second reason: their shrink order at a narrow window + /// is decided by their RESOLVED bases, so a mixed policy would let one Font setting overtake + /// another column. `repair_ui_font_delta` (`moon-core/src/config/schema.rs`) deliberately + /// preserves any finite hand-edited delta, well past the slider's +6, so "no reachable setting + /// reverses it" is only true when the comparison is scale-free. + TextScaled, } /// One column of the core table, as both the header and the row read it. @@ -74,10 +104,33 @@ pub(super) struct ConnCol { /// Flex basis in rendered pixels. Authoritative: every cell also carries `min_w_0()`, so a /// wider child paints over its neighbour instead of pushing it. pub(super) basis: f32, - /// Whether the column absorbs free space. Only columns holding user-typed text that - /// TRUNCATES do -- `h-name` and `h-group`. `h-key` deliberately does not: its content is - /// masked, so extra width buys more dots and nothing readable. + /// Whether the column absorbs free space. The three columns carrying user-typed text do -- + /// `h-name`, `h-key` and `h-group` -- and only `h-name` does so without a [`ConnCol::max`]. + /// + /// It is also what decides whether the column can SHRINK: `table.rs::cell` gives a + /// non-growing column `flex_shrink_0`, so `grow: false` means "this width is rigid in BOTH + /// directions". The default 860px Settings window does not fit this row, so every column that + /// can afford to give way there has to be ABLE to -- see [`ConnCol::max`] for the arithmetic. pub(super) grow: bool, + /// Upper bound on a GROWING column, in the same units as [`ConnCol::basis`] and resolved by + /// the same [`ConnCol::width`] policy. `None` means "grow without limit"; only `h-name` has it. + /// + /// GROW-WITH-A-CAP rather than `grow: false` is the load-bearing choice here, and the 860px + /// window is why. `settings/render.rs` leaves an 824px body inside its 18px padding there; the + /// rigid columns -- two checkboxes, the bundle field, the colour picker, the three Micro + /// dropdowns, the delete, reconnect and status glyphs -- plus the table inset, the scrollbar + /// gutter and twelve gaps take about 661px of it at the SHIPPED Font delta of +3, leaving + /// roughly 163px for three text columns whose bases resolve to about 487px. The default window + /// is therefore a SHRINK regime, and anything pinned `flex_shrink_0` in it is width the text + /// columns can never get back: a fixed 260px key would simply BE the widest column on a + /// default install while the name column collapsed to nothing. Growable, all three give way in + /// proportion to their RESOLVED bases instead -- which is why those bases are ordered, and why + /// all three carry the same [`ConnColWidth::TextScaled`] policy so that the ordering cannot + /// depend on the Font setting. + /// + /// The cap governs the other regime. Widen the window and Taffy freezes each capped item at + /// its cap, then hands the remaining free space to the only uncapped one -- `h-name`. + pub(super) max: Option, /// How [`ConnCol::basis`] becomes a rendered width. pub(super) width: ConnColWidth, /// Where the header label and the row control sit inside the cell. @@ -100,6 +153,7 @@ const CONN_COLS: [ConnCol; 13] = [ tip: Some("conn.tip.act"), basis: 34.0, grow: false, + max: None, width: ConnColWidth::Raw, align: ConnColAlign::Center, head_pad: 0.0, @@ -110,34 +164,52 @@ const CONN_COLS: [ConnCol; 13] = [ tip: Some("conn.tip.win"), basis: 34.0, grow: false, + max: None, width: ConnColWidth::Raw, align: ConnColAlign::Center, head_pad: 0.0, }, + // The ONLY uncapped column, which is what makes it the widest of the three on any window with + // free space to give: `h-key` and `h-group` freeze at their caps and Taffy hands what is left + // to whatever is still growable. Its 150 is also the largest of the three bases, and because + // all three share one width policy that ordering cannot be reversed by any Font setting -- so + // it stays the widest of them on a window too narrow for the table as well, which the default + // 860px one is. ConnCol { id: "h-name", label: Some("conn.col.name"), tip: Some("conn.tip.name"), basis: 150.0, grow: true, - width: ConnColWidth::Raw, + max: None, + width: ConnColWidth::TextScaled, align: ConnColAlign::Left, head_pad: 8.0, }, - // FIXED, not growing, at the basis it always had. The key field is MASKED and MoonUI draws - // one bullet per character of a variable-length key, so there is no "width of the masked - // value" to size to -- growth simply handed the column every spare pixel, ~480px of a 1791px - // window spent on identical dots while `Имя` and `Группа` truncated real text beside them. - // 200 keeps a usable text viewport next to the input's mask-toggle and clear affixes and the - // sibling Paste glyph (`table.rs::paste_key_affix`); a longer key scrolls inside the field, - // which is what it did before and what a masked field can afford. + // Capped rather than uncapped, because the key field is MASKED: MoonUI draws one bullet per + // character of a variable-length key, so there is no "width of the masked value" to size to, + // and uncapped growth simply handed the column every spare pixel -- ~480px of a 1791px window + // spent on identical dots while the name and group columns truncated real text beside them. + // + // 260 is what the field is actually asking for. At 200 the text viewport left over after the + // input's mask-toggle and clear affixes, its own padding and the sibling Paste glyph + // (`table.rs::paste_key_affix`) was still cramped, and that -- the VIEWPORT, not the key's + // length -- is what the user reported; a longer key scrolls inside the field and always did. + // + // The 140 basis is deliberately far BELOW the cap and below `h-name`'s. Shrinkage is + // proportional to the resolved basis, so on a window too narrow to hold the table -- and + // 860px, the default, is exactly such a window -- the bases alone decide the order the three + // text columns end up in, and 140 puts the key second. Pinning this column at a rigid 260 + // instead would have made the masked key the widest thing on screen there while the name + // column collapsed. ConnCol { id: "h-key", label: Some("conn.col.key"), tip: Some("conn.tip.key"), - basis: 200.0, - grow: false, - width: ConnColWidth::Raw, + basis: 140.0, + grow: true, + max: Some(260.0), + width: ConnColWidth::TextScaled, align: ConnColAlign::Left, head_pad: 8.0, }, @@ -150,6 +222,7 @@ const CONN_COLS: [ConnCol; 13] = [ tip: Some("conn.tip.proto"), basis: 52.0, grow: false, + max: None, width: ConnColWidth::MicroTrigger, align: ConnColAlign::Center, head_pad: 0.0, @@ -160,19 +233,30 @@ const CONN_COLS: [ConnCol; 13] = [ tip: Some("conn.tip.preset"), basis: 72.0, grow: false, + max: None, width: ConnColWidth::MicroTrigger, align: ConnColAlign::Center, head_pad: 0.0, }, - // Grows with `h-name`: both hold user-typed text that truncates, and the width the masked - // key gave up is exactly what they were missing. + // The lowest basis AND the lowest cap of the three, because a group name is one word. The 85 + // basis is what puts it last when the window is too narrow for the row; it is only ever + // compared against the other two, which share its policy, so the ordering is a property of the + // three literals and holds at every Font setting rather than only inside the slider's range. + // + // The cap: 140 is 126px of text viewport once the small input's 7px paddings are removed, + // about 24 lowercase glyphs at the design-reference font size -- comfortable for a name like + // "default" and for anything a user would actually type, and nothing like the ~420px this + // column was taking of a 1791px window while the name column truncated beside it. Uncapped + // growth here is what made the two growing columns split the free width evenly when only one + // of them holds a name long enough to need it. ConnCol { id: "h-group", label: Some("conn.col.group"), tip: Some("conn.tip.group"), - basis: 110.0, + basis: 85.0, grow: true, - width: ConnColWidth::Raw, + max: Some(140.0), + width: ConnColWidth::TextScaled, align: ConnColAlign::Left, head_pad: 8.0, }, @@ -182,6 +266,7 @@ const CONN_COLS: [ConnCol; 13] = [ tip: Some("conn.tip.bundle"), basis: 96.0, grow: false, + max: None, width: ConnColWidth::Raw, align: ConnColAlign::Left, head_pad: 8.0, @@ -192,6 +277,7 @@ const CONN_COLS: [ConnCol; 13] = [ tip: Some("conn.tip.flags"), basis: 52.0, grow: false, + max: None, width: ConnColWidth::MicroTrigger, align: ConnColAlign::Center, head_pad: 0.0, @@ -204,6 +290,7 @@ const CONN_COLS: [ConnCol; 13] = [ tip: Some("conn.tip.color"), basis: 128.0, grow: false, + max: None, width: ConnColWidth::Raw, align: ConnColAlign::Center, head_pad: 0.0, @@ -214,6 +301,7 @@ const CONN_COLS: [ConnCol; 13] = [ tip: None, basis: 24.0, grow: false, + max: None, width: ConnColWidth::Raw, align: ConnColAlign::Center, head_pad: 0.0, @@ -224,6 +312,7 @@ const CONN_COLS: [ConnCol; 13] = [ tip: None, basis: 24.0, grow: false, + max: None, width: ConnColWidth::Raw, align: ConnColAlign::Center, head_pad: 0.0, @@ -234,6 +323,7 @@ const CONN_COLS: [ConnCol; 13] = [ tip: None, basis: 16.0, grow: false, + max: None, width: ConnColWidth::Raw, align: ConnColAlign::Center, head_pad: 0.0, @@ -250,6 +340,10 @@ const CONN_COLS: [ConnCol; 13] = [ #[derive(Clone, Copy, Debug)] pub(super) struct MicroTriggerMetrics { /// Multiplier from a design-reference trigger width to its rendered width. + /// + /// It is `tokens.font(10) / 10`, the Font-slider ratio itself, which is why + /// [`ConnColWidth::TextScaled`] reads it too: a `MoonInput::small()` sizes its text at the + /// same `tokens.font(10)`. pub(super) scale: f32, /// Floor MoonUI clamps a scaled trigger up to. pub(super) min_width: f32, @@ -307,12 +401,46 @@ impl ConnColId { /// The width both the header cell and the row cell are laid out at. pub(super) fn width(self, micro: MicroTriggerMetrics) -> f32 { let spec = self.spec(); - match spec.width { - ConnColWidth::Raw => spec.basis, + Self::resolve(spec.width, spec.basis, micro) + } + + /// Upper bound this column may grow to, in pixels, or `None` for an uncapped one. + /// + /// Read by `table::SettingsView::cell` as `max_w`, which is what stops a growing column from + /// taking free width it has no readable content to spend. + /// + /// Args: + /// micro: The Micro dropdown trigger's rendered-width inputs, from + /// `table::micro_trigger_metrics`. + /// + /// Returns: + /// The cap both the header cell and the row cell are laid out against, if there is one. + pub(super) fn max_width(self, micro: MicroTriggerMetrics) -> Option { + let spec = self.spec(); + // Same policy as the basis on purpose: a cap in different units from the width it bounds + // is a cap that stops meaning what it says the moment a slider moves. + spec.max.map(|max| Self::resolve(spec.width, max, micro)) + } + + /// Turn one design reference into rendered pixels under a column's width policy. + /// + /// Args: + /// policy: The column's [`ConnColWidth`]. + /// reference: A basis or a cap, in that policy's units. + /// micro: The Micro dropdown trigger's rendered-width inputs. + /// + /// Returns: + /// The rendered pixel value. + fn resolve(policy: ConnColWidth, reference: f32, micro: MicroTriggerMetrics) -> f32 { + match policy { + ConnColWidth::Raw => reference, // MIRRORS MoonUI's own `max(scaled(basis), minimum_width)`. Dropping the floor let // a trigger outgrow its column at a large `ui_scale`; `min_w_0` would then have it // paint across its neighbour instead of moving it -- misaligned either way. - ConnColWidth::MicroTrigger => (spec.basis * micro.scale).max(micro.min_width), + ConnColWidth::MicroTrigger => (reference * micro.scale).max(micro.min_width), + // No floor: the floor beside `scale` is the dropdown trigger's own minimum, and an + // input has nothing to do with it. + ConnColWidth::TextScaled => reference * micro.scale, } } diff --git a/crates/moon-ui-gpui/src/settings/connections/columns/tests.rs b/crates/moon-ui-gpui/src/settings/connections/columns/tests.rs index efddf812..89a52158 100644 --- a/crates/moon-ui-gpui/src/settings/connections/columns/tests.rs +++ b/crates/moon-ui-gpui/src/settings/connections/columns/tests.rs @@ -46,24 +46,32 @@ fn indent_parts_match_the_header_inset() { ); } -/// `columns.rs:CONN_COLS` must preserve the three Micro-trigger columns. Replacing `h-preset`'s -/// 72px `MicroTrigger` policy with a 92px `Raw` width makes the control stop scaling with its -/// rendered trigger and lets the header and server row disagree at non-default font scales. +/// `columns.rs:CONN_COLS` must preserve each column's declared scaling policy. Replacing +/// any of `h-name`, `h-key` or `h-group`'s `TextScaled` policies with `Raw`, or `h-preset`'s +/// `MicroTrigger` policy with `Raw`, makes the header and server row disagree with their controls +/// at non-default font scales. #[test] fn widths_follow_the_frozen_per_column_policy() { const MICRO_COLUMNS: [ConnColId; 3] = [ConnColId::Proto, ConnColId::Preset, ConnColId::Data]; + const TEXT_SCALED_COLUMNS: [ConnColId; 3] = [ConnColId::Name, ConnColId::Key, ConnColId::Group]; const SCALES: [f32; 3] = [0.75, 1.0, 1.3]; for column in ConnColId::ALL { let basis = column.spec().basis; let is_micro = MICRO_COLUMNS.contains(&column); + let is_text_scaled = TEXT_SCALED_COLUMNS.contains(&column); let expected_policy = if is_micro { ConnColWidth::MicroTrigger + } else if is_text_scaled { + ConnColWidth::TextScaled } else { ConnColWidth::Raw }; for scale in SCALES { - let expected_width = if is_micro { basis * scale } else { basis }; + let expected_width = match expected_policy { + ConnColWidth::Raw => basis, + ConnColWidth::MicroTrigger | ConnColWidth::TextScaled => basis * scale, + }; assert_eq!( column.width(MicroTriggerMetrics { scale, @@ -82,9 +90,8 @@ fn widths_follow_the_frozen_per_column_policy() { } } -/// `columns.rs:CONN_COLS` must let only Name and Group absorb spare width: Key is excluded because -/// its masked content has no readable length to reward with width. Turning `h-key.grow` back on -/// wastes space on dots, while removing growth from Name or Group truncates user-entered text; +/// `columns.rs:CONN_COLS` must let Name, Key and Group absorb spare width without making any of +/// them rigid at the default window size. Removing growth from one truncates user-entered text; /// every visible header label also needs help text. #[test] fn growth_and_tooltips_match_the_text_column_contract() { @@ -92,7 +99,7 @@ fn growth_and_tooltips_match_the_text_column_contract() { .into_iter() .filter(|column| column.spec().grow) .collect(); - assert_eq!(growing, [ConnColId::Name, ConnColId::Group]); + assert_eq!(growing, [ConnColId::Name, ConnColId::Key, ConnColId::Group]); for column in ConnColId::ALL { let spec = column.spec(); @@ -104,3 +111,79 @@ fn growth_and_tooltips_match_the_text_column_contract() { } } } + +/// `columns.rs:CONN_COLS` must cap only Key and Group through their own policies. Removing either +/// cap lets that column consume wide-window space, while resolving either cap as raw pixels makes +/// its readable character count shrink when the user raises the Font setting. +#[test] +fn caps_match_the_text_column_contract_at_each_font_scale() { + const MICRO_COLUMNS: [ConnColId; 3] = [ConnColId::Proto, ConnColId::Preset, ConnColId::Data]; + const TEXT_SCALED_COLUMNS: [ConnColId; 3] = [ConnColId::Name, ConnColId::Key, ConnColId::Group]; + const SCALES: [f32; 3] = [0.75, 1.0, 1.3]; + + for column in ConnColId::ALL { + let expected_cap = match column { + ConnColId::Key => Some(260.0), + ConnColId::Group => Some(140.0), + _ => None, + }; + let expected_policy = if MICRO_COLUMNS.contains(&column) { + ConnColWidth::MicroTrigger + } else if TEXT_SCALED_COLUMNS.contains(&column) { + ConnColWidth::TextScaled + } else { + ConnColWidth::Raw + }; + + assert_eq!(column.spec().max, expected_cap, "{column:?} cap reference"); + for scale in SCALES { + let metrics = MicroTriggerMetrics { + scale, + min_width: 0.0, + }; + let expected_max = expected_cap.map(|cap| match expected_policy { + ConnColWidth::Raw => cap, + ConnColWidth::MicroTrigger | ConnColWidth::TextScaled => cap * scale, + }); + assert_eq!( + column.max_width(metrics), + expected_max, + "{column:?} cap at scale {scale}" + ); + + if let Some(expected_max) = expected_max { + let expected_basis = column.width(metrics); + assert!( + expected_max > expected_basis, + "{column:?} cap must exceed its basis at scale {scale}" + ); + } + } + } +} + +/// `columns.rs:CONN_COLS` must leave Name as the only uncapped growing column and resolve Name +/// wider than Key wider than Group at every finite Font scale. Changing `h-key` from `TextScaled` +/// to `Raw` makes Group outrank the masked Key at a hand-edited +10 Font delta, leaving less room +/// for the user-entered name. +#[test] +fn name_is_the_only_uncapped_growing_column_with_the_widest_narrow_width() { + let uncapped_growing: Vec<_> = ConnColId::ALL + .into_iter() + .filter(|column| column.spec().grow && column.spec().max.is_none()) + .collect(); + assert_eq!(uncapped_growing, [ConnColId::Name]); + + const SCALES: [f32; 5] = [0.75, 1.0, 1.3, 1.6, 2.0]; + for scale in SCALES { + let metrics = MicroTriggerMetrics { + scale, + min_width: 0.0, + }; + assert!( + ConnColId::Name.width(metrics) > ConnColId::Key.width(metrics) + && ConnColId::Key.width(metrics) > ConnColId::Group.width(metrics), + "Name, Key and Group must shrink in resolved-width order at scale {scale}" + ); + } +} diff --git a/crates/moon-ui-gpui/src/settings/connections/table.rs b/crates/moon-ui-gpui/src/settings/connections/table.rs index f0d113da..0df29f97 100644 --- a/crates/moon-ui-gpui/src/settings/connections/table.rs +++ b/crates/moon-ui-gpui/src/settings/connections/table.rs @@ -1039,6 +1039,13 @@ impl SettingsView { } else { d }; + // A cap, where a column has one, bounds growth WITHOUT freezing the column: it still + // shrinks when the window is too narrow for the row, which `flex_grow_0().flex_shrink_0()` + // below would not. + let d = match col.max_width(micro) { + Some(max) => d.max_w(px(max)), + None => d, + }; if spec.grow { d.flex_grow_1() } else { diff --git a/crates/moon-ui-gpui/tests/theme_contract/theme.rs b/crates/moon-ui-gpui/tests/theme_contract/theme.rs index 9244e0c2..b8c93807 100644 --- a/crates/moon-ui-gpui/tests/theme_contract/theme.rs +++ b/crates/moon-ui-gpui/tests/theme_contract/theme.rs @@ -383,3 +383,21 @@ fn popover_contents_do_not_paint_a_second_surface() { ); } } + +/// `settings/connections/table.rs:SettingsView::cell` must apply a column's resolved cap through +/// `max_w`. Deleting that match lets Group consume wide-window space past its 140px cap and +/// truncates the uncapped Name field. +#[test] +fn connections_cells_apply_their_resolved_growth_cap() { + let table = read_src("settings/connections/table.rs"); + let cell = code_only(braced_body( + &table, + "fn cell(col: ConnColId, micro: MicroTriggerMetrics) -> Div", + )); + + assert!( + cell.contains("let d = match col.max_width(micro) {") + && cell.contains("Some(max) => d.max_w(px(max)),"), + "SettingsView::cell must resolve ConnColId::max_width and apply it with Div::max_w" + ); +} From b6ab14eff021196d8badc277d5408b4831cd0417 Mon Sep 17 00:00:00 2001 From: kirillDevPro <113171057+kirillDevPro@users.noreply.github.com> Date: Thu, 3 Sep 2026 22:27:41 +0200 Subject: [PATCH 4/6] perf(db): make the Report row pass and distinct_strategies index-usable Two read paths in report_read.rs paid for a full sort or a full table read on every call. The Report row pass on the default DESC sort carried a redundant leading `(expr) IS NULL` term that kept SQLite off the index and forced a SORT over the whole source; the captured default-sort statement took 4.8 s and now takes 2.8 ms, -51% per call over a whole replayed run. distinct_strategies scanned the report once per arm; split into a normal-strategy arm and a liquidation arm over one WHERE it is -63% (1.81 s to 0.67 s). LIMIT is applied per source before the Rust merge, and SQL defines no order among rows tied on every ORDER BY term, so which tied trade survived the cut was whatever plan SQLite picked. The DESC arm now appends the source's primary key as a tie-break, which makes that order total: measured 0 of 25 limits differing with the tie-break against 20 of 25 without it. The db_read_timing example gains an EXPLAIN QUERY PLAN mode that registers stub scalar functions for the crate's own SQL functions, so plans resolve for every statement; gen_klines and tools/gen_replica.py build a realistic replica for the timing runs. No schema change: the candidate index was measured and deferred because its write cost could not be pinned down on this machine. --- crates/moon-core/examples/analytics_timing.rs | 254 ---- crates/moon-core/examples/db_read_timing.rs | 1180 +++++++++++++++++ crates/moon-core/examples/gen_klines.rs | 227 ++++ crates/moon-core/src/db/analytics/tests.rs | 62 + crates/moon-core/src/db/coin_lists/mod.rs | 55 +- crates/moon-core/src/db/integrity/mod.rs | 1 + crates/moon-core/src/db/mod.rs | 8 +- crates/moon-core/src/db/report_read.rs | 92 +- crates/moon-core/src/db/report_read/tests.rs | 128 +- crates/moon-core/src/db/trace.rs | 89 ++ .../moon-core/src/db/tuner/strategy_read.rs | 1 + crates/moon-core/src/db/valuation/mod.rs | 2 + crates/moon-core/src/market/kline_cache.rs | 4 + crates/moon-core/src/strat_db/mod.rs | 1 + crates/moon-core/src/strat_db/stats.rs | 1 + tools/gen_replica.py | 706 ++++++++-- 16 files changed, 2467 insertions(+), 344 deletions(-) delete mode 100644 crates/moon-core/examples/analytics_timing.rs create mode 100644 crates/moon-core/examples/db_read_timing.rs create mode 100644 crates/moon-core/examples/gen_klines.rs create mode 100644 crates/moon-core/src/db/trace.rs diff --git a/crates/moon-core/examples/analytics_timing.rs b/crates/moon-core/examples/analytics_timing.rs deleted file mode 100644 index 73529229..00000000 --- a/crates/moon-core/examples/analytics_timing.rs +++ /dev/null @@ -1,254 +0,0 @@ -//! Time the Analytics report reads against a throwaway synthetic replica. -//! -//! Measurement instrument for the "Analytics and the tuner take seconds to open" work: it -//! reproduces the exact read sequence one window open performs, so the wait can be attributed -//! to a named query instead of guessed at. The caller must supply a disposable data root; -//! `set_data_dir_override` makes this process resolve its data paths beneath that root. -//! -//! Usage: -//! cargo run --release -p moon-core --example analytics_timing -- [repeats] -//! -//! Build the data root first with `tools/gen_replica.py`, which writes a synthetic -//! `data/reports.sqlite` and `data/strategies.sqlite` at a chosen row and core count. - -use std::path::PathBuf; -use std::time::Instant; - -use moon_core::db::ProfitScope; -use moon_core::db::analytics::{ - GroupStat, Query, strategy_base_data, summary_data, undated_closes, -}; - -/// Run one closure `repeats` times and print its best and median wall-clock cost. -/// -/// Args: -/// label: ASCII name printed in the report column. -/// repeats: At least two timed runs; the first is reported separately as the cold one. -/// body: Work under measurement. -fn time(label: &str, repeats: usize, mut body: impl FnMut() -> T) { - let mut samples = Vec::with_capacity(repeats); - for _ in 0..repeats { - let started = Instant::now(); - let value = body(); - samples.push(started.elapsed().as_secs_f64() * 1000.0); - drop(value); - } - let cold = samples[0]; - let mut warm = samples[1..].to_vec(); - warm.sort_by(f64::total_cmp); - let median = warm.get(warm.len() / 2).copied().unwrap_or(cold); - println!("{label:<28} cold {cold:>8.1} ms warm-median {median:>8.1} ms"); -} - -/// Build the query one open Analytics window issues for a whole-history period. -/// -/// Args: -/// from: Inclusive period start in UTC seconds, or a negative all-history sentinel. -/// to: Exclusive period end in UTC seconds. -/// -/// Returns: -/// The default-filter query every tab shares. -fn query(from: i64, to: i64) -> Query { - Query { - axis: moon_core::db::ReportAxis::from_measured(Default::default(), chrono_tz::UTC), - previous_period_basis: Default::default(), - from, - to, - cores: Vec::new(), - side: Default::default(), - emulator: None, - strategies: Vec::new(), - strategy_name_mask: String::new(), - metric: Default::default(), - valuation: Default::default(), - prefer_usdt: false, - } -} - -/// Time selected raw SQL shapes to attribute the cost of Analytics reads. -/// -/// Args: -/// from: Inclusive period start, resolved from the all-history sentinel by the caller. -/// to: Exclusive period end. -/// name: ASCII period label. -/// repeats: Timed runs per shape. -fn probe_sql(from: i64, to: i64, name: &str, repeats: usize) { - let conn = match moon_core::db::open_reader() { - Ok(conn) => conn, - Err(error) => { - println!(" raw probes skipped: {error:?}"); - return; - } - }; - let from = if from < 0 { 1 } else { from }; - let period = "closedate >= ?1 AND closedate < ?2 AND closedate > 0 AND COALESCE(deleted,0) = 0"; - let count: i64 = conn - .query_row( - &format!("SELECT COUNT(*) FROM orders_rep WHERE {period}"), - rusqlite::params![from, to], - |row| row.get(0), - ) - .unwrap_or(-1); - println!(" rows in period: {count}"); - time(&format!(" raw COUNT(*) [{name}]"), repeats, || { - let value: i64 = conn - .query_row( - &format!("SELECT COUNT(*) FROM orders_rep WHERE {period}"), - rusqlite::params![from, to], - |row| row.get(0), - ) - .unwrap_or(-1); - value - }); - time(&format!(" raw group aggregate [{name}]"), repeats, || { - let mut stmt = conn - .prepare(&format!( - "SELECT CAST(strategyid AS TEXT) || '@' || CAST(core_uid AS TEXT) AS k, - SUM(profitbtc), COUNT(*) - FROM orders_rep WHERE {period} GROUP BY k ORDER BY 2 DESC, k" - )) - .expect("probe aggregate prepares"); - let rows = stmt - .query_map(rusqlite::params![from, to], |row| row.get::<_, i64>(2)) - .expect("probe aggregate runs") - .count(); - rows - }); - time(&format!(" raw full projection [{name}]"), repeats, || { - let mut stmt = conn - .prepare(&format!( - "SELECT closedate, buydate, profitbtc, spentbtc, core_uid, core_name, coin, - strategyid, isshort, emulator, basecurrency, boughtq, buyprice, - sellprice, sellreason - FROM orders_rep WHERE {period} ORDER BY closedate" - )) - .expect("probe projection prepares"); - let rows = stmt - .query_map(rusqlite::params![from, to], |row| row.get::<_, i64>(0)) - .expect("probe projection runs") - .count(); - rows - }); -} - -/// Print the compared fields of every strategy-base group row in source order. -/// -/// The A-B side of this instrument: run it before and after a change to the enrichment path and -/// diff the two dumps. Order is printed as it arrives, so a reordering shows up as a diff too. -/// -/// Args: -/// from: Inclusive period start, or the all-history sentinel. -/// to: Exclusive period end. -/// name: ASCII period label. -fn dump(from: i64, to: i64, name: &str) { - let q = query(from, to); - let read = strategy_base_data(&q, false); - let line = |kind: &str, index: usize, group: &GroupStat| { - println!( - "{name}|{kind}|{index}|{}|{}|{}|{}|{}|{:?}|{}|{:.6}|{}|{:.6}|{:.6}|{:.6}|{}|{}|{}|{:.6}|{:?}", - group.key, - group.name, - group.kind, - group.core, - group.cores_n, - group.alive, - group.n, - group.profit, - group.wins, - group.pf, - group.best, - group.worst, - group.lastedit, - group.bl, - group.wl, - group.raw_profit, - group.quote, - ); - }; - match read.data { - Ok(ProfitScope::Comparable { unit, data }) => { - println!( - "{name}|unit|{unit:?}|trades={}|{}|{}", - data.trades, data.from, data.to - ); - for (index, group) in data.strategies.iter().enumerate() { - line("strategy", index, group); - } - for (index, group) in data.coins.iter().enumerate() { - line("coin", index, group); - } - } - Ok(ProfitScope::Empty(data)) => println!("{name}|empty|trades={}", data.trades), - Ok(ProfitScope::Split(_)) => println!("{name}|split"), - Err(error) => println!("{name}|error|{error:?}"), - } -} - -/// Run timing measurements or a deterministic group dump for the supplied disposable replica. -fn main() { - let mut args = std::env::args().skip(1); - let root = PathBuf::from( - args.next() - .expect("usage: analytics_timing [repeats]"), - ); - let repeats = args - .next() - .and_then(|value| value.parse::().ok()) - .unwrap_or(5) - .max(2); - assert!( - moon_core::config::paths::set_data_dir_override(root.clone()), - "the data root must be installed before any path resolves" - ); - // `open_reader` refuses without the process-lifetime lease the application takes at startup. - let _permit = moon_core::db::report_recovery::prepare(); - println!("data root: {}", root.display()); - println!( - "reports: {}", - moon_core::config::paths::reports_db_path().display() - ); - - let end = 1_780_000_000i64; - let dump_only = std::env::args().any(|value| value == "--dump"); - let periods: [(&str, i64, i64); 3] = [ - ("month", end - 30 * 86_400, end), - ("year", end - 365 * 86_400, end), - ("all", -1, end), - ]; - - if dump_only { - for (name, from, to) in periods { - dump(from, to, name); - } - return; - } - - for (name, from, to) in periods { - println!("\n=== period: {name} ==="); - let q = query(from, to); - let probe = summary_data(&q, false); - println!( - " summary outcome: {}", - match &probe.data { - Ok(_) => "ok".to_string(), - Err(error) => format!("{error:?}"), - } - ); - drop(probe); - time(&format!("summary_data [{name}]"), repeats, || { - summary_data(&q, false) - }); - time(&format!("strategy_base_data [{name}]"), repeats, || { - strategy_base_data(&q, false) - }); - time(&format!("open then tuner [{name}]"), repeats, || { - let first = summary_data(&q, false); - let second = strategy_base_data(&q, false); - (first, second) - }); - time(&format!("undated_closes [{name}]"), repeats, || { - undated_closes(&q) - }); - probe_sql(from, to, name, repeats); - } -} diff --git a/crates/moon-core/examples/db_read_timing.rs b/crates/moon-core/examples/db_read_timing.rs new file mode 100644 index 00000000..552e0cc0 --- /dev/null +++ b/crates/moon-core/examples/db_read_timing.rs @@ -0,0 +1,1180 @@ +//! Time and dump every UI-visible SQLite read the terminal makes against a throwaway synthetic +//! replica, calling exactly what the UI calls with the UI's own default arguments. +//! +//! Measurement instrument for attributing Analytics/Report/tuner wait time to a named query +//! instead of guessing at it. Pairs with `db::trace`: this binary is the one place in the tree +//! that installs the read profiler, so every `open_reader`/`open_strategies`/`open_ro`/ +//! `KlineCache::open` connection born after that call reports its statement timings here. +//! +//! The caller must supply a disposable data root; `set_data_dir_override` makes this process +//! resolve its data paths beneath it. Build one with `tools/gen_replica.py +//! [span_days]`, which also writes the `data/fixture.json` manifest this harness reads. +//! +//! Usage: +//! cargo run --release -p moon-core --example db_read_timing -- [repeats] [--plan] [--dump] + +use std::collections::HashMap; +use std::path::PathBuf; +use std::sync::Mutex; +use std::time::Instant; + +use moon_core::db::analytics::{GroupStat, Query, calendar_data, strategy_base_data, summary_data}; +use moon_core::db::trace::{ProfiledStatement, install_read_profiler}; +use moon_core::db::tuner::threshold_search::{SearchHandle, SearchParams}; +use moon_core::db::tuner::{FIELDS, TimeAxes, Variant}; +use moon_core::db::valuation::ValuationMode; +use moon_core::db::{ProfitMetric, ReportFilter, RowScope}; +use moon_core::db::{ProfitScope, ReadFail}; + +// --------------------------------------------------------------------------------------------- +// The profiler sink. `install_read_profiler` takes a plain `fn`, so all state it can touch is +// process-global. A `Mutex>` rather than a thread-local: `KlineCache::open` moves its +// connection into a worker thread, and a thread-local sink would silently never see its +// statements. +// --------------------------------------------------------------------------------------------- + +static CAPTURED: Mutex> = Mutex::new(Vec::new()); + +/// Append one connection-local PROFILE event to the process-wide harness sink. +fn record_stmt(stmt: ProfiledStatement) { + CAPTURED + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .push(stmt); +} + +/// Take and clear everything captured since the last drain. +fn drain_captured() -> Vec { + std::mem::take( + &mut *CAPTURED + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()), + ) +} + +// --------------------------------------------------------------------------------------------- +// Fixture manifest — H1's contract, frozen in the spec. H2 never guesses these. +// --------------------------------------------------------------------------------------------- + +/// Shape and provenance recorded by `tools/gen_replica.py` for one disposable fixture. +#[derive(serde::Deserialize, Debug)] +struct FixtureManifest { + rows: i64, + cores: i64, + strategy_groups: i64, + coins: i64, + span_days: i64, + reports_bytes: i64, + analyzed: bool, + seed: i64, +} + +/// Read the fixture manifest below `root`, printing an actionable failure before exiting on error. +fn read_manifest(root: &std::path::Path) -> FixtureManifest { + let path = root.join("data").join("fixture.json"); + let text = std::fs::read_to_string(&path).unwrap_or_else(|error| { + eprintln!( + "[FAIL] fixture manifest unreadable at {}: {error}", + path.display() + ); + std::process::exit(1); + }); + serde_json::from_str(&text).unwrap_or_else(|error| { + eprintln!( + "[FAIL] fixture manifest malformed at {}: {error}", + path.display() + ); + std::process::exit(1); + }) +} + +// --------------------------------------------------------------------------------------------- +// Timing +// --------------------------------------------------------------------------------------------- + +/// One timed closure invocation and whether its captured SQL can be replayed for planning. +struct Sample { + wall_ms: f64, + sql_ms: f64, + replayable: bool, +} + +/// Run `body` `repeats` times, reporting wall time, summed SQL time and their difference for +/// both the first sample and the warm median. +/// +/// "First" is not "cold": no cache is reset between surfaces, so a later surface can inherit +/// pages an earlier one already warmed — see the process-wide note printed once in `main`. +/// +/// Args: +/// label: ASCII surface name printed in the report. +/// repeats: At least two timed runs. +/// body: Work under measurement. +/// outcome: Formats the FIRST call's result for the printed "outcome" line — run after +/// that call's timer already stopped, so it costs nothing against the first sample. +/// +/// Returns: +/// Every sample, and every statement captured across all `repeats` calls (for `--plan`). +fn measure( + label: &str, + repeats: usize, + mut body: impl FnMut() -> T, + outcome: impl FnOnce(&T) -> String, +) -> (Vec, Vec) { + let mut samples = Vec::with_capacity(repeats); + let mut all_stmts = Vec::new(); + let mut first_outcome = None; + let mut outcome = Some(outcome); + for i in 0..repeats { + drain_captured(); + let started = Instant::now(); + let value = body(); + let wall_ms = started.elapsed().as_secs_f64() * 1000.0; + let captured = drain_captured(); + let sql_ms: f64 = captured + .iter() + .map(|s| s.duration.as_secs_f64() * 1000.0) + .sum(); + let replayable = captured.iter().all(|s| s.expanded); + if i == 0 { + if let Some(outcome) = outcome.take() { + first_outcome = Some(outcome(&value)); + } + } + drop(value); + all_stmts.extend(captured); + samples.push(Sample { + wall_ms, + sql_ms, + replayable, + }); + } + let first = &samples[0]; + let mut warm: Vec<&Sample> = samples[1..].iter().collect(); + warm.sort_by(|a, b| a.wall_ms.total_cmp(&b.wall_ms)); + let median = warm.get(warm.len() / 2).copied().unwrap_or(first); + let note = |s: &Sample| { + if s.replayable { + "" + } else { + " [NOT REPLAYABLE]" + } + }; + println!( + "{label:<46} first {:>9.1} ms sql {:>9.1} ms diff {:>9.1} ms{}", + first.wall_ms, + first.sql_ms, + first.wall_ms - first.sql_ms, + note(first) + ); + println!( + "{:<46} warm {:>9.1} ms sql {:>9.1} ms diff {:>9.1} ms{}", + "", + median.wall_ms, + median.sql_ms, + median.wall_ms - median.sql_ms, + note(median) + ); + if let Some(outcome) = first_outcome { + println!("{:<46} outcome: {outcome}", ""); + } + (samples, all_stmts) +} + +/// Build the default query one open Analytics or tuner window issues for a period. +fn query(from: i64, to: i64, metric: ProfitMetric, cores: Vec) -> Query { + Query { + axis: moon_core::db::ReportAxis::from_measured(Default::default(), chrono_tz::UTC), + previous_period_basis: Default::default(), + from, + to, + cores, + side: Default::default(), + emulator: None, + strategies: Vec::new(), + strategy_name_mask: String::new(), + metric, + valuation: ValuationMode::Historical, + prefer_usdt: false, + } +} + +/// Format a read result for the harness's ASCII outcome column. +fn ok_or_err(result: &Result) -> String { + match result { + Ok(_) => "ok".to_string(), + Err(error) => format!("{error:?}"), + } +} + +// --------------------------------------------------------------------------------------------- +// Discovery — the harness never hardcodes a core, strategy or coin identity; it reads them from +// whatever fixture it was pointed at. +// --------------------------------------------------------------------------------------------- + +/// Fixture identities discovered from the replica instead of assumed by the harness. +struct Pool { + cores: Vec<(u64, String)>, + strategies: Vec, + /// The first `strategies` entry that names a REAL strategy — `strategy_id != 0` — for the + /// strat-db-backed calls (`versions_with_stats`, `strategy_purge_rows`), which know nothing + /// about `orders_rep`'s liquidation sentinel and legitimately find no rows for it. + first_real_strategy: Option, + coins: Vec, + kline_key: Option<(String, String, u32)>, +} + +/// Discover fixture identities while keeping discovery traffic out of the first timed sample. +fn discover_pool() -> Pool { + let cores = moon_core::db::open_reader() + .and_then(|conn| moon_core::db::distinct_cores(&conn)) + .unwrap_or_default(); + // NOT `db::distinct_strategies`: on this machine's fixture, `strat.strategies` carries a + // NULL `name` for at least one row, and `distinct_strategies` decodes that column + // unconditionally (`row.get::<_, String>(2)`), so the call fails with "Invalid column type + // Null at index: 2, name: name" — see the report. Pool discovery must not depend on that + // path; a direct scan of `orders_rep` gives the same `(core_uid, strategy_id)` identities. + // Deliberately UNFILTERED: this list's length feeds the fixture-drift guard below, which + // compares it against the manifest's own `groups_seen` count — including `strategyid=0`, + // `orders_rep`'s forced sentinel for a liquidation row with no owning strategy. Filtering it + // out here would desync the two counts and misfire the guard. See `first_real_strategy` + // below for the probe that skips the sentinel instead. + let strategies: Vec = moon_core::db::open_reader() + .map(|conn| { + conn.prepare("SELECT DISTINCT core_uid, strategyid FROM orders_rep") + .and_then(|mut stmt| { + stmt.query_map([], |row| { + Ok(moon_core::db::ReportStrategyKey { + core_uid: row.get::<_, i64>(0)? as u64, + strategy_id: row.get::<_, i64>(1)?, + }) + })? + .collect::, _>>() + }) + .unwrap_or_default() + }) + .unwrap_or_default(); + let coins = moon_core::db::open_reader() + .map(|conn| { + conn.prepare("SELECT DISTINCT coin FROM orders_rep LIMIT 5") + .and_then(|mut stmt| { + stmt.query_map([], |row| row.get::<_, String>(0))? + .collect::, _>>() + }) + .unwrap_or_default() + }) + .unwrap_or_default(); + let kline_key = { + let path = moon_core::config::paths::klines_db_path(); + // `chunks`/`chunks_v2` both name their third column `kind`, never `kind_min` — that + // name belongs only to `MergeItem`/`read_range`'s Rust-side parameter. `chunks` is also + // the wrong TABLE to probe: it is the legacy v1 store `gen_klines` seeds for only + // `LEGACY_MARKETS` markets on one kind and one day, while `chunks_v2` is the write + // target `merge_batch_blocking` actually fills for every market and kind the fixture + // claims to hold (`market/kline_cache.rs`'s own module doc). Reading `chunks_v2` here + // picks a key that is guaranteed to exist, straight from the cache's own contents. + rusqlite::Connection::open_with_flags(&path, rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY) + .ok() + .and_then(|conn| { + conn.query_row( + "SELECT exchange, market, kind FROM chunks_v2 LIMIT 1", + [], + |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, u32>(2)?, + )) + }, + ) + .ok() + }) + }; + let first_real_strategy = strategies.iter().find(|k| k.strategy_id != 0).copied(); + // The discovery reads above are real database traffic; they must not pollute the first + // measured surface's SQL sum. + drain_captured(); + Pool { + cores, + strategies, + first_real_strategy, + coins, + kline_key, + } +} + +// --------------------------------------------------------------------------------------------- +// Dump — arrival-order rows over the compared fields, at full `{:?}` precision, so two dumps +// from two binaries over one fixture diff to empty when a change is identity-preserving. +// --------------------------------------------------------------------------------------------- + +/// Print one ordered group collection in the dump format used for cross-binary comparisons. +fn dump_groups(tag: &str, kind: &str, groups: &[GroupStat]) { + for (index, group) in groups.iter().enumerate() { + println!("{tag}|{kind}|{index}|{group:?}"); + } +} + +/// Dump every comparable read result for one period without running the timing loop. +fn dump_all(period_name: &str, from: i64, to: i64, pool: &Pool) { + let q = |metric| query(from, to, metric, Vec::new()); + + let summary = summary_data(&q(ProfitMetric::Quote), false); + println!("{period_name}|summary|data|{:?}", summary.data); + + let strategies = strategy_base_data(&q(ProfitMetric::Quote), false); + match &strategies.data { + Ok(ProfitScope::Comparable { unit, data }) => { + println!( + "{period_name}|strategy_base|unit|{unit:?}|trades={}", + data.trades + ); + dump_groups(period_name, "strategy", &data.strategies); + dump_groups(period_name, "coin", &data.coins); + } + other => println!("{period_name}|strategy_base|{other:?}"), + } + + let calendar_day = calendar_data(&q(ProfitMetric::Quote), None, false, false); + println!("{period_name}|calendar_daily|{:?}", calendar_day.period); + let calendar_hour = calendar_data(&q(ProfitMetric::Quote), None, true, false); + println!("{period_name}|calendar_hourly|{:?}", calendar_hour.period); + + let monitor = moon_core::db::analytics::profit_monitor(&q(ProfitMetric::Quote)); + println!("{period_name}|profit_monitor|{monitor:?}"); + + if let Some(conn) = moon_core::db::open_reader().ok() { + if let Ok(snap) = moon_core::db::read_snapshot(&conn) { + let filter = ReportFilter::default(); + if let Ok(cores) = moon_core::db::distinct_cores(&snap) { + println!("{period_name}|distinct_cores|{cores:?}"); + } + if let Ok(strategies) = moon_core::db::distinct_strategies(&snap, &filter) { + println!("{period_name}|distinct_strategies|{strategies:?}"); + } + if let Ok(table) = moon_core::db::query_reports(&snap, &filter, "closedate", true, 200) + { + println!("{period_name}|report_cols|{:?}", table.cols); + for (index, row) in table.rows.iter().enumerate() { + println!("{period_name}|report_row|{index}|{row:?}"); + } + } + if let Ok(totals) = moon_core::db::query_totals(&snap, &filter) { + println!("{period_name}|report_totals|{totals:?}"); + } + } + } + + if let (Some((core_uid, _)), false) = (pool.cores.first(), pool.coins.is_empty()) { + if let Ok(conn) = moon_core::db::open_reader() { + if let Ok(history) = + moon_core::db::query_chart_trade_history(&conn, *core_uid, &pool.coins, None, 50) + { + for (index, record) in history.records.iter().enumerate() { + println!("{period_name}|chart_trade|{index}|{record:?}"); + } + } + } + } + + if let Some(key) = pool.first_real_strategy { + let versions = + moon_core::strat_db::stats::versions_with_stats(key.core_uid, key.strategy_id); + for (index, version) in versions.iter().enumerate() { + println!("{period_name}|version_stats|{index}|{version:?}"); + } + } +} + +// --------------------------------------------------------------------------------------------- +// Main +// --------------------------------------------------------------------------------------------- + +fn main() { + let mut args = std::env::args().skip(1); + let root = PathBuf::from( + args.next() + .expect("usage: db_read_timing [repeats] [--plan] [--dump]"), + ); + let repeats = args + .next() + .and_then(|value| value.parse::().ok()) + .unwrap_or(5) + .max(2); + let raw_args: Vec = std::env::args().collect(); + let plan_mode = raw_args.iter().any(|value| value == "--plan"); + let dump_mode = raw_args.iter().any(|value| value == "--dump"); + + assert!( + moon_core::config::paths::set_data_dir_override(root.clone()), + "the data root must be installed before any path resolves" + ); + let _permit = moon_core::db::report_recovery::prepare(); + let manifest = read_manifest(&root); + println!("data root: {}", root.display()); + println!( + "reports: {}", + moon_core::config::paths::reports_db_path().display() + ); + println!( + "fixture: rows={} cores={} strategy_groups={} coins={} span_days={} reports_bytes={} \ + analyzed={} seed={}", + manifest.rows, + manifest.cores, + manifest.strategy_groups, + manifest.coins, + manifest.span_days, + manifest.reports_bytes, + manifest.analyzed, + manifest.seed + ); + println!( + "note: no cache is reset between surfaces — every surface after the first inherits \ + pages warmed by whatever ran before it, so \"first\" below means \"first repetition of \ + this closure\", not \"never-warmed\"" + ); + + install_read_profiler(record_stmt); + let pool = discover_pool(); + + // --- Fixture drift guard -------------------------------------------------------------- + let mut drift: Vec = Vec::new(); + if pool.cores.len() as i64 != manifest.cores { + drift.push(format!( + "core count: fixture claims {}, replica shows {}", + manifest.cores, + pool.cores.len() + )); + } + if (pool.strategies.len() as i64) < manifest.strategy_groups { + drift.push(format!( + "strategy groups: fixture claims {}, replica shows only {}", + manifest.strategy_groups, + pool.strategies.len() + )); + } + let reports_bytes = std::fs::metadata(moon_core::config::paths::reports_db_path()) + .map(|meta| meta.len() as i64) + .unwrap_or(0); + let low = manifest.reports_bytes / 2; + let high = manifest.reports_bytes.saturating_mul(2).max(1); + if reports_bytes < low || reports_bytes > high { + drift.push(format!( + "reports.sqlite size: fixture claims {} bytes, on disk {} bytes (outside [{low}, {high}])", + manifest.reports_bytes, reports_bytes + )); + } + if !drift.is_empty() { + eprintln!("[FAIL] fixture drift detected:"); + for line in &drift { + eprintln!(" - {line}"); + } + std::process::exit(1); + } + + if dump_mode { + let end = 1_780_000_000i64; + let periods: [(&str, i64, i64); 4] = [ + ("today", end - 86_400, end), + ("month", end - 30 * 86_400, end), + ("year", end - 365 * 86_400, end), + ("all", -1, end), + ]; + for (name, from, to) in periods { + dump_all(name, from, to, &pool); + } + return; + } + + let end = 1_780_000_000i64; + let periods: [(&str, i64, i64); 4] = [ + ("today", end - 86_400, end), + ("month", end - 30 * 86_400, end), + ("year", end - 365 * 86_400, end), + ("all", -1, end), + ]; + let metrics = [ + ("quote", ProfitMetric::Quote), + ("percent", ProfitMetric::Percent), + ]; + + let mut plan_pool: Vec = Vec::new(); + let mut missing_guarantee: Vec = Vec::new(); + // Real production failures that happen to surface here — never fixture drift, so kept in a + // separate bucket the closing report never conflates with the guarantees above. + let mut production_failures: Vec = Vec::new(); + + let scoped_cores: Vec = pool.cores.iter().take(3).map(|(uid, _)| *uid).collect(); + + for (name, from, to) in periods { + println!("\n=== period: {name} ==="); + for (metric_name, metric) in metrics { + let label_suffix = format!("[{name}/{metric_name}]"); + let q = query(from, to, metric, Vec::new()); + let q_scoped = query(from, to, metric, scoped_cores.clone()); + + let (_, stmts) = measure( + &format!("summary_data {label_suffix}"), + repeats, + || summary_data(&q, false), + |r| { + match &r.data { + Ok(ProfitScope::Comparable { data, .. }) + if data.strategies.is_empty() && data.coins.is_empty() => + { + missing_guarantee + .push(format!("summary_data: no groups {label_suffix}")); + } + Err(_) => { + missing_guarantee.push(format!("summary_data: errored {label_suffix}")); + } + _ => {} + } + ok_or_err(&r.data) + }, + ); + plan_pool.extend(stmts); + + if metric_name == "quote" && !scoped_cores.is_empty() { + let (_, stmts) = measure( + &format!( + "summary_data [{name}/scoped-{}-of-{}]", + scoped_cores.len(), + pool.cores.len() + ), + repeats, + || summary_data(&q_scoped, false), + |r| ok_or_err(&r.data), + ); + plan_pool.extend(stmts); + } + + let (_, stmts) = measure( + &format!("strategy_base_data {label_suffix}"), + repeats, + || strategy_base_data(&q, false), + |r| { + match &r.data { + Ok(ProfitScope::Comparable { data, .. }) + if data.strategies.is_empty() && data.coins.is_empty() => + { + missing_guarantee + .push(format!("strategy_base_data: no groups {label_suffix}")); + } + Err(_) => missing_guarantee + .push(format!("strategy_base_data: errored {label_suffix}")), + _ => {} + } + ok_or_err(&r.data) + }, + ); + plan_pool.extend(stmts); + + let (_, stmts) = measure( + &format!("calendar_data daily {label_suffix}"), + repeats, + || calendar_data(&q, None, false, false), + |r| { + match &r.period { + Ok(ProfitScope::Comparable { data, .. }) if data.current.is_empty() => { + missing_guarantee + .push(format!("calendar_data daily: no cells {label_suffix}")); + } + Err(_) => missing_guarantee + .push(format!("calendar_data daily: errored {label_suffix}")), + _ => {} + } + ok_or_err(&r.period) + }, + ); + plan_pool.extend(stmts); + + let (_, stmts) = measure( + &format!("calendar_data hourly {label_suffix}"), + repeats, + || calendar_data(&q, None, true, false), + |r| { + match &r.period { + Ok(ProfitScope::Comparable { data, .. }) if data.current.is_empty() => { + missing_guarantee + .push(format!("calendar_data hourly: no cells {label_suffix}")); + } + Err(_) => missing_guarantee + .push(format!("calendar_data hourly: errored {label_suffix}")), + _ => {} + } + ok_or_err(&r.period) + }, + ); + plan_pool.extend(stmts); + + let (_, stmts) = measure( + &format!("profit_monitor {label_suffix}"), + repeats, + || moon_core::db::analytics::profit_monitor(&q), + |r| { + if r.is_err() { + missing_guarantee.push(format!("profit_monitor: errored {label_suffix}")); + } + ok_or_err(r) + }, + ); + plan_pool.extend(stmts); + + let variants = vec![Variant::default()]; + let field = FIELDS.first().map(|f| f.col).unwrap_or("profitbtc"); + + let (_, stmts) = measure( + &format!("filter_tuner_data {label_suffix}"), + repeats, + || moon_core::db::tuner::filter_tuner_data(&q, &variants, field, 20), + |r| { + match &r.stats { + Ok(v) if v.is_empty() => missing_guarantee + .push(format!("filter_tuner_data: no KPI rows {label_suffix}")), + Err(_) => missing_guarantee + .push(format!("filter_tuner_data: errored {label_suffix}")), + _ => {} + } + ok_or_err(&r.stats) + }, + ); + plan_pool.extend(stmts); + + let (_, stmts) = measure( + &format!("time_tuner_data {label_suffix}"), + repeats, + || moon_core::db::tuner::time_tuner_data(&q, &variants), + |r| { + match &r.stats { + Ok(v) if v.is_empty() => missing_guarantee + .push(format!("time_tuner_data: no KPI rows {label_suffix}")), + Err(_) => missing_guarantee + .push(format!("time_tuner_data: errored {label_suffix}")), + _ => {} + } + ok_or_err(&r.stats) + }, + ); + plan_pool.extend(stmts); + + let picked: Vec = pool.coins.first().cloned().into_iter().collect(); + let (_, stmts) = measure( + &format!("coin_tuner_data {label_suffix}"), + repeats, + || moon_core::db::tuner::coin_tuner_data(&q, &variants, &q, &picked), + |r| { + match &r.kpi { + Ok(v) if v.is_empty() => missing_guarantee + .push(format!("coin_tuner_data: no KPI rows {label_suffix}")), + Err(_) => missing_guarantee + .push(format!("coin_tuner_data: errored {label_suffix}")), + _ => {} + } + ok_or_err(&r.kpi) + }, + ); + plan_pool.extend(stmts); + + let (_, stmts) = measure( + &format!("variant_stats {label_suffix}"), + repeats, + || moon_core::db::tuner::variant_stats(&q, &variants), + ok_or_err, + ); + plan_pool.extend(stmts); + + let (_, stmts) = measure( + &format!("histogram {label_suffix}"), + repeats, + || moon_core::db::tuner::histogram(&q, field, 20), + ok_or_err, + ); + plan_pool.extend(stmts); + + let (_, stmts) = measure( + &format!("suggest_field {label_suffix}"), + repeats, + || moon_core::db::tuner::suggest_field(&q, field, 30, 20, true), + ok_or_err, + ); + plan_pool.extend(stmts); + + let (_, stmts) = measure( + &format!("suggest_time {label_suffix}"), + repeats, + || { + moon_core::db::tuner::suggest_time( + &q, + 30, + 20, + true, + TimeAxes { + week: true, + day: true, + hour: true, + ..Default::default() + }, + ) + }, + ok_or_err, + ); + plan_pool.extend(stmts); + + let locked = vec![None; FIELDS.len()]; + let (_, stmts) = measure( + &format!("threshold_search::suggest {label_suffix}"), + repeats, + || { + let handle = SearchHandle::new(); + moon_core::db::tuner::threshold_search::suggest( + &q, + SearchParams { + restarts: 8, + min_n: None, + locked: &locked, + edges: 20, + round: true, + seed: Some(20260903), + train_frac: 1.0, + compose: false, + }, + &handle, + ) + }, + ok_or_err, + ); + plan_pool.extend(stmts); + + if !pool.coins.is_empty() { + let (_, stmts) = measure( + &format!("strategies_for_coins {label_suffix}"), + repeats, + || moon_core::db::analytics::strategies_for_coins(&q, &pool.coins), + ok_or_err, + ); + plan_pool.extend(stmts); + } + + // --- The Report: mirrors run_report_query's own sequence ----------------------- + for (sort_name, sort_key, desc) in [ + ("closedate DESC", "closedate", true), + ("profitbtc DESC", "profitbtc", true), + ] { + let filter = ReportFilter { + date_from: if from < 0 { None } else { Some(from) }, + date_to: Some(to), + rows: RowScope::ClosedAndOpen, + ..ReportFilter::default() + }; + let (_, stmts) = measure( + &format!("report [{name}/{sort_name}]"), + repeats, + || -> Result<(usize, usize), ReadFail> { + let conn = moon_core::db::open_reader()?; + let snap = moon_core::db::read_snapshot(&conn)?; + let cores = moon_core::db::distinct_cores(&snap)?; + // Mirrors `run_report_query`'s own OPTIONAL treatment (`.transpose()` + // there, not `?`): a strategy-scope refresh failing must not sink the + // rows/totals read it rides beside. See the report for why this call + // fails against this fixture's `strategies.sqlite` (a NULL `name`). + let strategies = moon_core::db::distinct_strategies(&snap, &filter).ok(); + let table = + moon_core::db::query_reports(&snap, &filter, sort_key, desc, 500)?; + let totals = moon_core::db::query_totals(&snap, &filter)?; + let _ = (cores, strategies, totals); + Ok((table.rows.len(), 0)) + }, + |r| { + match r { + Ok((rows, _)) if *rows == 0 => missing_guarantee + .push(format!("report [{name}/{sort_name}]: no rows")), + Err(_) => missing_guarantee + .push(format!("report [{name}/{sort_name}]: errored")), + _ => {} + } + ok_or_err(r) + }, + ); + plan_pool.extend(stmts); + } + + let (_, stmts) = measure( + &format!("distinct_cores {label_suffix}"), + repeats, + || { + moon_core::db::open_reader() + .and_then(|conn| moon_core::db::distinct_cores(&conn)) + }, + ok_or_err, + ); + plan_pool.extend(stmts); + + let (_, stmts) = measure( + &format!("distinct_strategies {label_suffix}"), + repeats, + || { + moon_core::db::open_reader().and_then(|conn| { + moon_core::db::distinct_strategies(&conn, &ReportFilter::default()) + }) + }, + ok_or_err, + ); + plan_pool.extend(stmts); + + if let Some((core_uid, _)) = pool.cores.first() { + let (_, stmts) = measure( + &format!("query_chart_trade_history {label_suffix}"), + repeats, + || { + moon_core::db::open_reader().and_then(|conn| { + moon_core::db::query_chart_trade_history( + &conn, + *core_uid, + &pool.coins, + None, + 200, + ) + }) + }, + ok_or_err, + ); + plan_pool.extend(stmts); + + let sample_record = moon_core::db::open_reader().ok().and_then(|conn| { + moon_core::db::query_chart_trade_history(&conn, *core_uid, &pool.coins, None, 1) + .ok() + .and_then(|history| history.records.into_iter().next()) + }); + if let Some(record) = sample_record { + let (_, stmts) = measure( + &format!("query_trade_meta {label_suffix}"), + repeats, + || { + moon_core::db::open_reader() + .and_then(|conn| moon_core::db::query_trade_meta(&conn, &record)) + }, + ok_or_err, + ); + plan_pool.extend(stmts); + } + } + + if let Some(key) = pool.first_real_strategy { + let (_, stmts) = measure( + &format!("strategy_purge_rows {label_suffix}"), + repeats, + || { + moon_core::db::open_reader() + .and_then(|conn| moon_core::db::strategy_purge_rows(&conn, key)) + }, + ok_or_err, + ); + plan_pool.extend(stmts); + + let (_, stmts) = measure( + &format!("versions_with_stats {label_suffix}"), + repeats, + || { + moon_core::strat_db::stats::versions_with_stats( + key.core_uid, + key.strategy_id, + ) + }, + |v| { + if v.is_empty() { + missing_guarantee + .push(format!("versions_with_stats: no versions {label_suffix}")); + } + format!("{} versions", v.len()) + }, + ); + plan_pool.extend(stmts); + } + + let targets: Vec<(i64, Option)> = pool + .strategies + .iter() + .map(|k| (k.strategy_id, Some(k.core_uid))) + .collect(); + if !targets.is_empty() { + let (_, stmts) = measure( + &format!("coin_lists::coin_lists {label_suffix}"), + repeats, + || moon_core::db::coin_lists::coin_lists(&targets), + |r| { + match r { + Ok(rows) if rows.black.is_empty() && rows.white.is_empty() => { + missing_guarantee + .push(format!("coin_lists: no rows {label_suffix}")); + } + // `scope_sql`'s one-OR-term-per-strategy join hits SQLite's + // "Expression tree is too large" parser limit once enough strategies + // are selected — a real production bug (`db/coin_lists/mod.rs` + // `scope_sql`, recorded for the goal owner), not this fixture + // drifting from what the harness expects. Kept out of + // `missing_guarantee` so the two classes are never conflated. + Err(error) => { + let detail = format!("{error:?}"); + if detail.contains("too large") { + production_failures + .push(format!("coin_lists {label_suffix}: {detail}")); + } else { + missing_guarantee + .push(format!("coin_lists: errored {label_suffix}")); + } + } + _ => {} + } + ok_or_err(r) + }, + ); + plan_pool.extend(stmts); + } + + let (_, stmts) = measure( + &format!("open_reader alone {label_suffix}"), + repeats, + || moon_core::db::open_reader().map(|_| ()), + ok_or_err, + ); + plan_pool.extend(stmts); + + if let Some((exchange, market, kind_min)) = &pool.kline_key { + let path = moon_core::config::paths::klines_db_path(); + let cache = moon_core::market::kline_cache::KlineCache::open(path); + if let Some(cache) = cache { + let (_, stmts) = measure( + &format!("KlineCache::read_range {label_suffix}"), + repeats, + || cache.read_range(exchange, market, *kind_min, 0, i64::MAX), + |r| match r { + Some(rows) if rows.is_empty() => { + missing_guarantee.push(format!( + "KlineCache::read_range: no rows {label_suffix}" + )); + "0 rows".to_string() + } + Some(rows) => format!("{} rows", rows.len()), + None => { + missing_guarantee.push(format!( + "KlineCache::read_range: timed out {label_suffix}" + )); + "timed out".to_string() + } + }, + ); + plan_pool.extend(stmts); + } else { + missing_guarantee + .push(format!("KlineCache::open failed [{name}/{metric_name}]")); + } + } else if pool.kline_key.is_none() { + missing_guarantee.push(format!( + "KlineCache: no (exchange, market, kind_min) row in klines.sqlite [{name}]" + )); + } + } + } + + // Neither surface below can be called from this example crate: `report_quote_ordinals` is + // a private `fn` inside a private `mod worker` (`db/valuation/worker.rs:1953`, + // `db/valuation/mod.rs:12`), and `basis::probe` is a private `fn` inside the private + // `mod basis` of `db::analytics` (`db/analytics/basis.rs:94`, `db/analytics/mod.rs:25`) — + // both files are out of this branch's edit scope, and a copied SQL string is explicitly + // forbidden by the spec. Never made `pub` to reach them; kept first-class instead so the + // omission cannot be mistaken for "measured and cheap" — printed here AND carried into the + // `--plan` table itself (`run_plan_mode`), not only as this footnote. + for (label, reason) in NOT_MEASURED { + println!("\n[NOT MEASURED] {label} - {reason}"); + } + + // Print everything the run promised — the timing table above, `--plan`'s EQP dump below — + // BEFORE reporting the guard's own failures and exiting. A guard that hides the artifact it + // was meant to protect is worse than no guard; only the ORDER changed here, the exit code on + // a real drift is unchanged. + if plan_mode { + run_plan_mode(plan_pool, &NOT_MEASURED); + } + + if !production_failures.is_empty() { + eprintln!("[FAIL] production bug(s) hit while measuring (not fixture drift):"); + for line in &production_failures { + eprintln!(" - {line}"); + } + } + if !missing_guarantee.is_empty() { + eprintln!("[FAIL] guaranteed-non-empty surfaces did not deliver:"); + for line in &missing_guarantee { + eprintln!(" - {line}"); + } + } + if !production_failures.is_empty() || !missing_guarantee.is_empty() { + std::process::exit(1); + } +} + +/// Surfaces the deliverable promises a number for but this example genuinely cannot reach — +/// see the two reasons printed beside this constant's use in `main`. +const NOT_MEASURED: [(&str, &str); 2] = [ + ( + "valuation reconciliation row", + "report_quote_ordinals is private, unreachable from an example", + ), + ( + "basis::probe full scan", + "private fn in a private module, unreachable from an example, and db/analytics/mod.rs \ + is out of edit scope", + ), +]; + +/// Register STUB scalar functions matching the crate's own calendar/time-tuner SQL surface, so +/// `EXPLAIN QUERY PLAN` can resolve their names on this example's own reader connection. +/// `db::analytics::time_zone::install` is `pub(crate)` and installs the REAL bodies against a +/// captured `ReportAxis`; an example cannot call it, which is why these are stubs rather than +/// the genuine functions. `EXPLAIN QUERY PLAN` only resolves a function's name and declared +/// properties and never evaluates its body, so a stub returning a constant is correct for this +/// purpose — but the PLAN it enables is real, which is why every plan taken on this connection +/// is printed with an explicit marker (see `run_plan_mode`) rather than silently passed off as +/// having run against production functions. +/// +/// Registered with the SAME `SQLITE_UTF8 | SQLITE_DETERMINISTIC` flags the real functions use, +/// so the planner sees the same function properties the production connection would. +fn install_plan_stub_functions(conn: &rusqlite::Connection) -> rusqlite::Result<()> { + use rusqlite::functions::FunctionFlags; + let flags = FunctionFlags::SQLITE_UTF8 | FunctionFlags::SQLITE_DETERMINISTIC; + conn.create_scalar_function("mt_to_utc", 2, flags, |_ctx| Ok(0i64))?; + conn.create_scalar_function("mt_local_bucket", 3, flags, |_ctx| Ok(0i64))?; + conn.create_scalar_function("mt_minute_of_day", 2, flags, |_ctx| Ok(0i64))?; + conn.create_scalar_function("mt_minute_of_week", 2, flags, |_ctx| Ok(0i64))?; + conn.create_scalar_function("mt_core_minute_of_day", 1, flags, |_ctx| Ok(0i64))?; + conn.create_scalar_function("mt_core_minute_of_week", 1, flags, |_ctx| Ok(0i64))?; + // The real function returns its argument casefolded; a stub only needs to resolve, but + // returning the argument unchanged keeps its declared return type (TEXT) honest too. + conn.create_scalar_function("mt_unicode_casefold", 1, flags, |ctx| ctx.get::(0))?; + Ok(()) +} + +/// `--plan`: dedup captured statements by exact text, print total time, call count and +/// `EXPLAIN QUERY PLAN` for every replayable one; mark placeholder-only captures instead of +/// ranking them; and list every `not_measured` surface honestly instead of silently ranking +/// past it. +fn run_plan_mode(pool: Vec, not_measured: &[(&str, &str)]) { + struct Agg { + total_ms: f64, + count: usize, + replayable: bool, + } + let mut by_text: HashMap = HashMap::new(); + for stmt in pool { + let entry = by_text.entry(stmt.sql.clone()).or_insert(Agg { + total_ms: 0.0, + count: 0, + replayable: true, + }); + entry.total_ms += stmt.duration.as_secs_f64() * 1000.0; + entry.count += 1; + entry.replayable = entry.replayable && stmt.expanded; + } + let (mut ranked, mut not_replayable): (Vec<(String, Agg)>, Vec<(String, Agg)>) = + by_text.into_iter().partition(|(_, agg)| agg.replayable); + ranked.sort_by(|a, b| b.1.total_ms.total_cmp(&a.1.total_ms)); + not_replayable.sort_by(|a, b| b.1.total_ms.total_cmp(&a.1.total_ms)); + + println!( + "\n=== --plan: {} ranked statements ({} not replayable, {} not measured, see below) ===", + ranked.len(), + not_replayable.len(), + not_measured.len() + ); + let conn = match moon_core::db::open_reader() { + Ok(conn) => { + if let Err(error) = install_plan_stub_functions(&conn) { + println!("(cannot register plan-stub functions: {error:?})"); + } + Some(conn) + } + Err(error) => { + println!("(cannot open a reader for EXPLAIN QUERY PLAN: {error:?})"); + None + } + }; + // Statements whose EXPLAIN fails with "no such table" ran on the writer's own connection or + // the kline cache's own connection, neither of which this reader has attached — a different, + // unfixable gap from the mt_* function names the stubs above resolve. Collected here instead + // of printed inline, so the count of planless statements is explained under its own heading + // rather than merely reported one line at a time. + let mut no_such_table: Vec<(String, f64, usize)> = Vec::new(); + for (sql, agg) in ranked { + println!( + "\n-- total {:>9.1} ms calls {:>4}", + agg.total_ms, agg.count + ); + println!(" {sql}"); + let Some(conn) = &conn else { continue }; + match conn + .prepare(&format!("EXPLAIN QUERY PLAN {sql}")) + .and_then(|mut stmt| { + stmt.query_map([], |row| { + Ok(format!( + "id={} parent={} detail={}", + row.get::<_, i64>(0)?, + row.get::<_, i64>(1)?, + row.get::<_, String>(3)? + )) + })? + .collect::, _>>() + }) { + Ok(lines) => { + println!(" [stub-resolved: mt_* bodies are stubs, the PLAN is real]"); + for line in lines { + println!(" {line}"); + } + } + Err(error) => { + let text = error.to_string(); + if text.contains("no such table") { + no_such_table.push((sql, agg.total_ms, agg.count)); + } else { + println!(" plan failed: {error}"); + } + } + } + } + + println!( + "\n=== --plan: {} statement(s) with no plan (writer-only or kline-cache-only table, \ + not attached to this reader) ===", + no_such_table.len() + ); + for (sql, total_ms, count) in &no_such_table { + println!("\n-- total {total_ms:>9.1} ms calls {count:>4} NO PLAN (table not attached)"); + println!(" {sql}"); + } + + // Placeholder-only captures never had their real literal-bound shape recorded, so they + // cannot be replayed with EXPLAIN QUERY PLAN and must never drive the optimisation order + // above — printed here, visible but unranked, instead of being dropped silently. + println!( + "\n=== --plan: {} not-replayable captures (never ranked) ===", + not_replayable.len() + ); + for (sql, agg) in not_replayable { + println!( + "\n-- total {:>9.1} ms calls {:>4} NOT REPLAYABLE (placeholder capture)", + agg.total_ms, agg.count + ); + println!(" {sql}"); + } + + // Promised numbers this example can never produce, listed in the same ranked table rather + // than only as a footnote elsewhere — a reader scanning just this section still sees them + // flagged, never silently absent. + println!( + "\n=== --plan: {} not-measured surfaces (never ranked, never callable from here) ===", + not_measured.len() + ); + for (label, reason) in not_measured { + println!("\n-- NOT MEASURED {label}"); + println!(" {reason}"); + } +} diff --git a/crates/moon-core/examples/gen_klines.rs b/crates/moon-core/examples/gen_klines.rs new file mode 100644 index 00000000..303cb27b --- /dev/null +++ b/crates/moon-core/examples/gen_klines.rs @@ -0,0 +1,227 @@ +//! Deterministic `klines.sqlite` seeder for the report-replica measurement fixture +//! (`tools/gen_replica.py`), built through `KlineCache`'s own production write API rather than a +//! second binary-codec implementation. +//! +//! `chunks_v2`'s packed row codec (`pack_rows_v2`) is private to `market/kline_cache.rs` on +//! purpose: a Python re-implementation of it would be a second binary-format authority no drift +//! check could prove equivalent to the first (rejected in review). This helper instead opens the +//! cache and calls `KlineCache::merge_batch_blocking` directly, the same entry point the +//! background recorder uses. +//! +//! Usage: +//! cargo run --release -p moon-core --example gen_klines -- [seed] +//! +//! `KlineCache::open` prunes chunks older than each kind's retention window +//! (`retention_days`: 30 days for 1-minute, 15 for 5-minute, 3650 for everything coarser) the +//! moment it opens the database — including on a LATER reopen by a reader, not just this +//! process's own. Anchoring the generated series at a fixed historical timestamp (as +//! `gen_replica.py` does for `reports.sqlite`) would therefore make the 1- and 5-minute series +//! vanish the next time anything opens the cache, days after generation. To guarantee +//! `KlineCache::read_range` stays non-empty for every kind this fixture writes, every series here +//! is anchored at the REAL wall clock (`now_unix_ms`) instead: `klines.sqlite` is consequently the +//! one fixture file that is NOT byte-identical across reruns on different days, unlike +//! `reports.sqlite` / `strategies.sqlite` / `valuation.sqlite` (see the generator's own report for +//! why this is a deliberate deviation from the frozen contract's byte-identical clause). + +use std::path::PathBuf; + +use moon_core::market::ChartCandle; +use moon_core::market::kline_cache::{KlineCache, MergeItem}; + +const DAY_MS: i64 = 86_400_000; +const KINDS: [u32; 3] = [1, 5, 60]; +const MARKETS: usize = 50; +// FBinance (4) and ByBit (7) codes, per feed::types::ExchangeId's doc comment; dex=0 (no HIP-3 +// DEX), formatted exactly as market/source/read.rs's `exchange_key` does: `"{code}:{dex:08x}"`. +const EXCHANGE_CODES: [u8; 2] = [4, 7]; +// A handful of markets also get direct legacy `chunks` (v1) rows for one day, so `read_rows`' +// v1/v2 merge is exercised — never through a re-implemented v2 codec, only the documented +// 24-byte-per-row v1 layout (`u32 offset_ms + 5×f32`, no turnover; kline_cache.rs's module doc). +const LEGACY_MARKETS: usize = 5; + +/// Deterministic splitmix64 source for reproducible candle values despite the real-time anchor. +struct Rng(u64); + +impl Rng { + /// Advance the generator and return its next uniformly distributed `u64`. + fn next_u64(&mut self) -> u64 { + self.0 = self.0.wrapping_add(0x9E37_79B9_7F4A_7C15); + let mut z = self.0; + z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); + z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); + z ^ (z >> 31) + } + + /// Return the next generator value scaled to the half-open unit interval. + fn next_f64(&mut self) -> f64 { + (self.next_u64() >> 11) as f64 * (1.0 / (1u64 << 53) as f64) + } + + /// Return the next generator value scaled to the requested half-open numeric interval. + fn range(&mut self, lo: f64, hi: f64) -> f64 { + lo + self.next_f64() * (hi - lo) + } +} + +/// Build one synthetic market identity from its deterministic fixture ordinal. +fn market_name(i: usize) -> String { + format!("SYN{i:03}USDT") +} + +/// Build the persisted exchange key assigned to one synthetic market ordinal. +fn exchange_key(i: usize) -> String { + format!("{}:00000000", EXCHANGE_CODES[i % EXCHANGE_CODES.len()]) +} + +/// Pack one legacy v1 24-byte row: `u32 offset_ms LE + 5×f32 LE (open, high, low, close, volume)`. +fn pack_v1_row(offset_ms: u32, c: &ChartCandle) -> [u8; 24] { + let mut out = [0u8; 24]; + out[0..4].copy_from_slice(&offset_ms.to_le_bytes()); + out[4..8].copy_from_slice(&c.open.to_le_bytes()); + out[8..12].copy_from_slice(&c.high.to_le_bytes()); + out[12..16].copy_from_slice(&c.low.to_le_bytes()); + out[16..20].copy_from_slice(&c.close.to_le_bytes()); + out[20..24].copy_from_slice(&c.volume.to_le_bytes()); + out +} + +/// Generate one deterministic OHLCV series over `[from_ms, to_ms)` at `step_ms` spacing. +fn candle_series( + rng: &mut Rng, + from_ms: i64, + to_ms: i64, + step_ms: i64, + base_price: f64, +) -> Vec { + let mut out = Vec::new(); + let mut price = base_price; + let mut t = from_ms; + while t < to_ms { + let open = price; + let close = (open * (1.0 + rng.range(-0.01, 0.01))).max(0.0001); + let high = open.max(close) * (1.0 + rng.range(0.0, 0.004)); + let low = open.min(close) * (1.0 - rng.range(0.0, 0.004)); + let volume = rng.range(1.0, 5000.0); + out.push(ChartCandle { + t_open_ms: t as f64, + open: open as f32, + high: high as f32, + low: low as f32, + close: close as f32, + volume: volume as f32, + quote_volume: (volume * (open + close) / 2.0) as f32, + }); + price = close; + t += step_ms; + } + out +} + +/// Seed the requested disposable kline cache through the production write API. +fn main() { + let mut args = std::env::args().skip(1); + let data_dir = PathBuf::from( + args.next() + .expect("usage: gen_klines [seed]"), + ); + let span_days: i64 = args + .next() + .expect("span_days required") + .parse() + .expect("span_days must be an integer"); + let seed: u64 = args.next().and_then(|s| s.parse().ok()).unwrap_or(20260820); + + assert!( + moon_core::config::paths::set_data_dir_override(data_dir.clone()), + "the data root must be installed before any path resolves" + ); + let path = moon_core::config::paths::klines_db_path(); + for suffix in ["", "-wal", "-shm"] { + let _ = std::fs::remove_file(format!("{}{suffix}", path.display())); + } + + let end_ms = moon_core::util::now_unix_ms_i64(); + + let cache = KlineCache::open(path.clone()).expect("open kline cache"); + let mut rng = Rng(seed); + let mut written = 0usize; + for i in 0..MARKETS { + let market = market_name(i); + let exchange = exchange_key(i); + let base_price = rng.range(0.01, 70_000.0); + for kind in KINDS { + // `retention_days` prunes 1- and 5-minute chunks to 30 / 15 days on every open, so + // covering the full requested span at those kinds would just synthesize rows the very + // next reopen deletes — capped well inside each kind's own retention window instead. + // The coarse kind's 3650-day retention has room for the whole span. + let this_span_days = match kind { + 1 => span_days.min(20), + 5 => span_days.min(10), + _ => span_days, + }; + let from_ms = end_ms - this_span_days * DAY_MS; + let rows = candle_series(&mut rng, from_ms, end_ms, kind as i64 * 60_000, base_price); + written += rows.len(); + cache.merge_batch_blocking(vec![MergeItem { + exchange: exchange.clone(), + market: market.clone(), + kind_min: kind, + rows, + }]); + } + } + drop(cache); + // The worker thread's Sender was just dropped, which ends its `rx.recv()` loop; the last + // `merge_batch_blocking` above already waited for its own transaction to commit, so nothing + // further needs to settle before a second connection can write. + std::thread::sleep(std::time::Duration::from_millis(100)); + + // Direct legacy v1 rows for a handful of markets, one day each, so `read_rows`' v1/v2 merge + // is exercised on read. `updated_ms` is older than the v2 rows above, matching the normal + // (non-downgrade) case where v2 is the fresher write. + let legacy_conn = + rusqlite::Connection::open(&path).expect("open klines.sqlite for legacy rows"); + // Matches kind 60's own (uncapped) span, since the legacy row below is written under kind 60. + let legacy_day = (end_ms - span_days * DAY_MS) / DAY_MS; + let day_start = legacy_day * DAY_MS; + let mut legacy_rows = 0usize; + for i in 0..LEGACY_MARKETS { + let market = market_name(i); + let exchange = exchange_key(i); + let base_price = rng.range(0.01, 70_000.0); + let series = candle_series( + &mut rng, + day_start, + day_start + DAY_MS, + 60 * 60_000, + base_price, + ); + let mut blob = Vec::new(); + for c in &series { + let offset_ms = (c.t_open_ms as i64 - day_start) as u32; + blob.extend_from_slice(&pack_v1_row(offset_ms, c)); + } + legacy_rows += series.len(); + legacy_conn + .execute( + "INSERT OR REPLACE INTO chunks(exchange, market, kind, day, rows, updated_ms) + VALUES (?1, ?2, ?3, ?4, ?5, ?6)", + rusqlite::params![ + exchange, + market, + 60u32, + legacy_day, + blob, + day_start - DAY_MS + ], + ) + .expect("insert legacy v1 chunk"); + } + + println!( + "[OK] {}: {written} v2 rows across {MARKETS} markets x {} kinds, \ + {legacy_rows} legacy v1 rows across {LEGACY_MARKETS} markets", + path.display(), + KINDS.len() + ); +} diff --git a/crates/moon-core/src/db/analytics/tests.rs b/crates/moon-core/src/db/analytics/tests.rs index f66e3bb2..5339d791 100644 --- a/crates/moon-core/src/db/analytics/tests.rs +++ b/crates/moon-core/src/db/analytics/tests.rs @@ -1160,3 +1160,65 @@ fn min_closedate_uses_the_axis_converted_instant_not_the_smaller_raw_value() { remove_db(&path); } + +/// `db/analytics/mod.rs:min_closedate` must preserve the `closedate > 0` predicate and compare +/// each core after its axis conversion when C2 replaces the grouped query. Including a core whose +/// only rows are non-positive would make every all-history Analytics surface start at an invented +/// epoch instead of the first real trade; an empty replica must still use the documented floor 1. +#[test] +fn min_closedate_skips_non_positive_cores_and_keeps_the_empty_floor() { + const ONLY_NON_POSITIVE_CORE: u64 = 4; + const OFFSET_CORE: u64 = 8; + const OFFSET_SECS: i32 = -3_600; + const FIRST_TRUE_INSTANT: i64 = 1_700_300_000; + + let path = temp_db("min-closedate-positive"); + let conn = build_replica_multi_core( + &path, + &[ + (ONLY_NON_POSITIVE_CORE, -99, 1.0, "ZEROLESS"), + (ONLY_NON_POSITIVE_CORE, 0, 2.0, "ZEROLESS"), + ( + OFFSET_CORE, + FIRST_TRUE_INSTANT + i64::from(OFFSET_SECS), + 3.0, + "OFFSET", + ), + ( + OFFSET_CORE, + FIRST_TRUE_INSTANT + 86_400 + i64::from(OFFSET_SECS), + 4.0, + "OFFSET", + ), + ], + ); + let axis = crate::db::ReportAxis::from_measured( + std::collections::HashMap::from([( + OFFSET_CORE, + vec![crate::db::OffsetSegment { + from_utc: 0, + offset_secs: OFFSET_SECS, + }], + )]), + chrono_tz::UTC, + ); + + assert_eq!( + min_closedate(&conn, &axis).expect("resolve positive multi-core floor"), + FIRST_TRUE_INSTANT, + "only the offset core contributes a positive close, and its axis converts it to the independently seeded true instant" + ); + drop(conn); + remove_db(&path); + + let empty_path = temp_db("min-closedate-empty"); + let empty = build_replica(&empty_path, &[]); + assert_eq!( + min_closedate(&empty, &crate::db::ReportAxis::identity_core_local()) + .expect("resolve empty history floor"), + 1, + "an empty replica has no per-core minimum and therefore keeps the public all-history sentinel" + ); + drop(empty); + remove_db(&empty_path); +} diff --git a/crates/moon-core/src/db/coin_lists/mod.rs b/crates/moon-core/src/db/coin_lists/mod.rs index 1aa1365d..e66c2639 100644 --- a/crates/moon-core/src/db/coin_lists/mod.rs +++ b/crates/moon-core/src/db/coin_lists/mod.rs @@ -32,7 +32,7 @@ //! "creations" sharing one timestamp, some of them at unix epoch 1. "No later than X" is a //! fact; "added on X" from the same row is a lie the user cannot see through. -use std::collections::{BTreeSet, HashMap}; +use std::collections::{BTreeMap, BTreeSet, HashMap}; use std::time::Duration; use rusqlite::{Connection, OpenFlags}; @@ -324,17 +324,61 @@ struct Head { /// /// The ids are `i64`/`u64` formatted as decimals, so there is nothing here a string could /// inject; binding them instead would mean rebuilding the SQL per selection size anyway. +/// +/// GROUPED BY CORE, and that grouping is the whole point. SQLite parses `a OR b OR c` into a +/// LEFT-DEEP tree, so an `OR` chain with one term per selected strategy has a depth equal to +/// the selection size — and past `SQLITE_MAX_EXPR_DEPTH` (1000) the statement does not run +/// slowly, it fails to prepare: `Expression tree is too large`, the whole panel down. Measured +/// on a replica with 2 420 `strategyid@core_uid` groups, where every period failed. Factoring +/// the repeated `s.core_uid = C` out of each group makes the depth the number of DISTINCT +/// CORES (tens) instead, and an `IN (…)` right-hand side is a flat expression LIST that costs +/// no depth per element. +/// +/// One id in a group still emits `= id` rather than `IN (id)`: same plan, and it keeps the +/// generated SQL readable for the ordinary small selection. Groups and ids are emitted in +/// sorted order so one selection always renders one string. +/// +/// THE CEILING MOVED, IT DID NOT VANISH. What is joined by `OR` is now one term per distinct +/// core, so a selection spanning ~1000 distinct CORES would hit the same wall again. That is a +/// far higher ceiling on a variable a user cannot push — the core count is the fleet's shape, +/// not the size of a selection — but it is a bound, not a guarantee, and it is written here so +/// the next person reads it instead of rediscovering it. fn scope_sql(targets: &[(i64, Option)]) -> String { - let terms: Vec = targets + let mut by_core: BTreeMap, BTreeSet> = BTreeMap::new(); + for (sid, core) in targets { + by_core + .entry(core.map(|c| c as i64)) + .or_default() + .insert(*sid); + } + let terms: Vec = by_core .iter() - .map(|(sid, core)| match core { - Some(c) => format!("(s.strategy_id = {sid} AND s.core_uid = {})", *c as i64), - None => format!("s.strategy_id = {sid}"), + .map(|(core, sids)| { + let ids = strategy_id_predicate(sids); + match core { + Some(c) => format!("(s.core_uid = {c} AND s.strategy_id {ids})"), + None => format!("s.strategy_id {ids}"), + } }) .collect(); format!(" AND ({})", terms.join(" OR ")) } +/// `= id` for a lone strategy, `IN (…)` for several — the comparison half of [`scope_sql`]'s +/// per-core term. Never called with an empty set: a group exists only because a target put an +/// id in it. +fn strategy_id_predicate(sids: &BTreeSet) -> String { + if sids.len() == 1 { + let only = sids + .iter() + .next() + .expect("a set of len 1 has a first element"); + return format!("= {only}"); + } + let list: Vec = sids.iter().map(i64::to_string).collect(); + format!("IN ({})", list.join(", ")) +} + /// Read-only handle on the strategy database. /// /// An absent file is `NotReady` (the terminal has simply never synced strategies), while a @@ -351,6 +395,7 @@ fn open_strategies() -> ReadResult { // landing under our snapshot surfaces to the user as a read failure. Same 3s the // strategy store's own reader uses. let _ = conn.busy_timeout(Duration::from_secs(3)); + super::trace::install_on(&conn); Ok(conn) } diff --git a/crates/moon-core/src/db/integrity/mod.rs b/crates/moon-core/src/db/integrity/mod.rs index 5e8ca535..3e1aa370 100644 --- a/crates/moon-core/src/db/integrity/mod.rs +++ b/crates/moon-core/src/db/integrity/mod.rs @@ -234,6 +234,7 @@ pub(crate) fn run(path: &Path) -> Integrity { Err(e) => return classify_pragma_error(e, "открытие"), }; let _ = conn.busy_timeout(Duration::from_secs(30)); + super::trace::install_on(&conn); // HARD DEADLINE. `busy_timeout` bounds lock waiting, not statement runtime, // and the scan holds a WAL read snapshot for its whole duration — which diff --git a/crates/moon-core/src/db/mod.rs b/crates/moon-core/src/db/mod.rs index 965a033b..e83bd0ed 100644 --- a/crates/moon-core/src/db/mod.rs +++ b/crates/moon-core/src/db/mod.rs @@ -32,6 +32,7 @@ mod report_read; pub mod report_recovery; #[cfg(test)] mod test_support; +pub mod trace; mod trade_meta; pub mod tuner; pub mod valuation; @@ -995,6 +996,7 @@ pub fn open_reader() -> ReadResult { // before report-derived views can open readers; an absent file remains a normal not-ready state. let _ = valuation::attach(&conn)?; read_cancel::install_current(&conn)?; + trace::install_on(&conn); Ok(conn) } @@ -1282,11 +1284,13 @@ pub fn open_readonly() -> ReadResult { let path = paths::reports_db_path(); // Same reasoning as `open_reader`: only a genuine absence may report `NotReady`. metadata_gate(&path, "отчёты(ro): доступ к файлу")?; - Connection::open_with_flags( + let conn = Connection::open_with_flags( &path, rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY | rusqlite::OpenFlags::SQLITE_OPEN_NO_MUTEX, ) - .map_err(|e| read_fail("отчёты(ro)", e)) + .map_err(|e| read_fail("отчёты(ro)", e))?; + trace::install_on(&conn); + Ok(conn) } #[cfg(test)] diff --git a/crates/moon-core/src/db/report_read.rs b/crates/moon-core/src/db/report_read.rs index 716ee452..396bd038 100644 --- a/crates/moon-core/src/db/report_read.rs +++ b/crates/moon-core/src/db/report_read.rs @@ -1933,7 +1933,47 @@ fn run_row_pass( let rec_id_select = rec_id_expr(src); // Sort in SQL only if this source can express the column; otherwise source order is // irrelevant, because the merge below reorders everything anyway. + // + // The DESC arm drops the leading `({expression}) IS NULL` term: SQLite orders NULL + // lowest, so a plain `{expression} DESC` already places NULLs last, exactly what the + // `IS NULL` term bought. Without it that leading term forces a temp b-tree sort instead + // of using the source's index on `{expression}`. The ASC arm keeps the term — without it + // NULLs would sort FIRST there, a real result change — so do not drop it from that arm. + // + // The DESC arm then MUST carry the primary key as a tie-break, and that is not a + // refinement — it is what keeps the rewrite honest. `LIMIT` is applied per source + // BEFORE the merge below, and SQL guarantees nothing about the relative order of rows + // equal on every ORDER BY term. Dropping the leading term changes the plan from a temp + // sort to a backwards index walk, so a tie straddling the limit boundary would surface + // a DIFFERENT SET of rows, not merely a different order: measured on SQLite 3.50.4 with + // 20 rows sharing one `closedate`, 19 of 25 limits returned a different set. The key + // `(core_uid, newrecid)` is `rep.rs`'s own PRIMARY KEY, so it is total on the typed + // replica and the whole result becomes DEFINED rather than planner-chosen. It costs the + // sort only WITHIN each group of equal values — measured 2.4 ms -> 2.8 ms against + // 4804 ms before the rewrite, so the gain survives it intact. + // + // The key is taken PER SOURCE, because the two sources do not share one. The typed + // replica is keyed `(core_uid, newrecid)` (`rep.rs`'s own PRIMARY KEY) and the legacy + // `closed_sell_reports` is keyed `(core_uid, db_id)` (`db/mod.rs:10`) — and reaching for + // `rec_id_expr` here instead would be a trap twice over: it yields the LITERAL `0` on a + // source without `newrecid`, and SQLite reads a bare integer in ORDER BY as a COLUMN + // ORDINAL even parenthesised, so it would fail the whole query rather than order it. + // The same source-shape test `source_sort_expression` already uses for `id` is what + // picks the right column. A source offering neither keeps today's undefined tie order, + // which is no worse than before this rewrite. let order = match source_sort_expression(src, sort_col, valuation.as_ref()) { + Some(expression) if desc => { + let mut order = format!("{expression} {dir}"); + if src.cols.contains("core_uid") { + order.push_str(&format!(", r.core_uid {dir}")); + } + if src.cols.contains("newrecid") { + order.push_str(&format!(", r.newrecid {dir}")); + } else if src.legacy && src.cols.contains("db_id") { + order.push_str(&format!(", r.\"db_id\" {dir}")); + } + order + } Some(expression) => format!("({expression}) IS NULL, {expression} {dir}"), None => "1".to_string(), }; @@ -2429,6 +2469,13 @@ pub fn distinct_cores(conn: &Connection) -> ReadResult> { /// strategy predicates are deliberately removed so an active checkbox or name mask does not hide /// alternative strategies that match every other Report filter. /// +/// Split into a normal-strategy arm and a liquidation arm when the two differ. What the split +/// buys is NOT an index-only scan — `deleted` sits in no index, so both arms still read table +/// rows to apply the deletion predicate. It buys keeping the liquidation-attribution CASE and +/// its two correlated subqueries off the rows that can never satisfy it, which on the 600k-row +/// measurement fixture is 599 696 of 600 000. Measured there: 2057 ms as one statement, +/// 1236 ms as two arms. +/// /// Args: /// conn: Open report reader or snapshot with optional strategy attachment. /// filter: Active Report filter; every non-strategy predicate scopes discovery. @@ -2475,12 +2522,47 @@ pub fn distinct_strategies( } let strategy_id = super::analytics::effective_sid_expr("r", &src.cols, has_strategy_names); let (where_sql, params) = build_where(&scope, &src.cols, has_strategy_names); - let sql = format!( - "SELECT DISTINCT r.core_uid, COALESCE({strategy_id}, 0) FROM {} r{where_sql}", - src.table, - ); - let refs: Vec<&dyn rusqlite::types::ToSql> = + // `COALESCE(r."strategyid",0)` never yields NULL, so `<> 0` / `= 0` partition every row + // with no third case, and identity is set equality (this function sorts its whole output + // in Rust below, so SQL row order never matters). When `strategy_id` is already the plain + // column, both arms would be identical and the split would buy nothing — keep the single + // statement in that case. + // + // `UNION`, not `UNION ALL`, and this is MEASURED rather than reasoned. The arms are + // disjoint by construction and the caller below dedups every pair through `seen`, so + // `UNION ALL` looks like the free choice — it is the opposite. `UNION` lets SQLite + // dedup the compound ONCE, and its left arm then needs no `DISTINCT` of its own; + // `UNION ALL` makes each arm materialise its own `USE TEMP B-TREE FOR DISTINCT`. + // On the 600k-row measurement fixture: 2057 ms before this split, 1236 ms with `UNION`, + // 6599 ms with `UNION ALL` — three times WORSE than doing nothing at all. + // + // ONE `build_where`, its parameters bound TWICE. Calling it a second time would re-read + // the wall clock — `append_row_scope` resolves "does this window still reach the + // present" against `now`, and its own doc says that must be ONE reading for the whole + // predicate — so two calls can straddle a second boundary and scope the two arms to + // genuinely different row states. Binding the same vector twice also keeps the bound + // parameter count where it was instead of doubling it. + let two_arms = strategy_id != "r.\"strategyid\""; + let sql = if two_arms { + format!( + "SELECT DISTINCT r.core_uid, COALESCE(r.\"strategyid\",0) FROM {table} r{where_sql} AND COALESCE(r.\"strategyid\",0) <> 0 \ + UNION \ + SELECT DISTINCT r.core_uid, COALESCE({strategy_id}, 0) FROM {table} r{where_sql} AND COALESCE(r.\"strategyid\",0) = 0", + table = src.table, + ) + } else { + format!( + "SELECT DISTINCT r.core_uid, COALESCE({strategy_id}, 0) FROM {} r{where_sql}", + src.table, + ) + }; + let mut refs: Vec<&dyn rusqlite::types::ToSql> = params.iter().map(|value| value.as_ref()).collect(); + if two_arms { + let second: Vec<&dyn rusqlite::types::ToSql> = + params.iter().map(|value| value.as_ref()).collect(); + refs.extend(second); + } let mut stmt = conn.prepare(&sql).map_err(|e| read_fail(CTX, e))?; let rows = stmt .query_map(refs.as_slice(), |row| { diff --git a/crates/moon-core/src/db/report_read/tests.rs b/crates/moon-core/src/db/report_read/tests.rs index 809706b4..d6d1b824 100644 --- a/crates/moon-core/src/db/report_read/tests.rs +++ b/crates/moon-core/src/db/report_read/tests.rs @@ -1,7 +1,7 @@ //! Regression tests for exact Report strategy filtering. -use rusqlite::Connection; use rusqlite::types::Value; +use rusqlite::{Connection, params}; use super::{ QuoteCurrency, ReportFilter, ReportStrategyKey, RowScope, SideFilter, distinct_strategies, @@ -2591,3 +2591,129 @@ fn open_block_stays_newest_first_across_both_sources_under_an_ascending_sort() { proving the closed rows were never reversed, only the open block" ); } + +/// `db/report_read.rs:run_row_pass` must retain an ASC `NULLS LAST` equivalent while optimizing +/// its SQL order. Dropping the leading NULL discriminator without adding `NULLS LAST` promotes an +/// unset Profit % above completed trades in an ascending Report and CSV export, silently changing +/// the user's visible top rows. +#[test] +fn profit_percent_nulls_stay_last_in_both_report_directions() { + let conn = Connection::open_in_memory().expect("open null-order fixture"); + super::super::init_db(&conn).expect("initialize report metadata"); + conn.execute_batch( + "CREATE TABLE orders_rep ( + core_uid INTEGER NOT NULL, core_name TEXT NOT NULL, newrecid INTEGER NOT NULL, + closedate INTEGER, profitbtc REAL, spentbtc REAL, + PRIMARY KEY (core_uid, newrecid) + ); + INSERT INTO orders_rep VALUES + (1, 'CORE', 10, 100, 10.0, 100.0), + (1, 'CORE', 11, 200, 5.0, 100.0), + (1, 'CORE', 20, 300, 5.0, 0.0);", + ) + .expect("seed finite and undefined Profit percent rows"); + super::super::test_support::rep_init(&conn); + + for descending in [false, true] { + let table = query_reports( + &conn, + &ReportFilter { + rows: RowScope::Closed, + ..ReportFilter::default() + }, + super::PROFIT_PERCENT_COLUMN, + descending, + 2, + ) + .expect("read Profit percent report"); + + assert_eq!( + table.rec_ids, + if descending { + vec![10, 11] + } else { + vec![11, 10] + }, + "the limited Report must retain both finite percentages when descending={descending}" + ); + } +} + +/// `report_read.rs:run_row_pass` must retain `r.newrecid DESC` after the default closed-date +/// sort. Removing that tie-break lets SQLite choose which equal-date trades survive the +/// source-local LIMIT, so the Report grid and CSV export can silently show a different set after +/// an unrelated planner or schema change. +#[test] +fn closedate_desc_source_limit_uses_the_total_replica_key() { + let conn = Connection::open_in_memory().expect("open closed-date tie fixture"); + super::super::init_db(&conn).expect("initialize report metadata"); + conn.execute_batch( + "CREATE TABLE orders_rep ( + core_uid INTEGER NOT NULL, core_name TEXT NOT NULL, newrecid INTEGER NOT NULL, + closedate INTEGER, PRIMARY KEY (core_uid, newrecid) + );", + ) + .expect("create replica fixture"); + let seeded = vec![ + (3_i64, 32_i64, 1_000_i64), + (1, 20, 1_000), + (4, 53, 1_000), + (2, 11, 1_000), + (1, 50, 1_000), + (3, 12, 1_000), + (2, 51, 1_000), + (4, 23, 1_000), + (1, 30, 1_000), + (3, 52, 1_000), + (2, 21, 1_000), + (4, 43, 1_000), + (1, 10, 1_000), + (3, 42, 1_000), + (2, 41, 1_000), + (4, 13, 1_000), + (1, 40, 1_000), + (3, 22, 1_000), + (2, 31, 1_000), + (4, 33, 1_000), + ]; + let mut insert = conn + .prepare("INSERT INTO orders_rep (core_uid, core_name, newrecid, closedate) VALUES (?1, ?2, ?3, ?4)") + .expect("prepare tied replica rows"); + for &(core_uid, newrecid, closedate) in &seeded { + insert + .execute(params![ + core_uid, + format!("CORE-{core_uid}"), + newrecid, + closedate + ]) + .expect("insert tied replica row"); + } + drop(insert); + super::super::test_support::rep_init(&conn); + + let limit = 9; + let mut expected = seeded; + expected.sort_unstable_by(|left, right| right.cmp(left)); + let expected_rec_ids: Vec = expected + .into_iter() + .take(limit) + .map(|(_, newrecid, _)| newrecid) + .collect(); + let table = query_reports( + &conn, + &ReportFilter { + rows: RowScope::Closed, + ..ReportFilter::default() + }, + "closedate", + true, + limit, + ) + .expect("read default closed-date report"); + + assert_eq!( + table.rec_ids, expected_rec_ids, + "the source-local LIMIT must retain the rows selected by closedate, core uid, and record id" + ); +} diff --git a/crates/moon-core/src/db/trace.rs b/crates/moon-core/src/db/trace.rs new file mode 100644 index 00000000..ca2015d1 --- /dev/null +++ b/crates/moon-core/src/db/trace.rs @@ -0,0 +1,89 @@ +//! Opt-in statement profiler shared by every read connection this crate opens. +//! +//! This is a measurement instrument, not a feature: with no hook installed it costs one +//! `OnceLock::get` per connection birth and changes no query, no schema, and no result. It must +//! never gate behaviour — a caller that wants to branch on "is profiling on" is using this +//! module wrong. +//! +//! `trace_v2` accepts only a plain `fn` pointer, not a closure, so the hook itself carries no +//! state; a caller that needs to accumulate results supplies a `fn` that writes into its own +//! static or thread-local storage (see `crates/moon-core/examples/db_read_timing.rs` for the +//! harness that does exactly this). +//! +//! `trace_v2` is CONNECTION-LOCAL: this crate opens report/strategy/valuation/kline read +//! connections from ten call sites, and each one must call [`install_on`] itself. There is no +//! single choke point that sees every connection. + +use std::sync::OnceLock; +use std::time::Duration; + +use rusqlite::Connection; +use rusqlite::trace::{TraceEvent, TraceEventCodes}; + +/// One profiled SQLite statement, captured from a `trace_v2` PROFILE event. +#[derive(Clone, Debug)] +pub struct ProfiledStatement { + /// The statement text. Expanded (bound values substituted) when [`Self::expanded`] is + /// true; otherwise the raw text with `?N` placeholders. + pub sql: String, + /// Whether [`Self::sql`] is expanded. A placeholder-only capture cannot be replayed with + /// `EXPLAIN QUERY PLAN` against real literals and must never be ranked as if it could. + pub expanded: bool, + /// Wall time SQLite itself reports for running this statement. + pub duration: Duration, +} + +/// Process-wide profiler hook. `None` until [`install_read_profiler`] sets it. +static HOOK: OnceLock = OnceLock::new(); + +/// Install the process-wide read profiler. +/// +/// There can be only one; a later call never replaces an earlier one; see the memory ordering +/// of [`OnceLock::set`]. +/// +/// Args: +/// hook: Called once per profiled statement, on the thread that ran it. +/// +/// Returns: +/// Whether this call actually installed the hook (`false` when one was already set). +pub fn install_read_profiler(hook: fn(ProfiledStatement)) -> bool { + HOOK.set(hook).is_ok() +} + +/// Attach the profiler to one freshly opened connection, if a hook is installed. +/// +/// No-op when [`install_read_profiler`] was never called: one `OnceLock::get`, no query +/// touched, no behaviour changed. Call this from every connection-owning call site in this +/// crate — there is no single place that sees every connection (see the module docs). +/// +/// Args: +/// conn: Freshly opened connection, before it is handed to any caller. +pub(crate) fn install_on(conn: &Connection) { + if HOOK.get().is_some() { + conn.trace_v2( + TraceEventCodes::SQLITE_TRACE_PROFILE, + Some(on_profile_event), + ); + } +} + +/// `trace_v2` callback: build a [`ProfiledStatement`] from a PROFILE event and forward it to +/// the installed hook. A plain `fn`, per `trace_v2`'s signature — it carries no state of its +/// own and reads the current hook from [`HOOK`] on every call. +fn on_profile_event(event: TraceEvent<'_>) { + let TraceEvent::Profile(stmt, duration) = event else { + return; + }; + let Some(hook) = HOOK.get() else { + return; + }; + let (sql, expanded) = match stmt.expanded_sql() { + Some(sql) => (sql, true), + None => (stmt.sql().into_owned(), false), + }; + hook(ProfiledStatement { + sql, + expanded, + duration, + }); +} diff --git a/crates/moon-core/src/db/tuner/strategy_read.rs b/crates/moon-core/src/db/tuner/strategy_read.rs index 6db4a9f2..8fddd0af 100644 --- a/crates/moon-core/src/db/tuner/strategy_read.rs +++ b/crates/moon-core/src/db/tuner/strategy_read.rs @@ -14,6 +14,7 @@ fn open_strategies_ro() -> Option { let conn = Connection::open_with_flags(&path, rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY).ok()?; let _ = conn.busy_timeout(std::time::Duration::from_secs(3)); + crate::db::trace::install_on(&conn); Some(conn) } diff --git a/crates/moon-core/src/db/valuation/mod.rs b/crates/moon-core/src/db/valuation/mod.rs index 31d2636f..32838f13 100644 --- a/crates/moon-core/src/db/valuation/mod.rs +++ b/crates/moon-core/src/db/valuation/mod.rs @@ -967,6 +967,7 @@ fn existing_store_is_healthy(path: &Path) -> Result { let flags = OpenFlags::SQLITE_OPEN_READ_ONLY | OpenFlags::SQLITE_OPEN_URI; let conn = Connection::open_with_flags(uri, flags) .map_err(|error| format!("read-only open failed: {error}"))?; + super::trace::install_on(&conn); let check = conn.query_row("PRAGMA main.quick_check(1)", [], |row| { row.get::<_, String>(0) }); @@ -1353,6 +1354,7 @@ pub(crate) fn open_store(path: &Path) -> rusqlite::Result { CREATE INDEX IF NOT EXISTS idx_trade_values_inputs ON trade_values (algorithm_version, quote_ordinal, rate_minute_utc);", )?; + super::trace::install_on(&conn); Ok(conn) } diff --git a/crates/moon-core/src/market/kline_cache.rs b/crates/moon-core/src/market/kline_cache.rs index 33649e5a..ab33319b 100644 --- a/crates/moon-core/src/market/kline_cache.rs +++ b/crates/moon-core/src/market/kline_cache.rs @@ -148,6 +148,10 @@ impl KlineCache { log::warn!("kline cache schema failed {}: {e}", path.display()); return None; } + // This one connection also serves `Op::Merge`/`Op::MergeBatch` on the worker thread + // below, so a hook installed here captures background WRITE timings too, mixed in with + // the `Op::Read` ones — unlike every other call site, which only ever opens a reader. + crate::db::trace::install_on(&conn); let (tx, rx) = mpsc::channel::(); std::thread::Builder::new() .name("kline-cache".into()) diff --git a/crates/moon-core/src/strat_db/mod.rs b/crates/moon-core/src/strat_db/mod.rs index 4ab51606..1508757f 100644 --- a/crates/moon-core/src/strat_db/mod.rs +++ b/crates/moon-core/src/strat_db/mod.rs @@ -278,6 +278,7 @@ fn open_ro(path: &std::path::Path) -> rusqlite::Result { rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY | rusqlite::OpenFlags::SQLITE_OPEN_NO_MUTEX, )?; let _ = conn.busy_timeout(std::time::Duration::from_secs(3)); + crate::db::trace::install_on(&conn); Ok(conn) } diff --git a/crates/moon-core/src/strat_db/stats.rs b/crates/moon-core/src/strat_db/stats.rs index 59284601..19be6302 100644 --- a/crates/moon-core/src/strat_db/stats.rs +++ b/crates/moon-core/src/strat_db/stats.rs @@ -41,6 +41,7 @@ fn open_rw() -> Option { } let conn = Connection::open(&path).ok()?; let _ = conn.busy_timeout(std::time::Duration::from_secs(3)); + crate::db::trace::install_on(&conn); Some(conn) } diff --git a/tools/gen_replica.py b/tools/gen_replica.py index 1f239c3e..c7c205d9 100644 --- a/tools/gen_replica.py +++ b/tools/gen_replica.py @@ -1,90 +1,275 @@ """Build a synthetic MoonTerminal report replica for Analytics timing measurements. -Usage: python gen_replica.py [span_days] +Usage: python gen_replica.py [span_days] [--analyze] -Writes /data/reports.sqlite and /data/strategies.sqlite (the Windows -data-directory layout crates/moon-core/src/config/paths.rs resolves). -Schema mirrors crates/moon-core/src/db/rep.rs (orders_rep + REP_INDEXES) and the columns -crates/moon-core/src/db/analytics reads. The supplied data directory must be disposable: the -generator recreates its report and strategy database files. +Writes the complete Windows data-directory layout `crates/moon-core/src/config/paths.rs` +resolves: `/data/reports.sqlite`, `strategies.sqlite`, `valuation.sqlite` and +`klines.sqlite`, plus a `/data/fixture.json` manifest. The supplied data directory +must be disposable: the generator recreates every one of those files. + +`reports.sqlite`'s schema mirrors every column `crates/moon-core/src/db/report_read.rs`'s +`DISPLAY_COLUMNS` projects (see `crates/moon-core/src/db/rep.rs` for the real replica's own +create-table, which starts from three columns and grows the rest dynamically from the core's +`ReportSchema` — this generator's hand-written column set plays that same role for the fixture). +`strategies.sqlite` mirrors the full writer schema in `crates/moon-core/src/strat_db/write.rs`. +`valuation.sqlite` mirrors `crates/moon-core/src/db/valuation/mod.rs::open_store`'s DDL exactly. +`klines.sqlite` is seeded by a separate Rust helper (`crates/moon-core/examples/gen_klines.rs`) +through `KlineCache`'s own production write API — never re-implemented here, see `_build_klines`. + +`--analyze` is opt-in and defaults to off: production never runs SQLite `ANALYZE` on this +database, so a fixture that always carried `sqlite_stat1` statistics the user's replica does not +have would let every measured query plan diverge from the one shipped. + +After generation the fixture is checked against hard bounds (row/core/coin/group cardinality, the +`reports.sqlite` size band, and the special-case ratios below) and the process exits non-zero on a +miss instead of silently shipping an undersized or ratio-drifted replica. """ +import json import os import random import sqlite3 +import subprocess import sys -TUNER_COLS = "lev,dmark,pricebug,hvol,hvolf,dvol,vd1m,bvsvratio,d24h,d3h,da1m,d5s,btc1hdelta,exchange1hdelta,btc24hdelta,exchange24hdelta,btc5mdelta,dbtc1m,d1h,d15m,d5m,d1m,pump1h,dump1h".split(",") +TUNER_COLS = ["lev", "dmark", "pricebug", "hvol", "hvolf", "dvol", "vd1m", "bvsvratio", "d24h", "d3h", "da1m", "d5s", "btc1hdelta", "exchange1hdelta", "btc24hdelta", "exchange24hdelta", "btc5mdelta", "dbtc1m", "d1h", "d15m", "d5m", "d1m", "pump1h", "dump1h"] +# Every DISPLAY_COLUMNS entry that is a real stored column rather than one of report_read.rs's +# four SYNTHETIC (computed) columns (profitpct, valuation_profit_usdt, valuation_rate, +# valuation_rate_source) — see report_read.rs:18-73 and :108-125. +DATE_COLS = ["closedate", "buydate", "sellsetdate"] REAL_COLS = [ "profitbtc", "spentbtc", "boughtq", "buyprice", "sellprice", + "quantity", "gainedbtc", "takeprofitlag", ] -DATE_COLS = ["closedate", "buydate"] -INT_COLS = ["isshort", "emulator", "deleted", "strategyid", "basecurrency"] -TEXT_COLS = ["coin", "sellreason", "channelname", "signaltype", "comment"] +INT_COLS = [ + "isshort", "emulator", "deleted", "strategyid", "basecurrency", + "id", "taskid", "source", "channel", "status", "last_update_at", +] +TEXT_COLS = ["coin", "sellreason", "channelname", "signaltype", "comment", "exorderid", "fname"] + +STRATEGIES_PER_CORE = 120 +COINS_TARGET = 400 +_REAL_TICKERS = ["BTC", "ETH", "SOL", "XRP", "ADA", "DOGE", "AVAX", "LINK", "DOT", "MATIC", "LTC", "BCH", "ATOM", "UNI", "ETC", "FIL", "APT", "ARB", "OP", "NEAR"] +LEV_CHOICES = [1, 5, 10, 25] -COINS = [f"{a}USDT" for a in - "BTC ETH SOL XRP ADA DOGE AVAX LINK DOT MATIC LTC BCH ATOM UNI ETC FIL APT ARB OP NEAR".split()] +# Special-case ratios (1b): expected fraction of rows falling into each mutually exclusive +# non-normal bucket, drawn from one categorical roll per row so the buckets never overlap. +LIQUIDATION_RATIO = 0.0005 +FUNDING_RATIO = 0.003 +NULLDATE_RATIO = 0.0005 +EMULATOR_RATIO = 0.02 +DELETED_RATIO = 0.005 +LIQUIDATION_OWNER = "ExternalOwner" -def build(data_dir, cores, rows, span_days=400): - """Create a disposable report and strategy replica with deterministic synthetic data. + +def _coin_universe(n=COINS_TARGET): + """Build a deterministic coin ticker universe wide enough for cardinality targets. Args: - data_dir: Root whose `data` child receives recreated SQLite databases. - cores: Number of synthetic core identities to generate. - rows: Number of synthetic report rows to generate. - span_days: Number of days covered by the generated close times. + n: Total distinct coin identities to produce. + + Returns: + The 20 original real tickers (unchanged, for anything keying off them) followed by + synthetic fillers up to `n`. + """ + coins = [f"{a}USDT" for a in _REAL_TICKERS] + i = 0 + while len(coins) < n: + coins.append(f"ALT{i:03d}USDT") + i += 1 + return coins[:n] + + +def _core_name(core): + """Format: + core -> f"core-{core:03d}", the same label the real replica writes per core. + """ + return f"core-{core:03d}" + + +def _btc_cores(cores): + """Cores whose reports are BTC-quoted (`basecurrency = 0`); every other core is USDT (`1`). + + Args: + cores: Total synthetic core count. + + Returns: + Up to the first 3 core ids, verified against `db/quote.rs`'s `QuoteCurrency::btc()` + ordinal (0) and `QuoteCurrency::usdt()` ordinal (1). + """ + return {c for c in range(1, cores + 1) if c <= 3} + + +def _margin_cores(cores): + """Cores that write posted MARGIN (`notional / lev`) into `spentbtc` instead of full notional. + + Args: + cores: Total synthetic core count. + + Returns: + Up to 2 core ids distinct from `_btc_cores`, matching `db/analytics/basis.rs`'s + per-core margin-vs-notional probe. + """ + return {c for c in range(1, cores + 1) if 4 <= c <= 5} + + +def _rate_usdt_for_minute(minute_utc): + """Deterministic synthetic BTC/USDT spot rate for one closed minute. + + A pure function of `minute_utc` rather than an RNG draw, so the same minute always resolves + to the same rate across cores and reruns without needing a shared random-state thread through + the row loop. + + Args: + minute_utc: UTC minute start, in Unix seconds. + + Returns: + A plausible BTC/USDT rate that varies slowly with time. + """ + return 20_000.0 + float((minute_utc // 60) % 50_000) + + +def _build_reports(conn, cores, rows, span_days, rng, btc_cores, margin_cores): + """Populate `orders_rep` and collect everything the valuation pass and the shape gate need. + + Args: + conn: Open connection to the recreated `reports.sqlite`. + cores: Synthetic core count. + rows: Synthetic row count. + span_days: Days covered by generated close times. + rng: Seeded RNG shared with every other builder in this run. + btc_cores: Core ids whose rows are BTC-quoted. + margin_cores: Core ids that write margin instead of notional into `spentbtc`. + + Returns: + Stats dict consumed by `_build_valuation` and the shape gate: `cores_seen`, `coins_seen`, + `groups_seen`, per-special-case row counts, and the BTC-quoted closed rows to value. """ - db_dir = os.path.join(data_dir, "data") - os.makedirs(db_dir, exist_ok=True) - reports = os.path.join(db_dir, "reports.sqlite") - for suffix in ("", "-wal", "-shm"): - try: - os.remove(reports + suffix) - except OSError: - pass - conn = sqlite3.connect(reports) - conn.execute("PRAGMA journal_mode=WAL") - conn.execute("CREATE TABLE IF NOT EXISTS app_meta (key TEXT PRIMARY KEY, value TEXT NOT NULL)") - conn.execute("INSERT OR REPLACE INTO app_meta(key,value) VALUES('legacy_dropped','1')") cols = ["core_uid INTEGER NOT NULL", "core_name TEXT NOT NULL", "newrecid INTEGER NOT NULL"] cols += [f"{c} INTEGER" for c in DATE_COLS] cols += [f"{c} REAL" for c in REAL_COLS] cols += [f"{c} INTEGER" for c in INT_COLS] cols += [f"{c} TEXT" for c in TEXT_COLS] cols += [f"{c} REAL" for c in TUNER_COLS] - conn.execute( - "CREATE TABLE orders_rep (%s, PRIMARY KEY (core_uid, newrecid))" % ", ".join(cols) - ) + conn.execute("CREATE TABLE IF NOT EXISTS app_meta (key TEXT PRIMARY KEY, value TEXT NOT NULL)") + conn.execute("INSERT OR REPLACE INTO app_meta(key,value) VALUES('legacy_dropped','1')") + conn.execute(f"CREATE TABLE orders_rep ({', '.join(cols)}, PRIMARY KEY (core_uid, newrecid))") names = ["core_uid", "core_name", "newrecid"] + DATE_COLS + REAL_COLS + INT_COLS + TEXT_COLS + TUNER_COLS placeholders = ",".join("?" * len(names)) - sql = "INSERT INTO orders_rep (%s) VALUES (%s)" % (",".join(names), placeholders) + sql = f"INSERT INTO orders_rep ({','.join(names)}) VALUES ({placeholders})" - rng = random.Random(20260820) + coins = _coin_universe() end = 1_780_000_000 start = end - span_days * 86_400 # A real replica accumulates in close-date order, so its table pages carry that locality. # Reproduce it: without this the index range scan pays a random page fetch per row and the # measurement overstates every period read. closes = sorted(rng.randrange(start, end) for _ in range(rows)) - strategies_per_core = 12 + + stats = { + "cores_seen": set(), + "coins_seen": set(), + "groups_seen": set(), + "lev_seen": set(), + "n_emulator": 0, + "n_deleted": 0, + "n_funding": 0, + "n_liquidation": 0, + "n_nulldate": 0, + } + to_value = [] # (core_uid, newrecid, closedate, profitbtc, spentbtc, last_update_at) + batch = [] for i in range(rows): core = i % cores + 1 - close = closes[i] - buy = close - rng.randrange(60, 86_400) + stats["cores_seen"].add(core) + raw_close = closes[i] + + kind_roll = rng.random() + if kind_roll < LIQUIDATION_RATIO: + kind = "liquidation" + elif kind_roll < LIQUIDATION_RATIO + FUNDING_RATIO: + kind = "funding" + elif kind_roll < LIQUIDATION_RATIO + FUNDING_RATIO + NULLDATE_RATIO: + kind = "nulldate" + else: + kind = "normal" + + if kind == "funding": + buy = raw_close + sellreason = "Funding" + stats["n_funding"] += 1 + else: + buy = raw_close - rng.randrange(60, 86_400) + sellreason = "TakeProfit" + closedate_val = None if kind == "nulldate" else raw_close + if kind == "nulldate": + stats["n_nulldate"] += 1 + sellsetdate_val = raw_close + rng.randrange(0, 300) + profit = rng.gauss(0.4, 25.0) - spent = rng.uniform(20.0, 900.0) - sid = (i // cores) % strategies_per_core + 1 - row = [ - core, "core-%03d" % core, i, - close, buy, profit, spent, - rng.uniform(0.01, 100.0), rng.uniform(0.5, 70000.0), rng.uniform(0.5, 70000.0), - rng.randrange(2), 0, 0, sid, 1, - COINS[i % len(COINS)], "TakeProfit", "", "", "", + notional = rng.uniform(20.0, 900.0) + lev = rng.choice(LEV_CHOICES) + stats["lev_seen"].add(lev) + spent = notional / lev if core in margin_cores else notional + boughtq = rng.uniform(0.01, 100.0) + buyprice = rng.uniform(0.5, 70000.0) + sellprice = rng.uniform(0.5, 70000.0) + quantity = boughtq + gainedbtc = spent + profit + takeprofitlag = rng.uniform(0.0, 300.0) + + isshort = rng.randrange(2) + emulator = 1 if rng.random() < EMULATOR_RATIO else 0 + stats["n_emulator"] += emulator + deleted = 1 if rng.random() < DELETED_RATIO else 0 + stats["n_deleted"] += deleted + sid = (i // cores) % STRATEGIES_PER_CORE + 1 + channelname = "Core" + signaltype = "AutoSignal" + if kind == "liquidation": + sid = 0 + channelname = "LIQUIDATION" + signaltype = LIQUIDATION_OWNER + stats["n_liquidation"] += 1 + stats["groups_seen"].add((core, sid)) + basecurrency = 0 if core in btc_cores else 1 + taskid = rng.randrange(1, 999_999) + source_val = rng.randrange(0, 3) + channel_val = rng.randrange(0, 3) + status_val = rng.randrange(0, 4) + last_update_at = raw_close * 1000 + rng.randrange(0, 1000) + coin = coins[i % len(coins)] + stats["coins_seen"].add(coin) + # `exorderid`/`fname`/`comment` are sized to match the byte width real exchange order ids + # and order filenames carry (see the size-band gate below): a real replica's rows are not + # this narrow, and the row width is what the index range-scan measurement is timing. + exorderid = f"{(i + 1) * 137 + core:018d}" + fname = f"core{core:03d}_order_{i:08d}_synthetic_fixture_row_reference" + comment = ( + f"synthetic-fixture-note row={i} core={core} strategy={sid} " + f"closeref={raw_close} padding-{'x' * 96}" + ) + + tuner_vals = [ + float(lev) if col == "lev" else rng.uniform(-50.0, 50.0) for col in TUNER_COLS ] - row += [rng.uniform(-50.0, 50.0) for _ in TUNER_COLS] + + row = ( + [core, _core_name(core), i] + + [closedate_val, buy, sellsetdate_val] + + [profit, spent, boughtq, buyprice, sellprice, quantity, gainedbtc, takeprofitlag] + + [ + isshort, emulator, deleted, sid, basecurrency, + i, taskid, source_val, channel_val, status_val, last_update_at, + ] + + [coin, sellreason, channelname, signaltype, comment, exorderid, fname] + + tuner_vals + ) batch.append(row) + if basecurrency == 0 and closedate_val is not None: + to_value.append((core, i, closedate_val, profit, spent, last_update_at)) if len(batch) >= 20000: conn.executemany(sql, batch) batch = [] @@ -97,58 +282,425 @@ def build(data_dir, cores, rows, span_days=400): ("idx_rep_strat", "core_uid, strategyid, buydate"), ("idx_rep_strategy_close", "core_uid, strategyid, closedate"), ]: - conn.execute("CREATE INDEX IF NOT EXISTS %s ON orders_rep(%s)" % (name, index_cols)) - conn.commit() - conn.execute("ANALYZE") + conn.execute(f"CREATE INDEX IF NOT EXISTS {name} ON orders_rep({index_cols})") conn.commit() - conn.close() - strat_path = os.path.join(db_dir, "strategies.sqlite") + stats["to_value"] = to_value + return stats + + +def _build_strategies(strat_path, cores, rng): + """Recreate `strategies.sqlite` with the full production schema (`strat_db/write.rs`). + + Args: + strat_path: Destination `strategies.sqlite` path. + cores: Synthetic core count. + rng: Shared seeded RNG. + + Returns: + None. Keeps the 7 deliberate per-core sid special cases the enrichment path depends on: + 1 live+enabled, 2 live+disabled, 3 deleted, 4 nameless, 5 no current version, 6 two + versions, 7 no head row at all. + """ try: os.remove(strat_path) except OSError: pass s = sqlite3.connect(strat_path) s.execute( - "CREATE TABLE strategies (core_uid INTEGER, strategy_id INTEGER, name TEXT," - " deleted INTEGER DEFAULT 0, checked INTEGER DEFAULT 1," - " PRIMARY KEY (core_uid, strategy_id))" + "CREATE TABLE strategies (\n" + " core_uid INTEGER NOT NULL,\n" + " strategy_id INTEGER NOT NULL,\n" + " core_name TEXT NOT NULL DEFAULT '',\n" + " name TEXT NOT NULL DEFAULT '',\n" + " kind TEXT NOT NULL DEFAULT '',\n" + " kind_ordinal INTEGER NOT NULL DEFAULT 0,\n" + " folder_path TEXT NOT NULL DEFAULT '',\n" + " is_short INTEGER NOT NULL DEFAULT 0,\n" + " checked INTEGER NOT NULL DEFAULT 0,\n" + " server_ver INTEGER NOT NULL DEFAULT 0,\n" + " server_ms INTEGER NOT NULL DEFAULT 0,\n" + " deleted INTEGER NOT NULL DEFAULT 0,\n" + " content_hash INTEGER NOT NULL DEFAULT 0,\n" + " head_hash INTEGER NOT NULL DEFAULT 0,\n" + " updated_ms INTEGER NOT NULL DEFAULT 0,\n" + " PRIMARY KEY (core_uid, strategy_id))" ) + s.execute("CREATE INDEX IF NOT EXISTS idx_strat_name ON strategies(core_uid, name)") + s.execute("CREATE INDEX IF NOT EXISTS idx_strat_sid ON strategies(strategy_id)") s.execute( - "CREATE TABLE strategy_versions (core_uid INTEGER, strategy_id INTEGER," - " raw_json TEXT, valid_to INTEGER)" + "CREATE TABLE strategy_versions (\n" + " id INTEGER PRIMARY KEY,\n" + " core_uid INTEGER NOT NULL,\n" + " strategy_id INTEGER NOT NULL,\n" + " valid_from INTEGER NOT NULL,\n" + " valid_to INTEGER,\n" + " change_kind TEXT NOT NULL,\n" + " origin TEXT,\n" + " n_changed INTEGER NOT NULL DEFAULT 0,\n" + " ver_gap INTEGER NOT NULL DEFAULT 0,\n" + " server_ver INTEGER NOT NULL DEFAULT 0,\n" + " server_ms INTEGER NOT NULL DEFAULT 0,\n" + " checked_at INTEGER NOT NULL DEFAULT 0,\n" + " raw_json TEXT NOT NULL,\n" + " changed_json TEXT,\n" + " UNIQUE (core_uid, strategy_id, valid_from))" ) - # Deliberate variety, so an equivalence A/B over the enrichment path actually exercises it: - # sid 1 live+enabled, 2 live+disabled, 3 deleted (status but no lists), 4 NULL name (falls - # back to the bare id), 5 no current version at all, 6 TWO current versions, 7 no head row at - # all (traded, but the strategy database does not know it), the rest ordinary. + s.execute("CREATE TABLE app_meta (key TEXT PRIMARY KEY, value TEXT NOT NULL)") + heads = [] versions = [] + base_valid_from = 1_700_000_000_000 for core in range(1, cores + 1): - for sid in range(1, strategies_per_core + 1): + core_name = _core_name(core) + for sid in range(1, STRATEGIES_PER_CORE + 1): if sid == 7: - continue - name = None if sid == 4 else "Strat_%d_%d" % (core, sid) + continue # deliberate: traded, but the strategy database never learned of it + # Deliberate variety exercised by the enrichment path (kept from the earlier fixture): + # sid 1 live+enabled, 2 live+disabled, 3 deleted, 4 nameless (NOT NULL DEFAULT '' rules + # out a literal SQL NULL here; see the generator report for the resulting fallback + # caveat), 5 no current version, 6 two versions, the rest ordinary. + name = "" if sid == 4 else f"Strat_{core}_{sid}" deleted = 1 if sid == 3 else 0 checked = 0 if sid == 2 else 1 - heads.append((core, sid, name, deleted, checked)) + kind = "MoonShot" if sid % 2 else "Standard" + heads.append( + (core, sid, core_name, name, kind, 0, "", 0, checked, + sid, 0, deleted, 0, 0, 0) + ) if sid == 5: continue - raw = ('{"SignalType":"MoonShot","LastEditDate":"2026-01-0%d",' - '"CoinsBlackList":"BTC,ETH,btc_rp","CoinsWhiteList":"SOL"}' % (sid % 9 + 1)) - versions.append((core, sid, raw, None)) + valid_from = base_valid_from + core * 1_000_000 + sid * 1_000 + raw = ( + f'{{"SignalType":"MoonShot","LastEditDate":"2026-01-0{sid % 9 + 1}",' + f'"CoinsBlackList":"BTC,ETH,btc_rp","CoinsWhiteList":"SOL"}}' + ) if sid == 6: - versions.append((core, sid, - '{"SignalType":"Second","LastEditDate":"2025-12-31",' - '"CoinsBlackList":"XRP","CoinsWhiteList":""}', None)) - s.executemany("INSERT INTO strategies VALUES (?,?,?,?,?)", heads) - s.executemany("INSERT INTO strategy_versions VALUES (?,?,?,?)", versions) + second_valid_from = valid_from + 500 + versions.append( + (core, sid, valid_from, second_valid_from, "created", None, 0, 0, + 1, 0, 0, raw, None) + ) + second_raw = ('{"SignalType":"Second","LastEditDate":"2025-12-31",' + '"CoinsBlackList":"XRP","CoinsWhiteList":""}') + changed = '{"CoinsBlackList":{"old":"BTC,ETH,btc_rp","new":"XRP"}}' + versions.append( + (core, sid, second_valid_from, None, "params", "local", 1, 0, + 2, 0, 0, second_raw, changed) + ) + else: + versions.append( + (core, sid, valid_from, None, "created", None, 0, 0, 1, 0, 0, raw, None) + ) + s.executemany( + "INSERT INTO strategies (core_uid, strategy_id, core_name, name, kind, kind_ordinal," + " folder_path, is_short, checked, server_ver, server_ms, deleted, content_hash," + " head_hash, updated_ms) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)", + heads, + ) + s.executemany( + "INSERT INTO strategy_versions (core_uid, strategy_id, valid_from, valid_to," + " change_kind, origin, n_changed, ver_gap, server_ver, server_ms, checked_at," + " raw_json, changed_json) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?)", + versions, + ) s.commit() s.close() - size = os.path.getsize(reports) / 1024 / 1024 - print("[OK] %s: %d rows, %d cores, %.1f MB" % (reports, rows, cores, size)) + + +def _build_valuation(valuation_path, to_value): + """Recreate `valuation.sqlite` with the exact reader-facing DDL from `valuation/mod.rs::open_store`. + + Args: + valuation_path: Destination `valuation.sqlite` path. + to_value: `(core_uid, newrecid, closedate, profitbtc, spentbtc, last_update_at)` rows for + every closed BTC-quoted report row, collected while `_build_reports` ran. + + Returns: + None. Writes one `trade_values` row per input row, keyed `(0, core_uid, newrecid)` + (`TradeSource::Typed.code() == 0`), plus the `rates` row each one resolves against. + """ + for suffix in ("", "-wal", "-shm"): + try: + os.remove(valuation_path + suffix) + except OSError: + pass + v = sqlite3.connect(valuation_path) + v.execute("PRAGMA journal_mode=WAL") + v.executescript( + "CREATE TABLE IF NOT EXISTS rates (\n" + " algorithm_version INTEGER NOT NULL,\n" + " quote_ordinal INTEGER NOT NULL,\n" + " minute_utc INTEGER NOT NULL,\n" + " resolved_minute_utc INTEGER NOT NULL,\n" + " rate_usdt REAL NOT NULL,\n" + " price_basis INTEGER NOT NULL,\n" + " provider TEXT NOT NULL,\n" + " symbol TEXT NOT NULL,\n" + " orientation INTEGER NOT NULL,\n" + " candle_open_ms INTEGER NOT NULL,\n" + " candle_close_ms INTEGER NOT NULL,\n" + " leg1_rate REAL NOT NULL,\n" + " leg2_provider TEXT,\n" + " leg2_symbol TEXT,\n" + " leg2_orientation INTEGER,\n" + " leg2_rate REAL,\n" + " fetched_at_ms INTEGER NOT NULL,\n" + " PRIMARY KEY (algorithm_version, quote_ordinal, minute_utc)\n" + " );\n" + " CREATE TABLE IF NOT EXISTS rate_searches (\n" + " algorithm_version INTEGER NOT NULL,\n" + " quote_ordinal INTEGER NOT NULL,\n" + " minute_utc INTEGER NOT NULL,\n" + " searched_through_minute INTEGER NOT NULL,\n" + " next_retry_at_ms INTEGER NOT NULL,\n" + " attempts INTEGER NOT NULL,\n" + " updated_at_ms INTEGER NOT NULL,\n" + " PRIMARY KEY (algorithm_version, quote_ordinal, minute_utc)\n" + " );\n" + " CREATE INDEX IF NOT EXISTS idx_rate_searches_retry\n" + " ON rate_searches (algorithm_version, next_retry_at_ms);\n" + " CREATE TABLE IF NOT EXISTS trade_values (\n" + " source_kind INTEGER NOT NULL,\n" + " core_uid INTEGER NOT NULL,\n" + " row_id INTEGER NOT NULL,\n" + " algorithm_version INTEGER NOT NULL,\n" + " closedate INTEGER NOT NULL,\n" + " quote_ordinal INTEGER NOT NULL,\n" + " profit_quote REAL NOT NULL,\n" + " spent_quote REAL,\n" + " rate_minute_utc INTEGER NOT NULL,\n" + " rate_usdt REAL NOT NULL,\n" + " profit_usdt REAL NOT NULL,\n" + " spent_usdt REAL,\n" + " valued_at_ms INTEGER NOT NULL,\n" + " PRIMARY KEY (source_kind, core_uid, row_id)\n" + " );\n" + " CREATE INDEX IF NOT EXISTS idx_trade_values_inputs\n" + " ON trade_values (algorithm_version, quote_ordinal, rate_minute_utc);" + ) + + ALGORITHM_VERSION = 2 + QUOTE_ORDINAL_BTC = 0 + SOURCE_KIND_TYPED = 0 + + rate_rows = [] + trade_rows = [] + for core_uid, newrecid, closedate, profit, spent, last_update_at in to_value: + minute_utc = (closedate // 60) * 60 + rate_usdt = _rate_usdt_for_minute(minute_utc) + candle_open_ms = minute_utc * 1000 + rate_rows.append(( + ALGORITHM_VERSION, QUOTE_ORDINAL_BTC, minute_utc, minute_utc, rate_usdt, + 0, "synthetic", "BTCUSDT", 0, candle_open_ms, candle_open_ms + 59_999, + rate_usdt, None, None, None, None, last_update_at, + )) + profit_usdt = profit * rate_usdt + spent_usdt = spent * rate_usdt + trade_rows.append(( + SOURCE_KIND_TYPED, core_uid, newrecid, ALGORITHM_VERSION, closedate, + QUOTE_ORDINAL_BTC, profit, spent, minute_utc, rate_usdt, profit_usdt, + spent_usdt, last_update_at, + )) + + v.executemany( + "INSERT OR IGNORE INTO rates (algorithm_version, quote_ordinal, minute_utc," + " resolved_minute_utc, rate_usdt, price_basis, provider, symbol, orientation," + " candle_open_ms, candle_close_ms, leg1_rate, leg2_provider, leg2_symbol," + " leg2_orientation, leg2_rate, fetched_at_ms) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)", + rate_rows, + ) + v.executemany( + "INSERT INTO trade_values (source_kind, core_uid, row_id, algorithm_version," + " closedate, quote_ordinal, profit_quote, spent_quote, rate_minute_utc, rate_usdt," + " profit_usdt, spent_usdt, valued_at_ms) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?)", + trade_rows, + ) + v.commit() + v.close() + + +def _build_klines(data_dir, span_days): + """Seed `data/klines.sqlite` through `KlineCache`'s own production write API. + + Never re-implemented in Python: `chunks_v2`'s packed row codec (`pack_rows_v2`) is private to + `market/kline_cache.rs`, and a second Python encoder of that binary format would be a second + authority no drift check could prove equivalent to the first (rejected in review). Instead + this shells out to a small Rust example that opens the cache and calls + `KlineCache::merge_batch_blocking` directly. + + Args: + data_dir: Root whose `data` child receives the recreated kline cache. + span_days: Days the synthetic candle series should cover. + + Returns: + None. Raises `subprocess.CalledProcessError` if the helper build or run fails. + """ + repo_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + cmd = [ + "cargo", "run", "--release", "-p", "moon-core", "--example", "gen_klines", + "--", os.path.abspath(data_dir), str(span_days), + ] + subprocess.run(cmd, cwd=repo_root, check=True) + + +def _check_shape(rows, cores, span_days, analyze, reports_bytes, stats): + """Hard shape gate (1g): fail loudly rather than shipping an undersized or ratio-drifted + fixture that would still produce a ranking and a cut line. + + Bounds are derived from the CLI arguments actually supplied, not hardcoded to one invocation, + so a small development run does not spuriously fail; every check below — cardinality, + size-band and ratio — scales to `rows` and applies at every size, so a tiny sparse fixture + cannot pass the gate by virtue of being tiny. + + Args: + rows, cores, span_days, analyze: The generator's own resolved arguments. + reports_bytes: Size of the produced `reports.sqlite` in bytes. + stats: The dict `_build_reports` returned. + + Returns: + None on success. + + Raises: + SystemExit(1) with a description of the first bound the fixture missed. + """ + failures = [] + + if len(stats["cores_seen"]) != cores: + failures.append( + f"core cardinality: expected {cores}, got {len(stats['cores_seen'])}" + ) + expected_coins = min(COINS_TARGET, rows) + if len(stats["coins_seen"]) != expected_coins: + failures.append( + f"coin cardinality: expected {expected_coins}, got {len(stats['coins_seen'])}" + ) + expected_groups = cores * STRATEGIES_PER_CORE + # A liquidation row's forced strategyid=0 can add one extra (core, 0) group per core beyond + # the ordinary 1..STRATEGIES_PER_CORE groups; both the plain and +cores counts are accepted. + if not (expected_groups <= len(stats["groups_seen"]) <= expected_groups + cores): + failures.append( + f"strategy-group cardinality: expected ~{expected_groups}, " + f"got {len(stats['groups_seen'])}" + ) + if not set(stats["lev_seen"]) <= set(LEV_CHOICES): + failures.append(f"lev values outside {LEV_CHOICES}: {stats['lev_seen']}") + + min_bytes_per_row, max_bytes_per_row = 650, 1000 + low, high = rows * min_bytes_per_row, rows * max_bytes_per_row + if not (low <= reports_bytes <= high): + failures.append( + f"reports.sqlite size band: expected {low / 1e6:.0f}-{high / 1e6:.0f} MB " + f"at {rows} rows, got {reports_bytes / 1e6:.1f} MB" + ) + + def ratio_ok(count, ratio, label): + """Append a failure when one observed special-case count escapes its scaled band.""" + expected = rows * ratio + # Wide multiplicative tolerance: this gate exists to catch a broken generator (an + # off-by-a-lot bug), not to police RNG sampling noise at the target row count. The + # scale-independent "+5" floor keeps a small fixture's near-zero expected count from + # rejecting on ordinary RNG noise. + if not (expected * 0.3 <= count <= expected * 3.0 + 5): + failures.append( + f"{label} ratio: expected ~{expected:.0f} rows ({ratio:.3%}), got {count}" + ) + + ratio_ok(stats["n_emulator"], EMULATOR_RATIO, "emulator") + ratio_ok(stats["n_deleted"], DELETED_RATIO, "deleted") + ratio_ok(stats["n_funding"], FUNDING_RATIO, "Funding") + ratio_ok(stats["n_liquidation"], LIQUIDATION_RATIO, "LIQUIDATION") + ratio_ok(stats["n_nulldate"], NULLDATE_RATIO, "NULL closedate") + + if failures: + sys.exit("[FAIL] fixture shape gate:\n " + "\n ".join(failures)) + + +def build(data_dir, cores, rows, span_days=400, analyze=False): + """Create a disposable report, strategy, valuation and kline replica with deterministic data. + + Args: + data_dir: Root whose `data` child receives recreated SQLite databases. + cores: Number of synthetic core identities to generate. + rows: Number of synthetic report rows to generate. + span_days: Number of days covered by the generated close times. + analyze: Whether to run SQLite `ANALYZE` on the finished `reports.sqlite` (opt-in; off by + default, since production never runs it — see the module docstring). + + Returns: + None. Writes the fixture databases and manifest below `data_dir`. + + Raises: + subprocess.CalledProcessError: If the production-API kline helper fails. + SystemExit: If the generated fixture violates a required shape bound. + """ + db_dir = os.path.join(data_dir, "data") + os.makedirs(db_dir, exist_ok=True) + reports = os.path.join(db_dir, "reports.sqlite") + for suffix in ("", "-wal", "-shm"): + try: + os.remove(reports + suffix) + except OSError: + pass + + seed = 20260820 + rng = random.Random(seed) + conn = sqlite3.connect(reports) + conn.execute("PRAGMA journal_mode=WAL") + btc_cores = _btc_cores(cores) + margin_cores = _margin_cores(cores) + stats = _build_reports(conn, cores, rows, span_days, rng, btc_cores, margin_cores) + if analyze: + conn.execute("ANALYZE") + conn.commit() + conn.close() + + strat_path = os.path.join(db_dir, "strategies.sqlite") + _build_strategies(strat_path, cores, rng) + + valuation_path = os.path.join(db_dir, "valuation.sqlite") + _build_valuation(valuation_path, stats["to_value"]) + + _build_klines(data_dir, span_days) + + reports_bytes = os.path.getsize(reports) + manifest = { + "rows": rows, + "cores": cores, + "strategy_groups": len(stats["groups_seen"]), + "coins": len(stats["coins_seen"]), + "span_days": span_days, + "reports_bytes": reports_bytes, + "analyzed": analyze, + "seed": seed, + "btc_cores": sorted(btc_cores), + "margin_cores": sorted(margin_cores), + "n_emulator": stats["n_emulator"], + "n_deleted": stats["n_deleted"], + "n_funding": stats["n_funding"], + "n_liquidation": stats["n_liquidation"], + "n_nulldate": stats["n_nulldate"], + } + with open(os.path.join(db_dir, "fixture.json"), "w", encoding="utf-8") as f: + json.dump(manifest, f, indent=2) + + _check_shape(rows, cores, span_days, analyze, reports_bytes, stats) + + print( + f"[OK] {reports}: {rows} rows, {cores} cores, " + f"{reports_bytes / 1024 / 1024:.1f} MB, " + f"{manifest['strategy_groups']} strategy groups, {manifest['coins']} coins" + ) if __name__ == "__main__": - build(sys.argv[1], int(sys.argv[2]), int(sys.argv[3]), - int(sys.argv[4]) if len(sys.argv) > 4 else 400) + argv = sys.argv[1:] + analyze_flag = "--analyze" in argv + argv = [a for a in argv if a != "--analyze"] + if len(argv) < 3: + sys.exit("usage: gen_replica.py [span_days] [--analyze]") + build( + argv[0], int(argv[1]), int(argv[2]), + int(argv[3]) if len(argv) > 3 else 400, + analyze=analyze_flag, + ) From c39263ee189c78bce6532d0db132640512d36cc3 Mon Sep 17 00:00:00 2001 From: kirillDevPro <113171057+kirillDevPro@users.noreply.github.com> Date: Thu, 3 Sep 2026 22:30:19 +0200 Subject: [PATCH 5/6] fix(analytics): keep the window's body through every kind of catch-up The previous fix kept the Analytics body on screen only when a refresh failed with Busy. Three sibling paths still swapped or blanked it while trades kept landing: a valuation-coverage gap returned Ok(Split) and replaced the whole body with the quote-split note; the Busy preserve gave up after three retries and published the error on the fourth under sustained contention; and a machine-driven report-axis re-observation escalated into a manual reload that blanked the body and armed the dimming overlay. One rule now covers all of them. CatchUpOutcome classifies every read, preserve_on_catch_up keeps settled data through a superseded result as well as a busy one, and the axis observer retires the in-flight identities and asks the writer for a catch-up instead of reloading. Calendar and the tuner's filter, time and coins surfaces route through the same predicate. --- .../src/analytics/calendar/mod.rs | 73 ++++--- .../src/analytics/calendar/tests.rs | 45 +++++ crates/moon-ui-gpui/src/analytics/mod.rs | 179 ++++++++++-------- crates/moon-ui-gpui/src/analytics/refresh.rs | 126 ++++++++++-- .../src/analytics/refresh/tests.rs | 112 +++++++++-- .../src/analytics/tuner/coins/load.rs | 49 ++--- .../src/analytics/tuner/filter/mod.rs | 36 ++-- .../src/analytics/tuner/time/mod.rs | 41 ++-- .../tests/theme_contract/analytics.rs | 56 +++++- 9 files changed, 475 insertions(+), 242 deletions(-) diff --git a/crates/moon-ui-gpui/src/analytics/calendar/mod.rs b/crates/moon-ui-gpui/src/analytics/calendar/mod.rs index 14db1cac..cbe317b0 100644 --- a/crates/moon-ui-gpui/src/analytics/calendar/mod.rs +++ b/crates/moon-ui-gpui/src/analytics/calendar/mod.rs @@ -38,19 +38,20 @@ use moon_core::db::{ProfitScope, ProfitUnit, ReadResult}; /// previous: Previous-month aggregate UI load state. /// period_result: Current cells and optional comparison from one SQLite snapshot. /// has_previous: Whether the active mode requested a comparison period. -/// preserve_on_failure: Whether a same-scope catch-up failure should keep the visible days -/// and previous-period cells instead of publishing the classified error — decided by the -/// caller via `refresh::preserve_on_catch_up_failure` (transient contention with retry -/// allowance left; every other failure is still published). +/// preserve_transient: Whether a same-scope catch-up should keep the visible days and +/// previous-period cells instead of publishing the classified result — decided by the +/// caller via `refresh::preserve_on_catch_up` (a transient outcome with a scheduled +/// correction left; every other outcome is still published). /// /// Returns: -/// `true` when the shared period read failed. +/// `true` when the shared period read did not settle into a real replacement (a failure or a +/// split totals result, published or preserved alike), so `cal_dirty` stays set. fn apply_calendar_results( days: &mut ProfitLoadState>, previous: &mut LoadState>, period_result: ReadResult>, has_previous: bool, - preserve_on_failure: bool, + preserve_transient: bool, ) -> bool { match period_result { Ok(ProfitScope::Comparable { unit, data: period }) => { @@ -67,12 +68,16 @@ fn apply_calendar_results( false } Ok(ProfitScope::Split(totals)) => { - days.apply(Ok(ProfitScope::Split(totals))); - previous.apply(Ok(None)); - false + // Mirrors the `Err` arm: a Split result is never a real replacement, so `cal_dirty` + // stays set regardless of whether this pass published it or kept the prior snapshot. + if !preserve_transient { + days.apply(Ok(ProfitScope::Split(totals))); + previous.apply(Ok(None)); + } + true } Err(error) => { - if !preserve_on_failure { + if !preserve_transient { days.apply(Err(error.clone())); previous.apply(if has_previous { Err(error) } else { Ok(None) }); } @@ -676,7 +681,7 @@ impl AnalyticsView { let (from, to) = match self.cal_mode { CalMode::Month => month_range(self.cal_ym, self.bound_zone()), CalMode::Year => all_history_range(self.bound_zone()), - // "Day" loads a 7-day window (selected day centered/at the bottom). + // "Day" loads the 30-day window, with the selected day centered or at the bottom. CalMode::Day => { let (top, bottom) = day_window(self.cal_day, self.bound_zone()); let to = @@ -749,9 +754,9 @@ impl AnalyticsView { /// /// Args: /// after_report: Whether to preserve report-style catch-up and retry semantics, including - /// keeping the visible cells over a same-scope failure that is transient contention - /// with retry allowance left (`refresh::preserve_on_catch_up_failure`) — every other - /// failure still publishes the classified error. + /// keeping the visible cells over a same-scope outcome that is transient with a + /// scheduled correction left (retry allowance or a newer generation, + /// `refresh::preserve_on_catch_up`) — every other outcome still publishes. /// show_overlay: Whether this refresh must block interaction with visible progress feedback. /// cx: GPUI context used to execute and publish the reads. fn reload_calendar_inner( @@ -793,49 +798,41 @@ impl AnalyticsView { if this.cal_seq != req { return; // mode/filters already changed } - let retry = data - .period + // Computed before `data.period` moves into the call below. + let period_outcome = super::refresh::CatchUpOutcome::of_scope(&data.period); + let cores_outcome = data + .cores .as_ref() - .err() - .filter(|error| error.kind() == Some(moon_core::db::FailKind::Busy)) - .or_else(|| { - data.cores - .as_ref() - .and_then(|cores| cores.as_ref().err()) - .filter(|error| error.kind() == Some(moon_core::db::FailKind::Busy)) - }) - .or_else(|| { - data.undated - .as_ref() - .err() - .filter(|error| error.kind() == Some(moon_core::db::FailKind::Busy)) - }) - .cloned(); + .map(super::refresh::CatchUpOutcome::of_read); + let undated_outcome = super::refresh::CatchUpOutcome::of_read(&data.undated); + let transient = period_outcome.is_transient() + || cores_outcome.is_some_and(super::refresh::CatchUpOutcome::is_transient) + || undated_outcome.is_transient(); let metadata_failed = data.cores.as_ref().is_some_and(|cores| cores.is_err()) || data.undated.is_err(); - // Peek at `data.period`'s error before it moves into the call below: only - // transient contention with retry allowance left may keep the visible cells. - let preserve_on_failure = !matches!(this.cal_days, ProfitLoadState::Loading) - && this.keep_on_busy(after_report, data.period.as_ref().err()); + // Peek at `data.period`'s outcome before it moves into the call below: only a + // transient outcome with a scheduled correction left may keep the visible cells. + let preserve_transient = !matches!(this.cal_days, ProfitLoadState::Loading) + && this.keep_on_catch_up(after_report, period_outcome, report_req); let read_failed = apply_calendar_results( &mut this.cal_days, &mut this.cal_prev, data.period, has_previous, - preserve_on_failure, + preserve_transient, ); if let Some(Ok(cores)) = data.cores { this.cores = cores; this.last_cores_at = Some(std::time::Instant::now()); this.core_refresh_needed = false; } - this.apply_undated_result(data.undated, after_report); + this.apply_undated_result(data.undated, after_report, report_req); this.cal_dirty = super::refresh::report_result_is_stale( report_req, this.current_report_generation(), read_failed || metadata_failed, ); - this.settle_report_refresh_retry(retry.as_ref(), cx); + this.settle_report_refresh_retry(transient, cx); cx.notify(); }, ); diff --git a/crates/moon-ui-gpui/src/analytics/calendar/tests.rs b/crates/moon-ui-gpui/src/analytics/calendar/tests.rs index 4fe6e47f..dfa0ee73 100644 --- a/crates/moon-ui-gpui/src/analytics/calendar/tests.rs +++ b/crates/moon-ui-gpui/src/analytics/calendar/tests.rs @@ -85,6 +85,51 @@ fn report_catch_up_failure_preserves_current_and_previous_calendar_snapshots() { ); } +/// `analytics/calendar/mod.rs:apply_calendar_results` must preserve both settled Calendar +/// snapshots for a transient Split result and keep Calendar dirty. Publishing Split unconditionally +/// flashes incomplete cells, while returning false stops the catch-up that would replace them. +#[test] +fn transient_split_preserves_calendar_snapshots_and_requests_a_catch_up() { + let mut days = ProfitLoadState::Ready { + unit: None, + data: Arc::new(vec![DayCell { + start: 456, + ..Default::default() + }]), + }; + let mut previous = LoadState::Ready(Arc::new(Some(Default::default()))); + let original_days = days.data().expect("settled Calendar cells").clone(); + let original_previous = previous + .data() + .expect("settled previous-period total") + .clone(); + + assert!( + apply_calendar_results( + &mut days, + &mut previous, + Ok(moon_core::db::ProfitScope::Split(Default::default())), + true, + true, + ), + "a transient Split result must keep Calendar dirty for its scheduled correction" + ); + assert!( + Arc::ptr_eq( + &original_days, + days.data().expect("preserved Calendar cells") + ), + "a transient Split must retain the exact current-cell snapshot" + ); + assert!( + Arc::ptr_eq( + &original_previous, + previous.data().expect("preserved previous-period total") + ), + "a transient Split must retain the exact comparison snapshot" + ); +} + /// Replacing `calendar::hour_start`'s gap rejection with the shared picker clamp would map both /// Warsaw 02:00 and 03:00 to the same bucket and duplicate that hour's profit in the Day grid. #[test] diff --git a/crates/moon-ui-gpui/src/analytics/mod.rs b/crates/moon-ui-gpui/src/analytics/mod.rs index 9ad36108..eb57a00f 100644 --- a/crates/moon-ui-gpui/src/analytics/mod.rs +++ b/crates/moon-ui-gpui/src/analytics/mod.rs @@ -10,9 +10,9 @@ //! tab/mode entry and committed report generations use the shared quiet-period and maximum-wait //! gate. Hidden surfaces remain marked stale until entry, and automatic scans never overlap. //! -//! A writer-driven catch-up preserves a settled snapshot only for transient contention that a -//! bounded retry will correct. The final Busy failure still publishes, so stale data never looks -//! current after the retry budget is exhausted. +//! A writer-driven catch-up preserves a settled snapshot only for a transient outcome that a +//! scheduled correction — a bounded retry or a newer generation — will resolve. Any durable +//! outcome still publishes, so stale data never looks current with no correction path left. /// The shared spawn+overlay envelope of every background DB read. mod bg; @@ -58,11 +58,12 @@ use crate::controls::date_range::{self, Bound}; use moon_core::db::ReportAxis; use moon_core::db::analytics::{DayCell, PreviousPeriodBasis, Query, StrategyBase, Summary}; use moon_core::db::valuation::{ValuationMode, ValuationStatus}; -use moon_core::db::{FailKind, ProfitMetric, ProfitUnit, ReadFail, SideFilter}; +use moon_core::db::{ProfitMetric, ProfitUnit, ReadFail, SideFilter}; use crate::load_state::{LoadState, note_el}; use refresh::{ - BusyRetryBudget, RefreshGate, RefreshPlan, RefreshUrgency, VisibleRefresh, visible_refresh, + BusyRetryBudget, CatchUpOutcome, RefreshGate, RefreshPlan, RefreshUrgency, VisibleRefresh, + visible_refresh, }; const ANALYTICS_HEADER_H: f32 = 32.0; @@ -402,7 +403,7 @@ pub struct AnalyticsView { last_valuation_status_rev: u64, /// Debounce/max-wait state for automatic report-driven refreshes. report_refresh: RefreshGate, - /// Bounded automatic Busy retries for the active contention episode. + /// Bounded automatic retries for the active transient-outcome episode. report_busy_retries: BusyRetryBudget, tab: Tab, /// Period of the Summary tab (presets or the from/to range). @@ -1115,26 +1116,44 @@ impl AnalyticsView { cx.notify(); } - /// Adopt a newly measured core offset, reloading every surface that was computed on the old - /// one. + /// Adopt a newly measured core offset, catching up every surface that was computed on the + /// old one. /// /// Polled beside the valuation health rather than pushed, following this window's own idiom. /// Unlike health, an adoption DOES change rows — every date bucket, every calendar cell and - /// every period bound moves — so it reloads rather than merely repainting. + /// every period bound moves. But the SCOPE (period, filters) does not, so this is a + /// writer-driven catch-up, not a user reload: the visible snapshot stays on screen, with no + /// blocking overlay, until the replacement lands. The observer retires EVERY in-flight read + /// identity for the old axis — `seq`, `cal_seq`, `cancel_latest_reads`, plus `tuner`, + /// `time_tuner`, `coins` and `coin_lists` `invalidate()` for the axes that keep their own + /// request generations — because a cancelled read is not silently dropped: the DB layer + /// raises a real SQLite interrupt that gets classified as a durable `Settled` failure, so a + /// read whose identity was not retired would pass its own `seq != req` guard and publish that + /// failure as if it were a real result. Retiring the tuner's identity here also clears any + /// unsaved filter draft, matching this observer's behavior before it stopped calling + /// `reload()` for axis changes. /// /// Args: - /// cx: Analytics window context, reloaded only when the axis actually moved. + /// cx: Analytics window context used to schedule a catch-up only when the axis moved. /// /// Returns: - /// Nothing; an axis change marks calendar state dirty and reloads Analytics. + /// Nothing; an axis change retires every in-flight read identity and schedules a + /// writer-driven catch-up. fn observe_report_axis(&mut self, cx: &mut Context) { let axis = self.backend.read(cx).report_axis(self.display_zone); if axis == self.axis { return; } self.axis = axis; - self.cal_dirty = true; - self.reload(cx); + self.seq = self.seq.wrapping_add(1); + self.cal_seq = self.cal_seq.wrapping_add(1); + self.cancel_latest_reads(); + self.tuner.invalidate(); + self.time_tuner.invalidate(); + self.coins.invalidate(); + self.coin_lists.invalidate(); + self.mark_report_data_stale(); + self.request_report_refresh(RefreshUrgency::Writer, false, cx); cx.notify(); } @@ -1199,16 +1218,16 @@ impl AnalyticsView { .refresh_started(generation, std::time::Instant::now()); } - /// Settle the Busy retry episode and optionally schedule its next bounded attempt. + /// Settle the transient retry episode and optionally schedule its next bounded attempt. /// /// Permanent corruption and unclassified I/O failures remain visible instead of creating an /// endless full-history retry loop. /// /// Args: - /// error: Transient read failure, or `None` when the read escaped SQLite contention. + /// transient: Whether any part of the completed read classified as a transient outcome. /// cx: GPUI context used to arm the quiet-period retry. - fn settle_report_refresh_retry(&mut self, error: Option<&ReadFail>, cx: &mut Context) { - if error.and_then(ReadFail::kind) != Some(FailKind::Busy) { + fn settle_report_refresh_retry(&mut self, transient: bool, cx: &mut Context) { + if !transient { self.report_busy_retries.resolve(); return; } @@ -1216,8 +1235,8 @@ impl AnalyticsView { log::warn!("analytics: automatic database retry budget exhausted"); return; } - // WRITER urgency on purpose: a bounded Busy retry that fired immediately would hammer a - // database already under contention, which is the one thing the quiet period is for. + // WRITER urgency lets this follow-up share the quiet period with a fresh report + // generation instead of immediately starting another full-period read. self.report_refresh.request_refresh( std::time::Instant::now(), false, @@ -1226,25 +1245,33 @@ impl AnalyticsView { self.schedule_report_refresh(cx); } - /// Decide whether a writer-driven catch-up failure may keep a visible snapshot instead of + /// Decide whether a writer-driven catch-up outcome may keep a visible snapshot instead of /// publishing it. /// - /// Centralizing the predicate keeps every load surface aligned as their call sites evolve. + /// Centralizing the predicate keeps every load surface aligned as their call sites evolve. A + /// preserved snapshot is always covered by a scheduled correction: a transient outcome with a + /// scheduled correction (retry allowance or a newer generation) left, never a bare exhausted + /// budget. /// /// Args: /// after_report: Whether this is a writer-driven catch-up rather than a manual reload. - /// error: The classified failure this read completed with, if any. + /// outcome: The classified outcome this read completed with. + /// started_generation: Report generation captured immediately before the read started. /// /// Returns: - /// `true` only when the failure is transient contention AND a retry will actually follow. - fn keep_on_busy(&self, after_report: bool, error: Option<&ReadFail>) -> bool { - error.is_some_and(|error| { - refresh::preserve_on_catch_up_failure( - after_report, - error, - self.report_busy_retries.has_allowance(), - ) - }) + /// `true` only when the outcome is transient AND a correction is already scheduled. + fn keep_on_catch_up( + &self, + after_report: bool, + outcome: CatchUpOutcome, + started_generation: u64, + ) -> bool { + refresh::preserve_on_catch_up( + after_report, + outcome, + self.report_busy_retries.has_allowance(), + started_generation != self.current_report_generation(), + ) } /// Queue a visible catch-up behind any Analytics database work already in flight. @@ -1651,7 +1678,7 @@ impl AnalyticsView { /// /// Args: /// after_report: Whether this writer-driven catch-up may keep a settled snapshot while a - /// transient Busy retry remains. + /// transient outcome with a scheduled correction remains. /// show_overlay: Whether this refresh must block interaction with visible progress feedback. /// cx: GPUI context used to run and publish the shared background query. fn reload_summary(&mut self, after_report: bool, show_overlay: bool, cx: &mut Context) { @@ -1701,46 +1728,39 @@ impl AnalyticsView { let data = result.data; let undated = result.undated; let cores = result.cores; - let data_error = data.as_ref().err().cloned(); + // Computed before `data` moves into `apply` below. + let data_outcome = CatchUpOutcome::of_scope(&data); let undated_error = undated.as_ref().err().cloned(); let cores_error = cores .as_ref() .and_then(|cores| cores.as_ref().err()) .cloned(); - let retry_error = data_error - .as_ref() - .filter(|error| error.kind() == Some(FailKind::Busy)) - .or_else(|| { - undated - .as_ref() - .err() - .filter(|error| error.kind() == Some(FailKind::Busy)) - }) - .or_else(|| { - cores_error - .as_ref() - .filter(|error| error.kind() == Some(FailKind::Busy)) - }) - .cloned(); + let transient = data_outcome.is_transient() + || CatchUpOutcome::of_read(&undated).is_transient() + || cores + .as_ref() + .is_some_and(|cores| CatchUpOutcome::of_read(cores).is_transient()); if let Some(Ok(cores)) = cores { this.cores = cores; this.last_cores_at = Some(std::time::Instant::now()); this.core_refresh_needed = false; } - // A snapshot survives only while a Busy retry can replace it; otherwise stale - // values would look current with no remaining correction path. + // A snapshot survives only while a scheduled correction can replace it; otherwise + // stale values would look current with no remaining correction path. let preserve_snapshot = !matches!(this.data, ProfitLoadState::Loading) - && this.keep_on_busy(after_report, data_error.as_ref()); + && this.keep_on_catch_up(after_report, data_outcome, report_req); if !preserve_snapshot { this.data.apply(data); } this.data_dirty = refresh::report_result_is_stale( report_req, this.current_report_generation(), - data_error.is_some() || undated_error.is_some() || cores_error.is_some(), + !matches!(data_outcome, CatchUpOutcome::Replacement) + || undated_error.is_some() + || cores_error.is_some(), ); - this.apply_undated_result(undated, after_report); - this.settle_report_refresh_retry(retry_error.as_ref(), cx); + this.apply_undated_result(undated, after_report, report_req); + this.settle_report_refresh_retry(transient, cx); cx.notify(); }, ); @@ -1749,8 +1769,8 @@ impl AnalyticsView { /// Reload the compact Strategies base and optionally continue with its visible axis. /// /// Args: - /// after_report: Whether to preserve the visible snapshot while loading and on read - /// failure, and to use report-style catch-up and retry semantics. + /// after_report: Whether a transient outcome with a scheduled correction may preserve the + /// visible snapshot under report-style catch-up semantics. /// chain_visible_axis: Whether a successful base read should continue into the active axis. /// show_overlay: Whether this refresh must block interaction with visible progress feedback. /// cx: GPUI context used to run and publish the shared background query. @@ -1785,26 +1805,19 @@ impl AnalyticsView { let data = result.data; let undated = result.undated; let cores = result.cores; + // Computed before `data` moves into `apply` below. + let data_outcome = CatchUpOutcome::of_scope(&data); let data_error = data.as_ref().err().cloned(); let undated_error = undated.as_ref().err().cloned(); let cores_error = cores .as_ref() .and_then(|cores| cores.as_ref().err()) .cloned(); - let retry_error = data_error - .as_ref() - .filter(|error| error.kind() == Some(FailKind::Busy)) - .or_else(|| { - undated_error - .as_ref() - .filter(|error| error.kind() == Some(FailKind::Busy)) - }) - .or_else(|| { - cores_error - .as_ref() - .filter(|error| error.kind() == Some(FailKind::Busy)) - }) - .cloned(); + let transient = data_outcome.is_transient() + || CatchUpOutcome::of_read(&undated).is_transient() + || cores + .as_ref() + .is_some_and(|cores| CatchUpOutcome::of_read(cores).is_transient()); if let Some(Ok(cores)) = cores { this.cores = cores; this.last_cores_at = Some(std::time::Instant::now()); @@ -1816,11 +1829,12 @@ impl AnalyticsView { this.strategy_data.data().map(|d| d.strategies.as_slice()), tuner::published_groups(&data), ); - // Keep a settled same-scope snapshot only until a Busy retry replaces it. An - // initial `Loading` state must publish its failure rather than remain pending - // forever; `Split`, `NotReady`, and `Failed` are already settled snapshots. + // Keep a settled same-scope snapshot only until a scheduled correction replaces + // it. An initial `Loading` state must publish its failure rather than remain + // pending forever; `Split`, `NotReady`, and `Failed` are already settled + // snapshots unless still transient (see `CatchUpOutcome`). let preserve_snapshot = !matches!(this.strategy_data, ProfitLoadState::Loading) - && this.keep_on_busy(after_report, data_error.as_ref()); + && this.keep_on_catch_up(after_report, data_outcome, report_req); if !preserve_snapshot { this.strategy_data.apply(data); // Measuring every core name is expensive, so invalidate only when its input @@ -1838,9 +1852,11 @@ impl AnalyticsView { this.strategy_dirty = refresh::report_result_is_stale( report_req, this.current_report_generation(), - data_error.is_some() || undated_error.is_some() || cores_error.is_some(), + !matches!(data_outcome, CatchUpOutcome::Replacement) + || undated_error.is_some() + || cores_error.is_some(), ); - this.apply_undated_result(undated, after_report); + this.apply_undated_result(undated, after_report, report_req); let probe_took_over = probe_selects_strategy() && this.probe_select_first(cx); if refresh::strategy_base_allows_axis( data_error.is_some(), @@ -1853,14 +1869,14 @@ impl AnalyticsView { { this.reload_axis_after_report(this.strat_mode, show_overlay, cx); } - this.settle_report_refresh_retry(retry_error.as_ref(), cx); + this.settle_report_refresh_retry(transient, cx); cx.notify(); }, ); } - /// Apply an undated-close result without replacing a settled strip with a transient Busy - /// alert that the bounded retry will immediately remove. + /// Apply an undated-close result without replacing a settled strip with a transient alert + /// that a scheduled correction will immediately remove. /// /// Every other failure still publishes: otherwise stale counts would look current without a /// correction path. @@ -1868,18 +1884,21 @@ impl AnalyticsView { /// Args: /// result: Current undated-close read outcome. /// after_report: Whether this is a writer-driven catch-up rather than a manual reload. + /// started_generation: Report generation captured immediately before the read started. fn apply_undated_result( &mut self, result: moon_core::db::ReadResult, after_report: bool, + started_generation: u64, ) { + let outcome = CatchUpOutcome::of_read(&result); match result { Ok(undated) => { self.undated = Some(undated); self.undated_error = None; } Err(error) => { - if self.keep_on_busy(after_report, Some(&error)) { + if self.keep_on_catch_up(after_report, outcome, started_generation) { return; } self.undated = None; diff --git a/crates/moon-ui-gpui/src/analytics/refresh.rs b/crates/moon-ui-gpui/src/analytics/refresh.rs index 52897aad..2b94e76e 100644 --- a/crates/moon-ui-gpui/src/analytics/refresh.rs +++ b/crates/moon-ui-gpui/src/analytics/refresh.rs @@ -10,14 +10,17 @@ //! between a tuner tab that opens now and one that opens a second from now. The `db_active` //! interlock is unaffected: nothing here ever overlaps work already in flight. //! -//! [`preserve_on_catch_up_failure`] is the one rule every writer-driven completion handler -//! consults before publishing a failed read: only transient contention with retry allowance -//! left may keep a settled surface on screen instead of the classified error, because that is -//! the only kind of failure the bounded [`BusyRetryBudget`] retry will actually clear. +//! [`preserve_on_catch_up`] is the rule every writer-driven completion handler in the Analytics +//! window's own surfaces -- Summary, Strategies, Calendar and the three tuner axes -- consults +//! before publishing a failed or split read: only a transient outcome with a scheduled +//! correction (retry allowance or a newer generation) left may keep a settled surface on screen +//! instead of the classified result, because that is the only kind of outcome the bounded +//! [`BusyRetryBudget`] retry -- or the writer's own next generation -- will actually clear. The +//! Profit Monitor keeps its own separate Busy-retry flow and does not consult this rule. use std::time::{Duration, Instant}; -use moon_core::db::{FailKind, ReadFail}; +use moon_core::db::{FailKind, ProfitScope, ReadFail, ReadResult}; use super::Tab; @@ -33,7 +36,7 @@ const CORE_METADATA_INTERVAL: Duration = Duration::from_secs(60); /// Maximum automatic retries for one report generation under persistent SQLite contention. const MAX_BUSY_RETRIES: u8 = 3; -/// Bounded retry allowance for transient database contention. +/// Bounded retry allowance for a transient database outcome. #[derive(Default)] pub(super) struct BusyRetryBudget { attempts: u8, @@ -65,14 +68,14 @@ impl BusyRetryBudget { true } - /// Whether a Busy failure can still be corrected automatically. + /// Whether a transient outcome can still be corrected automatically. /// /// This is read-only because only `claim` may spend an attempt. pub(super) fn has_allowance(&self) -> bool { self.attempts < MAX_BUSY_RETRIES } - /// Close a retry episode after a read completes without SQLite contention. + /// Close a retry episode after a read completes without a transient outcome. pub(super) fn resolve(&mut self) { self.attempts = 0; self.active = false; @@ -147,30 +150,117 @@ pub(super) fn report_result_is_stale( read_failed || started_generation != current_generation } -/// Decide whether a writer-driven catch-up may keep a settled snapshot for a failed read. +/// Classification of a completed report-derived read for catch-up preservation. /// -/// Only a Busy failure with another retry may be hidden: any other result, including an exhausted -/// Busy budget, must publish so stale values never look current without a correction path. +/// `Transient` is deliberately wider than a bare `Busy` failure only in that a `Split` scope still +/// short of full valuation coverage joins it, correctable by the writer's own next generation. A +/// `NotReady` schema gap during a replica reset is a COMPLETED absence, not a momentary one: it +/// stays `Settled` and publishes now. See the module docs and `CatchUpOutcome::of_scope` for the +/// narrowed Split rule. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(super) enum CatchUpOutcome { + /// A real result: publish it. + Replacement, + /// Correctable by a scheduled retry or a newer generation; may be hidden behind the last + /// settled snapshot instead of publishing. + Transient, + /// Durable: no correction is coming, so it must publish now. + Settled, +} + +/// Classify a failed read into [`CatchUpOutcome::Transient`] or [`CatchUpOutcome::Settled`]. +/// +/// Only `Busy` is transient: it is the sole failure kind the bounded retry actually clears. +/// `NotReady` is a completed absence rather than proof of momentariness (see the module docs). +fn classify_fail(error: &ReadFail) -> CatchUpOutcome { + if error.kind() == Some(FailKind::Busy) { + CatchUpOutcome::Transient + } else { + CatchUpOutcome::Settled + } +} + +impl CatchUpOutcome { + /// Classify a plain fallible report read. + /// + /// Args: + /// result: The completed read. + /// + /// Returns: + /// `Replacement` for `Ok`, otherwise the failure's transient/settled classification. + pub(super) fn of_read(result: &Result) -> Self { + match result { + Ok(_) => CatchUpOutcome::Replacement, + Err(error) => classify_fail(error), + } + } + + /// Classify a scope-shaped report read. + /// + /// `Split` is transient only while the valuation worker demonstrably still has eligible rows + /// left to value; an unknown-quote identity or a worker that finished unroutable rows is a + /// durable safety verdict and must publish (see the module's CORRECTION A). + /// + /// Args: + /// result: The completed scope read. + /// + /// Returns: + /// `Replacement` for `Comparable`/`Empty`, the narrowed Split classification for `Split`, + /// otherwise the failure's transient/settled classification. + pub(super) fn of_scope(result: &ReadResult>) -> Self { + match result { + Ok(ProfitScope::Split(totals)) => { + if totals.unknown_orders == 0 + && totals.valuation.is_some_and(|coverage| { + coverage.valued_orders + coverage.unavailable_orders + < coverage.eligible_orders + }) + { + CatchUpOutcome::Transient + } else { + CatchUpOutcome::Settled + } + } + Ok(ProfitScope::Comparable { .. } | ProfitScope::Empty(_)) => { + CatchUpOutcome::Replacement + } + Err(error) => classify_fail(error), + } + } + + /// Whether this outcome is a scheduled-correction candidate. + pub(super) fn is_transient(self) -> bool { + matches!(self, CatchUpOutcome::Transient) + } +} + +/// Decide whether a writer-driven catch-up may keep a settled snapshot instead of publishing. +/// +/// Only a transient outcome with a scheduled correction still open may be hidden: any other +/// result, including an exhausted retry budget with no newer generation pending, must publish so +/// stale values never look current without a correction path. /// /// Args: /// after_report: Whether this is a writer-driven catch-up rather than a manual reload. -/// error: The classified failure this read completed with. -/// retry_allowance: Whether another automatic Busy retry remains for this episode. +/// outcome: The classified outcome this read completed with. +/// retry_allowance: Whether another automatic transient retry remains for this episode. +/// superseded: Whether a newer generation is already pending behind this read. /// /// Returns: -/// `true` only when the failure is transient contention AND a retry will actually follow. -pub(super) fn preserve_on_catch_up_failure( +/// `true` only when the outcome is transient AND a correction is already scheduled. +pub(super) fn preserve_on_catch_up( after_report: bool, - error: &ReadFail, + outcome: CatchUpOutcome, retry_allowance: bool, + superseded: bool, ) -> bool { - after_report && error.kind() == Some(FailKind::Busy) && retry_allowance + after_report && outcome == CatchUpOutcome::Transient && (retry_allowance || superseded) } /// Decide whether the compact Strategies base may continue into its visible axis. /// /// The axis is a child of the compound base refresh. Starting it after a partial base failure -/// wastes another scan and lets its success resolve the parent's active Busy retry episode. +/// wastes another scan and lets its success resolve the parent's active transient retry episode. /// /// Args: /// data_failed: Whether the strategy-list aggregate failed. diff --git a/crates/moon-ui-gpui/src/analytics/refresh/tests.rs b/crates/moon-ui-gpui/src/analytics/refresh/tests.rs index 3029e868..7daee7fb 100644 --- a/crates/moon-ui-gpui/src/analytics/refresh/tests.rs +++ b/crates/moon-ui-gpui/src/analytics/refresh/tests.rs @@ -3,12 +3,14 @@ use std::time::{Duration, Instant}; use super::{ - BusyRetryBudget, RefreshGate, RefreshPlan, RefreshUrgency, VisibleRefresh, core_metadata_wait, - preserve_on_catch_up_failure, report_result_is_stale, strategy_base_allows_axis, + BusyRetryBudget, CatchUpOutcome, RefreshGate, RefreshPlan, RefreshUrgency, VisibleRefresh, + core_metadata_wait, preserve_on_catch_up, report_result_is_stale, strategy_base_allows_axis, visible_refresh, }; use crate::analytics::Tab; -use moon_core::db::{FailKind, ReadFail}; +use moon_core::db::{ + FailKind, ProfitScope, ProfitUnit, QuoteBreakdown, ReadFail, ValuationCoverage, +}; use std::sync::Arc; /// Construct a classified database failure without depending on display text. @@ -19,31 +21,103 @@ fn failure(kind: FailKind) -> ReadFail { } } -/// `analytics/refresh.rs:preserve_on_catch_up_failure` must preserve only a Busy failure that an -/// automatic retry can still erase. Broadening the predicate lets a permanent failure, NotReady, -/// or the final exhausted Busy failure leave stale Analytics numbers looking current forever. +/// `analytics/refresh.rs:preserve_on_catch_up` must preserve only a transient result covered by a +/// retry allowance or a newer generation. Dropping that correction guard, or letting a settled +/// result through, leaves stale figures under a current period label with nothing scheduled to +/// correct them. #[test] -fn catch_up_preservation_requires_busy_and_an_available_retry() { - for (label, error, is_busy) in [ - ("Busy", failure(FailKind::Busy), true), - ("Corrupt", failure(FailKind::Corrupt), false), - ("Other", failure(FailKind::Other), false), - ("NotReady", ReadFail::NotReady, false), - ("IncomparableQuote", ReadFail::IncomparableQuote, false), - ("PeriodOutOfRange", ReadFail::PeriodOutOfRange, false), +fn catch_up_preservation_requires_a_transient_outcome_and_scheduled_correction() { + for (label, outcome, is_transient) in [ + ("Replacement", CatchUpOutcome::Replacement, false), + ("Transient", CatchUpOutcome::Transient, true), + ("Settled", CatchUpOutcome::Settled, false), ] { for after_report in [false, true] { for retry_allowance in [false, true] { - assert_eq!( - preserve_on_catch_up_failure(after_report, &error, retry_allowance), - after_report && is_busy && retry_allowance, - "{label}: after_report={after_report}, retry_allowance={retry_allowance}" - ); + for superseded in [false, true] { + assert_eq!( + preserve_on_catch_up(after_report, outcome, retry_allowance, superseded,), + after_report && is_transient && (retry_allowance || superseded), + "{label}: after_report={after_report}, retry_allowance={retry_allowance}, \ + superseded={superseded}" + ); + } } } } } +/// `analytics/refresh.rs:CatchUpOutcome::{of_scope,of_read}` must distinguish a valuation gap +/// from unknown-quote or settled results. Removing the unknown-quote guard hides money behind a +/// stale comparable scalar in the wrong unit, while moving Empty or Corrupt to transient hides a +/// legitimate purge result or permanent database failure. +#[test] +fn catch_up_outcome_classifies_scope_and_read_results() { + let incomplete_split = Ok::, ReadFail>(ProfitScope::Split(QuoteBreakdown { + unknown_orders: 0, + valuation: Some(ValuationCoverage { + eligible_orders: 5, + valued_orders: 3, + unavailable_orders: 1, + ..Default::default() + }), + ..Default::default() + })); + let unknown_quote_split = Ok::, ReadFail>(ProfitScope::Split(QuoteBreakdown { + unknown_orders: 1, + valuation: Some(ValuationCoverage { + eligible_orders: 5, + valued_orders: 3, + unavailable_orders: 1, + ..Default::default() + }), + ..Default::default() + })); + let settled_split = Ok::, ReadFail>(ProfitScope::Split(Default::default())); + let comparable = Ok(ProfitScope::Comparable { + unit: ProfitUnit::Percent, + data: (), + }); + let empty = Ok(ProfitScope::Empty(())); + + assert!(matches!( + CatchUpOutcome::of_scope(&incomplete_split), + CatchUpOutcome::Transient + )); + assert!(matches!( + CatchUpOutcome::of_scope(&unknown_quote_split), + CatchUpOutcome::Settled + )); + assert!(matches!( + CatchUpOutcome::of_scope(&settled_split), + CatchUpOutcome::Settled + )); + assert!(matches!( + CatchUpOutcome::of_scope(&comparable), + CatchUpOutcome::Replacement + )); + assert!(matches!( + CatchUpOutcome::of_scope(&empty), + CatchUpOutcome::Replacement + )); + + let cases: [(&str, Result<(), ReadFail>, bool); 6] = [ + ("Busy", Err(failure(FailKind::Busy)), true), + ("NotReady", Err(ReadFail::NotReady), false), + ("Corrupt", Err(failure(FailKind::Corrupt)), false), + ("Other", Err(failure(FailKind::Other)), false), + ("IncomparableQuote", Err(ReadFail::IncomparableQuote), false), + ("PeriodOutOfRange", Err(ReadFail::PeriodOutOfRange), false), + ]; + for (label, result, expected_transient) in cases { + assert_eq!( + matches!(CatchUpOutcome::of_read(&result), CatchUpOutcome::Transient), + expected_transient, + "{label} must be transient exactly when a scheduled correction can resolve it" + ); + } +} + /// `analytics/refresh.rs:BusyRetryBudget::has_allowance` must read without spending a retry. /// Letting this publication query increment attempts exhausts a transient Busy recovery budget /// before `claim` schedules the retry, leaving stale data or a false error. diff --git a/crates/moon-ui-gpui/src/analytics/tuner/coins/load.rs b/crates/moon-ui-gpui/src/analytics/tuner/coins/load.rs index c654b9e2..4b54c1f2 100644 --- a/crates/moon-ui-gpui/src/analytics/tuner/coins/load.rs +++ b/crates/moon-ui-gpui/src/analytics/tuner/coins/load.rs @@ -10,7 +10,7 @@ use gpui::*; use super::super::super::AnalyticsView; use super::state::{CoinLists, CoinPlan}; -use crate::analytics::refresh::report_result_is_stale; +use crate::analytics::refresh::{CatchUpOutcome, report_result_is_stale}; use moon_core::db::analytics::GroupStat; /// How long a burst of ticks is allowed to keep coalescing before the KPI is rescanned. @@ -28,7 +28,7 @@ impl AnalyticsView { self.reload_coins_inner(false, true, cx); } - /// Recompute report-stale coin data and retry transient database contention. + /// Recompute report-stale coin data through the writer-driven catch-up path. /// /// Args: /// show_overlay: Whether queued user work requires blocking progress feedback. @@ -44,7 +44,7 @@ impl AnalyticsView { /// Start one coin-axis snapshot with behavior selected by its reload cause. /// /// Args: - /// after_report: Whether transient database contention should re-arm automatic refresh. + /// after_report: Whether this is a writer-driven catch-up that may arm a follow-up refresh. /// show_overlay: Whether this refresh must block interaction with visible progress feedback. /// cx: GPUI context used to run and publish the combined background query. fn reload_coins_inner( @@ -130,7 +130,8 @@ impl AnalyticsView { ) }, move |this, (stats, kpi, lists, bl_n, wl_n, entries, picked_strategies), cx| { - let entries_error = entries.as_ref().err().cloned(); + let entries_outcome = CatchUpOutcome::of_read(&entries); + let picked_outcome = CatchUpOutcome::of_read(&picked_strategies); let picked_error = picked_strategies.as_ref().err().cloned(); // Guarded separately from the table below: the panels have their own // generation, so a scope change that retired only them still lands here @@ -165,34 +166,12 @@ impl AnalyticsView { // what says "the user has touched the working lists since this request // started" — so it must not be overwritten by a baseline read before it. let edited = this.coins.kpi_seq != kpi_req; - let retry = stats - .as_ref() - .err() - .filter(|error| error.kind() == Some(moon_core::db::FailKind::Busy)) - .or_else(|| { - if edited { - None - } else { - kpi.as_ref() - .err() - .filter(|error| error.kind() == Some(moon_core::db::FailKind::Busy)) - } - }) - .or_else(|| { - entries_error - .as_ref() - .filter(|error| error.kind() == Some(moon_core::db::FailKind::Busy)) - }) - .or_else(|| { - if picked_current { - picked_error - .as_ref() - .filter(|error| error.kind() == Some(moon_core::db::FailKind::Busy)) - } else { - None - } - }) - .cloned(); + let stats_outcome = CatchUpOutcome::of_read(&stats); + let kpi_outcome = CatchUpOutcome::of_read(&kpi); + let transient = stats_outcome.is_transient() + || (!edited && kpi_outcome.is_transient()) + || entries_outcome.is_transient() + || (picked_current && picked_outcome.is_transient()); // Clear "recompute me" only when everything this pass owed came back. The // KPI counts only when it is actually applied — a discarded request's // error says nothing about the table. An empty universe means the plan @@ -207,7 +186,7 @@ impl AnalyticsView { this.current_report_generation(), read_failed, ); - let keep_stats = this.keep_on_busy(after_report, stats.as_ref().err()); + let keep_stats = this.keep_on_catch_up(after_report, stats_outcome, report_req); this.coins.stats.apply_or_keep(stats, keep_stats); // Only a confirmed strategies snapshot may replace the baseline. On a failed // read, preserving both sets is safer than replaying the draft against a @@ -217,7 +196,7 @@ impl AnalyticsView { // left to show; fall back before the table renders "no matches". this.coins.settle_filter(); if !edited { - let keep_kpi = this.keep_on_busy(after_report, kpi.as_ref().err()); + let keep_kpi = this.keep_on_catch_up(after_report, kpi_outcome, report_req); this.coins.kpi.apply_or_keep(kpi, keep_kpi); this.coins.kpi_bl = bl_n; this.coins.kpi_wl = wl_n; @@ -227,7 +206,7 @@ impl AnalyticsView { this.arm_coin_kpi(cx); } if after_report { - this.settle_report_refresh_retry(retry.as_ref(), cx); + this.settle_report_refresh_retry(transient, cx); } cx.notify(); }, diff --git a/crates/moon-ui-gpui/src/analytics/tuner/filter/mod.rs b/crates/moon-ui-gpui/src/analytics/tuner/filter/mod.rs index 8f799219..e77ee6a4 100644 --- a/crates/moon-ui-gpui/src/analytics/tuner/filter/mod.rs +++ b/crates/moon-ui-gpui/src/analytics/tuner/filter/mod.rs @@ -25,6 +25,7 @@ use moon_ui::{ }; use rust_i18n::t; +use super::super::refresh::CatchUpOutcome; use super::super::{AnalyticsView, LoadState}; pub(in crate::analytics::tuner) use super::shared::{N_VAR, TunerKind, card, glyph_btn}; use crate::design; @@ -216,10 +217,12 @@ impl AnalyticsView { }, move |this, (stats, histogram, strat), cx| { let mut hist_error = None; + let mut hist_outcome = CatchUpOutcome::Replacement; if this.tuner.hist_seq == hist_req { this.tuner.hist_loading = false; + hist_outcome = CatchUpOutcome::of_read(&histogram); hist_error = histogram.as_ref().err().cloned(); - let keep_hist = this.keep_on_busy(after_report, hist_error.as_ref()); + let keep_hist = this.keep_on_catch_up(after_report, hist_outcome, report_req); this.tuner.hist.apply_or_keep(histogram, keep_hist); this.tuner.hist_dirty = super::super::refresh::report_result_is_stale( report_req, @@ -228,18 +231,19 @@ impl AnalyticsView { ); } if this.tuner.seq != req { - if let Some(error) = hist_error.as_ref() { - this.settle_report_refresh_retry(Some(error), cx); + if hist_error.is_some() { + this.settle_report_refresh_retry(hist_outcome.is_transient(), cx); } cx.notify(); return; } + let stats_outcome = CatchUpOutcome::of_read(&stats); let error = stats.as_ref().err().cloned(); - // A completed non-data result clears stale numbers, unless it is transient - // contention with retry allowance left (`keep_on_busy`) — values - // under a changed period label must belong to it, but a settled snapshot the - // bounded retry is about to replace anyway must not blink through it. - let keep_stats = this.keep_on_busy(after_report, error.as_ref()); + // A completed non-data result clears stale numbers, unless it is a transient + // outcome with a scheduled correction left (`keep_on_catch_up`) — values + // under a changed period label must belong to it, but a settled snapshot with a + // scheduled correction must not blink through the intermediate result. + let keep_stats = this.keep_on_catch_up(after_report, stats_outcome, report_req); this.tuner.stats.apply_or_keep(stats, keep_stats); // `strategy_filters` is intentionally lossy and reports an unreadable row as // `found=false`. An automatic report refresh must not turn that ambiguity into @@ -250,15 +254,8 @@ impl AnalyticsView { this.current_report_generation(), error.is_some(), ); - let retry_error = error - .as_ref() - .filter(|error| error.kind() == Some(moon_core::db::FailKind::Busy)) - .or_else(|| { - hist_error - .as_ref() - .filter(|error| error.kind() == Some(moon_core::db::FailKind::Busy)) - }); - this.settle_report_refresh_retry(retry_error, cx); + let transient = stats_outcome.is_transient() || hist_outcome.is_transient(); + this.settle_report_refresh_retry(transient, cx); cx.notify(); }, ); @@ -315,8 +312,9 @@ impl AnalyticsView { if this.tuner.seq != req { return; } + let stats_outcome = CatchUpOutcome::of_read(&stats); let error = stats.as_ref().err().cloned(); - let keep_stats = this.keep_on_busy(after_report, error.as_ref()); + let keep_stats = this.keep_on_catch_up(after_report, stats_outcome, report_req); this.tuner.stats.apply_or_keep(stats, keep_stats); this.tuner.apply_strategy_read(filters, after_report); this.tuner.dirty = super::super::refresh::report_result_is_stale( @@ -325,7 +323,7 @@ impl AnalyticsView { error.is_some(), ); if after_report { - this.settle_report_refresh_retry(error.as_ref(), cx); + this.settle_report_refresh_retry(stats_outcome.is_transient(), cx); } cx.notify(); }, diff --git a/crates/moon-ui-gpui/src/analytics/tuner/time/mod.rs b/crates/moon-ui-gpui/src/analytics/tuner/time/mod.rs index a42a01a9..05285de3 100644 --- a/crates/moon-ui-gpui/src/analytics/tuner/time/mod.rs +++ b/crates/moon-ui-gpui/src/analytics/tuner/time/mod.rs @@ -21,6 +21,7 @@ use moon_ui::{MoonPalette, h_flex, v_flex}; use rust_i18n::t; use super::super::AnalyticsView; +use super::super::refresh::CatchUpOutcome; use super::super::summary::{fmt_signed, sign_color}; use super::kpi::kpi_matrix_card; use crate::design; @@ -36,7 +37,7 @@ impl AnalyticsView { self.reload_time_inner(false, true, cx); } - /// Recompute report-stale time data and retry transient database contention. + /// Recompute report-stale time data through the writer-driven catch-up path. /// /// Args: /// show_overlay: Whether queued user work requires blocking progress feedback. @@ -127,23 +128,9 @@ impl AnalyticsView { if this.time_tuner.seq != req { return; // the period/filters/strategy/grid have already changed } - let retry = profiles - .as_ref() - .err() - .filter(|error| error.kind() == Some(moon_core::db::FailKind::Busy)) - .or_else(|| { - stats - .as_ref() - .err() - .filter(|error| error.kind() == Some(moon_core::db::FailKind::Busy)) - }) - .or_else(|| { - slider - .as_ref() - .err() - .filter(|error| error.kind() == Some(moon_core::db::FailKind::Busy)) - }) - .cloned(); + let profiles_outcome = CatchUpOutcome::of_read(&profiles); + let stats_outcome = CatchUpOutcome::of_read(&stats); + let slider_outcome = CatchUpOutcome::of_read(&slider); this.time_tuner.dirty = super::super::refresh::report_result_is_stale( report_req, this.current_report_generation(), @@ -151,12 +138,13 @@ impl AnalyticsView { ); // The profile, slider colors, and KPI are independent load surfaces, so each // retains its own classified error instead of collapsing it to "no data" — unless - // that error is transient contention with retry allowance left (`keep_on_busy`), - // in which case its own settled snapshot survives the catch-up instead of - // blinking through the failure. - let keep_profiles = this.keep_on_busy(after_report, profiles.as_ref().err()); - let keep_slider = this.keep_on_busy(after_report, slider.as_ref().err()); - let keep_stats = this.keep_on_busy(after_report, stats.as_ref().err()); + // that outcome is transient with a scheduled correction left + // (`keep_on_catch_up`), in which case its own settled snapshot survives the + // catch-up instead of blinking through the intermediate result. + let keep_profiles = + this.keep_on_catch_up(after_report, profiles_outcome, report_req); + let keep_slider = this.keep_on_catch_up(after_report, slider_outcome, report_req); + let keep_stats = this.keep_on_catch_up(after_report, stats_outcome, report_req); this.time_tuner .profiles .apply_or_keep(profiles, keep_profiles); @@ -164,7 +152,10 @@ impl AnalyticsView { this.time_tuner.stats.apply_or_keep(stats, keep_stats); this.time_tuner.apply_current_read(current, after_report); if after_report { - this.settle_report_refresh_retry(retry.as_ref(), cx); + let transient = profiles_outcome.is_transient() + || stats_outcome.is_transient() + || slider_outcome.is_transient(); + this.settle_report_refresh_retry(transient, cx); } cx.notify(); }, diff --git a/crates/moon-ui-gpui/tests/theme_contract/analytics.rs b/crates/moon-ui-gpui/tests/theme_contract/analytics.rs index 3638134b..7ebf7279 100644 --- a/crates/moon-ui-gpui/tests/theme_contract/analytics.rs +++ b/crates/moon-ui-gpui/tests/theme_contract/analytics.rs @@ -3,6 +3,46 @@ use super::support::*; +/// `analytics/mod.rs:observe_report_axis` must send a machine axis observation through the Writer +/// refresh path instead of reload, while `observe_valuation_mode` remains a real scope reload. +/// Merging these paths blanks and dims Analytics on every feed reconnect; removing the valuation +/// reload leaves an actual mode change under stale values. +#[test] +fn report_axis_observation_uses_writer_refresh_while_valuation_mode_reloads() { + let analytics = read_src("analytics/mod.rs"); + let report_axis = code_only(braced_body(&analytics, "fn observe_report_axis(")); + let valuation_mode = code_only(braced_body(&analytics, "fn observe_valuation_mode(")); + + assert!( + !report_axis.contains("reload("), + "a report-axis observation must not blank the settled surface through reload" + ); + assert!( + report_axis.contains("self.request_report_refresh(") + && report_axis.contains("RefreshUrgency::Writer,") + && report_axis.contains("false,"), + "a report-axis observation must request a nonblocking Writer refresh" + ); + for required in [ + "self.seq = self.seq.wrapping_add(1);", + "self.cal_seq = self.cal_seq.wrapping_add(1);", + "self.cancel_latest_reads();", + "self.tuner.invalidate();", + "self.time_tuner.invalidate();", + "self.coins.invalidate();", + "self.coin_lists.invalidate();", + ] { + assert!( + report_axis.contains(required), + "a report-axis observation must retire every stale read identity: {required}" + ); + } + assert!( + valuation_mode.contains("self.reload("), + "a valuation-mode change remains a real scope change and must reload" + ); +} + /// Every retained view that caches civil-time presentation must observe the one shared display /// zone revision published by the header clock. /// @@ -1425,7 +1465,7 @@ fn automatic_strategy_refresh_keeps_the_visible_snapshot() { manual.contains("self.reload_strategy_base(false, true, true, cx)"), "manual scope refresh must retire values from the previous scope" ); - let preserve_snapshot = "let preserve_snapshot = !matches!(this.strategy_data, ProfitLoadState::Loading)\n && this.keep_on_busy(after_report, data_error.as_ref());"; + let preserve_snapshot = "let preserve_snapshot = !matches!(this.strategy_data, ProfitLoadState::Loading)\n && this.keep_on_catch_up(after_report, data_outcome, report_req);"; assert!( reload.contains(preserve_snapshot), "only a settled strategy snapshot may survive an automatic report refresh" @@ -1437,7 +1477,7 @@ fn automatic_strategy_refresh_keeps_the_visible_snapshot() { "automatic strategy result publication", ); assert!( - automatic_result.contains("this.keep_on_busy(after_report, data_error.as_ref())") + automatic_result.contains("this.keep_on_catch_up(after_report, data_outcome, report_req)") && !automatic_result.contains("if !preserve_snapshot || data_error.is_none() {") && automatic_result.contains("this.strategy_data.apply(data)") && automatic_result.contains("this.strat_core_w = None;") @@ -1538,7 +1578,7 @@ fn summary_catch_up_preserves_only_a_retryable_busy_snapshot() { "Summary completion must distinguish automatic settled snapshots from initial loading" ); assert!( - decision.contains("this.keep_on_busy(after_report, data_error.as_ref())") + decision.contains("this.keep_on_catch_up(after_report, data_outcome, report_req)") && !decision.contains("if !preserve_snapshot || data_error.is_none() {") && decision.contains("this.data.apply(data);"), "only a retryable Busy failure may retain the Summary snapshot" @@ -1601,11 +1641,11 @@ fn report_generation_bookkeeping_has_no_repaint_and_each_completion_has_one() { } } -/// `analytics/mod.rs:apply_undated_result` must call `keep_on_busy` before it publishes an error. -/// Dropping that gate flashes a retryable Busy catch-up, while broadening it hides an exhausted -/// Busy failure and leaves the retained undated count falsely current forever. +/// `analytics/mod.rs:apply_undated_result` must call `keep_on_catch_up` before it publishes an +/// error. Dropping that gate flashes a retryable Busy catch-up, while broadening it hides an +/// exhausted Busy failure and leaves the retained undated count falsely current forever. #[test] -fn undated_busy_gate_runs_before_the_error_is_published() { +fn undated_catch_up_gate_runs_before_the_error_is_published() { let analytics = read_src("analytics/mod.rs"); let apply = braced_body(&analytics, "fn apply_undated_result("); let failure_arm = code_only(braced_body(apply, "Err(error) =>")); @@ -1617,7 +1657,7 @@ fn undated_busy_gate_runs_before_the_error_is_published() { ); assert!( - before_error.contains("self.keep_on_busy(after_report, Some(&error))"), + before_error.contains("self.keep_on_catch_up(after_report, outcome, started_generation)"), "the retryable-Busy gate must run before the undated error can replace a retained count" ); } From 674297d8e5ac32cba93593cfc84ae5670c01685d Mon Sep 17 00:00:00 2001 From: kirillDevPro <113171057+kirillDevPro@users.noreply.github.com> Date: Thu, 3 Sep 2026 22:34:11 +0200 Subject: [PATCH 6/6] fix(trade-replay): serve the ticks already fetched instead of abandoning the stage The tick stage walked the replay window in ascending time order and returned OverTickBudget the moment the running total passed 40 000 rows, discarding every page already paid for. The window pads a trade with up to an hour of lead context, so on a liquid Binance USD-M coin the budget was spent on context before the trade itself was reached, and the window silently fell back to 1-minute candles. The window now carries the trade's own span and the tick plan tiles it focus-first, so any prefix of the fetch is a contiguous interval around the entry and exit. A spent budget, a deadline or a venue refusal serves what was fetched: the exchange klines stay the bar layer for the whole window while tick points cover the fetched span, and an over-budget vector thins to real ticks rather than dropping to candles. Every remaining candle fallback names its reason in the caption (no trades route, retention, error, budget) in ru, en and es. --- crates/moon-core/src/market/candles.rs | 68 ++ crates/moon-core/src/market/candles/tests.rs | 70 ++ .../moon-core/src/market/trade_replay/mod.rs | 282 +++++++- .../src/market/trade_replay/tests.rs | 132 ++++ .../src/market/trade_replay/worker.rs | 601 ++++++++++++++---- .../src/market/trade_replay/worker/tests.rs | 222 +++++++ crates/moon-ui-gpui/src/trade_window/mod.rs | 69 +- .../moon-ui-gpui/src/trade_window/render.rs | 69 +- locales/trade_window.yml | 43 +- 9 files changed, 1401 insertions(+), 155 deletions(-) create mode 100644 crates/moon-core/src/market/trade_replay/worker/tests.rs diff --git a/crates/moon-core/src/market/candles.rs b/crates/moon-core/src/market/candles.rs index cce6f619..a98a6238 100644 --- a/crates/moon-core/src/market/candles.rs +++ b/crates/moon-core/src/market/candles.rs @@ -484,6 +484,74 @@ pub fn aggregate_trades(trades: &[Tick], tf_ms: i64, out: &mut Vec) } } +/// Thins a tick run to at most four REAL points per bucket, never inventing one. +/// +/// A candle can fabricate an OHLC body because nobody reads its open/close as "a trade happened +/// at that exact instant" — a chart POINT is read exactly that way, so a bucket-mid point with no +/// matching trade would be a lie about when the market moved. Keeping the first, highest, lowest +/// and last real tick per bucket instead preserves the shape a candle would draw (open/high/low/ +/// close) while every emitted point stays a value that actually traded. +/// +/// Args: +/// ticks: Ascending by time (the caller sorts). +/// bucket_ms: Bucket width; `<= 0` emits the input unchanged — there is nothing to thin to. +/// out: Cleared first, then filled ascending, the same discipline [`aggregate_trades`] uses. +pub(crate) fn thin_ticks(ticks: &[Tick], bucket_ms: i64, out: &mut Vec) { + out.clear(); + if bucket_ms <= 0 { + out.extend_from_slice(ticks); + return; + } + let mut open_ms: Option = None; + // Indices into `ticks` for the four roles of the bucket currently being collected. + let mut first_i = 0usize; + let mut last_i = 0usize; + let mut high_i = 0usize; + let mut low_i = 0usize; + // Emits one bucket's up-to-four representative ticks, ascending, each index kept once — a + // quiet bucket with a single trade collapses all four roles onto the same index. + let flush = + |first_i: usize, high_i: usize, low_i: usize, last_i: usize, out: &mut Vec| { + let mut idx = [first_i, high_i, low_i, last_i]; + idx.sort_unstable(); + let mut prev: Option = None; + for i in idx { + if prev != Some(i) { + out.push(ticks[i]); + prev = Some(i); + } + } + }; + + for (i, t) in ticks.iter().enumerate() { + let this_open = bucket_open_ms(t.time_ms, bucket_ms); + match open_ms { + Some(open) if open == this_open => { + last_i = i; + if t.price > ticks[high_i].price { + high_i = i; + } + if t.price < ticks[low_i].price { + low_i = i; + } + } + _ => { + if open_ms.is_some() { + flush(first_i, high_i, low_i, last_i, out); + } + open_ms = Some(this_open); + first_i = i; + last_i = i; + high_i = i; + low_i = i; + } + } + } + if open_ms.is_some() { + flush(first_i, high_i, low_i, last_i, out); + } +} + fn candle_from_tick(open_ms: f64, t: &Tick) -> ChartCandle { ChartCandle { t_open_ms: open_ms, diff --git a/crates/moon-core/src/market/candles/tests.rs b/crates/moon-core/src/market/candles/tests.rs index bb8f8726..fcd680b1 100644 --- a/crates/moon-core/src/market/candles/tests.rs +++ b/crates/moon-core/src/market/candles/tests.rs @@ -736,3 +736,73 @@ fn compose_with_coarse_preserves_empty_single_and_contiguous_series_inputs() { "empty layers leave an already complete fine series untouched" ); } + +/// `market/candles.rs:thin_ticks` synthesising bucket-mid points or dropping its role ordering +/// lies about when a market traded and can draw points out of chronological order. +#[test] +fn thin_ticks_keeps_only_real_ascending_points_and_copies_non_positive_buckets() { + let input = vec![ + tick(0.0, 10.0, 1.0), + tick(100.0, 14.0, 2.0), + tick(200.0, 8.0, 3.0), + tick(300.0, 12.0, 4.0), + tick(1_000.0, 20.0, 5.0), + tick(1_100.0, 18.0, 6.0), + tick(1_200.0, 22.0, 7.0), + tick(1_300.0, 19.0, 8.0), + tick(1_400.0, 21.0, 9.0), + ]; + let mut thinned = Vec::new(); + thin_ticks(&input, 1_000, &mut thinned); + + assert!( + thinned.iter().all(|point| input.iter().any(|source| { + source.time_ms == point.time_ms + && source.price == point.price + && source.qty == point.qty + })), + "every emitted point must be an exact time, price, and quantity observed in the input" + ); + assert!( + thinned + .windows(2) + .all(|pair| pair[0].time_ms <= pair[1].time_ms), + "thinned points must stay ascending for the chart consumer" + ); + for bucket in [0_i64, 1_i64] { + assert!( + thinned + .iter() + .filter(|point| (point.time_ms as i64).div_euclid(1_000) == bucket) + .count() + <= 4, + "a bucket may keep at most first, high, low, and last real points" + ); + } + + let mut copied = Vec::new(); + thin_ticks(&input, 0, &mut copied); + assert_eq!( + copied + .iter() + .map(|point| (point.time_ms, point.price, point.qty)) + .collect::>(), + input + .iter() + .map(|point| (point.time_ms, point.price, point.qty)) + .collect::>(), + "bucket_ms == 0 must preserve raw ticks unchanged" + ); + thin_ticks(&input, -1, &mut copied); + assert_eq!( + copied + .iter() + .map(|point| (point.time_ms, point.price, point.qty)) + .collect::>(), + input + .iter() + .map(|point| (point.time_ms, point.price, point.qty)) + .collect::>(), + "negative bucket widths also copy input unchanged" + ); +} diff --git a/crates/moon-core/src/market/trade_replay/mod.rs b/crates/moon-core/src/market/trade_replay/mod.rs index e2ef2057..36bbc2a0 100644 --- a/crates/moon-core/src/market/trade_replay/mod.rs +++ b/crates/moon-core/src/market/trade_replay/mod.rs @@ -83,6 +83,54 @@ const MAX_SPAN_MS: i64 = 7 * 24 * 60 * MINUTE_MS; /// exit — so a window clipped exactly to the position would answer the wrong question. const CONTEXT_FRACTION: f64 = 0.5; +/// Margin added around the trade's own span when computing [`ReplayWindow::focus`]. +/// +/// Wide enough that the entry and exit sit comfortably inside the focus tiles [`tick_plan`] +/// fetches first, rather than landing on the very edge of one; not wider, because every extra +/// millisecond here is lead/trail context pulled ahead of a slice that is actually IN the trade. +const FOCUS_MARGIN_MS: i64 = 5 * MINUTE_MS; + +/// Width of one tick-fetch tile in [`tick_plan`], before a route's own cap narrows it further. +/// +/// An 80-minute scalp window under Binance USD-M's one-hour query cap would otherwise tile into +/// two requests that both straddle the focus, making trade-priority ordering inert — chopping +/// finer than the route cap is what gives [`tick_plan`] something to actually prioritise. +const TICK_SLICE_MS: i64 = 10 * MINUTE_MS; + +/// Bucket widths [`fit_ticks`] tries in order, coarsest last. +/// +/// Each rung roughly doubles to triples the previous one, so a run that barely overflows the +/// budget loses little precision while a run that overflows it by orders of magnitude still +/// terminates in a handful of steps instead of walking one millisecond at a time. +const THIN_LADDER_MS: [i64; 9] = [ + 1_000, 2_000, 5_000, 10_000, 15_000, 30_000, 60_000, 120_000, 300_000, +]; + +/// How the tick stage for one window ended — the thing the window's caption NAMES. +/// +/// Each variant is a DIFFERENT sentence to show the user, and two of them (`NoRoute`, +/// `OutOfRetention`) are known before a single request is spent, so they cost nothing. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum TickStatus { + /// A tick stage is queued and has not answered yet. The first outcome of every window that + /// earns one carries this. + Pending, + /// This build knows no public trades route for the venue (Bybit, Hyperliquid). Retrying + /// cannot help. + NoRoute, + /// The window is older than the route's documented trade retention. + OutOfRetention { + /// How far back the venue's own tick retention actually reaches, in milliseconds. + retention_ms: i64, + }, + /// The venue answered and held no trade inside the window, while klines exist. + NoTrades, + /// The tick fetch itself did not produce an answer. + Failed, + /// Ticks were served. Only a [`TradeReplaySource::Ticks`] series carries this. + Served, +} + /// Which data a replay actually carries, so the window can say which it is showing. /// /// This is user-visible and load-bearing: a one-minute picture of a forty-second scalp is an @@ -169,6 +217,10 @@ pub struct ReplayWindow { pub from_ms: i64, /// Last millisecond the replay covers. pub to_ms: i64, + /// The trade's own open, in milliseconds — the position's real entry stamp, not [`Self::from_ms`]. + pub open_ms: i64, + /// The trade's own close, in milliseconds — the position's real exit stamp, not [`Self::to_ms`]. + pub close_ms: i64, /// Whether this window is WIDER than [`MAX_SPAN_MS`] because its floors demanded it. /// /// Renamed from `clipped`, and the rename is the point: the field used to mean "half the @@ -193,6 +245,28 @@ impl ReplayWindow { n => n, } } + + /// The sub-window closest to the trade itself, for ordering a tick fetch around it. + /// + /// [`tick_plan`] fetches this region FIRST and orders every other tile by distance to it, so + /// a fetch cut short by budget or deadline still lands the trade's own span rather than an + /// hour of lead context nobody asked to see before it. + /// + /// Returns: + /// `(left, right)` inclusive, clamped into `[Self::from_ms, Self::to_ms]` on both ends — + /// independently. Every constructor of this type preserves `open_ms <= close_ms`; a + /// hand-built window that violates it is out of this function's contract and can yield an + /// inverted `(left, right)` rather than a usable focus (no guard here — that state is + /// unreachable today, per house style). + pub(crate) fn focus(self) -> (i64, i64) { + let left = (self.open_ms - FOCUS_MARGIN_MS) + .max(self.from_ms) + .min(self.to_ms); + let right = (self.close_ms + FOCUS_MARGIN_MS) + .min(self.to_ms) + .max(self.from_ms); + (left, right) + } } /// Compute the window to fetch around one trade. @@ -246,7 +320,10 @@ pub fn replay_window(buy_date_s: i64, close_date_s: i64) -> Option // fetch it, retry", never a chart quietly missing its own entry and exit. let over_budget = to_ms - from_ms > MAX_SPAN_MS; // A pre-epoch left edge is meaningless to every venue and would be sent as a negative - // `startTime`; pull it forward instead of asking for it. + // `startTime`; pull it forward instead of asking for it. `open_ms`/`close_ms` are left alone + // by this shift: they are the trade's own REAL stamps, not a fetch bound, and a pre-epoch + // trade is already rejected above by the `close_date_s < buy_date_s` / non-positive guard + // long before this shift ever runs, so nothing here has reason to move them. if from_ms < 0 { to_ms = to_ms.saturating_add(-from_ms); from_ms = 0; @@ -254,6 +331,8 @@ pub fn replay_window(buy_date_s: i64, close_date_s: i64) -> Option Some(ReplayWindow { from_ms, to_ms, + open_ms, + close_ms, over_budget, }) } @@ -333,6 +412,188 @@ pub fn time_slices(window: ReplayWindow, max_span_ms: Option) -> Vec<(i64, out } +/// One ordered decomposition of a [`ReplayWindow`] into tick-fetch tiles. +/// +/// [`Self::slices`] is ordered so that ANY PREFIX is a CONTIGUOUS span with no gap between the +/// sorted first-k slices, for every k — [`worker::paginate_ticks`] leans on exactly this to +/// report a fetch truncated by budget or a deadline as ONE covered interval rather than a comb of +/// holes. A later "tidy" that resorts these tiles purely by `from_ms` would keep them contiguous +/// too, but back in CLOCK order — which throws away the whole reason this type exists, so preserve +/// the ORDER here, not merely the contiguity. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct TickPlan { + /// Every tile to fetch, in fetch-priority order: the trade's own focus first, then outward. + pub slices: Vec<(i64, i64)>, + /// How many leading entries of [`Self::slices`] cover [`ReplayWindow::focus`]. + pub focus_len: usize, +} + +/// Tile a window into tick-fetch requests ordered around the TRADE instead of around the clock. +/// +/// The window is chopped into three independent regions — before the focus, the focus itself, +/// after the focus — each tiled by [`time_slices`] at `min(TICK_SLICE_MS, max_query_ms)`. The +/// focus tiles are placed first, ascending; every remaining tile then follows by distance to the +/// focus, nearest first, tied-break by `from_ms` (so the tile just before the focus outranks the +/// tile just after it, since it sits at a smaller time). This is what lets a walk that runs out of +/// budget or time abandon the FARTHEST tiles rather than the nearest ones — the defect this module +/// exists to fix in the first place. +/// +/// A route's documented retention makes the LEFT edge of every region a moving floor: rows older +/// than `now_ms - retention_ms` cannot be fetched regardless of what the window asks for. Judging +/// retention against the padded window (as the naive check does) refuses a trade whose own span +/// is well inside retention the moment its OPTIONAL lead context crosses the boundary — the ticks +/// the user actually wants are available and get skipped anyway. So this clips instead of +/// refusing outright: every region drops the part of itself older than `earliest_ms`, and if the +/// FOCUS itself — the trade, not its context — falls entirely before it, the whole plan is empty +/// rather than a lead-less scrap of trail. The caller reads an empty plan as "nothing worth +/// fetching" and reports `OutOfRetention`. +/// +/// Args: +/// window: The window to cover. +/// max_query_ms: The route's own cap on one request's span, or `None`/non-positive when it +/// documents none, in which case [`TICK_SLICE_MS`] alone tiles the window. +/// earliest_ms: The oldest millisecond the route's retention can still answer for, or `None` +/// when the route documents no retention limit. +/// +/// Returns: +/// A [`TickPlan`] whose prefix-contiguity invariant (see [`TickPlan`]) holds for every k, even +/// after retention clipping — clipping only ever shrinks the LEAD region from the near edge +/// inward, and whenever it shrinks the focus's own start too, the entire lead region (being +/// strictly older) is guaranteed to fall before `earliest_ms` as well and is dropped whole, so +/// no clipped region can end up separated from its neighbour by a gap. A region that ends up +/// empty or inverted contributes no tiles — [`time_slices`] already returns none for an +/// inverted span, so no extra per-region check is needed here. +pub(crate) fn tick_plan( + window: ReplayWindow, + max_query_ms: Option, + earliest_ms: Option, +) -> TickPlan { + let span = match max_query_ms { + Some(cap) if cap > 0 => TICK_SLICE_MS.min(cap), + _ => TICK_SLICE_MS, + }; + let (focus_from, focus_to) = window.focus(); + + // The trade itself is the one thing worth fetching; a route whose retention does not even + // reach the trade has nothing this plan can usefully prioritise. + if let Some(earliest) = earliest_ms { + if focus_to < earliest { + return TickPlan { + slices: Vec::new(), + focus_len: 0, + }; + } + } + let clip_from = |from: i64| match earliest_ms { + Some(earliest) => from.max(earliest), + None => from, + }; + + let focus_slices = time_slices( + ReplayWindow { + from_ms: clip_from(focus_from), + to_ms: focus_to, + ..window + }, + Some(span), + ); + let lead_slices = time_slices( + ReplayWindow { + from_ms: clip_from(window.from_ms), + to_ms: focus_from - 1, + ..window + }, + Some(span), + ); + let trail_slices = time_slices( + ReplayWindow { + from_ms: clip_from(focus_to + 1), + to_ms: window.to_ms, + ..window + }, + Some(span), + ); + + let focus_len = focus_slices.len(); + let mut rest: Vec<(i64, i64)> = lead_slices.into_iter().chain(trail_slices).collect(); + // Ascending distance first; `from_ms` breaks the tie between the one lead tile and the one + // trail tile that can sit exactly as close on either side of the focus. + rest.sort_by_key(|&(from, to)| { + let distance = if to < focus_from { + focus_from - to + } else { + from - focus_to + }; + (distance, from) + }); + + let mut slices = focus_slices; + slices.extend(rest); + TickPlan { slices, focus_len } +} + +/// Thin a tick run down to a render/remember budget, coarsening only as far as needed. +/// +/// Walks [`THIN_LADDER_MS`] in order and takes the FIRST bucket width whose thinned output fits +/// `budget` via [`super::candles::thin_ticks`], so a run that already fits pays no thinning at +/// all. A position held long enough makes even the coarsest rung's 300-second buckets outnumber +/// the budget — `THIN_LADDER_MS`'s terminal rung is a RATE, not a ceiling — so past it a final +/// uniform stride picks `budget` points evenly spaced across the coarsest rung's output, +/// including its first and last tick. Either path keeps every point a REAL tick, never a +/// synthesised one. +/// +/// **Contract:** `result.len() <= budget`, unconditionally — this is the one property every +/// caller relies on ([`worker::TICK_BUDGET`] bounds both the composed series and the GPU point +/// ring), not a best effort. +/// +/// Args: +/// ticks: Ascending by time (the caller sorts). +/// budget: Largest tick count the caller will draw or remember; `0` always returns nothing. +/// +/// Returns: +/// `(ticks, 0)` unchanged when `ticks.len() <= budget`; otherwise `(thinned, bucket_ms)`, +/// `bucket_ms` being the ladder rung that produced it — the coarsest rung when the final +/// stride also had to run. +pub(crate) fn fit_ticks(ticks: Vec, budget: usize) -> (Vec, i64) { + if ticks.len() <= budget { + return (ticks, 0); + } + if budget == 0 { + return ( + Vec::new(), + *THIN_LADDER_MS.last().expect("non-empty ladder"), + ); + } + let mut out = Vec::new(); + let mut bucket_ms = 0; + for &rung in THIN_LADDER_MS.iter() { + bucket_ms = rung; + crate::market::candles::thin_ticks(&ticks, rung, &mut out); + if out.len() <= budget { + return (out, bucket_ms); + } + } + // The coarsest rung still overflows. A stride of `ceil(last_idx / (budget - 1))`, walked from + // index 0 and always closed off by the true last index, is what keeps the bound UNCONDITIONAL: + // re-walking forward in fixed `ceil(len / budget)` steps and appending the last tick + // afterwards — the naive reading — can land `budget + 1` points whenever the true last index + // is not itself a multiple of that step (e.g. 10 points into a budget of 3: steps of 4 land + // 0/4/8, none of which is index 9, so appending it makes four). + let last_idx = out.len() - 1; + if budget == 1 { + return (vec![out[last_idx]], bucket_ms); + } + let stride = (last_idx as f64 / (budget - 1) as f64).ceil() as usize; + let mut strided = Vec::with_capacity(budget); + let mut i = 0usize; + while i < last_idx { + strided.push(out[i]); + i += stride; + } + strided.push(out[last_idx]); + (strided, bucket_ms) +} + /// Whether cached rows already cover a window densely enough to skip the network. /// /// COVERAGE, not presence, is the question. A partial prefix is exactly what a previously @@ -448,16 +709,27 @@ pub struct TradeReplaySeries { /// Timeframe of [`Self::candles`] in milliseconds; one minute for every current route. pub tf_ms: i64, /// Bars in ascending open time. A [`TradeReplaySource::Klines1m`] series carries these alone; - /// a [`TradeReplaySource::Ticks`] series carries these TOO — its own aggregated candles, - /// computed from [`Self::ticks`] via [`crate::market::candles::aggregate_trades`] — alongside - /// the points, never instead of them. + /// a [`TradeReplaySource::Ticks`] series carries these TOO — the EXCHANGE's own one-minute + /// klines, not bars aggregated from [`Self::ticks`], so the bar layer covers the WHOLE window + /// even where the points, per [`Self::partial`], do not. pub candles: Vec, /// Trade points in ascending time. Empty when [`Self::source`] is /// [`TradeReplaySource::Klines1m`]; carried alongside [`Self::candles`] for - /// [`TradeReplaySource::Ticks`], never in place of them. + /// [`TradeReplaySource::Ticks`] — never in place of them, and per [`Self::partial`] possibly + /// covering only part of [`Self::window`] while the bars cover all of it. pub ticks: Vec, /// Stable discriminator feeding [`replay_revision`], so two open windows never collide. pub identity: u64, + /// How the tick attempt for this window ended. `Served` on a [`TradeReplaySource::Ticks`] + /// series; every other variant is a reason the bar layer is all the window has, and the + /// window PRINTS it. + pub tick_status: TickStatus, + /// Bucket the points were thinned to, in ms; `0` means raw, untouched ticks. Meaningless (and + /// always `0`) on a [`TradeReplaySource::Klines1m`] series. + pub bucket_ms: i64, + /// Whether [`Self::ticks`] covers only PART of [`Self::window`] — the bars always cover all + /// of it. Always `false` on a [`TradeReplaySource::Klines1m`] series. + pub partial: bool, } impl TradeReplaySeries { diff --git a/crates/moon-core/src/market/trade_replay/tests.rs b/crates/moon-core/src/market/trade_replay/tests.rs index df0e6e41..c1bbbd72 100644 --- a/crates/moon-core/src/market/trade_replay/tests.rs +++ b/crates/moon-core/src/market/trade_replay/tests.rs @@ -21,6 +21,8 @@ fn bars_only_series() -> TradeReplaySeries { window: ReplayWindow { from_ms: 0, to_ms: 2 * MINUTE_MS, + open_ms: 0, + close_ms: 2 * MINUTE_MS, over_budget: false, }, tf_ms: MINUTE_MS, @@ -31,6 +33,9 @@ fn bars_only_series() -> TradeReplaySeries { ], ticks: Vec::new(), identity: 42, + tick_status: TickStatus::Pending, + bucket_ms: 0, + partial: false, } } @@ -158,6 +163,8 @@ fn cache_coverage_rejects_prefixes_and_oversized_holes() { let window = ReplayWindow { from_ms: 0, to_ms: 5 * MINUTE_MS, + open_ms: 0, + close_ms: 5 * MINUTE_MS, over_budget: false, }; let exact = [0, 1, 2, 3, 4, 5] @@ -400,6 +407,8 @@ fn time_slices_keeps_unbounded_windows_whole_and_bounded_windows_gap_free() { let window = ReplayWindow { from_ms: 1_000, to_ms: 7_200_999, + open_ms: 1_000, + close_ms: 7_200_999, over_budget: false, }; @@ -432,3 +441,126 @@ fn time_slices_keeps_unbounded_windows_whole_and_bounded_windows_gap_free() { "adjacent requests neither leave a market-data gap nor re-fetch a boundary millisecond" ); } + +/// `market/trade_replay/mod.rs:tick_plan` sorting tiles by clock time instead of focus-first +/// spends the budget on lead context and makes a partial replay omit the trade itself. +#[test] +fn tick_plan_prioritizes_focus_and_keeps_every_prefix_contiguous_after_clipping() { + let window = replay_window(100_000, 100_000).expect("a same-second scalp has floor context"); + let earliest_ms = window.from_ms + 20 * MINUTE_MS; + let plan = tick_plan(window, Some(60 * MINUTE_MS), Some(earliest_ms)); + let focus = window.focus(); + + assert!( + plan.focus_len > 0, + "the retained focus must have at least one slice" + ); + assert!( + plan.slices[0].0 <= window.open_ms && window.open_ms <= plan.slices[0].1, + "the 80-minute scalp's entry belongs to the very first fetched slice" + ); + let focus_slices = &plan.slices[..plan.focus_len]; + assert_eq!( + focus_slices.first().map(|slice| slice.0), + Some(focus.0), + "the focus prefix begins at the independently derived focus edge" + ); + assert_eq!( + focus_slices.last().map(|slice| slice.1), + Some(focus.1), + "the focus prefix reaches the independently derived focus edge" + ); + assert!( + plan.slices.iter().all(|(from, _)| *from >= earliest_ms), + "retention clipping must exclude tiles older than the route can answer" + ); + for prefix_len in 1..=plan.slices.len() { + let mut prefix = plan.slices[..prefix_len].to_vec(); + prefix.sort_unstable(); + assert!( + prefix.windows(2).all(|pair| pair[0].1 + 1 == pair[1].0), + "prefix {prefix_len} must form one gap-free covered interval rather than a comb" + ); + } +} + +/// `market/trade_replay/mod.rs:fit_ticks` dropping its terminal stride or thinning an already +/// fitting input can overflow the GPU ring or alter raw trade points without need. +#[test] +fn fit_ticks_obeys_every_budget_boundary_and_preserves_raw_inputs_that_fit() { + let raw = (0..10) + .map(|index| crate::feed::types::Tick { + time_ms: (index * 100) as f64, + price: 100.0 + index as f32, + qty: 1.0, + side: crate::feed::types::Side::Buy, + }) + .collect::>(); + + for budget in [0, 1, 2, 3] { + let (result, _) = fit_ticks(raw.clone(), budget); + assert!( + result.len() <= budget, + "a ten-row input must never exceed requested budget {budget}" + ); + } + for budget in [10, 11] { + let (result, bucket_ms) = fit_ticks(raw.clone(), budget); + assert_eq!( + result + .iter() + .map(|tick| (tick.time_ms, tick.price, tick.qty)) + .collect::>(), + raw.iter() + .map(|tick| (tick.time_ms, tick.price, tick.qty)) + .collect::>(), + "a raw vector that fits budget {budget} stays unchanged" + ); + assert_eq!(bucket_ms, 0, "an already fitting vector reports raw ticks"); + } + let (empty, bucket_ms) = fit_ticks(Vec::new(), 0); + assert!(empty.is_empty(), "empty input stays empty at zero budget"); + assert_eq!(bucket_ms, 0, "empty input already fits and remains raw"); +} + +/// `market/trade_replay/mod.rs:tick_identity_salt` including tick-status payload makes identical +/// candle rows re-upload merely because a tick attempt changed from pending to failed. +#[test] +fn kline_tick_statuses_keep_the_same_chart_revision_while_ticks_change_it() { + let pending = bars_only_series(); + let mut failed = pending.clone(); + failed.tick_status = TickStatus::Failed; + let mut tick_upgrade = pending.clone(); + tick_upgrade.source = TradeReplaySource::Ticks; + tick_upgrade.tick_status = TickStatus::Served; + tick_upgrade.ticks = vec![crate::feed::types::Tick { + time_ms: MINUTE_MS as f64, + price: 101.0, + qty: 1.0, + side: crate::feed::types::Side::Buy, + }]; + + let read_revision = |series: &TradeReplaySeries| { + let mut out = ChartHistoryBuffers::default(); + series + .read_into( + 0.0, + 0.0, + (2 * MINUTE_MS) as f32, + Some(&candle_params(0)), + &mut out, + ) + .revision + }; + + assert_eq!( + read_revision(&pending), + read_revision(&failed), + "pending and failed candle fallbacks carry identical rows and therefore one revision" + ); + assert_ne!( + read_revision(&pending), + read_revision(&tick_upgrade), + "a tick upgrade must have its own revision so the pane uploads its new points" + ); +} diff --git a/crates/moon-core/src/market/trade_replay/worker.rs b/crates/moon-core/src/market/trade_replay/worker.rs index bfee5898..047be1f9 100644 --- a/crates/moon-core/src/market/trade_replay/worker.rs +++ b/crates/moon-core/src/market/trade_replay/worker.rs @@ -17,6 +17,18 @@ //! worker, keyed by the exact question asked. That second cache is what actually satisfies "the //! second open of the same trade costs nothing": the SQLite cache cannot hold ticks at all, and //! nothing else in the process remembers that a given window was already answered. +//! +//! # The degrade ladder +//! +//! A tick stage never throws away what it already paid for. Cancellation is the one thing that +//! discards everything collected so far, because the window itself is gone; every other stop — +//! the page or tick budget, the job deadline, a venue's own refusal — instead SERVES what was +//! already fetched and names the reason in [`TradeReplaySeries::tick_status`], rather than +//! abandoning the whole stage and falling back to bars with no explanation. Only a harvest that +//! ends up genuinely empty reaches the candles-only outcome, and even then the bar layer drawn is +//! never blank: [`TickStage::candles`] carries the exchange's own one-minute klines forward from +//! the candle stage that ran first, so the window always has SOMETHING to show while the reasoned +//! caption explains what is missing and why. use std::collections::VecDeque; use std::sync::atomic::{AtomicBool, Ordering}; @@ -27,8 +39,8 @@ use std::time::{Duration, Instant}; use super::gate::ReplayGate; use super::venue_caps::{TradeRoute, bybit_category, kline_route, trade_route}; use super::{ - ReplayWindow, TradeReplayEmpty, TradeReplayFailure, TradeReplayOutcome, TradeReplaySeries, - TradeReplaySource, pages, rest, time_slices, + ReplayWindow, TickPlan, TickStatus, TradeReplayEmpty, TradeReplayFailure, TradeReplayOutcome, + TradeReplaySeries, TradeReplaySource, fit_ticks, pages, rest, tick_plan, }; use crate::feed::types::Tick; use crate::market::candles::ChartCandle; @@ -68,7 +80,11 @@ const OUTCOME_CACHE_LEN: usize = 8; /// rather than immediately discarded and re-fetched. const OUTCOME_CACHE_MAX_TICKS: usize = 2 * TICK_BUDGET; -/// Bounds MEMORY and the GPU point ring for one tick series. +/// Bounds the COMPOSED series and the outcome ring for one tick series — never the in-flight +/// fetch, which is bounded instead by [`TICK_PAGE_BUDGET`] times a route's own page size. A +/// budget crossed while paginating STOPS the walk and serves what is already held rather than +/// discarding it (see the module header's degrade ladder), so this constant ceilings what gets +/// drawn and remembered, not what a stage may fetch before giving up. /// /// Sits under the live chart's default `trades_limit` of 50 000 (`candles.rs:93`), so a tick /// replay never asks the point ring for more than the main chart already draws. @@ -76,8 +92,12 @@ pub(crate) const TICK_BUDGET: usize = 40_000; /// Bounds WALL TIME on the single worker thread for one tick stage. /// -/// 60 pages at [`super::gate::ReplayGate::pace`]'s 100 ms floor plus a ~250 ms round trip is -/// about 21 s, inside [`JOB_DEADLINE`] with room for a slow venue. +/// 60 pages at [`super::gate::ReplayGate::pace`]'s 100 ms floor plus a ~250 ms round trip is an +/// ORDER-OF-MAGNITUDE bound of a few tens of seconds, inside [`JOB_DEADLINE`] with room for a slow +/// venue. Not a precise figure: [`tick_plan`] now tiles the window into many small slices rather +/// than the one-or-two wide ones this constant was first sized against, and a quiet-market tile +/// still costs one round trip apiece, so the true page count for a given window depends on how +/// many tiles it takes as much as on how much data each holds. const TICK_PAGE_BUDGET: usize = 60; /// What one answered question is remembered as. @@ -121,17 +141,23 @@ struct OutcomeKey { to_ms: i64, } -/// Which route and cache key a queued tick stage answers. +/// Which route, cache key and bar layer a queued tick stage answers. /// -/// Carried on [`Job::Ticks`] alongside the original [`TradeReplayRequest`] rather than -/// re-derived when the stage finally runs, so a stage queued behind a long line of candle jobs -/// answers the same question `serve` decided it should — never a re-lookup that could disagree. +/// Carried on [`Job::Ticks`] alongside the original [`TradeReplayRequest`] rather than re-derived +/// when the stage finally runs, so a stage queued behind a long line of candle jobs answers the +/// same question `serve` decided it should — never a re-lookup that could disagree. `candles` is +/// the exchange klines `serve` already composed for this window: carrying them here removes the +/// dependence on the outcome ring not having evicted this key's entry between the two jobs, and is +/// what lets a tick outcome keep the bar layer whole even where its own points, per +/// [`TradeReplaySeries::partial`], cover only part of the window. #[derive(Clone, Debug)] pub(crate) struct TickStage { /// Which venue endpoint to ask. route: TradeRoute, /// The ring key this stage's answer replaces on success. key: OutcomeKey, + /// The exchange klines to carry forward as the bar layer of the eventual tick series. + candles: Vec, } /// One unit of the worker's internal priority queue. @@ -159,9 +185,12 @@ fn next_job(queue: &mut VecDeque) -> Option { } } -/// Why a tick stage produced nothing, so a caller and a test can each name the reason precisely. +/// Why a tick stage's walk stopped without a usable harvest, or was cancelled outright. /// -/// Each arm is a DIFFERENT log line and a different test; see [`paginate_ticks`]. +/// `Cancelled` throws away whatever was collected because the window itself closed. Every other +/// arm here is reached only when the harvest that stopped for that reason turned out EMPTY — +/// a non-empty one is served instead, whatever the stop reason was; see [`paginate_ticks`]. Each +/// arm is a DIFFERENT log line and a different test. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub(crate) enum TickAbandon { Cancelled, @@ -169,7 +198,6 @@ pub(crate) enum TickAbandon { Transient, Empty, UnknownSymbol, - OverTickBudget, OverPageBudget, /// The stage's own [`TickObserver::claim`] was refused: an active refusal is already /// recorded for this host by some other request, and the tick stage must respect it rather @@ -177,14 +205,32 @@ pub(crate) enum TickAbandon { RateLimited, } -/// What one tick stage produced. +/// What one tick stage's walk produced when it did produce something. +#[derive(Debug)] +pub(crate) struct TickHarvest { + /// Ticks collected, in the vendor's own per-page order within each slice — the global sort + /// and the clip to [`Self::covered`] both happen in [`serve_ticks`] after this returns, so a + /// test can hand in DESCENDING pages and observe that the SORT, not the pagination, is what + /// fixes them. + pub ticks: Vec, + /// The inclusive time range [`Self::ticks`] is guaranteed exhaustive over — [`serve_ticks`] + /// clips to this rather than to the request window, since a walk cut short still holds a + /// complete answer for the slices it actually finished. + pub covered: (i64, i64), + /// Whether every slice of the plan was walked to completion. + pub complete: bool, + /// Whether the walk stopped because the venue itself refused (`Transient`/`UnknownSymbol`), + /// as opposed to our own budget or the caller cancelling — see [`serve_ticks`]'s gate-clear. + pub venue_refused: bool, +} + +/// What one tick stage's walk produced. /// -/// `Ready` carries the ticks EXACTLY as fetched, page order untouched: the global sort and the -/// window clip happen in [`serve_ticks`] after this returns, so a test can hand in DESCENDING -/// pages and observe that the SORT, not the pagination, is what fixes them. +/// `Ready` carries the harvest exactly as walked; `Abandoned` carries the reason nothing usable +/// resulted. See [`TickHarvest`] and [`TickAbandon`]. #[derive(Debug)] pub(crate) enum TickVerdict { - Ready(Vec), + Ready(TickHarvest), Abandoned(TickAbandon), } @@ -327,17 +373,47 @@ fn run(rx: &Receiver) { if request.cancel.load(Ordering::Relaxed) { continue; } - if let Some(series) = serve_ticks(&agent, &gate, &request, &stage) { - remember_store( - &cache, - stage.key, - Remembered::Ready { - series: series.clone(), - ticks_settled: true, - }, - ); - // Normal, not an error, for the same reason as the candle send above. - let _ = request.reply.send(TradeReplayOutcome::Ready(series)); + match serve_ticks(&agent, &gate, &request, &stage) { + Ok((series, venue_refused)) => { + // A PARTIAL harvest is still a COMPLETE run of the stage: the walk is + // done deciding what it can serve, so a reopen must not re-ask for ticks + // it already answered, whether or not `series.partial` is set — UNLESS + // the venue's own account of why it stopped was `Transient`: serving the + // partial rows is right (throwing away paid-for data is what this goal + // removes), but calling that answer SETTLED is not, because the venue + // called its own refusal transient. An exact-key reopen must retry it + // rather than pin the same partial answer until unrelated cache eviction. + remember_store( + &cache, + stage.key, + Remembered::Ready { + series: series.clone(), + ticks_settled: !venue_refused, + }, + ); + // Normal, not an error, for the same reason as the candle send above. + let _ = request.reply.send(TradeReplayOutcome::Ready(series)); + } + Err(Some(status)) => { + let mut series = compose(&request, request.address.venue, stage.candles); + series.tick_status = status; + // `NoTrades` is authoritative — the venue answered and held nothing — and + // is remembered settled exactly like a `Ready` harvest. `Failed` is not: + // the fetch itself did not produce an answer, so a reopen must retry it. + if status == TickStatus::NoTrades { + remember_store( + &cache, + stage.key, + Remembered::Ready { + series: series.clone(), + ticks_settled: true, + }, + ); + } + let _ = request.reply.send(TradeReplayOutcome::Ready(series)); + } + // The window closed; there is no one left to send a second outcome to. + Err(None) => {} } } } @@ -359,6 +435,10 @@ struct Served { /// gate is refusing — a user in backoff still sees the real chart rather than a countdown. Only /// then is a permit taken. /// +/// Each of the three points that produces a fresh candle answer (a non-settled ring hit, a +/// SQLite hit, a completed network fetch) also decides the tick stage for it and stamps the +/// outgoing series' [`TradeReplaySeries::tick_status`] to match, via [`stage_and_stamp`]. +/// /// Args: /// agent: Shared HTTP client. /// gate: Per-host pacing and backoff. @@ -390,9 +470,20 @@ fn serve( match remember_lookup(cache, &key, request.identity) { Some(Remembered::Ready { series, - ticks_settled, + ticks_settled: true, + }) => { + // Sent exactly as stored: its own fields already carry the final answer, so no + // stage is re-decided and none is queued. + return Served { + outcome: TradeReplayOutcome::Ready(series), + tick_stage: None, + }; + } + Some(Remembered::Ready { + mut series, + ticks_settled: false, }) => { - let tick_stage = tick_stage_for(venue, request.window, &key, ticks_settled); + let tick_stage = stage_and_stamp(venue, request.window, &key, &mut series); return Served { outcome: TradeReplayOutcome::Ready(series), tick_stage, @@ -409,16 +500,20 @@ fn serve( // The SQLite cache is read first and unconditionally: it costs no request and is not gated. if let Some(rows) = read_cached_bars(request.address.cache.as_ref(), request) { - let series = compose(request, venue, rows); + let mut series = compose(request, venue, rows); + let tick_stage = stage_and_stamp(venue, request.window, &key, &mut series); + // Settled exactly when NO stage was queued: `stage_and_stamp` already stamped a TERMINAL + // status (`NoRoute`/`OutOfRetention`) in that case, and both are stable facts a reopen + // would only re-derive identically — a queued stage, by contrast, is still `Pending` and + // must be re-decided (or answered) on the next open. remember_store( cache, key.clone(), Remembered::Ready { series: series.clone(), - ticks_settled: false, + ticks_settled: tick_stage.is_none(), }, ); - let tick_stage = tick_stage_for(venue, request.window, &key, false); return Served { outcome: TradeReplayOutcome::Ready(series), tick_stage, @@ -434,15 +529,22 @@ fn serve( let category = bybit_category(venue, &request.market); let deadline = Instant::now() + JOB_DEADLINE; let mut rows: Vec = Vec::new(); - // Whether every page of the window was actually fetched. A cancelled run keeps its rows — they - // were paid for — but must NOT be remembered as this window's answer. + // Whether every page of the window was actually fetched. Two independent things can make this + // false, and only `cancelled` below may still be true when this is — see the tick-stage + // decision after the forming-bar drop for why the two must not be read as one fact. A + // cancelled run keeps its rows — they were paid for — but must NOT be remembered as this + // window's answer. let mut complete = true; + // Whether the WINDOW ITSELF closed mid-fetch, as opposed to `complete` going false for the + // forming-bar drop below: only this one discards the tick upgrade outright. + let mut cancelled = false; for (from_ms, to_ms) in pages(request.window, BAR_MS, route.max_rows()) { if request.cancel.load(Ordering::Relaxed) { // The window is gone, or a Retry superseded this request. Whatever was fetched is // still worth merging into the shared cache, so fall through rather than discarding a // page already paid for. complete = false; + cancelled = true; break; } if Instant::now() >= deadline { @@ -529,23 +631,37 @@ fn serve( tick_stage: None, }; } - let series = compose(request, venue, rows); + let mut series = compose(request, venue, rows); // Only a COMPLETE run may be remembered. Pages are issued left to right, so a cancelled run // holds the window's left-hand prefix — typically missing exactly the bars around the exit — // and the in-memory ring, unlike the SQLite path, has no coverage re-check to catch that on // read. Storing it would serve a silently truncated chart as `Ready` for the life of the // entry. The SQLite merge above is unaffected: `cache_covers` re-checks it on every read. - let tick_stage = match complete { - true => tick_stage_for(venue, request.window, &key, false), - false => None, + // + // The tick stage is gated on `cancelled` alone, NOT on `complete`: a forming-bar drop leaves + // `complete` false too, but the window is fine and its ticks are fetched independently of the + // bar layer — queuing the stage is the whole point of this feature, and skipping it here is + // exactly what used to leave a freshly closed trade stuck on "tics ещё грузятся" forever. + // CANCELLED is the one reason to skip it outright: the window itself is gone. + let tick_stage = if cancelled { + // No stage is queued, so the status must be TERMINAL: `Pending` (compose()'s default) + // asserts a stage is in flight, and none is. `Failed` reads honestly — whatever a retry + // would have answered, it never ran. + series.tick_status = TickStatus::Failed; + None + } else { + stage_and_stamp(venue, request.window, &key, &mut series) }; if complete { + // Settled exactly when no stage was queued — see the SQLite-hit branch above for why a + // terminal `stage_and_stamp` result never needs re-deciding, while a queued stage's + // `Pending` must be. remember_store( cache, key, Remembered::Ready { series: series.clone(), - ticks_settled: false, + ticks_settled: tick_stage.is_none(), }, ); } @@ -555,15 +671,46 @@ fn serve( } } -/// Decide whether a just-served CANDLE outcome earns a queued tick upgrade. +/// Decide the tick stage for a just-built candle series, and stamp its own `tick_status` in place +/// to match — `Pending` when a stage is queued, or the reason it is not, via [`tick_stage_for`]. /// -/// `Some` only when every one of four conditions holds: a trade route exists for this venue -/// ([`trade_route`]), the ring entry backing this key does not already carry a SETTLED tick -/// series, and the window is inside the route's own documented retention -/// ([`inside_retention`]) — evaluated here, BEFORE any request is spent, so an out-of-retention -/// window costs zero. The fourth condition, "the outcome was `Ready{Klines1m}`", is enforced by -/// every call site: this is reached only from a branch that just composed or looked up a candle -/// `Ready` outcome. +/// One helper for the three sites in [`serve`] that each produce a fresh candle answer: the +/// ring-hit-but-not-settled branch, the SQLite-cache-hit branch, and the completed-network-fetch +/// branch — the last of these calls it only on its NON-CANCELLED path; the cancelled sub-branch +/// skips it entirely and stamps [`TickStatus::Failed`] directly, since there is no fresh route +/// decision to make for a window that is already gone. `series.tick_status` already reads +/// `Pending` from [`compose`], so this only overwrites it when a stage is NOT queued. +/// +/// Args: +/// venue: Venue the candles came from. +/// window: The window the candles cover. +/// key: The ring key this stage would replace on success. +/// series: The just-built series; its `tick_status` is overwritten in place when no stage is +/// queued for it. +/// +/// Returns: +/// The stage to queue, or `None`. +fn stage_and_stamp( + venue: crate::venue::Venue, + window: ReplayWindow, + key: &OutcomeKey, + series: &mut TradeReplaySeries, +) -> Option { + match tick_stage_for(venue, window, key, &series.candles) { + Ok(stage) => Some(stage), + Err(status) => { + series.tick_status = status; + None + } + } +} + +/// Decide whether a just-built CANDLE series earns a queued tick upgrade, or the reason it does +/// not. +/// +/// The "already settled" short-circuit this used to take as a parameter no longer lives here: it +/// is checked once, in [`serve`]'s ring-hit branch, before this is ever called — a settled entry +/// is sent exactly as stored, with no stage queued and nothing here re-decided. /// /// A clock that cannot be read (`now_unix_ms_i64` answering `0`) is treated as INSIDE retention /// rather than refused, the same permissive default [`serve`] already applies to the closed-bar @@ -573,32 +720,44 @@ fn serve( /// venue: Venue the candles came from. /// window: The window the candles cover. /// key: The ring key this stage would replace on success. -/// ticks_settled: Whether the ring entry for this key already carries a settled tick series. +/// candles: The exchange klines just composed, carried forward as the eventual tick series' +/// bar layer — see [`TickStage::candles`]. /// /// Returns: -/// The stage to queue, or `None`. +/// The stage to queue, or the reason it is not queued. fn tick_stage_for( venue: crate::venue::Venue, window: ReplayWindow, key: &OutcomeKey, - ticks_settled: bool, -) -> Option { - if ticks_settled { - return None; - } - let route = trade_route(venue)?; + candles: &[ChartCandle], +) -> Result { + let route = trade_route(venue).ok_or(TickStatus::NoRoute)?; let now_ms = crate::util::time::now_unix_ms_i64(); if now_ms > 0 && !inside_retention(route, window, now_ms) { - return None; + // `inside_retention` is false here only when the route documents a retention: it is + // unconditionally true otherwise, so this default is never actually reached — see its own + // doc comment. + let retention_ms = route.retention_ms().unwrap_or(0); + return Err(TickStatus::OutOfRetention { retention_ms }); } - Some(TickStage { + Ok(TickStage { route, key: key.clone(), + candles: candles.to_vec(), }) } /// Whether a window is within a trade route's own documented retention. /// +/// Judges the FOCUS's own right edge — the trade's EXIT ([`ReplayWindow::focus`]) — never the +/// window's padded `from_ms` (D2-3), and never the focus's left edge either: the lead context is +/// optional padding, but the trade itself is not, and [`tick_plan`]'s own retention clip already +/// asks only that the exit be inside retention, clipping everything older. Judging the entry +/// instead is a STRICTLY STRONGER check that runs first, in [`tick_stage_for`], and made +/// `tick_plan`'s whole retention-clipping recovery path unreachable for exactly the windows it was +/// written to rescue: a Binance futures trade held ~10 h and closed 40 h ago (retention 48 h) was +/// refused outright although its exit's ticks were comfortably inside retention. +/// /// Free, and evaluated BEFORE any request is spent — see [`tick_stage_for`], the only caller. /// /// Args: @@ -607,12 +766,12 @@ fn tick_stage_for( /// now_ms: Current Unix time in milliseconds. /// /// Returns: -/// `true` when the route documents no retention limit, or when the window's left edge falls -/// inside the one it does document. +/// `true` when the route documents no retention limit, or when the focus's own right edge +/// falls inside the one it does document. pub(crate) fn inside_retention(route: TradeRoute, window: ReplayWindow, now_ms: i64) -> bool { route .retention_ms() - .is_none_or(|r| window.from_ms >= now_ms - r) + .is_none_or(|r| window.focus().1 >= now_ms - r) } /// Run one queued tick stage to completion. @@ -621,27 +780,48 @@ pub(crate) fn inside_retention(route: TradeRoute, window: ReplayWindow, now_ms: /// agent: Shared HTTP client. /// gate: Per-host pacing. /// request: The original request this stage upgrades. -/// stage: Which route and cache key this stage answers. +/// stage: Which route, cache key and bar layer this stage answers. /// /// Returns: -/// The tick series to send as the SECOND outcome, or `None` when it could not be produced — -/// in which case the candle picture already sent stands and nothing more is sent. +/// `Ok((series, venue_refused))` with the tick series to send as the SECOND outcome — +/// `venue_refused` is [`TickHarvest::venue_refused`], carried out here so the caller can +/// decide whether this harvest is safe to remember settled (see [`run`]'s `Job::Ticks` arm). +/// `Err(Some(status))` when the stage ended for a reason the window must print instead — the +/// caller composes that second outcome from [`TickStage::candles`] carrying `status`. +/// `Err(None)` only for a cancelled window, where the requester is already gone and nothing +/// more is sent. fn serve_ticks( agent: &ureq::Agent, gate: &ReplayGate, request: &TradeReplayRequest, stage: &TickStage, -) -> Option { +) -> Result<(TradeReplaySeries, bool), Option> { let route = stage.route; let deadline = Instant::now() + JOB_DEADLINE; - let slices = time_slices(request.window, route.max_query_ms()); + // Re-derived rather than trusted from `tick_stage_for`'s own permissive pass: that check ran + // BEFORE this stage was even queued, and a clock that could not be read then still cannot + // prove the window is too old now, so the same `now_ms > 0` guard applies here. + let now_ms = crate::util::time::now_unix_ms_i64(); + let earliest_ms = match now_ms > 0 { + true => route.retention_ms().map(|r| now_ms - r), + false => None, + }; + let plan = tick_plan(request.window, route.max_query_ms(), earliest_ms); + if plan.slices.is_empty() { + // The FOCUS itself — the trade, not its optional context — lies entirely before + // `earliest_ms`: nothing worth fetching remains, so this is reported as retention rather + // than as an empty venue answer. + return Err(Some(TickStatus::OutOfRetention { + retention_ms: route.retention_ms().unwrap_or(0), + })); + } let mut observer = GateObserver { gate, host: route.host(), }; let verdict = paginate_ticks( route, - &slices, + &plan, TICK_BUDGET, TICK_PAGE_BUDGET, || request.cancel.load(Ordering::Relaxed), @@ -651,15 +831,8 @@ fn serve_ticks( rest::fetch_trades(agent, route, &request.market, from_ms, to_ms, cursor) }, ); - let mut ticks = match verdict { - TickVerdict::Ready(ticks) => { - // The venue answered every slice, so its refusal history is stale whatever the rows - // say — exactly the candle stage's own `gate.clear` above. A `Transient` page never - // reaches here: it abandons through the branch below, and the refusal it recorded - // must stay. - gate.clear(route.host()); - ticks - } + let harvest = match verdict { + TickVerdict::Ready(harvest) => harvest, TickVerdict::Abandoned(reason) => { // RELEASE THE PERMIT THIS STAGE TOOK, unless the venue is the reason we stopped. // @@ -685,66 +858,121 @@ fn serve_ticks( | TickAbandon::Deadline | TickAbandon::Empty | TickAbandon::UnknownSymbol - | TickAbandon::OverTickBudget | TickAbandon::OverPageBudget => gate.clear(route.host()), } log::info!( "[x] trade-replay tick stage abandoned on {}: {reason:?}", route.host() ); - return None; + return Err(match reason { + TickAbandon::Cancelled => None, + TickAbandon::Empty => Some(TickStatus::NoTrades), + _ => Some(TickStatus::Failed), + }); } }; - // The complete tick vector, stably sorted ascending and clipped to the window, BEFORE - // anything else: per-page sorting is NOT enough for the two BACKWARD-paginating venues - // (Bitget, OKX), whose concatenated pages walk backwards across every page boundary, and - // `aggregate_trades`'s own trailing arm drops a trade older than the whole series rather than - // reordering it — a tick chart missing whole minutes, with no error anywhere. + let TickHarvest { + mut ticks, + covered, + complete, + venue_refused, + } = harvest; + // The venue answered without refusing anywhere along the walk, so its refusal history is + // stale — exactly the candle stage's own `gate.clear` above. A refusal it gave us mid-walk + // (`venue_refused`) must stand, or the next request to this host sends blind into a burst it + // just declined. + if !venue_refused { + gate.clear(route.host()); + } + // Stably sorted ascending, BEFORE anything else: per-page sorting is NOT enough for the two + // BACKWARD-paginating venues (Bitget, OKX), whose concatenated pages walk backwards across + // every page boundary. ticks.sort_by(|a, b| { a.time_ms .partial_cmp(&b.time_ms) .unwrap_or(std::cmp::Ordering::Equal) }); + // Clipped to what the walk actually finished (`covered`), not to the request window: a walk + // cut short still holds a complete answer for the slices it actually walked, and clipping to + // the wider window would let a stray page-overshoot outside `covered` back in. ticks.retain(|t| { - t.time_ms.is_finite() - && (t.time_ms as i64) >= request.window.from_ms - && (t.time_ms as i64) <= request.window.to_ms + t.time_ms.is_finite() && (t.time_ms as i64) >= covered.0 && (t.time_ms as i64) <= covered.1 }); + // `partial` must reflect what `covered` actually spans, not merely whether the walk finished. + // `tick_plan`'s own `earliest_ms` clip can make the PLAN narrower than `request.window` before + // the walk even starts, so a retention-clipped plan that completes still leaves the served + // ticks short of the requested window on one or both edges. + let partial = + !complete || covered.0 > request.window.from_ms || covered.1 < request.window.to_ms; + let (ticks, bucket_ms) = fit_ticks(ticks, TICK_BUDGET); + // An empty harvest after the coverage clip is not a success: some slice genuinely produced + // rows (`paginate_ticks` already refuses an empty one, above), so a caption of "Served" with + // zero points would be the lying-chart failure this module exists to remove. `Failed`, not + // `NoTrades` — a retry is honest here, while `NoTrades` claims an authoritative empty. if ticks.is_empty() { - log::info!( - "[x] trade-replay tick stage abandoned on {}: no ticks left inside the window after clipping", - route.host() - ); - return None; + return Err(Some(TickStatus::Failed)); } - Some(compose_ticks(request, request.address.venue, ticks)) + Ok(( + compose_ticks( + request, + request.address.venue, + ticks, + bucket_ms, + partial, + stage.candles.clone(), + ), + venue_refused, + )) } -/// Walk every slice of one tick stage, paginating each with the venue's own cursor, until the -/// stage completes, is abandoned, or either budget is spent. +/// Walk every tile of one tick stage's [`TickPlan`], paginating each with the venue's own cursor, +/// until the plan is exhausted or a stop condition is reached — and, unlike a candle job, a stop +/// never discards what was already collected except when the window itself closed. /// /// `fetch` is the injected seam — no network, no clock, no gate inside this function, which is -/// what makes it testable with a fake fetcher and a fake [`TickObserver`]. Cancellation and the -/// deadline are checked before every page, ahead of the page budget, ahead of the tick budget; -/// an empty result is recognised only once every slice has been walked with nothing collected. +/// what makes it testable with a fake fetcher and a fake [`TickObserver`]. +/// +/// The loop rule, in order of precedence (D2-2, replacing this function's earlier design in +/// full): +/// - [`cancelled`] stops EVERYTHING, always, and discards whatever was collected — the window is +/// gone and there is no one left to serve it to. +/// - The job deadline and the page budget each stop the WALK without abandoning it, at any point: +/// the harvest collected so far is kept, and the loop moves straight to the verdict. This is +/// the whole point of this function's redesign — a budget or a deadline crossed an hour of lead +/// context away from the trade must never throw away the tiles around the trade that were +/// already paid for. +/// - A venue's own answer — `Transient`/`UnknownSymbol` — also stops the walk rather than the +/// whole stage, and marks the harvest [`TickHarvest::venue_refused`], so [`serve_ticks`] knows +/// not to clear a refusal the venue just gave it. +/// - The FOCUS tiles (the leading [`TickPlan::focus_len`] entries) are never truncated by the +/// tick budget: it is checked only around a NON-focus tile, before it starts and again once it +/// finishes, so a tile is walked whole or not at all — never cut mid-body. +/// - Every page is clipped to the SLICE it was fetched for before it is counted toward any budget +/// (D2-1): Binance's forward pager and OKX's backward one both routinely return a page that +/// overshoots its own slice edge, and [`Tick`] carries no exchange trade id, so an unclipped +/// overlap between two adjacent slices is undetectable once concatenated, not merely unnoticed. +/// The aggregate clip to [`TickHarvest::covered`] in [`serve_ticks`] is the OUTER bound and does +/// not replace this inner one. +/// - The verdict is one question: is the harvest empty? A non-empty one is always `Ready`, +/// whatever stopped the walk; only an empty one reaches [`TickVerdict::Abandoned`], carrying +/// whichever reason actually stopped it. /// /// Args: /// route: Which venue endpoint this stage answers. -/// slices: The window's own [`time_slices`] output; one venue-side pagination loop runs per -/// slice. -/// tick_budget: Ceiling on the total ticks collected across every slice. -/// page_budget: Ceiling on the total pages fetched across every slice. +/// plan: The window's own [`tick_plan`] output — tiles in fetch-priority order, with the +/// first [`TickPlan::focus_len`] of them being the trade's own focus. +/// tick_budget: Ceiling on the total ticks collected before a non-focus tile is skipped. +/// page_budget: Ceiling on the total pages fetched across every tile. /// cancelled: Answers whether the requester's window has closed. /// expired: Answers whether this stage's own deadline has passed. /// observer: Records the `claim`/`pace` calls this stage makes. /// fetch: Fetches one page for a given slice and cursor. /// /// Returns: -/// The collected ticks in the vendor's own per-page order (unsorted across pages), or the -/// reason nothing was collected. +/// The harvest, or the reason nothing was collected. pub(crate) fn paginate_ticks( route: TradeRoute, - slices: &[(i64, i64)], + plan: &TickPlan, tick_budget: usize, page_budget: usize, cancelled: impl Fn() -> bool, @@ -756,7 +984,7 @@ where F: FnMut(i64, i64, Option) -> Result, O: TickObserver, { - if slices.is_empty() { + if plan.slices.is_empty() { return TickVerdict::Abandoned(TickAbandon::Empty); } if observer.claim(route.host()).is_err() { @@ -764,54 +992,163 @@ where } let mut ticks: Vec = Vec::new(); let mut pages_fetched = 0usize; - for &(slice_from, slice_to) in slices { + let mut covered: Option<(i64, i64)> = None; + let mut complete = true; + let mut venue_refused = false; + let mut stop_reason: Option = None; + // The tile still being walked when a `break 'walk` fired mid-body — its own bounds, where its + // rows begin in `ticks`, and the cursor most recently used for it — so its paid-for rows can + // extend `covered` afterward rather than being reclaimed by the clip in `serve_ticks` (F2). + // `None` whenever every stop happened BETWEEN tiles (or the walk was cancelled outright, which + // returns before this is ever read). + let mut interrupted: Option<(i64, i64, usize, Option)> = None; + + 'walk: for (index, &(slice_from, slice_to)) in plan.slices.iter().enumerate() { + let is_focus = index < plan.focus_len; + // The tick budget never truncates a focus slice — checked only around a NON-focus one, so + // a slice is whole or absent rather than cut mid-body. See the after-check below for the + // other half of this rule. + if !is_focus && ticks.len() >= tick_budget { + complete = false; + break; + } + let start_len = ticks.len(); let mut cursor: Option = None; loop { if cancelled() { + // The window is gone; nothing collected so far is worth keeping. return TickVerdict::Abandoned(TickAbandon::Cancelled); } if expired() { - return TickVerdict::Abandoned(TickAbandon::Deadline); + complete = false; + stop_reason = Some(TickAbandon::Deadline); + interrupted = Some((slice_from, slice_to, start_len, cursor)); + break 'walk; } if pages_fetched >= page_budget { - return TickVerdict::Abandoned(TickAbandon::OverPageBudget); + complete = false; + stop_reason = Some(TickAbandon::OverPageBudget); + interrupted = Some((slice_from, slice_to, start_len, cursor)); + break 'walk; } observer.pace(route.host()); let page = match fetch(slice_from, slice_to, cursor) { Ok(page) => page, Err(rest::FetchError::UnknownSymbol) => { - return TickVerdict::Abandoned(TickAbandon::UnknownSymbol); + complete = false; + venue_refused = true; + stop_reason = Some(TickAbandon::UnknownSymbol); + interrupted = Some((slice_from, slice_to, start_len, cursor)); + break 'walk; } Err(rest::FetchError::Transient(_)) => { - return TickVerdict::Abandoned(TickAbandon::Transient); + complete = false; + venue_refused = true; + stop_reason = Some(TickAbandon::Transient); + interrupted = Some((slice_from, slice_to, start_len, cursor)); + break 'walk; } }; pages_fetched += 1; - ticks.extend(page.ticks); - if ticks.len() > tick_budget { - return TickVerdict::Abandoned(TickAbandon::OverTickBudget); - } + let mut rows = page.ticks; + // D2-1: clip THIS page to the slice it was fetched for, before extending or counting + // toward the budget — see this function's own doc comment for the vendor evidence. + rows.retain(|t| { + t.time_ms.is_finite() + && (t.time_ms as i64) >= slice_from + && (t.time_ms as i64) <= slice_to + }); + ticks.extend(rows); match page.next { Some(next_cursor) => cursor = Some(next_cursor), None => break, } } + // The slice's own pagination completed. For a non-focus slice only, a tick budget crossed + // during it removes the WHOLE slice rather than leaving it half-drawn. + if !is_focus && ticks.len() > tick_budget { + ticks.truncate(start_len); + complete = false; + break; + } + covered = Some(match covered { + None => (slice_from, slice_to), + Some((c_from, c_to)) => (c_from.min(slice_from), c_to.max(slice_to)), + }); } - match ticks.is_empty() { - true => TickVerdict::Abandoned(TickAbandon::Empty), - false => TickVerdict::Ready(ticks), + + if ticks.is_empty() { + return TickVerdict::Abandoned(stop_reason.unwrap_or(TickAbandon::Empty)); + } + // Extend `covered` by the interrupted tile's own paid-for rows — but ONLY when its pagination + // direction actually reached the edge touching `covered`, never unconditionally. Within one + // slice a paginated run is contiguous, but its direction is per-venue: Binance's `FromId` + // cursor walks FORWARD from the tile's own `slice_from` (a prefix of the tile); Bitget/OKX's + // `LessThanId` walks BACKWARD from `slice_to` (a suffix). A tile to the RIGHT of `covered` only + // touches the shared edge under a FORWARD cursor (it starts at `slice_from`, which sits right + // beside `covered`); a tile to the LEFT only under a BACKWARD one (it starts at `slice_to`, + // beside `covered` on that side). Gate's `Page`/`Offset` cursors carry an UNDOCUMENTED order + // (`venue_caps.rs`), so neither side ever trusts them. Getting this wrong would union in a + // stretch of the tile that was never actually fetched — the exact false "the market was quiet + // here" gap this whole design exists to prevent. + if let (Some((c_from, c_to)), Some((slice_from, slice_to, start, cursor))) = + (covered, interrupted) + { + if start < ticks.len() { + let (lo, hi) = ticks[start..] + .iter() + .fold((i64::MAX, i64::MIN), |(lo, hi), t| { + let time_ms = t.time_ms as i64; + (lo.min(time_ms), hi.max(time_ms)) + }); + // `AfterMs` is excluded from both: its own doc says no current route ever emits it, so + // there is no evidence for which edge it would touch. + let forward = matches!(cursor, Some(rest::TradeCursor::FromId(_))); + let backward = matches!(cursor, Some(rest::TradeCursor::LessThanId(_))); + covered = Some(if slice_from > c_to && forward { + (c_from, c_to.max(hi)) + } else if slice_to < c_from && backward { + (c_from.min(lo), c_to) + } else { + (c_from, c_to) + }); + } } + let covered = covered.unwrap_or_else(|| { + // No slice ever reached natural completion, yet a stop mid-walk still left partial pages + // in `ticks` — this function serves what is held rather than discarding it (D2-2). The + // observed extremes of what was actually fetched can only UNDER-state true coverage, + // never claim more than was really walked, which the interrupted slice's own nominal + // bounds could. + let (lo, hi) = ticks.iter().fold((i64::MAX, i64::MIN), |(lo, hi), t| { + let time_ms = t.time_ms as i64; + (lo.min(time_ms), hi.max(time_ms)) + }); + (lo, hi) + }); + TickVerdict::Ready(TickHarvest { + ticks, + covered, + complete, + venue_refused, + }) } /// Build the frozen TICK series one tick stage answers with. /// -/// `ticks` must already be globally sorted ascending and clipped to `request.window` — this -/// function does neither; [`serve_ticks`] does both before calling it. +/// `ticks` must already be globally sorted ascending and clipped to the harvest's own +/// [`TickHarvest::covered`] range — this function does neither; [`serve_ticks`] does both before +/// calling it. `candles` is the EXCHANGE'S OWN klines carried forward from the candle stage that +/// ran first ([`TickStage::candles`]), never aggregated from `ticks`: the bar layer covers the +/// whole window even where the points, per `partial`, cover only part of it. /// /// Args: /// request: The request being served. /// venue: Venue the ticks came from. -/// ticks: Trade points, ascending, already clipped to the window. +/// ticks: Trade points, ascending, already clipped to the harvest's covered range. +/// bucket_ms: The bucket [`fit_ticks`] thinned the points to; `0` means raw. +/// partial: Whether `ticks` covers only part of `request.window`. +/// candles: The exchange klines to carry as the bar layer. /// /// Returns: /// The series to hand the chart. @@ -819,9 +1156,10 @@ fn compose_ticks( request: &TradeReplayRequest, venue: crate::venue::Venue, ticks: Vec, + bucket_ms: i64, + partial: bool, + candles: Vec, ) -> TradeReplaySeries { - let mut candles = Vec::new(); - crate::market::candles::aggregate_trades(&ticks, BAR_MS, &mut candles); TradeReplaySeries { source: TradeReplaySource::Ticks, venue, @@ -830,16 +1168,21 @@ fn compose_ticks( candles, ticks, identity: request.identity, + tick_status: TickStatus::Served, + bucket_ms, + partial, } } /// Answer the rows one cache write may actually carry, keyed on the series it came from. /// -/// The SQLite isolation seam (acceptance criterion 7): a tick series' own aggregated candles must +/// The SQLite isolation seam (acceptance criterion 7): a [`TradeReplaySource::Ticks`] series must /// NEVER reach [`write_cached_bars`], because that table is the SHARED kline cache the live -/// recorder writes too, and a candle aggregated from ticks is not a genuine exchange bar. [`serve`] -/// routes its cache write through this and nothing else, so the invariant is enforced at the one -/// call site rather than merely honoured by omission. +/// recorder writes too. In practice no call site ever offers one this way — [`serve`] is the only +/// caller and always passes [`TradeReplaySource::Klines1m`], since [`serve_ticks`] writes nothing +/// back to SQLite at all — but the guard is keyed on the TYPE rather than on that fact, so the +/// invariant survives a future call site instead of depending on every one of them getting it +/// right by omission. /// /// Args: /// source: Which kind of series `rows` was built for. @@ -877,6 +1220,9 @@ fn compose( candles: rows, ticks: Vec::new(), identity: request.identity, + tick_status: TickStatus::Pending, + bucket_ms: 0, + partial: false, } } @@ -1028,3 +1374,6 @@ fn write_cached_bars( rows: rows.to_vec(), }]); } + +#[cfg(test)] +mod tests; diff --git a/crates/moon-core/src/market/trade_replay/worker/tests.rs b/crates/moon-core/src/market/trade_replay/worker/tests.rs new file mode 100644 index 00000000..1891d0c4 --- /dev/null +++ b/crates/moon-core/src/market/trade_replay/worker/tests.rs @@ -0,0 +1,222 @@ +use std::cell::Cell; + +use super::*; +use crate::feed::Side; +use crate::market::trade_replay::rest::{FetchError, TradePage}; + +/// Records the pagination seam without touching a real host gate. +#[derive(Default)] +struct FakeObserver { + claims: usize, + paces: usize, +} + +impl TickObserver for FakeObserver { + fn claim(&mut self, _host: &str) -> Result<(), u32> { + self.claims += 1; + Ok(()) + } + + fn pace(&mut self, _host: &str) { + self.paces += 1; + } +} + +/// Builds one real-looking trade row for a deterministic fake page. +fn tick(time_ms: i64, price: f32) -> Tick { + Tick { + time_ms: time_ms as f64, + price, + qty: 1.0, + side: Side::Buy, + } +} + +/// Returns a completed fake page with the supplied rows. +fn page(ticks: Vec) -> Result { + Ok(TradePage { ticks, next: None }) +} + +/// `market/trade_replay/worker.rs:rows_for_cache` returning rows for both arms would file +/// tick-derived data as shared settled klines for every core. +#[test] +fn ticks_are_never_eligible_for_the_shared_kline_cache() { + let rows = [ChartCandle { + t_open_ms: 10_000.0, + open: 10.0, + high: 12.0, + low: 9.0, + close: 11.0, + volume: 5.0, + quote_volume: 55.0, + }]; + + assert!( + rows_for_cache(TradeReplaySource::Ticks, &rows).is_empty(), + "a tick series must never write a candle-shaped row into shared klines.sqlite" + ); + assert_eq!( + rows_for_cache(TradeReplaySource::Klines1m, &rows), + &rows, + "genuine exchange klines remain cacheable" + ); +} + +/// `market/trade_replay/worker.rs:inside_retention` judging `window.from_ms` instead of the +/// focus rejects a still-retained trade merely because optional lead context is older. +#[test] +fn retention_is_decided_from_the_trade_focus_not_padded_context() { + const HOUR_MS: i64 = 3_600_000; + let now_ms = 1_000 * HOUR_MS; + let window = ReplayWindow { + from_ms: now_ms - 60 * HOUR_MS, + to_ms: now_ms - 10 * HOUR_MS, + open_ms: now_ms - 47 * HOUR_MS, + close_ms: now_ms - 46 * HOUR_MS, + over_budget: false, + }; + + assert!( + inside_retention(TradeRoute::BinanceUsdMAggTrades, window, now_ms), + "the focus begins inside Binance USD-M's 48-hour retention despite older optional lead" + ); +} + +/// `market/trade_replay/worker.rs:paginate_ticks` restoring an over-budget abandonment would +/// throw away a non-empty focus harvest and fall back to candles. +#[test] +fn budget_stops_after_whole_focus_slices_and_serves_their_harvest() { + let plan = TickPlan { + slices: vec![(100, 199), (200, 299), (0, 99)], + focus_len: 2, + }; + let mut observer = FakeObserver::default(); + + let verdict = paginate_ticks( + TradeRoute::BinanceUsdMAggTrades, + &plan, + 40_000, + 10, + || false, + || false, + &mut observer, + |from_ms, _, _| { + page( + (0..30_000) + .map(|n| tick(from_ms + (n % 100), n as f32)) + .collect(), + ) + }, + ); + + let TickVerdict::Ready(harvest) = verdict else { + panic!("a non-empty harvest stopped by the tick budget must be served") + }; + assert_eq!( + harvest.ticks.len(), + 60_000, + "the 40,000 budget must not truncate either 30,000-row focus slice" + ); + assert_eq!( + harvest.covered, + (100, 299), + "only the two completed focus slices are covered after the budget stops the walk" + ); + assert!( + !harvest.complete, + "skipping the non-focus slice is a partial, not a complete, harvest" + ); + assert_eq!(observer.claims, 1, "a stage takes one host permit"); +} + +/// `market/trade_replay/worker.rs:paginate_ticks` returning `Abandoned(Deadline)` after any +/// fetched page discards usable ticks and replaces the user's trade with candles. +#[test] +fn deadline_with_a_non_empty_harvest_is_ready_not_abandoned() { + let plan = TickPlan { + slices: vec![(100, 199), (0, 99)], + focus_len: 1, + }; + let fetched = Cell::new(false); + let mut observer = FakeObserver::default(); + + let verdict = paginate_ticks( + TradeRoute::BinanceUsdMAggTrades, + &plan, + 40_000, + 10, + || false, + || fetched.get(), + &mut observer, + |from_ms, _, _| { + fetched.set(true); + page(vec![tick(from_ms + 5, 10.0)]) + }, + ); + + let TickVerdict::Ready(harvest) = verdict else { + panic!("a deadline after a fetched focus page must preserve the non-empty harvest") + }; + assert_eq!( + harvest + .ticks + .iter() + .map(|tick| (tick.time_ms as i64, tick.price, tick.qty)) + .collect::>(), + vec![(105, 10.0, 1.0)], + "the deadline preserves the fetched tick's time, price, and quantity" + ); + assert_eq!(harvest.covered, (100, 199)); + assert!( + !harvest.complete, + "the deadline leaves remaining slices unwalked" + ); +} + +/// `market/trade_replay/worker.rs:paginate_ticks` dropping its per-page retain lets adjacent +/// slices duplicate overshot exchange trades and spend the tick budget twice. +#[test] +fn each_fetched_page_is_clipped_to_its_own_slice_before_collection() { + let plan = TickPlan { + slices: vec![(100, 199), (200, 299)], + focus_len: 1, + }; + let mut observer = FakeObserver::default(); + let verdict = paginate_ticks( + TradeRoute::BinanceUsdMAggTrades, + &plan, + 40_000, + 10, + || false, + || false, + &mut observer, + |from_ms, _, _| match from_ms { + 100 => page(vec![ + tick(99, 1.0), + tick(100, 2.0), + tick(199, 3.0), + tick(200, 4.0), + ]), + 200 => page(vec![ + tick(199, 5.0), + tick(200, 6.0), + tick(299, 7.0), + tick(300, 8.0), + ]), + _ => unreachable!("the plan contains only two slices"), + }, + ); + + let TickVerdict::Ready(harvest) = verdict else { + panic!("the fake pages contain in-slice ticks") + }; + assert_eq!( + harvest + .ticks + .iter() + .map(|tick| tick.time_ms as i64) + .collect::>(), + vec![100, 199, 200, 299], + "only rows inside their own requested slice may enter the aggregate harvest" + ); +} diff --git a/crates/moon-ui-gpui/src/trade_window/mod.rs b/crates/moon-ui-gpui/src/trade_window/mod.rs index b8b2084a..a7814a88 100644 --- a/crates/moon-ui-gpui/src/trade_window/mod.rs +++ b/crates/moon-ui-gpui/src/trade_window/mod.rs @@ -50,10 +50,11 @@ use gpui::*; use moon_core::db::{ChartTradeRecord, TradeMeta}; use moon_core::market::trade_replay::worker::{self, TradeReplayRequest}; use moon_core::market::trade_replay::{ - TradeReplayEmpty, TradeReplayFailure, TradeReplayOutcome, TradeReplaySeries, TradeReplaySource, - replay_window, + TickStatus, TradeReplayEmpty, TradeReplayFailure, TradeReplayOutcome, TradeReplaySeries, + TradeReplaySource, replay_window, }; use moon_core::session::CoreId; +use moon_core::venue::Brand; pub(crate) use window::open_trade_window; @@ -73,9 +74,27 @@ pub(crate) enum TradeWindowState { /// five-minute bucket — a wrong label over real-money data, which is worse than an honest gap. /// A typed number rather than a formatted string, because this value is produced beside /// `moon-core` data and only the UI can localize the sentence around it. + /// + /// The next four fields exist for the same reason: the caption cannot ask the network again + /// for a fact only the fetch answer holds, so each is read off the series once, in `apply`, + /// before it is moved into the panel. Ready { source: TradeReplaySource, tf_min: u16, + /// How the tick attempt for this window ended — what a `Klines1m` caption NAMES as its + /// reason. `Served` only ever rides a `Ticks` source. + tick_status: TickStatus, + /// Bucket the `Ticks` points were thinned to, in ms; `0` means raw. Meaningless (and + /// always `0`) on `Klines1m`. + bucket_ms: i64, + /// Whether the `Ticks` points cover only part of the window. Always `false` on + /// `Klines1m`. + partial: bool, + /// Brand of the venue the rows came from, so a `NoRoute` caption can name it. Read off + /// `series.venue` rather than kept as a `TradeWindowView` field: `window.rs`'s + /// construction of that struct is outside this branch's file bounds, so nothing here may + /// add a field it would have to initialize. + brand: Brand, }, /// Nothing to draw, for a reason the window names. Empty(TradeReplayEmpty), @@ -233,10 +252,9 @@ pub(super) fn trade_labels( /// test without a GPUI harness — nothing in this repo renders GPUI in tests. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub(super) struct Fold { - /// Whether `outcome` replaces `state` at all. A drawn window regressing to an error or an - /// empty message is never wanted, so once `state` is [`TradeWindowState::Ready`] only a - /// strictly better answer — a tick series for the exact request already on screen — is let - /// through. + /// Whether `outcome` replaces `state` at all. A drawn window never regresses to an error or + /// empty message; once it is [`TradeWindowState::Ready`], only a tick upgrade or a terminal + /// answer that resolves its provisional `Pending` caption is let through. pub accept: bool, /// Whether the user's own candle mode should be restored before this outcome is drawn. pub restore_candle_mode: bool, @@ -261,14 +279,22 @@ pub(super) fn fold_outcome( outcome: &TradeReplayOutcome, ) -> Fold { let accept = match state { - // Protocol item 3 guarantees the worker never sends a second `Empty` or `Failed`, so this - // arm only ever meets a tick `Ready`, but the rule is stated for what it means rather than - // for what the worker happens to send: a chart already showing something never regresses. - TradeWindowState::Ready { .. } => matches!( - outcome, - TradeReplayOutcome::Ready(series) - if series.source == TradeReplaySource::Ticks && !series.is_empty() - ), + // The rule is stated for what it means rather than for what the worker happens to send + // (the existing comment's own standard): a chart already showing something never + // regresses, but a caption that is still PROVISIONAL may be resolved. So a `Ready` state + // whose tick status has not answered yet accepts a second `Ready` outcome once that + // outcome's own status has moved past `Pending` and actually carries rows — the only + // thing that turns "пробуем тики…" into a stated reason. A state whose status is already + // terminal (a settled `Ticks` series, or a `Klines1m` reason already printed) accepts + // nothing. + TradeWindowState::Ready { tick_status, .. } => { + *tick_status == TickStatus::Pending + && matches!( + outcome, + TradeReplayOutcome::Ready(series) + if series.tick_status != TickStatus::Pending && !series.is_empty() + ) + } // Loading, Empty or Failed has nothing on screen to protect. TradeWindowState::Loading | TradeWindowState::Empty(_) | TradeWindowState::Failed(_) => { true @@ -635,6 +661,12 @@ impl TradeWindowView { // range: a floor of one minute for anything finer, and no `as` wrap for anything // absurdly coarse. let tf_min = (series.tf_ms / 60_000).clamp(1, i64::from(u16::MAX)) as u16; + // Same reasoning, same moment: the caption's four remaining facts are the series' + // alone to give. + let tick_status = series.tick_status; + let bucket_ms = series.bucket_ms; + let partial = series.partial; + let brand = series.venue.brand; if fold.restore_candle_mode { self.restore_candle_mode(cx); } @@ -642,7 +674,14 @@ impl TradeWindowView { if fold.frame { self.framed_this_sequence = true; } - TradeWindowState::Ready { source, tf_min } + TradeWindowState::Ready { + source, + tf_min, + tick_status, + bucket_ms, + partial, + brand, + } } TradeReplayOutcome::Empty(empty) => TradeWindowState::Empty(empty), TradeReplayOutcome::Failed(failure) => { diff --git a/crates/moon-ui-gpui/src/trade_window/render.rs b/crates/moon-ui-gpui/src/trade_window/render.rs index 8fed9be8..bd443696 100644 --- a/crates/moon-ui-gpui/src/trade_window/render.rs +++ b/crates/moon-ui-gpui/src/trade_window/render.rs @@ -7,7 +7,9 @@ use gpui::prelude::FluentBuilder; use gpui::*; -use moon_core::market::trade_replay::{TradeReplayEmpty, TradeReplayFailure, TradeReplaySource}; +use moon_core::market::trade_replay::{ + TickStatus, TradeReplayEmpty, TradeReplayFailure, TradeReplaySource, +}; use moon_ui::{ MoonButton, MoonButtonSize, MoonPalette, MoonWindowFrame, MoonWindowFrameControls, h_flex, v_flex, @@ -183,11 +185,70 @@ impl TradeWindowView { let caption = match &self.state { TradeWindowState::Ready { source: TradeReplaySource::Ticks, + tf_min, + bucket_ms, + partial, .. - } => t!("trade_window.source.ticks").to_string(), - TradeWindowState::Ready { tf_min, .. } => { - t!("trade_window.source.candles_tf", min = tf_min).to_string() + } => { + let base = if *bucket_ms == 0 { + t!("trade_window.source.ticks").to_string() + } else { + t!( + "trade_window.source.ticks_bucketed", + secs = bucket_ms / 1_000 + ) + .to_string() + }; + if *partial { + // The join happens IN CODE, so no locale value carries a separator glyph. + // Every sibling Russian caption in `trade_window.yml` joins its two halves + // with an em dash, never the ASCII hyphen the other locales use — so the + // glyph itself must follow the active locale, not just the words either + // side of it. + let edges = t!("trade_window.source.ticks_edges", min = tf_min).to_string(); + let sep = match rust_i18n::locale().as_ref() { + "ru" => "—", + _ => "-", + }; + format!("{base} {sep} {edges}") + } else { + base + } } + TradeWindowState::Ready { + source: TradeReplaySource::Klines1m, + tf_min, + tick_status, + brand, + .. + } => match tick_status { + TickStatus::Pending => { + t!("trade_window.source.candles_ticks_pending", min = tf_min).to_string() + } + TickStatus::NoRoute => t!( + "trade_window.source.candles_no_route", + min = tf_min, + brand = brand.display() + ) + .to_string(), + TickStatus::OutOfRetention { retention_ms } => t!( + "trade_window.source.candles_retention", + min = tf_min, + hours = retention_ms / 3_600_000 + ) + .to_string(), + TickStatus::NoTrades => { + t!("trade_window.source.candles_no_trades", min = tf_min).to_string() + } + TickStatus::Failed => { + t!("trade_window.source.candles_failed", min = tf_min).to_string() + } + // Unreachable: `Served` only ever rides a `Ticks` source (see `TickStatus`'s + // own doc comment), never a `Klines1m` one. Answered rather than panicked, the + // way the `Loading | Empty | Failed` arm below answers its own unreachable + // case. + TickStatus::Served => String::new(), + }, // Unreachable: the caller checked `overlays_chart` first. Answered rather than // panicked, for the same reason `overlay_message` answers its own unreachable arm. TradeWindowState::Loading diff --git a/locales/trade_window.yml b/locales/trade_window.yml index 9ddfd5a1..61912c12 100644 --- a/locales/trade_window.yml +++ b/locales/trade_window.yml @@ -43,12 +43,45 @@ trade_window.source.ticks: en: "exchange ticks" es: "ticks del exchange" +trade_window.source.ticks_bucketed: + ru: "тики с биржи, шаг %{secs} с" + en: "exchange ticks, %{secs} s step" + es: "ticks del exchange, paso de %{secs} s" + +# Присоединяется к одной из двух строк выше через " - " прямо в коде — глиф-разделитель в +# словаре не хранится. +trade_window.source.ticks_edges: + ru: "края закрыты свечами %{min} мин" + en: "edges filled by %{min}-minute candles" + es: "bordes cubiertos por velas de %{min} min" + # Таймфрейм подставляется из САМИХ строк, а не заявляется константой: раньше здесь стояла -# фиксированная «минутные свечи», пока панель пересобирала минутки в пятиминутки. -trade_window.source.candles_tf: - ru: "свечи %{min} мин" - en: "%{min}-minute candles" - es: "velas de %{min} min" +# фиксированная «минутные свечи», пока панель пересобирала минутки в пятиминутки. Свечи теперь +# всегда называют причину, по которой окно показывает их вместо тиков. +trade_window.source.candles_ticks_pending: + ru: "свечи %{min} мин, тики ещё грузятся" + en: "%{min}-minute candles, still fetching ticks" + es: "velas de %{min} min, cargando ticks todavía" + +trade_window.source.candles_no_route: + ru: "свечи %{min} мин — у %{brand} нет тиков" + en: "%{min}-minute candles - %{brand} has no public ticks" + es: "velas de %{min} min - %{brand} no ofrece ticks" + +trade_window.source.candles_retention: + ru: "свечи %{min} мин — биржа хранит тики только %{hours} ч" + en: "%{min}-minute candles - the exchange keeps ticks for only %{hours} h" + es: "velas de %{min} min - el exchange guarda ticks solo %{hours} h" + +trade_window.source.candles_no_trades: + ru: "свечи %{min} мин — сделок за это время не было" + en: "%{min}-minute candles - the exchange had no trades here" + es: "velas de %{min} min - el exchange no tuvo operaciones aquí" + +trade_window.source.candles_failed: + ru: "свечи %{min} мин — тики не загрузились" + en: "%{min}-minute candles - could not load ticks" + es: "velas de %{min} min - no se pudieron cargar los ticks" trade_window.empty.no_data: ru: "За это время по рынку нет данных."