diff --git a/crates/moon-core/examples/analytics_timing.rs b/crates/moon-core/examples/analytics_timing.rs deleted file mode 100644 index 73529229..00000000 --- a/crates/moon-core/examples/analytics_timing.rs +++ /dev/null @@ -1,254 +0,0 @@ -//! Time the Analytics report reads against a throwaway synthetic replica. -//! -//! Measurement instrument for the "Analytics and the tuner take seconds to open" work: it -//! reproduces the exact read sequence one window open performs, so the wait can be attributed -//! to a named query instead of guessed at. The caller must supply a disposable data root; -//! `set_data_dir_override` makes this process resolve its data paths beneath that root. -//! -//! Usage: -//! cargo run --release -p moon-core --example analytics_timing -- [repeats] -//! -//! Build the data root first with `tools/gen_replica.py`, which writes a synthetic -//! `data/reports.sqlite` and `data/strategies.sqlite` at a chosen row and core count. - -use std::path::PathBuf; -use std::time::Instant; - -use moon_core::db::ProfitScope; -use moon_core::db::analytics::{ - GroupStat, Query, strategy_base_data, summary_data, undated_closes, -}; - -/// Run one closure `repeats` times and print its best and median wall-clock cost. -/// -/// Args: -/// label: ASCII name printed in the report column. -/// repeats: At least two timed runs; the first is reported separately as the cold one. -/// body: Work under measurement. -fn time(label: &str, repeats: usize, mut body: impl FnMut() -> T) { - let mut samples = Vec::with_capacity(repeats); - for _ in 0..repeats { - let started = Instant::now(); - let value = body(); - samples.push(started.elapsed().as_secs_f64() * 1000.0); - drop(value); - } - let cold = samples[0]; - let mut warm = samples[1..].to_vec(); - warm.sort_by(f64::total_cmp); - let median = warm.get(warm.len() / 2).copied().unwrap_or(cold); - println!("{label:<28} cold {cold:>8.1} ms warm-median {median:>8.1} ms"); -} - -/// Build the query one open Analytics window issues for a whole-history period. -/// -/// Args: -/// from: Inclusive period start in UTC seconds, or a negative all-history sentinel. -/// to: Exclusive period end in UTC seconds. -/// -/// Returns: -/// The default-filter query every tab shares. -fn query(from: i64, to: i64) -> Query { - Query { - axis: moon_core::db::ReportAxis::from_measured(Default::default(), chrono_tz::UTC), - previous_period_basis: Default::default(), - from, - to, - cores: Vec::new(), - side: Default::default(), - emulator: None, - strategies: Vec::new(), - strategy_name_mask: String::new(), - metric: Default::default(), - valuation: Default::default(), - prefer_usdt: false, - } -} - -/// Time selected raw SQL shapes to attribute the cost of Analytics reads. -/// -/// Args: -/// from: Inclusive period start, resolved from the all-history sentinel by the caller. -/// to: Exclusive period end. -/// name: ASCII period label. -/// repeats: Timed runs per shape. -fn probe_sql(from: i64, to: i64, name: &str, repeats: usize) { - let conn = match moon_core::db::open_reader() { - Ok(conn) => conn, - Err(error) => { - println!(" raw probes skipped: {error:?}"); - return; - } - }; - let from = if from < 0 { 1 } else { from }; - let period = "closedate >= ?1 AND closedate < ?2 AND closedate > 0 AND COALESCE(deleted,0) = 0"; - let count: i64 = conn - .query_row( - &format!("SELECT COUNT(*) FROM orders_rep WHERE {period}"), - rusqlite::params![from, to], - |row| row.get(0), - ) - .unwrap_or(-1); - println!(" rows in period: {count}"); - time(&format!(" raw COUNT(*) [{name}]"), repeats, || { - let value: i64 = conn - .query_row( - &format!("SELECT COUNT(*) FROM orders_rep WHERE {period}"), - rusqlite::params![from, to], - |row| row.get(0), - ) - .unwrap_or(-1); - value - }); - time(&format!(" raw group aggregate [{name}]"), repeats, || { - let mut stmt = conn - .prepare(&format!( - "SELECT CAST(strategyid AS TEXT) || '@' || CAST(core_uid AS TEXT) AS k, - SUM(profitbtc), COUNT(*) - FROM orders_rep WHERE {period} GROUP BY k ORDER BY 2 DESC, k" - )) - .expect("probe aggregate prepares"); - let rows = stmt - .query_map(rusqlite::params![from, to], |row| row.get::<_, i64>(2)) - .expect("probe aggregate runs") - .count(); - rows - }); - time(&format!(" raw full projection [{name}]"), repeats, || { - let mut stmt = conn - .prepare(&format!( - "SELECT closedate, buydate, profitbtc, spentbtc, core_uid, core_name, coin, - strategyid, isshort, emulator, basecurrency, boughtq, buyprice, - sellprice, sellreason - FROM orders_rep WHERE {period} ORDER BY closedate" - )) - .expect("probe projection prepares"); - let rows = stmt - .query_map(rusqlite::params![from, to], |row| row.get::<_, i64>(0)) - .expect("probe projection runs") - .count(); - rows - }); -} - -/// Print the compared fields of every strategy-base group row in source order. -/// -/// The A-B side of this instrument: run it before and after a change to the enrichment path and -/// diff the two dumps. Order is printed as it arrives, so a reordering shows up as a diff too. -/// -/// Args: -/// from: Inclusive period start, or the all-history sentinel. -/// to: Exclusive period end. -/// name: ASCII period label. -fn dump(from: i64, to: i64, name: &str) { - let q = query(from, to); - let read = strategy_base_data(&q, false); - let line = |kind: &str, index: usize, group: &GroupStat| { - println!( - "{name}|{kind}|{index}|{}|{}|{}|{}|{}|{:?}|{}|{:.6}|{}|{:.6}|{:.6}|{:.6}|{}|{}|{}|{:.6}|{:?}", - group.key, - group.name, - group.kind, - group.core, - group.cores_n, - group.alive, - group.n, - group.profit, - group.wins, - group.pf, - group.best, - group.worst, - group.lastedit, - group.bl, - group.wl, - group.raw_profit, - group.quote, - ); - }; - match read.data { - Ok(ProfitScope::Comparable { unit, data }) => { - println!( - "{name}|unit|{unit:?}|trades={}|{}|{}", - data.trades, data.from, data.to - ); - for (index, group) in data.strategies.iter().enumerate() { - line("strategy", index, group); - } - for (index, group) in data.coins.iter().enumerate() { - line("coin", index, group); - } - } - Ok(ProfitScope::Empty(data)) => println!("{name}|empty|trades={}", data.trades), - Ok(ProfitScope::Split(_)) => println!("{name}|split"), - Err(error) => println!("{name}|error|{error:?}"), - } -} - -/// Run timing measurements or a deterministic group dump for the supplied disposable replica. -fn main() { - let mut args = std::env::args().skip(1); - let root = PathBuf::from( - args.next() - .expect("usage: analytics_timing [repeats]"), - ); - let repeats = args - .next() - .and_then(|value| value.parse::().ok()) - .unwrap_or(5) - .max(2); - assert!( - moon_core::config::paths::set_data_dir_override(root.clone()), - "the data root must be installed before any path resolves" - ); - // `open_reader` refuses without the process-lifetime lease the application takes at startup. - let _permit = moon_core::db::report_recovery::prepare(); - println!("data root: {}", root.display()); - println!( - "reports: {}", - moon_core::config::paths::reports_db_path().display() - ); - - let end = 1_780_000_000i64; - let dump_only = std::env::args().any(|value| value == "--dump"); - let periods: [(&str, i64, i64); 3] = [ - ("month", end - 30 * 86_400, end), - ("year", end - 365 * 86_400, end), - ("all", -1, end), - ]; - - if dump_only { - for (name, from, to) in periods { - dump(from, to, name); - } - return; - } - - for (name, from, to) in periods { - println!("\n=== period: {name} ==="); - let q = query(from, to); - let probe = summary_data(&q, false); - println!( - " summary outcome: {}", - match &probe.data { - Ok(_) => "ok".to_string(), - Err(error) => format!("{error:?}"), - } - ); - drop(probe); - time(&format!("summary_data [{name}]"), repeats, || { - summary_data(&q, false) - }); - time(&format!("strategy_base_data [{name}]"), repeats, || { - strategy_base_data(&q, false) - }); - time(&format!("open then tuner [{name}]"), repeats, || { - let first = summary_data(&q, false); - let second = strategy_base_data(&q, false); - (first, second) - }); - time(&format!("undated_closes [{name}]"), repeats, || { - undated_closes(&q) - }); - probe_sql(from, to, name, repeats); - } -} diff --git a/crates/moon-core/examples/db_read_timing.rs b/crates/moon-core/examples/db_read_timing.rs new file mode 100644 index 00000000..552e0cc0 --- /dev/null +++ b/crates/moon-core/examples/db_read_timing.rs @@ -0,0 +1,1180 @@ +//! Time and dump every UI-visible SQLite read the terminal makes against a throwaway synthetic +//! replica, calling exactly what the UI calls with the UI's own default arguments. +//! +//! Measurement instrument for attributing Analytics/Report/tuner wait time to a named query +//! instead of guessing at it. Pairs with `db::trace`: this binary is the one place in the tree +//! that installs the read profiler, so every `open_reader`/`open_strategies`/`open_ro`/ +//! `KlineCache::open` connection born after that call reports its statement timings here. +//! +//! The caller must supply a disposable data root; `set_data_dir_override` makes this process +//! resolve its data paths beneath it. Build one with `tools/gen_replica.py +//! [span_days]`, which also writes the `data/fixture.json` manifest this harness reads. +//! +//! Usage: +//! cargo run --release -p moon-core --example db_read_timing -- [repeats] [--plan] [--dump] + +use std::collections::HashMap; +use std::path::PathBuf; +use std::sync::Mutex; +use std::time::Instant; + +use moon_core::db::analytics::{GroupStat, Query, calendar_data, strategy_base_data, summary_data}; +use moon_core::db::trace::{ProfiledStatement, install_read_profiler}; +use moon_core::db::tuner::threshold_search::{SearchHandle, SearchParams}; +use moon_core::db::tuner::{FIELDS, TimeAxes, Variant}; +use moon_core::db::valuation::ValuationMode; +use moon_core::db::{ProfitMetric, ReportFilter, RowScope}; +use moon_core::db::{ProfitScope, ReadFail}; + +// --------------------------------------------------------------------------------------------- +// The profiler sink. `install_read_profiler` takes a plain `fn`, so all state it can touch is +// process-global. A `Mutex>` rather than a thread-local: `KlineCache::open` moves its +// connection into a worker thread, and a thread-local sink would silently never see its +// statements. +// --------------------------------------------------------------------------------------------- + +static CAPTURED: Mutex> = Mutex::new(Vec::new()); + +/// Append one connection-local PROFILE event to the process-wide harness sink. +fn record_stmt(stmt: ProfiledStatement) { + CAPTURED + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .push(stmt); +} + +/// Take and clear everything captured since the last drain. +fn drain_captured() -> Vec { + std::mem::take( + &mut *CAPTURED + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()), + ) +} + +// --------------------------------------------------------------------------------------------- +// Fixture manifest — H1's contract, frozen in the spec. H2 never guesses these. +// --------------------------------------------------------------------------------------------- + +/// Shape and provenance recorded by `tools/gen_replica.py` for one disposable fixture. +#[derive(serde::Deserialize, Debug)] +struct FixtureManifest { + rows: i64, + cores: i64, + strategy_groups: i64, + coins: i64, + span_days: i64, + reports_bytes: i64, + analyzed: bool, + seed: i64, +} + +/// Read the fixture manifest below `root`, printing an actionable failure before exiting on error. +fn read_manifest(root: &std::path::Path) -> FixtureManifest { + let path = root.join("data").join("fixture.json"); + let text = std::fs::read_to_string(&path).unwrap_or_else(|error| { + eprintln!( + "[FAIL] fixture manifest unreadable at {}: {error}", + path.display() + ); + std::process::exit(1); + }); + serde_json::from_str(&text).unwrap_or_else(|error| { + eprintln!( + "[FAIL] fixture manifest malformed at {}: {error}", + path.display() + ); + std::process::exit(1); + }) +} + +// --------------------------------------------------------------------------------------------- +// Timing +// --------------------------------------------------------------------------------------------- + +/// One timed closure invocation and whether its captured SQL can be replayed for planning. +struct Sample { + wall_ms: f64, + sql_ms: f64, + replayable: bool, +} + +/// Run `body` `repeats` times, reporting wall time, summed SQL time and their difference for +/// both the first sample and the warm median. +/// +/// "First" is not "cold": no cache is reset between surfaces, so a later surface can inherit +/// pages an earlier one already warmed — see the process-wide note printed once in `main`. +/// +/// Args: +/// label: ASCII surface name printed in the report. +/// repeats: At least two timed runs. +/// body: Work under measurement. +/// outcome: Formats the FIRST call's result for the printed "outcome" line — run after +/// that call's timer already stopped, so it costs nothing against the first sample. +/// +/// Returns: +/// Every sample, and every statement captured across all `repeats` calls (for `--plan`). +fn measure( + label: &str, + repeats: usize, + mut body: impl FnMut() -> T, + outcome: impl FnOnce(&T) -> String, +) -> (Vec, Vec) { + let mut samples = Vec::with_capacity(repeats); + let mut all_stmts = Vec::new(); + let mut first_outcome = None; + let mut outcome = Some(outcome); + for i in 0..repeats { + drain_captured(); + let started = Instant::now(); + let value = body(); + let wall_ms = started.elapsed().as_secs_f64() * 1000.0; + let captured = drain_captured(); + let sql_ms: f64 = captured + .iter() + .map(|s| s.duration.as_secs_f64() * 1000.0) + .sum(); + let replayable = captured.iter().all(|s| s.expanded); + if i == 0 { + if let Some(outcome) = outcome.take() { + first_outcome = Some(outcome(&value)); + } + } + drop(value); + all_stmts.extend(captured); + samples.push(Sample { + wall_ms, + sql_ms, + replayable, + }); + } + let first = &samples[0]; + let mut warm: Vec<&Sample> = samples[1..].iter().collect(); + warm.sort_by(|a, b| a.wall_ms.total_cmp(&b.wall_ms)); + let median = warm.get(warm.len() / 2).copied().unwrap_or(first); + let note = |s: &Sample| { + if s.replayable { + "" + } else { + " [NOT REPLAYABLE]" + } + }; + println!( + "{label:<46} first {:>9.1} ms sql {:>9.1} ms diff {:>9.1} ms{}", + first.wall_ms, + first.sql_ms, + first.wall_ms - first.sql_ms, + note(first) + ); + println!( + "{:<46} warm {:>9.1} ms sql {:>9.1} ms diff {:>9.1} ms{}", + "", + median.wall_ms, + median.sql_ms, + median.wall_ms - median.sql_ms, + note(median) + ); + if let Some(outcome) = first_outcome { + println!("{:<46} outcome: {outcome}", ""); + } + (samples, all_stmts) +} + +/// Build the default query one open Analytics or tuner window issues for a period. +fn query(from: i64, to: i64, metric: ProfitMetric, cores: Vec) -> Query { + Query { + axis: moon_core::db::ReportAxis::from_measured(Default::default(), chrono_tz::UTC), + previous_period_basis: Default::default(), + from, + to, + cores, + side: Default::default(), + emulator: None, + strategies: Vec::new(), + strategy_name_mask: String::new(), + metric, + valuation: ValuationMode::Historical, + prefer_usdt: false, + } +} + +/// Format a read result for the harness's ASCII outcome column. +fn ok_or_err(result: &Result) -> String { + match result { + Ok(_) => "ok".to_string(), + Err(error) => format!("{error:?}"), + } +} + +// --------------------------------------------------------------------------------------------- +// Discovery — the harness never hardcodes a core, strategy or coin identity; it reads them from +// whatever fixture it was pointed at. +// --------------------------------------------------------------------------------------------- + +/// Fixture identities discovered from the replica instead of assumed by the harness. +struct Pool { + cores: Vec<(u64, String)>, + strategies: Vec, + /// The first `strategies` entry that names a REAL strategy — `strategy_id != 0` — for the + /// strat-db-backed calls (`versions_with_stats`, `strategy_purge_rows`), which know nothing + /// about `orders_rep`'s liquidation sentinel and legitimately find no rows for it. + first_real_strategy: Option, + coins: Vec, + kline_key: Option<(String, String, u32)>, +} + +/// Discover fixture identities while keeping discovery traffic out of the first timed sample. +fn discover_pool() -> Pool { + let cores = moon_core::db::open_reader() + .and_then(|conn| moon_core::db::distinct_cores(&conn)) + .unwrap_or_default(); + // NOT `db::distinct_strategies`: on this machine's fixture, `strat.strategies` carries a + // NULL `name` for at least one row, and `distinct_strategies` decodes that column + // unconditionally (`row.get::<_, String>(2)`), so the call fails with "Invalid column type + // Null at index: 2, name: name" — see the report. Pool discovery must not depend on that + // path; a direct scan of `orders_rep` gives the same `(core_uid, strategy_id)` identities. + // Deliberately UNFILTERED: this list's length feeds the fixture-drift guard below, which + // compares it against the manifest's own `groups_seen` count — including `strategyid=0`, + // `orders_rep`'s forced sentinel for a liquidation row with no owning strategy. Filtering it + // out here would desync the two counts and misfire the guard. See `first_real_strategy` + // below for the probe that skips the sentinel instead. + let strategies: Vec = moon_core::db::open_reader() + .map(|conn| { + conn.prepare("SELECT DISTINCT core_uid, strategyid FROM orders_rep") + .and_then(|mut stmt| { + stmt.query_map([], |row| { + Ok(moon_core::db::ReportStrategyKey { + core_uid: row.get::<_, i64>(0)? as u64, + strategy_id: row.get::<_, i64>(1)?, + }) + })? + .collect::, _>>() + }) + .unwrap_or_default() + }) + .unwrap_or_default(); + let coins = moon_core::db::open_reader() + .map(|conn| { + conn.prepare("SELECT DISTINCT coin FROM orders_rep LIMIT 5") + .and_then(|mut stmt| { + stmt.query_map([], |row| row.get::<_, String>(0))? + .collect::, _>>() + }) + .unwrap_or_default() + }) + .unwrap_or_default(); + let kline_key = { + let path = moon_core::config::paths::klines_db_path(); + // `chunks`/`chunks_v2` both name their third column `kind`, never `kind_min` — that + // name belongs only to `MergeItem`/`read_range`'s Rust-side parameter. `chunks` is also + // the wrong TABLE to probe: it is the legacy v1 store `gen_klines` seeds for only + // `LEGACY_MARKETS` markets on one kind and one day, while `chunks_v2` is the write + // target `merge_batch_blocking` actually fills for every market and kind the fixture + // claims to hold (`market/kline_cache.rs`'s own module doc). Reading `chunks_v2` here + // picks a key that is guaranteed to exist, straight from the cache's own contents. + rusqlite::Connection::open_with_flags(&path, rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY) + .ok() + .and_then(|conn| { + conn.query_row( + "SELECT exchange, market, kind FROM chunks_v2 LIMIT 1", + [], + |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, u32>(2)?, + )) + }, + ) + .ok() + }) + }; + let first_real_strategy = strategies.iter().find(|k| k.strategy_id != 0).copied(); + // The discovery reads above are real database traffic; they must not pollute the first + // measured surface's SQL sum. + drain_captured(); + Pool { + cores, + strategies, + first_real_strategy, + coins, + kline_key, + } +} + +// --------------------------------------------------------------------------------------------- +// Dump — arrival-order rows over the compared fields, at full `{:?}` precision, so two dumps +// from two binaries over one fixture diff to empty when a change is identity-preserving. +// --------------------------------------------------------------------------------------------- + +/// Print one ordered group collection in the dump format used for cross-binary comparisons. +fn dump_groups(tag: &str, kind: &str, groups: &[GroupStat]) { + for (index, group) in groups.iter().enumerate() { + println!("{tag}|{kind}|{index}|{group:?}"); + } +} + +/// Dump every comparable read result for one period without running the timing loop. +fn dump_all(period_name: &str, from: i64, to: i64, pool: &Pool) { + let q = |metric| query(from, to, metric, Vec::new()); + + let summary = summary_data(&q(ProfitMetric::Quote), false); + println!("{period_name}|summary|data|{:?}", summary.data); + + let strategies = strategy_base_data(&q(ProfitMetric::Quote), false); + match &strategies.data { + Ok(ProfitScope::Comparable { unit, data }) => { + println!( + "{period_name}|strategy_base|unit|{unit:?}|trades={}", + data.trades + ); + dump_groups(period_name, "strategy", &data.strategies); + dump_groups(period_name, "coin", &data.coins); + } + other => println!("{period_name}|strategy_base|{other:?}"), + } + + let calendar_day = calendar_data(&q(ProfitMetric::Quote), None, false, false); + println!("{period_name}|calendar_daily|{:?}", calendar_day.period); + let calendar_hour = calendar_data(&q(ProfitMetric::Quote), None, true, false); + println!("{period_name}|calendar_hourly|{:?}", calendar_hour.period); + + let monitor = moon_core::db::analytics::profit_monitor(&q(ProfitMetric::Quote)); + println!("{period_name}|profit_monitor|{monitor:?}"); + + if let Some(conn) = moon_core::db::open_reader().ok() { + if let Ok(snap) = moon_core::db::read_snapshot(&conn) { + let filter = ReportFilter::default(); + if let Ok(cores) = moon_core::db::distinct_cores(&snap) { + println!("{period_name}|distinct_cores|{cores:?}"); + } + if let Ok(strategies) = moon_core::db::distinct_strategies(&snap, &filter) { + println!("{period_name}|distinct_strategies|{strategies:?}"); + } + if let Ok(table) = moon_core::db::query_reports(&snap, &filter, "closedate", true, 200) + { + println!("{period_name}|report_cols|{:?}", table.cols); + for (index, row) in table.rows.iter().enumerate() { + println!("{period_name}|report_row|{index}|{row:?}"); + } + } + if let Ok(totals) = moon_core::db::query_totals(&snap, &filter) { + println!("{period_name}|report_totals|{totals:?}"); + } + } + } + + if let (Some((core_uid, _)), false) = (pool.cores.first(), pool.coins.is_empty()) { + if let Ok(conn) = moon_core::db::open_reader() { + if let Ok(history) = + moon_core::db::query_chart_trade_history(&conn, *core_uid, &pool.coins, None, 50) + { + for (index, record) in history.records.iter().enumerate() { + println!("{period_name}|chart_trade|{index}|{record:?}"); + } + } + } + } + + if let Some(key) = pool.first_real_strategy { + let versions = + moon_core::strat_db::stats::versions_with_stats(key.core_uid, key.strategy_id); + for (index, version) in versions.iter().enumerate() { + println!("{period_name}|version_stats|{index}|{version:?}"); + } + } +} + +// --------------------------------------------------------------------------------------------- +// Main +// --------------------------------------------------------------------------------------------- + +fn main() { + let mut args = std::env::args().skip(1); + let root = PathBuf::from( + args.next() + .expect("usage: db_read_timing [repeats] [--plan] [--dump]"), + ); + let repeats = args + .next() + .and_then(|value| value.parse::().ok()) + .unwrap_or(5) + .max(2); + let raw_args: Vec = std::env::args().collect(); + let plan_mode = raw_args.iter().any(|value| value == "--plan"); + let dump_mode = raw_args.iter().any(|value| value == "--dump"); + + assert!( + moon_core::config::paths::set_data_dir_override(root.clone()), + "the data root must be installed before any path resolves" + ); + let _permit = moon_core::db::report_recovery::prepare(); + let manifest = read_manifest(&root); + println!("data root: {}", root.display()); + println!( + "reports: {}", + moon_core::config::paths::reports_db_path().display() + ); + println!( + "fixture: rows={} cores={} strategy_groups={} coins={} span_days={} reports_bytes={} \ + analyzed={} seed={}", + manifest.rows, + manifest.cores, + manifest.strategy_groups, + manifest.coins, + manifest.span_days, + manifest.reports_bytes, + manifest.analyzed, + manifest.seed + ); + println!( + "note: no cache is reset between surfaces — every surface after the first inherits \ + pages warmed by whatever ran before it, so \"first\" below means \"first repetition of \ + this closure\", not \"never-warmed\"" + ); + + install_read_profiler(record_stmt); + let pool = discover_pool(); + + // --- Fixture drift guard -------------------------------------------------------------- + let mut drift: Vec = Vec::new(); + if pool.cores.len() as i64 != manifest.cores { + drift.push(format!( + "core count: fixture claims {}, replica shows {}", + manifest.cores, + pool.cores.len() + )); + } + if (pool.strategies.len() as i64) < manifest.strategy_groups { + drift.push(format!( + "strategy groups: fixture claims {}, replica shows only {}", + manifest.strategy_groups, + pool.strategies.len() + )); + } + let reports_bytes = std::fs::metadata(moon_core::config::paths::reports_db_path()) + .map(|meta| meta.len() as i64) + .unwrap_or(0); + let low = manifest.reports_bytes / 2; + let high = manifest.reports_bytes.saturating_mul(2).max(1); + if reports_bytes < low || reports_bytes > high { + drift.push(format!( + "reports.sqlite size: fixture claims {} bytes, on disk {} bytes (outside [{low}, {high}])", + manifest.reports_bytes, reports_bytes + )); + } + if !drift.is_empty() { + eprintln!("[FAIL] fixture drift detected:"); + for line in &drift { + eprintln!(" - {line}"); + } + std::process::exit(1); + } + + if dump_mode { + let end = 1_780_000_000i64; + let periods: [(&str, i64, i64); 4] = [ + ("today", end - 86_400, end), + ("month", end - 30 * 86_400, end), + ("year", end - 365 * 86_400, end), + ("all", -1, end), + ]; + for (name, from, to) in periods { + dump_all(name, from, to, &pool); + } + return; + } + + let end = 1_780_000_000i64; + let periods: [(&str, i64, i64); 4] = [ + ("today", end - 86_400, end), + ("month", end - 30 * 86_400, end), + ("year", end - 365 * 86_400, end), + ("all", -1, end), + ]; + let metrics = [ + ("quote", ProfitMetric::Quote), + ("percent", ProfitMetric::Percent), + ]; + + let mut plan_pool: Vec = Vec::new(); + let mut missing_guarantee: Vec = Vec::new(); + // Real production failures that happen to surface here — never fixture drift, so kept in a + // separate bucket the closing report never conflates with the guarantees above. + let mut production_failures: Vec = Vec::new(); + + let scoped_cores: Vec = pool.cores.iter().take(3).map(|(uid, _)| *uid).collect(); + + for (name, from, to) in periods { + println!("\n=== period: {name} ==="); + for (metric_name, metric) in metrics { + let label_suffix = format!("[{name}/{metric_name}]"); + let q = query(from, to, metric, Vec::new()); + let q_scoped = query(from, to, metric, scoped_cores.clone()); + + let (_, stmts) = measure( + &format!("summary_data {label_suffix}"), + repeats, + || summary_data(&q, false), + |r| { + match &r.data { + Ok(ProfitScope::Comparable { data, .. }) + if data.strategies.is_empty() && data.coins.is_empty() => + { + missing_guarantee + .push(format!("summary_data: no groups {label_suffix}")); + } + Err(_) => { + missing_guarantee.push(format!("summary_data: errored {label_suffix}")); + } + _ => {} + } + ok_or_err(&r.data) + }, + ); + plan_pool.extend(stmts); + + if metric_name == "quote" && !scoped_cores.is_empty() { + let (_, stmts) = measure( + &format!( + "summary_data [{name}/scoped-{}-of-{}]", + scoped_cores.len(), + pool.cores.len() + ), + repeats, + || summary_data(&q_scoped, false), + |r| ok_or_err(&r.data), + ); + plan_pool.extend(stmts); + } + + let (_, stmts) = measure( + &format!("strategy_base_data {label_suffix}"), + repeats, + || strategy_base_data(&q, false), + |r| { + match &r.data { + Ok(ProfitScope::Comparable { data, .. }) + if data.strategies.is_empty() && data.coins.is_empty() => + { + missing_guarantee + .push(format!("strategy_base_data: no groups {label_suffix}")); + } + Err(_) => missing_guarantee + .push(format!("strategy_base_data: errored {label_suffix}")), + _ => {} + } + ok_or_err(&r.data) + }, + ); + plan_pool.extend(stmts); + + let (_, stmts) = measure( + &format!("calendar_data daily {label_suffix}"), + repeats, + || calendar_data(&q, None, false, false), + |r| { + match &r.period { + Ok(ProfitScope::Comparable { data, .. }) if data.current.is_empty() => { + missing_guarantee + .push(format!("calendar_data daily: no cells {label_suffix}")); + } + Err(_) => missing_guarantee + .push(format!("calendar_data daily: errored {label_suffix}")), + _ => {} + } + ok_or_err(&r.period) + }, + ); + plan_pool.extend(stmts); + + let (_, stmts) = measure( + &format!("calendar_data hourly {label_suffix}"), + repeats, + || calendar_data(&q, None, true, false), + |r| { + match &r.period { + Ok(ProfitScope::Comparable { data, .. }) if data.current.is_empty() => { + missing_guarantee + .push(format!("calendar_data hourly: no cells {label_suffix}")); + } + Err(_) => missing_guarantee + .push(format!("calendar_data hourly: errored {label_suffix}")), + _ => {} + } + ok_or_err(&r.period) + }, + ); + plan_pool.extend(stmts); + + let (_, stmts) = measure( + &format!("profit_monitor {label_suffix}"), + repeats, + || moon_core::db::analytics::profit_monitor(&q), + |r| { + if r.is_err() { + missing_guarantee.push(format!("profit_monitor: errored {label_suffix}")); + } + ok_or_err(r) + }, + ); + plan_pool.extend(stmts); + + let variants = vec![Variant::default()]; + let field = FIELDS.first().map(|f| f.col).unwrap_or("profitbtc"); + + let (_, stmts) = measure( + &format!("filter_tuner_data {label_suffix}"), + repeats, + || moon_core::db::tuner::filter_tuner_data(&q, &variants, field, 20), + |r| { + match &r.stats { + Ok(v) if v.is_empty() => missing_guarantee + .push(format!("filter_tuner_data: no KPI rows {label_suffix}")), + Err(_) => missing_guarantee + .push(format!("filter_tuner_data: errored {label_suffix}")), + _ => {} + } + ok_or_err(&r.stats) + }, + ); + plan_pool.extend(stmts); + + let (_, stmts) = measure( + &format!("time_tuner_data {label_suffix}"), + repeats, + || moon_core::db::tuner::time_tuner_data(&q, &variants), + |r| { + match &r.stats { + Ok(v) if v.is_empty() => missing_guarantee + .push(format!("time_tuner_data: no KPI rows {label_suffix}")), + Err(_) => missing_guarantee + .push(format!("time_tuner_data: errored {label_suffix}")), + _ => {} + } + ok_or_err(&r.stats) + }, + ); + plan_pool.extend(stmts); + + let picked: Vec = pool.coins.first().cloned().into_iter().collect(); + let (_, stmts) = measure( + &format!("coin_tuner_data {label_suffix}"), + repeats, + || moon_core::db::tuner::coin_tuner_data(&q, &variants, &q, &picked), + |r| { + match &r.kpi { + Ok(v) if v.is_empty() => missing_guarantee + .push(format!("coin_tuner_data: no KPI rows {label_suffix}")), + Err(_) => missing_guarantee + .push(format!("coin_tuner_data: errored {label_suffix}")), + _ => {} + } + ok_or_err(&r.kpi) + }, + ); + plan_pool.extend(stmts); + + let (_, stmts) = measure( + &format!("variant_stats {label_suffix}"), + repeats, + || moon_core::db::tuner::variant_stats(&q, &variants), + ok_or_err, + ); + plan_pool.extend(stmts); + + let (_, stmts) = measure( + &format!("histogram {label_suffix}"), + repeats, + || moon_core::db::tuner::histogram(&q, field, 20), + ok_or_err, + ); + plan_pool.extend(stmts); + + let (_, stmts) = measure( + &format!("suggest_field {label_suffix}"), + repeats, + || moon_core::db::tuner::suggest_field(&q, field, 30, 20, true), + ok_or_err, + ); + plan_pool.extend(stmts); + + let (_, stmts) = measure( + &format!("suggest_time {label_suffix}"), + repeats, + || { + moon_core::db::tuner::suggest_time( + &q, + 30, + 20, + true, + TimeAxes { + week: true, + day: true, + hour: true, + ..Default::default() + }, + ) + }, + ok_or_err, + ); + plan_pool.extend(stmts); + + let locked = vec![None; FIELDS.len()]; + let (_, stmts) = measure( + &format!("threshold_search::suggest {label_suffix}"), + repeats, + || { + let handle = SearchHandle::new(); + moon_core::db::tuner::threshold_search::suggest( + &q, + SearchParams { + restarts: 8, + min_n: None, + locked: &locked, + edges: 20, + round: true, + seed: Some(20260903), + train_frac: 1.0, + compose: false, + }, + &handle, + ) + }, + ok_or_err, + ); + plan_pool.extend(stmts); + + if !pool.coins.is_empty() { + let (_, stmts) = measure( + &format!("strategies_for_coins {label_suffix}"), + repeats, + || moon_core::db::analytics::strategies_for_coins(&q, &pool.coins), + ok_or_err, + ); + plan_pool.extend(stmts); + } + + // --- The Report: mirrors run_report_query's own sequence ----------------------- + for (sort_name, sort_key, desc) in [ + ("closedate DESC", "closedate", true), + ("profitbtc DESC", "profitbtc", true), + ] { + let filter = ReportFilter { + date_from: if from < 0 { None } else { Some(from) }, + date_to: Some(to), + rows: RowScope::ClosedAndOpen, + ..ReportFilter::default() + }; + let (_, stmts) = measure( + &format!("report [{name}/{sort_name}]"), + repeats, + || -> Result<(usize, usize), ReadFail> { + let conn = moon_core::db::open_reader()?; + let snap = moon_core::db::read_snapshot(&conn)?; + let cores = moon_core::db::distinct_cores(&snap)?; + // Mirrors `run_report_query`'s own OPTIONAL treatment (`.transpose()` + // there, not `?`): a strategy-scope refresh failing must not sink the + // rows/totals read it rides beside. See the report for why this call + // fails against this fixture's `strategies.sqlite` (a NULL `name`). + let strategies = moon_core::db::distinct_strategies(&snap, &filter).ok(); + let table = + moon_core::db::query_reports(&snap, &filter, sort_key, desc, 500)?; + let totals = moon_core::db::query_totals(&snap, &filter)?; + let _ = (cores, strategies, totals); + Ok((table.rows.len(), 0)) + }, + |r| { + match r { + Ok((rows, _)) if *rows == 0 => missing_guarantee + .push(format!("report [{name}/{sort_name}]: no rows")), + Err(_) => missing_guarantee + .push(format!("report [{name}/{sort_name}]: errored")), + _ => {} + } + ok_or_err(r) + }, + ); + plan_pool.extend(stmts); + } + + let (_, stmts) = measure( + &format!("distinct_cores {label_suffix}"), + repeats, + || { + moon_core::db::open_reader() + .and_then(|conn| moon_core::db::distinct_cores(&conn)) + }, + ok_or_err, + ); + plan_pool.extend(stmts); + + let (_, stmts) = measure( + &format!("distinct_strategies {label_suffix}"), + repeats, + || { + moon_core::db::open_reader().and_then(|conn| { + moon_core::db::distinct_strategies(&conn, &ReportFilter::default()) + }) + }, + ok_or_err, + ); + plan_pool.extend(stmts); + + if let Some((core_uid, _)) = pool.cores.first() { + let (_, stmts) = measure( + &format!("query_chart_trade_history {label_suffix}"), + repeats, + || { + moon_core::db::open_reader().and_then(|conn| { + moon_core::db::query_chart_trade_history( + &conn, + *core_uid, + &pool.coins, + None, + 200, + ) + }) + }, + ok_or_err, + ); + plan_pool.extend(stmts); + + let sample_record = moon_core::db::open_reader().ok().and_then(|conn| { + moon_core::db::query_chart_trade_history(&conn, *core_uid, &pool.coins, None, 1) + .ok() + .and_then(|history| history.records.into_iter().next()) + }); + if let Some(record) = sample_record { + let (_, stmts) = measure( + &format!("query_trade_meta {label_suffix}"), + repeats, + || { + moon_core::db::open_reader() + .and_then(|conn| moon_core::db::query_trade_meta(&conn, &record)) + }, + ok_or_err, + ); + plan_pool.extend(stmts); + } + } + + if let Some(key) = pool.first_real_strategy { + let (_, stmts) = measure( + &format!("strategy_purge_rows {label_suffix}"), + repeats, + || { + moon_core::db::open_reader() + .and_then(|conn| moon_core::db::strategy_purge_rows(&conn, key)) + }, + ok_or_err, + ); + plan_pool.extend(stmts); + + let (_, stmts) = measure( + &format!("versions_with_stats {label_suffix}"), + repeats, + || { + moon_core::strat_db::stats::versions_with_stats( + key.core_uid, + key.strategy_id, + ) + }, + |v| { + if v.is_empty() { + missing_guarantee + .push(format!("versions_with_stats: no versions {label_suffix}")); + } + format!("{} versions", v.len()) + }, + ); + plan_pool.extend(stmts); + } + + let targets: Vec<(i64, Option)> = pool + .strategies + .iter() + .map(|k| (k.strategy_id, Some(k.core_uid))) + .collect(); + if !targets.is_empty() { + let (_, stmts) = measure( + &format!("coin_lists::coin_lists {label_suffix}"), + repeats, + || moon_core::db::coin_lists::coin_lists(&targets), + |r| { + match r { + Ok(rows) if rows.black.is_empty() && rows.white.is_empty() => { + missing_guarantee + .push(format!("coin_lists: no rows {label_suffix}")); + } + // `scope_sql`'s one-OR-term-per-strategy join hits SQLite's + // "Expression tree is too large" parser limit once enough strategies + // are selected — a real production bug (`db/coin_lists/mod.rs` + // `scope_sql`, recorded for the goal owner), not this fixture + // drifting from what the harness expects. Kept out of + // `missing_guarantee` so the two classes are never conflated. + Err(error) => { + let detail = format!("{error:?}"); + if detail.contains("too large") { + production_failures + .push(format!("coin_lists {label_suffix}: {detail}")); + } else { + missing_guarantee + .push(format!("coin_lists: errored {label_suffix}")); + } + } + _ => {} + } + ok_or_err(r) + }, + ); + plan_pool.extend(stmts); + } + + let (_, stmts) = measure( + &format!("open_reader alone {label_suffix}"), + repeats, + || moon_core::db::open_reader().map(|_| ()), + ok_or_err, + ); + plan_pool.extend(stmts); + + if let Some((exchange, market, kind_min)) = &pool.kline_key { + let path = moon_core::config::paths::klines_db_path(); + let cache = moon_core::market::kline_cache::KlineCache::open(path); + if let Some(cache) = cache { + let (_, stmts) = measure( + &format!("KlineCache::read_range {label_suffix}"), + repeats, + || cache.read_range(exchange, market, *kind_min, 0, i64::MAX), + |r| match r { + Some(rows) if rows.is_empty() => { + missing_guarantee.push(format!( + "KlineCache::read_range: no rows {label_suffix}" + )); + "0 rows".to_string() + } + Some(rows) => format!("{} rows", rows.len()), + None => { + missing_guarantee.push(format!( + "KlineCache::read_range: timed out {label_suffix}" + )); + "timed out".to_string() + } + }, + ); + plan_pool.extend(stmts); + } else { + missing_guarantee + .push(format!("KlineCache::open failed [{name}/{metric_name}]")); + } + } else if pool.kline_key.is_none() { + missing_guarantee.push(format!( + "KlineCache: no (exchange, market, kind_min) row in klines.sqlite [{name}]" + )); + } + } + } + + // Neither surface below can be called from this example crate: `report_quote_ordinals` is + // a private `fn` inside a private `mod worker` (`db/valuation/worker.rs:1953`, + // `db/valuation/mod.rs:12`), and `basis::probe` is a private `fn` inside the private + // `mod basis` of `db::analytics` (`db/analytics/basis.rs:94`, `db/analytics/mod.rs:25`) — + // both files are out of this branch's edit scope, and a copied SQL string is explicitly + // forbidden by the spec. Never made `pub` to reach them; kept first-class instead so the + // omission cannot be mistaken for "measured and cheap" — printed here AND carried into the + // `--plan` table itself (`run_plan_mode`), not only as this footnote. + for (label, reason) in NOT_MEASURED { + println!("\n[NOT MEASURED] {label} - {reason}"); + } + + // Print everything the run promised — the timing table above, `--plan`'s EQP dump below — + // BEFORE reporting the guard's own failures and exiting. A guard that hides the artifact it + // was meant to protect is worse than no guard; only the ORDER changed here, the exit code on + // a real drift is unchanged. + if plan_mode { + run_plan_mode(plan_pool, &NOT_MEASURED); + } + + if !production_failures.is_empty() { + eprintln!("[FAIL] production bug(s) hit while measuring (not fixture drift):"); + for line in &production_failures { + eprintln!(" - {line}"); + } + } + if !missing_guarantee.is_empty() { + eprintln!("[FAIL] guaranteed-non-empty surfaces did not deliver:"); + for line in &missing_guarantee { + eprintln!(" - {line}"); + } + } + if !production_failures.is_empty() || !missing_guarantee.is_empty() { + std::process::exit(1); + } +} + +/// Surfaces the deliverable promises a number for but this example genuinely cannot reach — +/// see the two reasons printed beside this constant's use in `main`. +const NOT_MEASURED: [(&str, &str); 2] = [ + ( + "valuation reconciliation row", + "report_quote_ordinals is private, unreachable from an example", + ), + ( + "basis::probe full scan", + "private fn in a private module, unreachable from an example, and db/analytics/mod.rs \ + is out of edit scope", + ), +]; + +/// Register STUB scalar functions matching the crate's own calendar/time-tuner SQL surface, so +/// `EXPLAIN QUERY PLAN` can resolve their names on this example's own reader connection. +/// `db::analytics::time_zone::install` is `pub(crate)` and installs the REAL bodies against a +/// captured `ReportAxis`; an example cannot call it, which is why these are stubs rather than +/// the genuine functions. `EXPLAIN QUERY PLAN` only resolves a function's name and declared +/// properties and never evaluates its body, so a stub returning a constant is correct for this +/// purpose — but the PLAN it enables is real, which is why every plan taken on this connection +/// is printed with an explicit marker (see `run_plan_mode`) rather than silently passed off as +/// having run against production functions. +/// +/// Registered with the SAME `SQLITE_UTF8 | SQLITE_DETERMINISTIC` flags the real functions use, +/// so the planner sees the same function properties the production connection would. +fn install_plan_stub_functions(conn: &rusqlite::Connection) -> rusqlite::Result<()> { + use rusqlite::functions::FunctionFlags; + let flags = FunctionFlags::SQLITE_UTF8 | FunctionFlags::SQLITE_DETERMINISTIC; + conn.create_scalar_function("mt_to_utc", 2, flags, |_ctx| Ok(0i64))?; + conn.create_scalar_function("mt_local_bucket", 3, flags, |_ctx| Ok(0i64))?; + conn.create_scalar_function("mt_minute_of_day", 2, flags, |_ctx| Ok(0i64))?; + conn.create_scalar_function("mt_minute_of_week", 2, flags, |_ctx| Ok(0i64))?; + conn.create_scalar_function("mt_core_minute_of_day", 1, flags, |_ctx| Ok(0i64))?; + conn.create_scalar_function("mt_core_minute_of_week", 1, flags, |_ctx| Ok(0i64))?; + // The real function returns its argument casefolded; a stub only needs to resolve, but + // returning the argument unchanged keeps its declared return type (TEXT) honest too. + conn.create_scalar_function("mt_unicode_casefold", 1, flags, |ctx| ctx.get::(0))?; + Ok(()) +} + +/// `--plan`: dedup captured statements by exact text, print total time, call count and +/// `EXPLAIN QUERY PLAN` for every replayable one; mark placeholder-only captures instead of +/// ranking them; and list every `not_measured` surface honestly instead of silently ranking +/// past it. +fn run_plan_mode(pool: Vec, not_measured: &[(&str, &str)]) { + struct Agg { + total_ms: f64, + count: usize, + replayable: bool, + } + let mut by_text: HashMap = HashMap::new(); + for stmt in pool { + let entry = by_text.entry(stmt.sql.clone()).or_insert(Agg { + total_ms: 0.0, + count: 0, + replayable: true, + }); + entry.total_ms += stmt.duration.as_secs_f64() * 1000.0; + entry.count += 1; + entry.replayable = entry.replayable && stmt.expanded; + } + let (mut ranked, mut not_replayable): (Vec<(String, Agg)>, Vec<(String, Agg)>) = + by_text.into_iter().partition(|(_, agg)| agg.replayable); + ranked.sort_by(|a, b| b.1.total_ms.total_cmp(&a.1.total_ms)); + not_replayable.sort_by(|a, b| b.1.total_ms.total_cmp(&a.1.total_ms)); + + println!( + "\n=== --plan: {} ranked statements ({} not replayable, {} not measured, see below) ===", + ranked.len(), + not_replayable.len(), + not_measured.len() + ); + let conn = match moon_core::db::open_reader() { + Ok(conn) => { + if let Err(error) = install_plan_stub_functions(&conn) { + println!("(cannot register plan-stub functions: {error:?})"); + } + Some(conn) + } + Err(error) => { + println!("(cannot open a reader for EXPLAIN QUERY PLAN: {error:?})"); + None + } + }; + // Statements whose EXPLAIN fails with "no such table" ran on the writer's own connection or + // the kline cache's own connection, neither of which this reader has attached — a different, + // unfixable gap from the mt_* function names the stubs above resolve. Collected here instead + // of printed inline, so the count of planless statements is explained under its own heading + // rather than merely reported one line at a time. + let mut no_such_table: Vec<(String, f64, usize)> = Vec::new(); + for (sql, agg) in ranked { + println!( + "\n-- total {:>9.1} ms calls {:>4}", + agg.total_ms, agg.count + ); + println!(" {sql}"); + let Some(conn) = &conn else { continue }; + match conn + .prepare(&format!("EXPLAIN QUERY PLAN {sql}")) + .and_then(|mut stmt| { + stmt.query_map([], |row| { + Ok(format!( + "id={} parent={} detail={}", + row.get::<_, i64>(0)?, + row.get::<_, i64>(1)?, + row.get::<_, String>(3)? + )) + })? + .collect::, _>>() + }) { + Ok(lines) => { + println!(" [stub-resolved: mt_* bodies are stubs, the PLAN is real]"); + for line in lines { + println!(" {line}"); + } + } + Err(error) => { + let text = error.to_string(); + if text.contains("no such table") { + no_such_table.push((sql, agg.total_ms, agg.count)); + } else { + println!(" plan failed: {error}"); + } + } + } + } + + println!( + "\n=== --plan: {} statement(s) with no plan (writer-only or kline-cache-only table, \ + not attached to this reader) ===", + no_such_table.len() + ); + for (sql, total_ms, count) in &no_such_table { + println!("\n-- total {total_ms:>9.1} ms calls {count:>4} NO PLAN (table not attached)"); + println!(" {sql}"); + } + + // Placeholder-only captures never had their real literal-bound shape recorded, so they + // cannot be replayed with EXPLAIN QUERY PLAN and must never drive the optimisation order + // above — printed here, visible but unranked, instead of being dropped silently. + println!( + "\n=== --plan: {} not-replayable captures (never ranked) ===", + not_replayable.len() + ); + for (sql, agg) in not_replayable { + println!( + "\n-- total {:>9.1} ms calls {:>4} NOT REPLAYABLE (placeholder capture)", + agg.total_ms, agg.count + ); + println!(" {sql}"); + } + + // Promised numbers this example can never produce, listed in the same ranked table rather + // than only as a footnote elsewhere — a reader scanning just this section still sees them + // flagged, never silently absent. + println!( + "\n=== --plan: {} not-measured surfaces (never ranked, never callable from here) ===", + not_measured.len() + ); + for (label, reason) in not_measured { + println!("\n-- NOT MEASURED {label}"); + println!(" {reason}"); + } +} diff --git a/crates/moon-core/examples/gen_klines.rs b/crates/moon-core/examples/gen_klines.rs new file mode 100644 index 00000000..303cb27b --- /dev/null +++ b/crates/moon-core/examples/gen_klines.rs @@ -0,0 +1,227 @@ +//! Deterministic `klines.sqlite` seeder for the report-replica measurement fixture +//! (`tools/gen_replica.py`), built through `KlineCache`'s own production write API rather than a +//! second binary-codec implementation. +//! +//! `chunks_v2`'s packed row codec (`pack_rows_v2`) is private to `market/kline_cache.rs` on +//! purpose: a Python re-implementation of it would be a second binary-format authority no drift +//! check could prove equivalent to the first (rejected in review). This helper instead opens the +//! cache and calls `KlineCache::merge_batch_blocking` directly, the same entry point the +//! background recorder uses. +//! +//! Usage: +//! cargo run --release -p moon-core --example gen_klines -- [seed] +//! +//! `KlineCache::open` prunes chunks older than each kind's retention window +//! (`retention_days`: 30 days for 1-minute, 15 for 5-minute, 3650 for everything coarser) the +//! moment it opens the database — including on a LATER reopen by a reader, not just this +//! process's own. Anchoring the generated series at a fixed historical timestamp (as +//! `gen_replica.py` does for `reports.sqlite`) would therefore make the 1- and 5-minute series +//! vanish the next time anything opens the cache, days after generation. To guarantee +//! `KlineCache::read_range` stays non-empty for every kind this fixture writes, every series here +//! is anchored at the REAL wall clock (`now_unix_ms`) instead: `klines.sqlite` is consequently the +//! one fixture file that is NOT byte-identical across reruns on different days, unlike +//! `reports.sqlite` / `strategies.sqlite` / `valuation.sqlite` (see the generator's own report for +//! why this is a deliberate deviation from the frozen contract's byte-identical clause). + +use std::path::PathBuf; + +use moon_core::market::ChartCandle; +use moon_core::market::kline_cache::{KlineCache, MergeItem}; + +const DAY_MS: i64 = 86_400_000; +const KINDS: [u32; 3] = [1, 5, 60]; +const MARKETS: usize = 50; +// FBinance (4) and ByBit (7) codes, per feed::types::ExchangeId's doc comment; dex=0 (no HIP-3 +// DEX), formatted exactly as market/source/read.rs's `exchange_key` does: `"{code}:{dex:08x}"`. +const EXCHANGE_CODES: [u8; 2] = [4, 7]; +// A handful of markets also get direct legacy `chunks` (v1) rows for one day, so `read_rows`' +// v1/v2 merge is exercised — never through a re-implemented v2 codec, only the documented +// 24-byte-per-row v1 layout (`u32 offset_ms + 5×f32`, no turnover; kline_cache.rs's module doc). +const LEGACY_MARKETS: usize = 5; + +/// Deterministic splitmix64 source for reproducible candle values despite the real-time anchor. +struct Rng(u64); + +impl Rng { + /// Advance the generator and return its next uniformly distributed `u64`. + fn next_u64(&mut self) -> u64 { + self.0 = self.0.wrapping_add(0x9E37_79B9_7F4A_7C15); + let mut z = self.0; + z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); + z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); + z ^ (z >> 31) + } + + /// Return the next generator value scaled to the half-open unit interval. + fn next_f64(&mut self) -> f64 { + (self.next_u64() >> 11) as f64 * (1.0 / (1u64 << 53) as f64) + } + + /// Return the next generator value scaled to the requested half-open numeric interval. + fn range(&mut self, lo: f64, hi: f64) -> f64 { + lo + self.next_f64() * (hi - lo) + } +} + +/// Build one synthetic market identity from its deterministic fixture ordinal. +fn market_name(i: usize) -> String { + format!("SYN{i:03}USDT") +} + +/// Build the persisted exchange key assigned to one synthetic market ordinal. +fn exchange_key(i: usize) -> String { + format!("{}:00000000", EXCHANGE_CODES[i % EXCHANGE_CODES.len()]) +} + +/// Pack one legacy v1 24-byte row: `u32 offset_ms LE + 5×f32 LE (open, high, low, close, volume)`. +fn pack_v1_row(offset_ms: u32, c: &ChartCandle) -> [u8; 24] { + let mut out = [0u8; 24]; + out[0..4].copy_from_slice(&offset_ms.to_le_bytes()); + out[4..8].copy_from_slice(&c.open.to_le_bytes()); + out[8..12].copy_from_slice(&c.high.to_le_bytes()); + out[12..16].copy_from_slice(&c.low.to_le_bytes()); + out[16..20].copy_from_slice(&c.close.to_le_bytes()); + out[20..24].copy_from_slice(&c.volume.to_le_bytes()); + out +} + +/// Generate one deterministic OHLCV series over `[from_ms, to_ms)` at `step_ms` spacing. +fn candle_series( + rng: &mut Rng, + from_ms: i64, + to_ms: i64, + step_ms: i64, + base_price: f64, +) -> Vec { + let mut out = Vec::new(); + let mut price = base_price; + let mut t = from_ms; + while t < to_ms { + let open = price; + let close = (open * (1.0 + rng.range(-0.01, 0.01))).max(0.0001); + let high = open.max(close) * (1.0 + rng.range(0.0, 0.004)); + let low = open.min(close) * (1.0 - rng.range(0.0, 0.004)); + let volume = rng.range(1.0, 5000.0); + out.push(ChartCandle { + t_open_ms: t as f64, + open: open as f32, + high: high as f32, + low: low as f32, + close: close as f32, + volume: volume as f32, + quote_volume: (volume * (open + close) / 2.0) as f32, + }); + price = close; + t += step_ms; + } + out +} + +/// Seed the requested disposable kline cache through the production write API. +fn main() { + let mut args = std::env::args().skip(1); + let data_dir = PathBuf::from( + args.next() + .expect("usage: gen_klines [seed]"), + ); + let span_days: i64 = args + .next() + .expect("span_days required") + .parse() + .expect("span_days must be an integer"); + let seed: u64 = args.next().and_then(|s| s.parse().ok()).unwrap_or(20260820); + + assert!( + moon_core::config::paths::set_data_dir_override(data_dir.clone()), + "the data root must be installed before any path resolves" + ); + let path = moon_core::config::paths::klines_db_path(); + for suffix in ["", "-wal", "-shm"] { + let _ = std::fs::remove_file(format!("{}{suffix}", path.display())); + } + + let end_ms = moon_core::util::now_unix_ms_i64(); + + let cache = KlineCache::open(path.clone()).expect("open kline cache"); + let mut rng = Rng(seed); + let mut written = 0usize; + for i in 0..MARKETS { + let market = market_name(i); + let exchange = exchange_key(i); + let base_price = rng.range(0.01, 70_000.0); + for kind in KINDS { + // `retention_days` prunes 1- and 5-minute chunks to 30 / 15 days on every open, so + // covering the full requested span at those kinds would just synthesize rows the very + // next reopen deletes — capped well inside each kind's own retention window instead. + // The coarse kind's 3650-day retention has room for the whole span. + let this_span_days = match kind { + 1 => span_days.min(20), + 5 => span_days.min(10), + _ => span_days, + }; + let from_ms = end_ms - this_span_days * DAY_MS; + let rows = candle_series(&mut rng, from_ms, end_ms, kind as i64 * 60_000, base_price); + written += rows.len(); + cache.merge_batch_blocking(vec![MergeItem { + exchange: exchange.clone(), + market: market.clone(), + kind_min: kind, + rows, + }]); + } + } + drop(cache); + // The worker thread's Sender was just dropped, which ends its `rx.recv()` loop; the last + // `merge_batch_blocking` above already waited for its own transaction to commit, so nothing + // further needs to settle before a second connection can write. + std::thread::sleep(std::time::Duration::from_millis(100)); + + // Direct legacy v1 rows for a handful of markets, one day each, so `read_rows`' v1/v2 merge + // is exercised on read. `updated_ms` is older than the v2 rows above, matching the normal + // (non-downgrade) case where v2 is the fresher write. + let legacy_conn = + rusqlite::Connection::open(&path).expect("open klines.sqlite for legacy rows"); + // Matches kind 60's own (uncapped) span, since the legacy row below is written under kind 60. + let legacy_day = (end_ms - span_days * DAY_MS) / DAY_MS; + let day_start = legacy_day * DAY_MS; + let mut legacy_rows = 0usize; + for i in 0..LEGACY_MARKETS { + let market = market_name(i); + let exchange = exchange_key(i); + let base_price = rng.range(0.01, 70_000.0); + let series = candle_series( + &mut rng, + day_start, + day_start + DAY_MS, + 60 * 60_000, + base_price, + ); + let mut blob = Vec::new(); + for c in &series { + let offset_ms = (c.t_open_ms as i64 - day_start) as u32; + blob.extend_from_slice(&pack_v1_row(offset_ms, c)); + } + legacy_rows += series.len(); + legacy_conn + .execute( + "INSERT OR REPLACE INTO chunks(exchange, market, kind, day, rows, updated_ms) + VALUES (?1, ?2, ?3, ?4, ?5, ?6)", + rusqlite::params![ + exchange, + market, + 60u32, + legacy_day, + blob, + day_start - DAY_MS + ], + ) + .expect("insert legacy v1 chunk"); + } + + println!( + "[OK] {}: {written} v2 rows across {MARKETS} markets x {} kinds, \ + {legacy_rows} legacy v1 rows across {LEGACY_MARKETS} markets", + path.display(), + KINDS.len() + ); +} diff --git a/crates/moon-core/src/db/analytics/tests.rs b/crates/moon-core/src/db/analytics/tests.rs index f66e3bb2..5339d791 100644 --- a/crates/moon-core/src/db/analytics/tests.rs +++ b/crates/moon-core/src/db/analytics/tests.rs @@ -1160,3 +1160,65 @@ fn min_closedate_uses_the_axis_converted_instant_not_the_smaller_raw_value() { remove_db(&path); } + +/// `db/analytics/mod.rs:min_closedate` must preserve the `closedate > 0` predicate and compare +/// each core after its axis conversion when C2 replaces the grouped query. Including a core whose +/// only rows are non-positive would make every all-history Analytics surface start at an invented +/// epoch instead of the first real trade; an empty replica must still use the documented floor 1. +#[test] +fn min_closedate_skips_non_positive_cores_and_keeps_the_empty_floor() { + const ONLY_NON_POSITIVE_CORE: u64 = 4; + const OFFSET_CORE: u64 = 8; + const OFFSET_SECS: i32 = -3_600; + const FIRST_TRUE_INSTANT: i64 = 1_700_300_000; + + let path = temp_db("min-closedate-positive"); + let conn = build_replica_multi_core( + &path, + &[ + (ONLY_NON_POSITIVE_CORE, -99, 1.0, "ZEROLESS"), + (ONLY_NON_POSITIVE_CORE, 0, 2.0, "ZEROLESS"), + ( + OFFSET_CORE, + FIRST_TRUE_INSTANT + i64::from(OFFSET_SECS), + 3.0, + "OFFSET", + ), + ( + OFFSET_CORE, + FIRST_TRUE_INSTANT + 86_400 + i64::from(OFFSET_SECS), + 4.0, + "OFFSET", + ), + ], + ); + let axis = crate::db::ReportAxis::from_measured( + std::collections::HashMap::from([( + OFFSET_CORE, + vec![crate::db::OffsetSegment { + from_utc: 0, + offset_secs: OFFSET_SECS, + }], + )]), + chrono_tz::UTC, + ); + + assert_eq!( + min_closedate(&conn, &axis).expect("resolve positive multi-core floor"), + FIRST_TRUE_INSTANT, + "only the offset core contributes a positive close, and its axis converts it to the independently seeded true instant" + ); + drop(conn); + remove_db(&path); + + let empty_path = temp_db("min-closedate-empty"); + let empty = build_replica(&empty_path, &[]); + assert_eq!( + min_closedate(&empty, &crate::db::ReportAxis::identity_core_local()) + .expect("resolve empty history floor"), + 1, + "an empty replica has no per-core minimum and therefore keeps the public all-history sentinel" + ); + drop(empty); + remove_db(&empty_path); +} diff --git a/crates/moon-core/src/db/coin_lists/mod.rs b/crates/moon-core/src/db/coin_lists/mod.rs index 1aa1365d..e66c2639 100644 --- a/crates/moon-core/src/db/coin_lists/mod.rs +++ b/crates/moon-core/src/db/coin_lists/mod.rs @@ -32,7 +32,7 @@ //! "creations" sharing one timestamp, some of them at unix epoch 1. "No later than X" is a //! fact; "added on X" from the same row is a lie the user cannot see through. -use std::collections::{BTreeSet, HashMap}; +use std::collections::{BTreeMap, BTreeSet, HashMap}; use std::time::Duration; use rusqlite::{Connection, OpenFlags}; @@ -324,17 +324,61 @@ struct Head { /// /// The ids are `i64`/`u64` formatted as decimals, so there is nothing here a string could /// inject; binding them instead would mean rebuilding the SQL per selection size anyway. +/// +/// GROUPED BY CORE, and that grouping is the whole point. SQLite parses `a OR b OR c` into a +/// LEFT-DEEP tree, so an `OR` chain with one term per selected strategy has a depth equal to +/// the selection size — and past `SQLITE_MAX_EXPR_DEPTH` (1000) the statement does not run +/// slowly, it fails to prepare: `Expression tree is too large`, the whole panel down. Measured +/// on a replica with 2 420 `strategyid@core_uid` groups, where every period failed. Factoring +/// the repeated `s.core_uid = C` out of each group makes the depth the number of DISTINCT +/// CORES (tens) instead, and an `IN (…)` right-hand side is a flat expression LIST that costs +/// no depth per element. +/// +/// One id in a group still emits `= id` rather than `IN (id)`: same plan, and it keeps the +/// generated SQL readable for the ordinary small selection. Groups and ids are emitted in +/// sorted order so one selection always renders one string. +/// +/// THE CEILING MOVED, IT DID NOT VANISH. What is joined by `OR` is now one term per distinct +/// core, so a selection spanning ~1000 distinct CORES would hit the same wall again. That is a +/// far higher ceiling on a variable a user cannot push — the core count is the fleet's shape, +/// not the size of a selection — but it is a bound, not a guarantee, and it is written here so +/// the next person reads it instead of rediscovering it. fn scope_sql(targets: &[(i64, Option)]) -> String { - let terms: Vec = targets + let mut by_core: BTreeMap, BTreeSet> = BTreeMap::new(); + for (sid, core) in targets { + by_core + .entry(core.map(|c| c as i64)) + .or_default() + .insert(*sid); + } + let terms: Vec = by_core .iter() - .map(|(sid, core)| match core { - Some(c) => format!("(s.strategy_id = {sid} AND s.core_uid = {})", *c as i64), - None => format!("s.strategy_id = {sid}"), + .map(|(core, sids)| { + let ids = strategy_id_predicate(sids); + match core { + Some(c) => format!("(s.core_uid = {c} AND s.strategy_id {ids})"), + None => format!("s.strategy_id {ids}"), + } }) .collect(); format!(" AND ({})", terms.join(" OR ")) } +/// `= id` for a lone strategy, `IN (…)` for several — the comparison half of [`scope_sql`]'s +/// per-core term. Never called with an empty set: a group exists only because a target put an +/// id in it. +fn strategy_id_predicate(sids: &BTreeSet) -> String { + if sids.len() == 1 { + let only = sids + .iter() + .next() + .expect("a set of len 1 has a first element"); + return format!("= {only}"); + } + let list: Vec = sids.iter().map(i64::to_string).collect(); + format!("IN ({})", list.join(", ")) +} + /// Read-only handle on the strategy database. /// /// An absent file is `NotReady` (the terminal has simply never synced strategies), while a @@ -351,6 +395,7 @@ fn open_strategies() -> ReadResult { // landing under our snapshot surfaces to the user as a read failure. Same 3s the // strategy store's own reader uses. let _ = conn.busy_timeout(Duration::from_secs(3)); + super::trace::install_on(&conn); Ok(conn) } diff --git a/crates/moon-core/src/db/integrity/mod.rs b/crates/moon-core/src/db/integrity/mod.rs index 5e8ca535..3e1aa370 100644 --- a/crates/moon-core/src/db/integrity/mod.rs +++ b/crates/moon-core/src/db/integrity/mod.rs @@ -234,6 +234,7 @@ pub(crate) fn run(path: &Path) -> Integrity { Err(e) => return classify_pragma_error(e, "открытие"), }; let _ = conn.busy_timeout(Duration::from_secs(30)); + super::trace::install_on(&conn); // HARD DEADLINE. `busy_timeout` bounds lock waiting, not statement runtime, // and the scan holds a WAL read snapshot for its whole duration — which diff --git a/crates/moon-core/src/db/mod.rs b/crates/moon-core/src/db/mod.rs index 965a033b..e83bd0ed 100644 --- a/crates/moon-core/src/db/mod.rs +++ b/crates/moon-core/src/db/mod.rs @@ -32,6 +32,7 @@ mod report_read; pub mod report_recovery; #[cfg(test)] mod test_support; +pub mod trace; mod trade_meta; pub mod tuner; pub mod valuation; @@ -995,6 +996,7 @@ pub fn open_reader() -> ReadResult { // before report-derived views can open readers; an absent file remains a normal not-ready state. let _ = valuation::attach(&conn)?; read_cancel::install_current(&conn)?; + trace::install_on(&conn); Ok(conn) } @@ -1282,11 +1284,13 @@ pub fn open_readonly() -> ReadResult { let path = paths::reports_db_path(); // Same reasoning as `open_reader`: only a genuine absence may report `NotReady`. metadata_gate(&path, "отчёты(ro): доступ к файлу")?; - Connection::open_with_flags( + let conn = Connection::open_with_flags( &path, rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY | rusqlite::OpenFlags::SQLITE_OPEN_NO_MUTEX, ) - .map_err(|e| read_fail("отчёты(ro)", e)) + .map_err(|e| read_fail("отчёты(ro)", e))?; + trace::install_on(&conn); + Ok(conn) } #[cfg(test)] diff --git a/crates/moon-core/src/db/report_read.rs b/crates/moon-core/src/db/report_read.rs index 716ee452..396bd038 100644 --- a/crates/moon-core/src/db/report_read.rs +++ b/crates/moon-core/src/db/report_read.rs @@ -1933,7 +1933,47 @@ fn run_row_pass( let rec_id_select = rec_id_expr(src); // Sort in SQL only if this source can express the column; otherwise source order is // irrelevant, because the merge below reorders everything anyway. + // + // The DESC arm drops the leading `({expression}) IS NULL` term: SQLite orders NULL + // lowest, so a plain `{expression} DESC` already places NULLs last, exactly what the + // `IS NULL` term bought. Without it that leading term forces a temp b-tree sort instead + // of using the source's index on `{expression}`. The ASC arm keeps the term — without it + // NULLs would sort FIRST there, a real result change — so do not drop it from that arm. + // + // The DESC arm then MUST carry the primary key as a tie-break, and that is not a + // refinement — it is what keeps the rewrite honest. `LIMIT` is applied per source + // BEFORE the merge below, and SQL guarantees nothing about the relative order of rows + // equal on every ORDER BY term. Dropping the leading term changes the plan from a temp + // sort to a backwards index walk, so a tie straddling the limit boundary would surface + // a DIFFERENT SET of rows, not merely a different order: measured on SQLite 3.50.4 with + // 20 rows sharing one `closedate`, 19 of 25 limits returned a different set. The key + // `(core_uid, newrecid)` is `rep.rs`'s own PRIMARY KEY, so it is total on the typed + // replica and the whole result becomes DEFINED rather than planner-chosen. It costs the + // sort only WITHIN each group of equal values — measured 2.4 ms -> 2.8 ms against + // 4804 ms before the rewrite, so the gain survives it intact. + // + // The key is taken PER SOURCE, because the two sources do not share one. The typed + // replica is keyed `(core_uid, newrecid)` (`rep.rs`'s own PRIMARY KEY) and the legacy + // `closed_sell_reports` is keyed `(core_uid, db_id)` (`db/mod.rs:10`) — and reaching for + // `rec_id_expr` here instead would be a trap twice over: it yields the LITERAL `0` on a + // source without `newrecid`, and SQLite reads a bare integer in ORDER BY as a COLUMN + // ORDINAL even parenthesised, so it would fail the whole query rather than order it. + // The same source-shape test `source_sort_expression` already uses for `id` is what + // picks the right column. A source offering neither keeps today's undefined tie order, + // which is no worse than before this rewrite. let order = match source_sort_expression(src, sort_col, valuation.as_ref()) { + Some(expression) if desc => { + let mut order = format!("{expression} {dir}"); + if src.cols.contains("core_uid") { + order.push_str(&format!(", r.core_uid {dir}")); + } + if src.cols.contains("newrecid") { + order.push_str(&format!(", r.newrecid {dir}")); + } else if src.legacy && src.cols.contains("db_id") { + order.push_str(&format!(", r.\"db_id\" {dir}")); + } + order + } Some(expression) => format!("({expression}) IS NULL, {expression} {dir}"), None => "1".to_string(), }; @@ -2429,6 +2469,13 @@ pub fn distinct_cores(conn: &Connection) -> ReadResult> { /// strategy predicates are deliberately removed so an active checkbox or name mask does not hide /// alternative strategies that match every other Report filter. /// +/// Split into a normal-strategy arm and a liquidation arm when the two differ. What the split +/// buys is NOT an index-only scan — `deleted` sits in no index, so both arms still read table +/// rows to apply the deletion predicate. It buys keeping the liquidation-attribution CASE and +/// its two correlated subqueries off the rows that can never satisfy it, which on the 600k-row +/// measurement fixture is 599 696 of 600 000. Measured there: 2057 ms as one statement, +/// 1236 ms as two arms. +/// /// Args: /// conn: Open report reader or snapshot with optional strategy attachment. /// filter: Active Report filter; every non-strategy predicate scopes discovery. @@ -2475,12 +2522,47 @@ pub fn distinct_strategies( } let strategy_id = super::analytics::effective_sid_expr("r", &src.cols, has_strategy_names); let (where_sql, params) = build_where(&scope, &src.cols, has_strategy_names); - let sql = format!( - "SELECT DISTINCT r.core_uid, COALESCE({strategy_id}, 0) FROM {} r{where_sql}", - src.table, - ); - let refs: Vec<&dyn rusqlite::types::ToSql> = + // `COALESCE(r."strategyid",0)` never yields NULL, so `<> 0` / `= 0` partition every row + // with no third case, and identity is set equality (this function sorts its whole output + // in Rust below, so SQL row order never matters). When `strategy_id` is already the plain + // column, both arms would be identical and the split would buy nothing — keep the single + // statement in that case. + // + // `UNION`, not `UNION ALL`, and this is MEASURED rather than reasoned. The arms are + // disjoint by construction and the caller below dedups every pair through `seen`, so + // `UNION ALL` looks like the free choice — it is the opposite. `UNION` lets SQLite + // dedup the compound ONCE, and its left arm then needs no `DISTINCT` of its own; + // `UNION ALL` makes each arm materialise its own `USE TEMP B-TREE FOR DISTINCT`. + // On the 600k-row measurement fixture: 2057 ms before this split, 1236 ms with `UNION`, + // 6599 ms with `UNION ALL` — three times WORSE than doing nothing at all. + // + // ONE `build_where`, its parameters bound TWICE. Calling it a second time would re-read + // the wall clock — `append_row_scope` resolves "does this window still reach the + // present" against `now`, and its own doc says that must be ONE reading for the whole + // predicate — so two calls can straddle a second boundary and scope the two arms to + // genuinely different row states. Binding the same vector twice also keeps the bound + // parameter count where it was instead of doubling it. + let two_arms = strategy_id != "r.\"strategyid\""; + let sql = if two_arms { + format!( + "SELECT DISTINCT r.core_uid, COALESCE(r.\"strategyid\",0) FROM {table} r{where_sql} AND COALESCE(r.\"strategyid\",0) <> 0 \ + UNION \ + SELECT DISTINCT r.core_uid, COALESCE({strategy_id}, 0) FROM {table} r{where_sql} AND COALESCE(r.\"strategyid\",0) = 0", + table = src.table, + ) + } else { + format!( + "SELECT DISTINCT r.core_uid, COALESCE({strategy_id}, 0) FROM {} r{where_sql}", + src.table, + ) + }; + let mut refs: Vec<&dyn rusqlite::types::ToSql> = params.iter().map(|value| value.as_ref()).collect(); + if two_arms { + let second: Vec<&dyn rusqlite::types::ToSql> = + params.iter().map(|value| value.as_ref()).collect(); + refs.extend(second); + } let mut stmt = conn.prepare(&sql).map_err(|e| read_fail(CTX, e))?; let rows = stmt .query_map(refs.as_slice(), |row| { diff --git a/crates/moon-core/src/db/report_read/tests.rs b/crates/moon-core/src/db/report_read/tests.rs index 809706b4..d6d1b824 100644 --- a/crates/moon-core/src/db/report_read/tests.rs +++ b/crates/moon-core/src/db/report_read/tests.rs @@ -1,7 +1,7 @@ //! Regression tests for exact Report strategy filtering. -use rusqlite::Connection; use rusqlite::types::Value; +use rusqlite::{Connection, params}; use super::{ QuoteCurrency, ReportFilter, ReportStrategyKey, RowScope, SideFilter, distinct_strategies, @@ -2591,3 +2591,129 @@ fn open_block_stays_newest_first_across_both_sources_under_an_ascending_sort() { proving the closed rows were never reversed, only the open block" ); } + +/// `db/report_read.rs:run_row_pass` must retain an ASC `NULLS LAST` equivalent while optimizing +/// its SQL order. Dropping the leading NULL discriminator without adding `NULLS LAST` promotes an +/// unset Profit % above completed trades in an ascending Report and CSV export, silently changing +/// the user's visible top rows. +#[test] +fn profit_percent_nulls_stay_last_in_both_report_directions() { + let conn = Connection::open_in_memory().expect("open null-order fixture"); + super::super::init_db(&conn).expect("initialize report metadata"); + conn.execute_batch( + "CREATE TABLE orders_rep ( + core_uid INTEGER NOT NULL, core_name TEXT NOT NULL, newrecid INTEGER NOT NULL, + closedate INTEGER, profitbtc REAL, spentbtc REAL, + PRIMARY KEY (core_uid, newrecid) + ); + INSERT INTO orders_rep VALUES + (1, 'CORE', 10, 100, 10.0, 100.0), + (1, 'CORE', 11, 200, 5.0, 100.0), + (1, 'CORE', 20, 300, 5.0, 0.0);", + ) + .expect("seed finite and undefined Profit percent rows"); + super::super::test_support::rep_init(&conn); + + for descending in [false, true] { + let table = query_reports( + &conn, + &ReportFilter { + rows: RowScope::Closed, + ..ReportFilter::default() + }, + super::PROFIT_PERCENT_COLUMN, + descending, + 2, + ) + .expect("read Profit percent report"); + + assert_eq!( + table.rec_ids, + if descending { + vec![10, 11] + } else { + vec![11, 10] + }, + "the limited Report must retain both finite percentages when descending={descending}" + ); + } +} + +/// `report_read.rs:run_row_pass` must retain `r.newrecid DESC` after the default closed-date +/// sort. Removing that tie-break lets SQLite choose which equal-date trades survive the +/// source-local LIMIT, so the Report grid and CSV export can silently show a different set after +/// an unrelated planner or schema change. +#[test] +fn closedate_desc_source_limit_uses_the_total_replica_key() { + let conn = Connection::open_in_memory().expect("open closed-date tie fixture"); + super::super::init_db(&conn).expect("initialize report metadata"); + conn.execute_batch( + "CREATE TABLE orders_rep ( + core_uid INTEGER NOT NULL, core_name TEXT NOT NULL, newrecid INTEGER NOT NULL, + closedate INTEGER, PRIMARY KEY (core_uid, newrecid) + );", + ) + .expect("create replica fixture"); + let seeded = vec![ + (3_i64, 32_i64, 1_000_i64), + (1, 20, 1_000), + (4, 53, 1_000), + (2, 11, 1_000), + (1, 50, 1_000), + (3, 12, 1_000), + (2, 51, 1_000), + (4, 23, 1_000), + (1, 30, 1_000), + (3, 52, 1_000), + (2, 21, 1_000), + (4, 43, 1_000), + (1, 10, 1_000), + (3, 42, 1_000), + (2, 41, 1_000), + (4, 13, 1_000), + (1, 40, 1_000), + (3, 22, 1_000), + (2, 31, 1_000), + (4, 33, 1_000), + ]; + let mut insert = conn + .prepare("INSERT INTO orders_rep (core_uid, core_name, newrecid, closedate) VALUES (?1, ?2, ?3, ?4)") + .expect("prepare tied replica rows"); + for &(core_uid, newrecid, closedate) in &seeded { + insert + .execute(params![ + core_uid, + format!("CORE-{core_uid}"), + newrecid, + closedate + ]) + .expect("insert tied replica row"); + } + drop(insert); + super::super::test_support::rep_init(&conn); + + let limit = 9; + let mut expected = seeded; + expected.sort_unstable_by(|left, right| right.cmp(left)); + let expected_rec_ids: Vec = expected + .into_iter() + .take(limit) + .map(|(_, newrecid, _)| newrecid) + .collect(); + let table = query_reports( + &conn, + &ReportFilter { + rows: RowScope::Closed, + ..ReportFilter::default() + }, + "closedate", + true, + limit, + ) + .expect("read default closed-date report"); + + assert_eq!( + table.rec_ids, expected_rec_ids, + "the source-local LIMIT must retain the rows selected by closedate, core uid, and record id" + ); +} diff --git a/crates/moon-core/src/db/trace.rs b/crates/moon-core/src/db/trace.rs new file mode 100644 index 00000000..ca2015d1 --- /dev/null +++ b/crates/moon-core/src/db/trace.rs @@ -0,0 +1,89 @@ +//! Opt-in statement profiler shared by every read connection this crate opens. +//! +//! This is a measurement instrument, not a feature: with no hook installed it costs one +//! `OnceLock::get` per connection birth and changes no query, no schema, and no result. It must +//! never gate behaviour — a caller that wants to branch on "is profiling on" is using this +//! module wrong. +//! +//! `trace_v2` accepts only a plain `fn` pointer, not a closure, so the hook itself carries no +//! state; a caller that needs to accumulate results supplies a `fn` that writes into its own +//! static or thread-local storage (see `crates/moon-core/examples/db_read_timing.rs` for the +//! harness that does exactly this). +//! +//! `trace_v2` is CONNECTION-LOCAL: this crate opens report/strategy/valuation/kline read +//! connections from ten call sites, and each one must call [`install_on`] itself. There is no +//! single choke point that sees every connection. + +use std::sync::OnceLock; +use std::time::Duration; + +use rusqlite::Connection; +use rusqlite::trace::{TraceEvent, TraceEventCodes}; + +/// One profiled SQLite statement, captured from a `trace_v2` PROFILE event. +#[derive(Clone, Debug)] +pub struct ProfiledStatement { + /// The statement text. Expanded (bound values substituted) when [`Self::expanded`] is + /// true; otherwise the raw text with `?N` placeholders. + pub sql: String, + /// Whether [`Self::sql`] is expanded. A placeholder-only capture cannot be replayed with + /// `EXPLAIN QUERY PLAN` against real literals and must never be ranked as if it could. + pub expanded: bool, + /// Wall time SQLite itself reports for running this statement. + pub duration: Duration, +} + +/// Process-wide profiler hook. `None` until [`install_read_profiler`] sets it. +static HOOK: OnceLock = OnceLock::new(); + +/// Install the process-wide read profiler. +/// +/// There can be only one; a later call never replaces an earlier one; see the memory ordering +/// of [`OnceLock::set`]. +/// +/// Args: +/// hook: Called once per profiled statement, on the thread that ran it. +/// +/// Returns: +/// Whether this call actually installed the hook (`false` when one was already set). +pub fn install_read_profiler(hook: fn(ProfiledStatement)) -> bool { + HOOK.set(hook).is_ok() +} + +/// Attach the profiler to one freshly opened connection, if a hook is installed. +/// +/// No-op when [`install_read_profiler`] was never called: one `OnceLock::get`, no query +/// touched, no behaviour changed. Call this from every connection-owning call site in this +/// crate — there is no single place that sees every connection (see the module docs). +/// +/// Args: +/// conn: Freshly opened connection, before it is handed to any caller. +pub(crate) fn install_on(conn: &Connection) { + if HOOK.get().is_some() { + conn.trace_v2( + TraceEventCodes::SQLITE_TRACE_PROFILE, + Some(on_profile_event), + ); + } +} + +/// `trace_v2` callback: build a [`ProfiledStatement`] from a PROFILE event and forward it to +/// the installed hook. A plain `fn`, per `trace_v2`'s signature — it carries no state of its +/// own and reads the current hook from [`HOOK`] on every call. +fn on_profile_event(event: TraceEvent<'_>) { + let TraceEvent::Profile(stmt, duration) = event else { + return; + }; + let Some(hook) = HOOK.get() else { + return; + }; + let (sql, expanded) = match stmt.expanded_sql() { + Some(sql) => (sql, true), + None => (stmt.sql().into_owned(), false), + }; + hook(ProfiledStatement { + sql, + expanded, + duration, + }); +} diff --git a/crates/moon-core/src/db/tuner/strategy_read.rs b/crates/moon-core/src/db/tuner/strategy_read.rs index 6db4a9f2..8fddd0af 100644 --- a/crates/moon-core/src/db/tuner/strategy_read.rs +++ b/crates/moon-core/src/db/tuner/strategy_read.rs @@ -14,6 +14,7 @@ fn open_strategies_ro() -> Option { let conn = Connection::open_with_flags(&path, rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY).ok()?; let _ = conn.busy_timeout(std::time::Duration::from_secs(3)); + crate::db::trace::install_on(&conn); Some(conn) } diff --git a/crates/moon-core/src/db/valuation/mod.rs b/crates/moon-core/src/db/valuation/mod.rs index 31d2636f..32838f13 100644 --- a/crates/moon-core/src/db/valuation/mod.rs +++ b/crates/moon-core/src/db/valuation/mod.rs @@ -967,6 +967,7 @@ fn existing_store_is_healthy(path: &Path) -> Result { let flags = OpenFlags::SQLITE_OPEN_READ_ONLY | OpenFlags::SQLITE_OPEN_URI; let conn = Connection::open_with_flags(uri, flags) .map_err(|error| format!("read-only open failed: {error}"))?; + super::trace::install_on(&conn); let check = conn.query_row("PRAGMA main.quick_check(1)", [], |row| { row.get::<_, String>(0) }); @@ -1353,6 +1354,7 @@ pub(crate) fn open_store(path: &Path) -> rusqlite::Result { CREATE INDEX IF NOT EXISTS idx_trade_values_inputs ON trade_values (algorithm_version, quote_ordinal, rate_minute_utc);", )?; + super::trace::install_on(&conn); Ok(conn) } diff --git a/crates/moon-core/src/market/candles.rs b/crates/moon-core/src/market/candles.rs index cce6f619..a98a6238 100644 --- a/crates/moon-core/src/market/candles.rs +++ b/crates/moon-core/src/market/candles.rs @@ -484,6 +484,74 @@ pub fn aggregate_trades(trades: &[Tick], tf_ms: i64, out: &mut Vec) } } +/// Thins a tick run to at most four REAL points per bucket, never inventing one. +/// +/// A candle can fabricate an OHLC body because nobody reads its open/close as "a trade happened +/// at that exact instant" — a chart POINT is read exactly that way, so a bucket-mid point with no +/// matching trade would be a lie about when the market moved. Keeping the first, highest, lowest +/// and last real tick per bucket instead preserves the shape a candle would draw (open/high/low/ +/// close) while every emitted point stays a value that actually traded. +/// +/// Args: +/// ticks: Ascending by time (the caller sorts). +/// bucket_ms: Bucket width; `<= 0` emits the input unchanged — there is nothing to thin to. +/// out: Cleared first, then filled ascending, the same discipline [`aggregate_trades`] uses. +pub(crate) fn thin_ticks(ticks: &[Tick], bucket_ms: i64, out: &mut Vec) { + out.clear(); + if bucket_ms <= 0 { + out.extend_from_slice(ticks); + return; + } + let mut open_ms: Option = None; + // Indices into `ticks` for the four roles of the bucket currently being collected. + let mut first_i = 0usize; + let mut last_i = 0usize; + let mut high_i = 0usize; + let mut low_i = 0usize; + // Emits one bucket's up-to-four representative ticks, ascending, each index kept once — a + // quiet bucket with a single trade collapses all four roles onto the same index. + let flush = + |first_i: usize, high_i: usize, low_i: usize, last_i: usize, out: &mut Vec| { + let mut idx = [first_i, high_i, low_i, last_i]; + idx.sort_unstable(); + let mut prev: Option = None; + for i in idx { + if prev != Some(i) { + out.push(ticks[i]); + prev = Some(i); + } + } + }; + + for (i, t) in ticks.iter().enumerate() { + let this_open = bucket_open_ms(t.time_ms, bucket_ms); + match open_ms { + Some(open) if open == this_open => { + last_i = i; + if t.price > ticks[high_i].price { + high_i = i; + } + if t.price < ticks[low_i].price { + low_i = i; + } + } + _ => { + if open_ms.is_some() { + flush(first_i, high_i, low_i, last_i, out); + } + open_ms = Some(this_open); + first_i = i; + last_i = i; + high_i = i; + low_i = i; + } + } + } + if open_ms.is_some() { + flush(first_i, high_i, low_i, last_i, out); + } +} + fn candle_from_tick(open_ms: f64, t: &Tick) -> ChartCandle { ChartCandle { t_open_ms: open_ms, diff --git a/crates/moon-core/src/market/candles/tests.rs b/crates/moon-core/src/market/candles/tests.rs index bb8f8726..fcd680b1 100644 --- a/crates/moon-core/src/market/candles/tests.rs +++ b/crates/moon-core/src/market/candles/tests.rs @@ -736,3 +736,73 @@ fn compose_with_coarse_preserves_empty_single_and_contiguous_series_inputs() { "empty layers leave an already complete fine series untouched" ); } + +/// `market/candles.rs:thin_ticks` synthesising bucket-mid points or dropping its role ordering +/// lies about when a market traded and can draw points out of chronological order. +#[test] +fn thin_ticks_keeps_only_real_ascending_points_and_copies_non_positive_buckets() { + let input = vec![ + tick(0.0, 10.0, 1.0), + tick(100.0, 14.0, 2.0), + tick(200.0, 8.0, 3.0), + tick(300.0, 12.0, 4.0), + tick(1_000.0, 20.0, 5.0), + tick(1_100.0, 18.0, 6.0), + tick(1_200.0, 22.0, 7.0), + tick(1_300.0, 19.0, 8.0), + tick(1_400.0, 21.0, 9.0), + ]; + let mut thinned = Vec::new(); + thin_ticks(&input, 1_000, &mut thinned); + + assert!( + thinned.iter().all(|point| input.iter().any(|source| { + source.time_ms == point.time_ms + && source.price == point.price + && source.qty == point.qty + })), + "every emitted point must be an exact time, price, and quantity observed in the input" + ); + assert!( + thinned + .windows(2) + .all(|pair| pair[0].time_ms <= pair[1].time_ms), + "thinned points must stay ascending for the chart consumer" + ); + for bucket in [0_i64, 1_i64] { + assert!( + thinned + .iter() + .filter(|point| (point.time_ms as i64).div_euclid(1_000) == bucket) + .count() + <= 4, + "a bucket may keep at most first, high, low, and last real points" + ); + } + + let mut copied = Vec::new(); + thin_ticks(&input, 0, &mut copied); + assert_eq!( + copied + .iter() + .map(|point| (point.time_ms, point.price, point.qty)) + .collect::>(), + input + .iter() + .map(|point| (point.time_ms, point.price, point.qty)) + .collect::>(), + "bucket_ms == 0 must preserve raw ticks unchanged" + ); + thin_ticks(&input, -1, &mut copied); + assert_eq!( + copied + .iter() + .map(|point| (point.time_ms, point.price, point.qty)) + .collect::>(), + input + .iter() + .map(|point| (point.time_ms, point.price, point.qty)) + .collect::>(), + "negative bucket widths also copy input unchanged" + ); +} diff --git a/crates/moon-core/src/market/kline_cache.rs b/crates/moon-core/src/market/kline_cache.rs index 33649e5a..ab33319b 100644 --- a/crates/moon-core/src/market/kline_cache.rs +++ b/crates/moon-core/src/market/kline_cache.rs @@ -148,6 +148,10 @@ impl KlineCache { log::warn!("kline cache schema failed {}: {e}", path.display()); return None; } + // This one connection also serves `Op::Merge`/`Op::MergeBatch` on the worker thread + // below, so a hook installed here captures background WRITE timings too, mixed in with + // the `Op::Read` ones — unlike every other call site, which only ever opens a reader. + crate::db::trace::install_on(&conn); let (tx, rx) = mpsc::channel::(); std::thread::Builder::new() .name("kline-cache".into()) diff --git a/crates/moon-core/src/market/trade_replay/mod.rs b/crates/moon-core/src/market/trade_replay/mod.rs index a2bebb2a..36bbc2a0 100644 --- a/crates/moon-core/src/market/trade_replay/mod.rs +++ b/crates/moon-core/src/market/trade_replay/mod.rs @@ -83,6 +83,54 @@ const MAX_SPAN_MS: i64 = 7 * 24 * 60 * MINUTE_MS; /// exit — so a window clipped exactly to the position would answer the wrong question. const CONTEXT_FRACTION: f64 = 0.5; +/// Margin added around the trade's own span when computing [`ReplayWindow::focus`]. +/// +/// Wide enough that the entry and exit sit comfortably inside the focus tiles [`tick_plan`] +/// fetches first, rather than landing on the very edge of one; not wider, because every extra +/// millisecond here is lead/trail context pulled ahead of a slice that is actually IN the trade. +const FOCUS_MARGIN_MS: i64 = 5 * MINUTE_MS; + +/// Width of one tick-fetch tile in [`tick_plan`], before a route's own cap narrows it further. +/// +/// An 80-minute scalp window under Binance USD-M's one-hour query cap would otherwise tile into +/// two requests that both straddle the focus, making trade-priority ordering inert — chopping +/// finer than the route cap is what gives [`tick_plan`] something to actually prioritise. +const TICK_SLICE_MS: i64 = 10 * MINUTE_MS; + +/// Bucket widths [`fit_ticks`] tries in order, coarsest last. +/// +/// Each rung roughly doubles to triples the previous one, so a run that barely overflows the +/// budget loses little precision while a run that overflows it by orders of magnitude still +/// terminates in a handful of steps instead of walking one millisecond at a time. +const THIN_LADDER_MS: [i64; 9] = [ + 1_000, 2_000, 5_000, 10_000, 15_000, 30_000, 60_000, 120_000, 300_000, +]; + +/// How the tick stage for one window ended — the thing the window's caption NAMES. +/// +/// Each variant is a DIFFERENT sentence to show the user, and two of them (`NoRoute`, +/// `OutOfRetention`) are known before a single request is spent, so they cost nothing. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum TickStatus { + /// A tick stage is queued and has not answered yet. The first outcome of every window that + /// earns one carries this. + Pending, + /// This build knows no public trades route for the venue (Bybit, Hyperliquid). Retrying + /// cannot help. + NoRoute, + /// The window is older than the route's documented trade retention. + OutOfRetention { + /// How far back the venue's own tick retention actually reaches, in milliseconds. + retention_ms: i64, + }, + /// The venue answered and held no trade inside the window, while klines exist. + NoTrades, + /// The tick fetch itself did not produce an answer. + Failed, + /// Ticks were served. Only a [`TradeReplaySource::Ticks`] series carries this. + Served, +} + /// Which data a replay actually carries, so the window can say which it is showing. /// /// This is user-visible and load-bearing: a one-minute picture of a forty-second scalp is an @@ -169,6 +217,10 @@ pub struct ReplayWindow { pub from_ms: i64, /// Last millisecond the replay covers. pub to_ms: i64, + /// The trade's own open, in milliseconds — the position's real entry stamp, not [`Self::from_ms`]. + pub open_ms: i64, + /// The trade's own close, in milliseconds — the position's real exit stamp, not [`Self::to_ms`]. + pub close_ms: i64, /// Whether this window is WIDER than [`MAX_SPAN_MS`] because its floors demanded it. /// /// Renamed from `clipped`, and the rename is the point: the field used to mean "half the @@ -193,6 +245,28 @@ impl ReplayWindow { n => n, } } + + /// The sub-window closest to the trade itself, for ordering a tick fetch around it. + /// + /// [`tick_plan`] fetches this region FIRST and orders every other tile by distance to it, so + /// a fetch cut short by budget or deadline still lands the trade's own span rather than an + /// hour of lead context nobody asked to see before it. + /// + /// Returns: + /// `(left, right)` inclusive, clamped into `[Self::from_ms, Self::to_ms]` on both ends — + /// independently. Every constructor of this type preserves `open_ms <= close_ms`; a + /// hand-built window that violates it is out of this function's contract and can yield an + /// inverted `(left, right)` rather than a usable focus (no guard here — that state is + /// unreachable today, per house style). + pub(crate) fn focus(self) -> (i64, i64) { + let left = (self.open_ms - FOCUS_MARGIN_MS) + .max(self.from_ms) + .min(self.to_ms); + let right = (self.close_ms + FOCUS_MARGIN_MS) + .min(self.to_ms) + .max(self.from_ms); + (left, right) + } } /// Compute the window to fetch around one trade. @@ -246,7 +320,10 @@ pub fn replay_window(buy_date_s: i64, close_date_s: i64) -> Option // fetch it, retry", never a chart quietly missing its own entry and exit. let over_budget = to_ms - from_ms > MAX_SPAN_MS; // A pre-epoch left edge is meaningless to every venue and would be sent as a negative - // `startTime`; pull it forward instead of asking for it. + // `startTime`; pull it forward instead of asking for it. `open_ms`/`close_ms` are left alone + // by this shift: they are the trade's own REAL stamps, not a fetch bound, and a pre-epoch + // trade is already rejected above by the `close_date_s < buy_date_s` / non-positive guard + // long before this shift ever runs, so nothing here has reason to move them. if from_ms < 0 { to_ms = to_ms.saturating_add(-from_ms); from_ms = 0; @@ -254,6 +331,8 @@ pub fn replay_window(buy_date_s: i64, close_date_s: i64) -> Option Some(ReplayWindow { from_ms, to_ms, + open_ms, + close_ms, over_budget, }) } @@ -291,6 +370,230 @@ pub fn pages(window: ReplayWindow, bar_ms: i64, max_rows: usize) -> Vec<(i64, i6 out } +/// Split one window into requests no larger than a trade route's documented query span. +/// +/// The frozen shape [`super::worker`] and every `rest::::fetch_trades` build against: +/// `None` answers one slice covering the whole window; `Some(span)` tiles the window into +/// requests of at most `span` ms with no gap and no overlap, the same tiling discipline [`pages`] +/// uses for the kline pager, just keyed on a request's time SPAN rather than its row count — +/// several trade routes ([`super::venue_caps::TradeRoute::max_query_ms`]) cap a request's window +/// rather than its row count. +/// +/// A non-positive span returns no slices: unlike [`pages`], which derives its step from a +/// `bar_ms * max_rows` product that route constants keep positive, a caller here supplies the +/// span directly. [`super::venue_caps::TradeRoute::max_query_ms`] represents an unbounded route +/// as `None`, so every `Some` value remains a finite vendor-imposed request window. +/// +/// Args: +/// window: The window to cover. +/// max_span_ms: Widest span one request may cover, or `None` when the route documents no +/// cap. +/// +/// Returns: +/// Inclusive `(from_ms, to_ms)` pairs in ascending order; empty when `max_span_ms` is +/// `Some(n)` with `n <= 0`, which no route reports. +pub fn time_slices(window: ReplayWindow, max_span_ms: Option) -> Vec<(i64, i64)> { + let Some(span) = max_span_ms else { + return vec![(window.from_ms, window.to_ms)]; + }; + if span <= 0 { + return Vec::new(); + } + let mut out = Vec::new(); + let mut cursor = window.from_ms; + while cursor <= window.to_ms { + let end = cursor.saturating_add(span - 1).min(window.to_ms); + out.push((cursor, end)); + if end == window.to_ms { + break; + } + cursor = end + 1; + } + out +} + +/// One ordered decomposition of a [`ReplayWindow`] into tick-fetch tiles. +/// +/// [`Self::slices`] is ordered so that ANY PREFIX is a CONTIGUOUS span with no gap between the +/// sorted first-k slices, for every k — [`worker::paginate_ticks`] leans on exactly this to +/// report a fetch truncated by budget or a deadline as ONE covered interval rather than a comb of +/// holes. A later "tidy" that resorts these tiles purely by `from_ms` would keep them contiguous +/// too, but back in CLOCK order — which throws away the whole reason this type exists, so preserve +/// the ORDER here, not merely the contiguity. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct TickPlan { + /// Every tile to fetch, in fetch-priority order: the trade's own focus first, then outward. + pub slices: Vec<(i64, i64)>, + /// How many leading entries of [`Self::slices`] cover [`ReplayWindow::focus`]. + pub focus_len: usize, +} + +/// Tile a window into tick-fetch requests ordered around the TRADE instead of around the clock. +/// +/// The window is chopped into three independent regions — before the focus, the focus itself, +/// after the focus — each tiled by [`time_slices`] at `min(TICK_SLICE_MS, max_query_ms)`. The +/// focus tiles are placed first, ascending; every remaining tile then follows by distance to the +/// focus, nearest first, tied-break by `from_ms` (so the tile just before the focus outranks the +/// tile just after it, since it sits at a smaller time). This is what lets a walk that runs out of +/// budget or time abandon the FARTHEST tiles rather than the nearest ones — the defect this module +/// exists to fix in the first place. +/// +/// A route's documented retention makes the LEFT edge of every region a moving floor: rows older +/// than `now_ms - retention_ms` cannot be fetched regardless of what the window asks for. Judging +/// retention against the padded window (as the naive check does) refuses a trade whose own span +/// is well inside retention the moment its OPTIONAL lead context crosses the boundary — the ticks +/// the user actually wants are available and get skipped anyway. So this clips instead of +/// refusing outright: every region drops the part of itself older than `earliest_ms`, and if the +/// FOCUS itself — the trade, not its context — falls entirely before it, the whole plan is empty +/// rather than a lead-less scrap of trail. The caller reads an empty plan as "nothing worth +/// fetching" and reports `OutOfRetention`. +/// +/// Args: +/// window: The window to cover. +/// max_query_ms: The route's own cap on one request's span, or `None`/non-positive when it +/// documents none, in which case [`TICK_SLICE_MS`] alone tiles the window. +/// earliest_ms: The oldest millisecond the route's retention can still answer for, or `None` +/// when the route documents no retention limit. +/// +/// Returns: +/// A [`TickPlan`] whose prefix-contiguity invariant (see [`TickPlan`]) holds for every k, even +/// after retention clipping — clipping only ever shrinks the LEAD region from the near edge +/// inward, and whenever it shrinks the focus's own start too, the entire lead region (being +/// strictly older) is guaranteed to fall before `earliest_ms` as well and is dropped whole, so +/// no clipped region can end up separated from its neighbour by a gap. A region that ends up +/// empty or inverted contributes no tiles — [`time_slices`] already returns none for an +/// inverted span, so no extra per-region check is needed here. +pub(crate) fn tick_plan( + window: ReplayWindow, + max_query_ms: Option, + earliest_ms: Option, +) -> TickPlan { + let span = match max_query_ms { + Some(cap) if cap > 0 => TICK_SLICE_MS.min(cap), + _ => TICK_SLICE_MS, + }; + let (focus_from, focus_to) = window.focus(); + + // The trade itself is the one thing worth fetching; a route whose retention does not even + // reach the trade has nothing this plan can usefully prioritise. + if let Some(earliest) = earliest_ms { + if focus_to < earliest { + return TickPlan { + slices: Vec::new(), + focus_len: 0, + }; + } + } + let clip_from = |from: i64| match earliest_ms { + Some(earliest) => from.max(earliest), + None => from, + }; + + let focus_slices = time_slices( + ReplayWindow { + from_ms: clip_from(focus_from), + to_ms: focus_to, + ..window + }, + Some(span), + ); + let lead_slices = time_slices( + ReplayWindow { + from_ms: clip_from(window.from_ms), + to_ms: focus_from - 1, + ..window + }, + Some(span), + ); + let trail_slices = time_slices( + ReplayWindow { + from_ms: clip_from(focus_to + 1), + to_ms: window.to_ms, + ..window + }, + Some(span), + ); + + let focus_len = focus_slices.len(); + let mut rest: Vec<(i64, i64)> = lead_slices.into_iter().chain(trail_slices).collect(); + // Ascending distance first; `from_ms` breaks the tie between the one lead tile and the one + // trail tile that can sit exactly as close on either side of the focus. + rest.sort_by_key(|&(from, to)| { + let distance = if to < focus_from { + focus_from - to + } else { + from - focus_to + }; + (distance, from) + }); + + let mut slices = focus_slices; + slices.extend(rest); + TickPlan { slices, focus_len } +} + +/// Thin a tick run down to a render/remember budget, coarsening only as far as needed. +/// +/// Walks [`THIN_LADDER_MS`] in order and takes the FIRST bucket width whose thinned output fits +/// `budget` via [`super::candles::thin_ticks`], so a run that already fits pays no thinning at +/// all. A position held long enough makes even the coarsest rung's 300-second buckets outnumber +/// the budget — `THIN_LADDER_MS`'s terminal rung is a RATE, not a ceiling — so past it a final +/// uniform stride picks `budget` points evenly spaced across the coarsest rung's output, +/// including its first and last tick. Either path keeps every point a REAL tick, never a +/// synthesised one. +/// +/// **Contract:** `result.len() <= budget`, unconditionally — this is the one property every +/// caller relies on ([`worker::TICK_BUDGET`] bounds both the composed series and the GPU point +/// ring), not a best effort. +/// +/// Args: +/// ticks: Ascending by time (the caller sorts). +/// budget: Largest tick count the caller will draw or remember; `0` always returns nothing. +/// +/// Returns: +/// `(ticks, 0)` unchanged when `ticks.len() <= budget`; otherwise `(thinned, bucket_ms)`, +/// `bucket_ms` being the ladder rung that produced it — the coarsest rung when the final +/// stride also had to run. +pub(crate) fn fit_ticks(ticks: Vec, budget: usize) -> (Vec, i64) { + if ticks.len() <= budget { + return (ticks, 0); + } + if budget == 0 { + return ( + Vec::new(), + *THIN_LADDER_MS.last().expect("non-empty ladder"), + ); + } + let mut out = Vec::new(); + let mut bucket_ms = 0; + for &rung in THIN_LADDER_MS.iter() { + bucket_ms = rung; + crate::market::candles::thin_ticks(&ticks, rung, &mut out); + if out.len() <= budget { + return (out, bucket_ms); + } + } + // The coarsest rung still overflows. A stride of `ceil(last_idx / (budget - 1))`, walked from + // index 0 and always closed off by the true last index, is what keeps the bound UNCONDITIONAL: + // re-walking forward in fixed `ceil(len / budget)` steps and appending the last tick + // afterwards — the naive reading — can land `budget + 1` points whenever the true last index + // is not itself a multiple of that step (e.g. 10 points into a budget of 3: steps of 4 land + // 0/4/8, none of which is index 9, so appending it makes four). + let last_idx = out.len() - 1; + if budget == 1 { + return (vec![out[last_idx]], bucket_ms); + } + let stride = (last_idx as f64 / (budget - 1) as f64).ceil() as usize; + let mut strided = Vec::with_capacity(budget); + let mut i = 0usize; + while i < last_idx { + strided.push(out[i]); + i += stride; + } + strided.push(out[last_idx]); + (strided, bucket_ms) +} + /// Whether cached rows already cover a window densely enough to skip the network. /// /// COVERAGE, not presence, is the question. A partial prefix is exactly what a previously @@ -370,6 +673,30 @@ pub fn replay_revision(identity: u64, tf_ms: i64, from_bucket: i64, to_bucket: i hash.max(1) } +/// Per-source salt for [`replay_revision`], so a tick series and its sibling candle series of the +/// same identity, timeframe and window never share a revision. +/// +/// The bug this exists to prevent: [`TradeReplaySeries::read_into`] derives `revision` from +/// `(identity, tf_ms, from_bucket, to_bucket)`, and a tick upgrade shares every one of those four +/// with the kline series it replaces — same `identity` ([`super::worker`] never changes it +/// between the two outcomes), same window, same `tf_ms == 60_000`. Unsalted, the upgrade's +/// revision would equal the one the pane already shipped, `read_into`'s `candles_changed` would +/// stay `false`, and the pane would keep drawing exchange klines forever under the new tick +/// points. +/// +/// Args: +/// source: Which kind of series is being read. +/// +/// Returns: +/// `0` for [`TradeReplaySource::Klines1m`], so every existing revision stays bit-identical to +/// today's; a fixed non-zero constant for [`TradeReplaySource::Ticks`]. +pub fn tick_identity_salt(source: TradeReplaySource) -> u64 { + match source { + TradeReplaySource::Klines1m => 0, + TradeReplaySource::Ticks => 0x9E37_79B9_7F4A_7C15, + } +} + /// One trade's frozen market history, ready to be drawn. #[derive(Clone, Debug)] pub struct TradeReplaySeries { @@ -381,12 +708,28 @@ pub struct TradeReplaySeries { pub window: ReplayWindow, /// Timeframe of [`Self::candles`] in milliseconds; one minute for every current route. pub tf_ms: i64, - /// Bars in ascending open time; empty when [`Self::source`] is [`TradeReplaySource::Ticks`]. + /// Bars in ascending open time. A [`TradeReplaySource::Klines1m`] series carries these alone; + /// a [`TradeReplaySource::Ticks`] series carries these TOO — the EXCHANGE's own one-minute + /// klines, not bars aggregated from [`Self::ticks`], so the bar layer covers the WHOLE window + /// even where the points, per [`Self::partial`], do not. pub candles: Vec, - /// Trade points in ascending time; empty when the source is bars. + /// Trade points in ascending time. Empty when [`Self::source`] is + /// [`TradeReplaySource::Klines1m`]; carried alongside [`Self::candles`] for + /// [`TradeReplaySource::Ticks`] — never in place of them, and per [`Self::partial`] possibly + /// covering only part of [`Self::window`] while the bars cover all of it. pub ticks: Vec, /// Stable discriminator feeding [`replay_revision`], so two open windows never collide. pub identity: u64, + /// How the tick attempt for this window ended. `Served` on a [`TradeReplaySource::Ticks`] + /// series; every other variant is a reason the bar layer is all the window has, and the + /// window PRINTS it. + pub tick_status: TickStatus, + /// Bucket the points were thinned to, in ms; `0` means raw, untouched ticks. Meaningless (and + /// always `0`) on a [`TradeReplaySource::Klines1m`] series. + pub bucket_ms: i64, + /// Whether [`Self::ticks`] covers only PART of [`Self::window`] — the bars always cover all + /// of it. Always `false` on a [`TradeReplaySource::Klines1m`] series. + pub partial: bool, } impl TradeReplaySeries { @@ -459,8 +802,13 @@ impl TradeReplaySeries { let to_ms = ((epoch_ms + f64::from(to_rel_ms.max(from_rel_ms))).round() as i64) .max(from_ms.saturating_add(1)); let tf_ms = candle_params.map_or(self.tf_ms, |p| p.tf_ms).max(1); + // Salted by source: a tick series and its sibling candle series otherwise share + // `(identity, tf_ms, from_bucket, to_bucket)` bit-for-bit (§4 of the tick-replay plan), + // so the tick upgrade's revision would equal the one the pane already shipped and + // `candles_changed` below would stay false forever. See `tick_identity_salt`. + let salted_identity = self.identity ^ tick_identity_salt(self.source); let revision = replay_revision( - self.identity, + salted_identity, tf_ms, from_ms.div_euclid(tf_ms), to_ms.div_euclid(tf_ms), @@ -469,7 +817,8 @@ impl TradeReplaySeries { read.candles_revision = revision; // Points first: they are re-emitted whole on every read, because a frozen series has no - // live edge to drain incrementally and the whole window is a few hundred rows at most. + // live edge to drain incrementally and the whole window is bounded — a few hundred rows + // for a candle-only series, or up to `worker::TICK_BUDGET` for a tick one. out.ticks.extend( self.ticks .iter() diff --git a/crates/moon-core/src/market/trade_replay/rest/binance.rs b/crates/moon-core/src/market/trade_replay/rest/binance.rs index eecaf75e..3d053114 100644 --- a/crates/moon-core/src/market/trade_replay/rest/binance.rs +++ b/crates/moon-core/src/market/trade_replay/rest/binance.rs @@ -5,9 +5,10 @@ use serde_json::Value; -use super::{FetchError, cell_f32}; +use super::{FetchError, TradeCursor, TradePage, cell_f32}; +use crate::feed::types::{Side, Tick}; use crate::market::candles::{ChartCandle, estimate_quote_volume}; -use crate::market::trade_replay::venue_caps::KlineRoute; +use crate::market::trade_replay::venue_caps::{KlineRoute, TradeRoute}; /// Positional index of the cell holding QUOTE-asset turnover on a spot or USD-M row. /// @@ -149,3 +150,151 @@ fn parse_row(row: &Value, quote_source: QuoteSource) -> Option { quote_volume, }) } + +/// Fetch one page of public aggregate trades and return the decoded body. +/// +/// # Continuation is by id, and the FIRST page is the only one carrying a time range +/// +/// The first page of a slice is asked with `startTime`/`endTime`. Every later page instead +/// carries `fromId` ALONE, Binance's own forward cursor — an aggregate-trade id is unambiguous +/// where a millisecond timestamp is not, since more than one trade can share a millisecond — and +/// `startTime`/`endTime` are dropped rather than kept alongside it. That is the vendor's own +/// documented contract, not a preference: Binance's `aggTrades` page states "Sending both +/// startTime/endTime and fromId might cause response timeout, please send either fromId or +/// startTime/endTime." A dense slice needing a second page would otherwise take a documented +/// timeout-prone path on exactly the busy markets where ticks matter most. The client-side clip +/// against `to_ms` in [`parse_agg_trades`] is what bounds a continuation page instead. +/// +/// **Resolves the earlier open assumption** ("nothing says `endTime` is rejected alongside +/// `fromId`"): the vendor's own page does warn against the combination, so the assumption is +/// refuted by the docs, not merely tidied away. +/// +/// Args: +/// agent: Shared client. +/// route: One of the three Binance trade routes. +/// market: Exchange-native market name. +/// from_ms: Left edge of this slice, inclusive; used only on the first page. +/// to_ms: Right edge of this slice, inclusive; used only on the first page. +/// max_rows: Row cap for this request. +/// cursor: Continuation from a previous page, or `None` for the first page. +/// +/// Returns: +/// The decoded response, or a classified failure. +pub(super) fn fetch_trades( + agent: &ureq::Agent, + route: TradeRoute, + market: &str, + from_ms: i64, + to_ms: i64, + max_rows: usize, + cursor: Option, +) -> Result { + let request = agent + .get(route.url()) + .query("symbol", market) + .query("limit", max_rows.to_string()); + let request = match cursor { + Some(TradeCursor::FromId(id)) => request.query("fromId", id.to_string()), + None => request + .query("startTime", from_ms.to_string()) + .query("endTime", to_ms.to_string()), + Some(_) => { + debug_assert!(false, "binance trade routes hand back only FromId"); + request + .query("startTime", from_ms.to_string()) + .query("endTime", to_ms.to_string()) + } + }; + let response = request + .call() + .map_err(|error| FetchError::Transient(error.to_string()))?; + super::decode_and_classify(response, "binance", classify) +} + +/// Parse a Binance aggregate-trade array into a page of ticks. +/// +/// Args: +/// body: Decoded response. +/// to_ms: Right edge of the slice this page belongs to, so completeness is judged from the +/// last row's own timestamp rather than assumed from the row count alone. +/// max_rows: Row cap that was sent, so a FULL page can be told from a short, final one. +/// +/// Returns: +/// The page, or a failure when the envelope is not an array. +pub(super) fn parse_agg_trades( + body: &Value, + to_ms: i64, + max_rows: usize, +) -> Result { + let rows = body + .as_array() + .ok_or_else(|| FetchError::Transient("binance: response is not an array".to_string()))?; + let ticks: Vec = rows.iter().filter_map(parse_trade_row).collect(); + // A page holding a malformed row alongside valid ones can still finish pagination with a + // non-empty tick vector that is silently missing rows — dropping one bad row in a thousand is + // fine for candles, where a missing bar is visible, but not here, where a hole must send the + // window to candles instead of drawing a partial tape as if it were whole. + if ticks.len() < rows.len() { + return Err(FetchError::Transient(format!( + "binance: page held {} unparseable row(s) of {} (parsed {})", + rows.len() - ticks.len(), + rows.len(), + ticks.len() + ))); + } + let last_id = rows.last().and_then(|r| r.get("a")).and_then(Value::as_u64); + let last_time_ms = rows.last().and_then(|r| r.get("T")).and_then(Value::as_i64); + let full = rows.len() >= max_rows; + let covered = last_time_ms.is_some_and(|t| t >= to_ms); + let next = match (full, covered, last_id) { + (true, false, Some(id)) => Some(TradeCursor::FromId(id + 1)), + // The page is full and the window is not yet covered, but the last row's own `a` (trade + // id) did not parse: the cursor to continue from is unknowable. Treating this as + // completion would silently ship a truncated window as a whole one — every field name + // here is inferred with no fixture behind it, so a wrong name yields exactly this shape + // on every page. + (true, false, None) => { + return Err(FetchError::Transient( + "binance: full page, window not covered, but the last row's `a` did not parse" + .to_string(), + )); + } + _ => None, + }; + Ok(TradePage { ticks, next }) +} + +/// Parse one Binance aggregate-trade row. +/// +/// **COIN-M unit note**: on a `BinanceCoinMAggTrades` row, `q` is a CONTRACT count, not a +/// base-currency amount — `aggTrades` exposes no `baseQty` alternative for COIN-M, the same fact +/// [`COIN_M_BASE_VOLUME_CELL`] already states for klines. `Tick::qty` therefore holds contracts +/// rather than base currency for that one route; no base-asset amount is available from this +/// endpoint. The chart's volume bars stay shape-correct regardless: the chart normalises the +/// visible window against its own maximum `qty`, and a per-instrument contract multiplier is a +/// constant that cancels out of that normalisation — only an ABSOLUTE volume figure would be +/// wrong, and a tick series' aggregated candles never reach the shared SQLite kline cache where +/// one could be read. See `venue_caps.rs`'s trade-route table for the same note. +/// +/// Args: +/// row: One element of the response array. +/// +/// Returns: +/// The tick, or `None` when the row is malformed. +fn parse_trade_row(row: &Value) -> Option { + let price = cell_f32(row.get("p")?)?; + let qty = cell_f32(row.get("q")?)?; + let time_ms = row.get("T")?.as_i64()? as f64; + // `m == true`: the BUYER was the maker, meaning the TAKER — the side this tape reports — SOLD. + // Time is read from `T`, never from the aggregate id `a`. + let side = match row.get("m")?.as_bool()? { + true => Side::Sell, + false => Side::Buy, + }; + Some(Tick { + time_ms, + price, + qty, + side, + }) +} diff --git a/crates/moon-core/src/market/trade_replay/rest/bitget.rs b/crates/moon-core/src/market/trade_replay/rest/bitget.rs index 43ae30b5..94feefe2 100644 --- a/crates/moon-core/src/market/trade_replay/rest/bitget.rs +++ b/crates/moon-core/src/market/trade_replay/rest/bitget.rs @@ -17,9 +17,10 @@ use serde_json::Value; -use super::{FetchError, cell_f32, cell_i64}; +use super::{FetchError, TradeCursor, TradePage, cell_f32, cell_i64}; +use crate::feed::types::{Side, Tick}; use crate::market::candles::ChartCandle; -use crate::market::trade_replay::venue_caps::KlineRoute; +use crate::market::trade_replay::venue_caps::{KlineRoute, TradeRoute}; /// Fetch one page and return the decoded body. /// @@ -147,5 +148,167 @@ fn parse_row(row: &Value) -> Option { }) } +/// Fetch one page of public fills and return the decoded body. +/// +/// Args: +/// agent: Shared client. +/// route: [`TradeRoute::BitgetSpotFills`] or [`TradeRoute::BitgetMixFills`]. +/// market: Exchange-native market name, e.g. `BTCUSDT`. +/// from_ms: First millisecond of this slice, inclusive. +/// to_ms: Last millisecond of this slice, inclusive; the vendor caps one request's span at +/// 7 days, see [`super::super::venue_caps::TradeRoute::max_query_ms`]. +/// cursor: Continuation from a previous page, or `None` for the first page. +/// +/// Returns: +/// The decoded response, or a classified failure. +pub(super) fn fetch_trades( + agent: &ureq::Agent, + route: TradeRoute, + market: &str, + from_ms: i64, + to_ms: i64, + cursor: Option, +) -> Result { + let futures = matches!(route, TradeRoute::BitgetMixFills); + let mut request = agent + .get(route.url()) + .query("symbol", market) + .query("startTime", from_ms.to_string()) + .query("endTime", to_ms.to_string()) + .query("limit", route.max_rows().to_string()); + if futures { + request = request.query("productType", "USDT-FUTURES"); + } + if let Some(TradeCursor::LessThanId(id)) = cursor { + request = request.query("idLessThan", id.to_string()); + } + let response = request + .call() + .map_err(|error| FetchError::Transient(error.to_string()))?; + super::decode_and_classify(response, "bitget", classify_fills) +} + +/// Classify a BitGet fills-history response by status and envelope `code`. +/// +/// **Assumption, not settled by vendor docs read for this task**: `fills-history` shares its +/// unknown-symbol codes (`40034` mix, `400172` spot) with `history-candles`. No vendor page for +/// this specific endpoint documents its own unknown-symbol code the way the candle endpoint's +/// pages do — see [`classify`] for the pair this borrows. If a later reader finds this endpoint +/// answers a different code for an unknown symbol, that market falls through to `Transient` +/// below rather than being misclassified as permanent, so the failure mode of a wrong guess here +/// is a retry loop, not a silently wrong "this market does not exist". +/// +/// Args: +/// status: HTTP status. +/// body: Decoded response. +/// +/// Returns: +/// `Ok(())` on success, or the classified failure. +pub(super) fn classify_fills(status: u16, body: &Value) -> Result<(), FetchError> { + if !(200..300).contains(&status) { + return Err(FetchError::Transient(format!("bitget HTTP {status}"))); + } + let code = body.get("code").and_then(Value::as_str).unwrap_or_default(); + match code { + "00000" => Ok(()), + "40034" | "400172" => Err(FetchError::UnknownSymbol), + "" => Err(FetchError::Transient(format!( + "bitget HTTP {status}: empty code" + ))), + other => { + let message = body.get("msg").and_then(Value::as_str).unwrap_or("unknown"); + Err(FetchError::Transient(format!("bitget {other}: {message}"))) + } + } +} + +/// Parse a BitGet fills envelope into a page of ticks. +/// +/// Rows arrive DESCENDING (newest first, per the vendor's own doc), so the oldest row in this +/// page is the LAST one — that is what both the next cursor and the window-covered check key on. +/// +/// Args: +/// body: Decoded response. +/// max_rows: Row cap that was sent, so a FULL page can be told from a short, final one. +/// from_ms: Left edge of the slice, so completeness is judged from the oldest row's own +/// timestamp rather than assumed from the row count alone. +/// +/// Returns: +/// The page, or a failure when the envelope is missing. +pub(super) fn parse_fills( + body: &Value, + max_rows: usize, + from_ms: i64, +) -> Result { + let rows = body + .get("data") + .and_then(Value::as_array) + .ok_or_else(|| FetchError::Transient("bitget: missing data".to_string()))?; + let ticks: Vec = rows.iter().filter_map(parse_fill_row).collect(); + // A page holding a malformed row alongside valid ones can still finish pagination with a + // non-empty tick vector that is silently missing rows — a hole must send the window to + // candles instead of drawing a partial tape as if it were whole. + if ticks.len() < rows.len() { + return Err(FetchError::Transient(format!( + "bitget: page held {} unparseable row(s) of {} (parsed {})", + rows.len() - ticks.len(), + rows.len(), + ticks.len() + ))); + } + let oldest_id = rows + .last() + .and_then(|r| r.get("tradeId")) + .and_then(cell_i64) + .map(|v| v as u64); + let oldest_time_ms = rows.last().and_then(|r| r.get("ts")).and_then(cell_i64); + let full = rows.len() >= max_rows; + let covered = oldest_time_ms.is_some_and(|t| t <= from_ms); + let next = match (full, covered, oldest_id) { + (true, false, Some(id)) => Some(TradeCursor::LessThanId(id)), + // Full page, window not covered, but the oldest row's own `tradeId` did not parse: the + // cursor to continue from is unknowable. Treating this as completion would silently ship + // a truncated window as a whole one. + (true, false, None) => { + return Err(FetchError::Transient( + "bitget: full page, window not covered, but the oldest row's tradeId did not parse" + .to_string(), + )); + } + _ => None, + }; + Ok(TradePage { ticks, next }) +} + +/// Parse one BitGet fills row. +/// +/// The vendor's own `side` casing contradicts itself between its field table and its worked +/// example, so it is read case-insensitively here rather than trusted literally. +/// +/// Args: +/// row: One element of `data`. +/// +/// Returns: +/// The tick, or `None` when the row is malformed. +fn parse_fill_row(row: &Value) -> Option { + let price = cell_f32(row.get("price")?)?; + let qty = cell_f32(row.get("size")?)?; + let time_ms = cell_i64(row.get("ts")?)? as f64; + let side = match row + .get("side") + .and_then(Value::as_str)? + .eq_ignore_ascii_case("sell") + { + true => Side::Sell, + false => Side::Buy, + }; + Some(Tick { + time_ms, + price, + qty, + side, + }) +} + #[cfg(test)] mod tests; diff --git a/crates/moon-core/src/market/trade_replay/rest/bitget/tests.rs b/crates/moon-core/src/market/trade_replay/rest/bitget/tests.rs index b707b011..15b19218 100644 --- a/crates/moon-core/src/market/trade_replay/rest/bitget/tests.rs +++ b/crates/moon-core/src/market/trade_replay/rest/bitget/tests.rs @@ -1,5 +1,7 @@ use super::*; +use serde_json::json; + fn fixture(name: &str) -> Value { let text = match name { "spot" => include_str!("fixtures/spot_klines.json"), @@ -62,3 +64,37 @@ fn bitget_unknown_symbol_codes_are_permanent() { Err(FetchError::UnknownSymbol) ); } + +/// `rest/bitget.rs:parse_fills` treating a full, uncovered page with an unparseable oldest +/// `tradeId` as complete silently ships a truncated tick window as a complete chart. +#[test] +fn bitget_rejects_a_full_uncovered_page_without_a_cursor() { + let body = json!({ + "data": [ + {"price": "100", "size": "1", "ts": "2000", "side": "buy", "tradeId": "101"}, + {"price": "99", "size": "1", "ts": "1000", "side": "sell", "tradeId": "not-an-id"} + ] + }); + + assert!( + parse_fills(&body, 2, 0).is_err(), + "an uncovered full page cannot be complete when its continuation cursor is unknowable" + ); +} + +/// `rest/bitget.rs:parse_fills` dropping its raw-row guard accepts a partly malformed page and +/// draws a gap-ridden tick series as complete market history. +#[test] +fn bitget_rejects_a_page_with_any_unparseable_row() { + let body = json!({ + "data": [ + {"price": "100", "size": "1", "ts": "2000", "side": "buy", "tradeId": "101"}, + {"size": "1", "ts": "1000", "side": "sell", "tradeId": "100"} + ] + }); + + assert!( + parse_fills(&body, 2, 0).is_err(), + "a response with a malformed row is incomplete even when another row parsed" + ); +} diff --git a/crates/moon-core/src/market/trade_replay/rest/gateio.rs b/crates/moon-core/src/market/trade_replay/rest/gateio.rs index b19182ec..37ae1d89 100644 --- a/crates/moon-core/src/market/trade_replay/rest/gateio.rs +++ b/crates/moon-core/src/market/trade_replay/rest/gateio.rs @@ -10,9 +10,10 @@ use serde_json::Value; -use super::{FetchError, cell_f32, cell_i64}; +use super::{FetchError, TradeCursor, TradePage, cell_f32, cell_i64}; +use crate::feed::types::{Side, Tick}; use crate::market::candles::ChartCandle; -use crate::market::trade_replay::venue_caps::KlineRoute; +use crate::market::trade_replay::venue_caps::{KlineRoute, TradeRoute}; /// Milliseconds per second, for Gate's second-resolution window and timestamps. const MS_PER_S: i64 = 1_000; @@ -203,5 +204,245 @@ fn parse_futures_row(row: &Value) -> Option { }) } +/// Largest page number the SPOT trades endpoint's own cap allows. +/// +/// Gate documents `limit*(page-1) <= 100000` for `/spot/trades`; a page beyond that is refused. +/// Futures pagination is by row `offset` and carries no such cap. +const SPOT_PAGE_ROW_CAP: usize = 100_000; + +/// Fetch one page of public trades and return the decoded body. +/// +/// # Order is UNDOCUMENTED here, unlike the candle endpoint +/// +/// Both routes' pagination below therefore keys on the ROW COUNT alone, never on a row's own +/// timestamp: nothing here assumes ascending or descending order, and +/// [`super::super::worker::serve_ticks`] is what sorts the complete tick vector once every page +/// of a stage is in. +/// +/// Args: +/// agent: Shared client. +/// route: [`TradeRoute::GateSpotTrades`] or [`TradeRoute::GateFuturesTrades`]. +/// market: Exchange-native market name, e.g. `BTC_USDT`. +/// from_ms: First millisecond of this slice, inclusive. +/// to_ms: Last millisecond of this slice, inclusive. +/// cursor: Continuation from a previous page, or `None` for the first page. +/// +/// Returns: +/// The decoded response, or a classified failure. +pub(super) fn fetch_trades( + agent: &ureq::Agent, + route: TradeRoute, + market: &str, + from_ms: i64, + to_ms: i64, + cursor: Option, +) -> Result { + let futures = matches!(route, TradeRoute::GateFuturesTrades); + let market_param = match futures { + true => "contract", + false => "currency_pair", + }; + let mut request = agent + .get(route.url()) + .query(market_param, market) + .query("limit", route.max_rows().to_string()) + .query("from", (from_ms / MS_PER_S).to_string()) + .query("to", (to_ms / MS_PER_S).to_string()); + request = match (futures, cursor) { + (false, Some(TradeCursor::Page(page))) => request.query("page", page.to_string()), + (false, _) => request.query("page", "1"), + (true, Some(TradeCursor::Offset(offset))) => request.query("offset", offset.to_string()), + (true, _) => request.query("offset", "0"), + }; + let response = request + .call() + .map_err(|error| FetchError::Transient(error.to_string()))?; + super::decode_and_classify(response, "gate", classify) +} + +/// Parse a Gate SPOT trades array into a page of ticks. +/// +/// Args: +/// body: Decoded response. +/// max_rows: Row cap that was sent, so a FULL page can be told from a short, final one. +/// cursor: The cursor this request was sent with, so the NEXT page number is derived from it +/// rather than reconstructed. +/// +/// Returns: +/// The page, or a failure when the envelope is not an array. +pub(super) fn parse_spot_trades( + body: &Value, + max_rows: usize, + cursor: Option, +) -> Result { + let rows = body + .as_array() + .ok_or_else(|| FetchError::Transient("gate: spot response is not an array".to_string()))?; + let ticks: Vec = rows.iter().filter_map(parse_spot_trade_row).collect(); + // A page holding a malformed row alongside valid ones can still finish pagination with a + // non-empty tick vector that is silently missing rows — a hole must send the window to + // candles instead of drawing a partial tape as if it were whole. + if ticks.len() < rows.len() { + return Err(FetchError::Transient(format!( + "gate: spot page held {} unparseable row(s) of {} (parsed {})", + rows.len() - ticks.len(), + rows.len(), + ticks.len() + ))); + } + let page = match cursor { + Some(TradeCursor::Page(page)) => page, + _ => 1, + }; + let next_page = page.saturating_add(1); + let within_cap = + (next_page.saturating_sub(1) as usize).saturating_mul(max_rows) <= SPOT_PAGE_ROW_CAP; + let next = match rows.len() >= max_rows && within_cap { + true => Some(TradeCursor::Page(next_page)), + false => None, + }; + Ok(TradePage { ticks, next }) +} + +/// Parse one Gate spot trade row. +/// +/// Args: +/// row: One element of the response array. +/// +/// Returns: +/// The tick, or `None` when the row is malformed. +fn parse_spot_trade_row(row: &Value) -> Option { + let price = cell_f32(row.get("price")?)?; + let qty = cell_f32(row.get("amount")?)?; + // Prefer the sub-second `create_time_ms`; fall back to `create_time`, which is SECONDS. + let time_ms = row + .get("create_time_ms") + .and_then(Value::as_str) + .and_then(|s| s.parse::().ok()) + .or_else(|| { + row.get("create_time") + .and_then(cell_i64) + .map(|s| (s * MS_PER_S) as f64) + })?; + let side = match row.get("side").and_then(Value::as_str)? { + "sell" => Side::Sell, + _ => Side::Buy, + }; + Some(Tick { + time_ms, + price, + qty, + side, + }) +} + +/// Parse a Gate USDT-perpetual trades array into a page of ticks. +/// +/// # No `side` field — the SIGN of `size` is the side +/// +/// Positive `size` is a buyer-taker fill and negative a seller-taker one; the magnitude is the +/// contract quantity. **Assumption, not settled by vendor docs read for this task**: which sign +/// means which side. Every other Gate/Binance/Bitget/OKX parser in this module reads an explicit +/// `side`/`m` field; this is the one route with no such field at all, so the mapping below is +/// inferred from the vendor's own field name (`size`, signed) rather than confirmed against a +/// recorded response, and it is the first place to look if a Gate futures tick chart shows every +/// trade on the wrong side. +/// +/// A FULL page is ALWAYS treated as incomplete regardless of any other signal: this endpoint +/// truncates SILENTLY at `limit` with no error, so a full page means "ask again", never "that was +/// all" — see [`super::super::rest::TradePage::next`]'s own doc for why that rule is frozen. +/// +/// Args: +/// body: Decoded response. +/// max_rows: Row cap that was sent. +/// cursor: The cursor this request was sent with, so the next `offset` is derived from it. +/// +/// Returns: +/// The page, or a failure when the envelope is not an array. +pub(super) fn parse_futures_trades( + body: &Value, + max_rows: usize, + cursor: Option, +) -> Result { + let rows = body.as_array().ok_or_else(|| { + FetchError::Transient("gate: futures response is not an array".to_string()) + })?; + let ticks: Vec = rows.iter().filter_map(parse_futures_trade_row).collect(); + // A page holding a malformed row alongside valid ones can still finish pagination with a + // non-empty tick vector that is silently missing rows — a hole must send the window to + // candles instead of drawing a partial tape as if it were whole. + if ticks.len() < rows.len() { + return Err(FetchError::Transient(format!( + "gate: futures page held {} unparseable row(s) of {} (parsed {})", + rows.len() - ticks.len(), + rows.len(), + ticks.len() + ))); + } + let offset = match cursor { + Some(TradeCursor::Offset(offset)) => offset, + _ => 0, + }; + let next = match rows.len() >= max_rows { + true => Some(TradeCursor::Offset( + offset.saturating_add(rows.len() as u32), + )), + false => None, + }; + Ok(TradePage { ticks, next }) +} + +/// Parse one Gate futures trade row. +/// +/// **Unit note**: `size` is a signed CONTRACT count, not a base-currency amount — its base +/// amount depends on the contract's `quanto_multiplier`, served by a different endpoint, so no +/// base-asset amount is available here and `Tick::qty` holds contracts for this route. The +/// chart's volume bars stay shape-correct regardless: the chart normalises the visible window +/// against its own maximum `qty`, and a per-instrument contract multiplier is a constant that +/// cancels out of that normalisation — only an ABSOLUTE volume figure would be wrong, and a tick +/// series' aggregated candles never reach the shared SQLite kline cache where one could be read. +/// See `venue_caps.rs`'s trade-route table for the same note. +/// +/// `size` is checked for finiteness and sign on the NARROWED `f32`, not the `f64` behind it, +/// exactly as [`cell_f32`] does and for the same reason: narrowing is itself a way to become +/// non-finite, so a check applied before the cast can pass a value that becomes infinite after +/// it. +/// +/// Args: +/// row: One element of the response array. +/// +/// Returns: +/// The tick, or `None` when the row is malformed. +fn parse_futures_trade_row(row: &Value) -> Option { + let price = cell_f32(row.get("price")?)?; + let size_raw = match row.get("size")? { + Value::String(text) => text.parse::().ok()?, + other => other.as_f64()?, + }; + let qty = size_raw.abs() as f32; + if !(qty.is_finite() && qty > 0.0) { + return None; + } + let side = match size_raw.is_sign_negative() { + true => Side::Sell, + false => Side::Buy, + }; + let time_ms = row + .get("create_time_ms") + .and_then(cell_i64) + .map(|v| v as f64) + .or_else(|| { + row.get("create_time") + .and_then(cell_i64) + .map(|s| (s * MS_PER_S) as f64) + })?; + Some(Tick { + time_ms, + price, + qty, + side, + }) +} + #[cfg(test)] mod tests; diff --git a/crates/moon-core/src/market/trade_replay/rest/mod.rs b/crates/moon-core/src/market/trade_replay/rest/mod.rs index 85b51f33..75dfff42 100644 --- a/crates/moon-core/src/market/trade_replay/rest/mod.rs +++ b/crates/moon-core/src/market/trade_replay/rest/mod.rs @@ -38,7 +38,8 @@ use std::time::Duration; use serde_json::Value; -use super::venue_caps::KlineRoute; +use super::venue_caps::{KlineRoute, TradeRoute}; +use crate::feed::types::Tick; use crate::market::candles::ChartCandle; /// Bounded lifetime of one HTTP request. @@ -149,6 +150,100 @@ pub fn fetch_klines( Ok(rows) } +/// Continuation token for one public-trade page. +/// +/// FIVE variants, because the venues genuinely paginate five ways and abusing one venue's +/// semantics for another silently truncates a window: Binance walks forward by aggregate-trade +/// id; OKX and Bitget walk BACKWARD by trade id; Gate spot +/// walks by 1-based `page`; Gate futures walks by row `offset`. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum TradeCursor { + /// Binance: next aggregate-trade id to ask for, walking forward. + FromId(u64), + /// Timestamp-bound `after` cursor. No current trade route emits it; OKX switches to + /// [`Self::LessThanId`] after its initial timestamp-bound request. + AfterMs(i64), + /// Bitget: next `idLessThan` bound, walking backward. + LessThanId(u64), + /// Gate spot: next 1-based page number. + Page(u32), + /// Gate futures: next row offset. + Offset(u32), +} + +/// One fetched page of public trades. +#[derive(Clone, Debug)] +pub struct TradePage { + /// Rows in the vendor's OWN order — never sorted here. The global ascending sort and window + /// clip belong to [`super::worker::serve_ticks`], after every page of a stage is in. + pub ticks: Vec, + /// Continuation, Some ONLY when this page was FULL and the window is not yet covered. + /// + /// A full Gate futures page is NEVER accepted as complete: that endpoint truncates silently + /// at `limit` with no error, so a full page means "ask again", never "that was all". + pub next: Option, +} + +/// Fetch one page of public trades. +/// +/// The match below is deliberately exhaustive and carries no `_` arm, matching +/// [`fetch_klines`]'s own discipline. No `category` parameter, deliberately: [`fetch_klines`] +/// carries one only for [`KlineRoute::Bybit`], Bybit has NO trade route, and Bitget derives its +/// required `productType` from the route itself. A parameter with no valid consumer is a +/// boundary leak, not future-proofing. +/// +/// Args: +/// agent: Shared client. +/// route: Which endpoint family to ask. +/// market: Exchange-native market name, as the core reports it. +/// from_ms: First millisecond of this request's slice, inclusive. +/// to_ms: Last millisecond of this request's slice, inclusive. +/// cursor: Continuation from a previous page of this same slice, or `None` for the first. +/// +/// Returns: +/// One page of ticks in the vendor's own order, or a classified failure. +pub fn fetch_trades( + agent: &ureq::Agent, + route: TradeRoute, + market: &str, + from_ms: i64, + to_ms: i64, + cursor: Option, +) -> Result { + match route { + TradeRoute::BinanceSpotAggTrades + | TradeRoute::BinanceUsdMAggTrades + | TradeRoute::BinanceCoinMAggTrades => { + let value = binance::fetch_trades( + agent, + route, + market, + from_ms, + to_ms, + route.max_rows(), + cursor, + )?; + binance::parse_agg_trades(&value, to_ms, route.max_rows()) + } + TradeRoute::GateSpotTrades => { + let value = gateio::fetch_trades(agent, route, market, from_ms, to_ms, cursor)?; + gateio::parse_spot_trades(&value, route.max_rows(), cursor) + } + TradeRoute::GateFuturesTrades => { + let value = gateio::fetch_trades(agent, route, market, from_ms, to_ms, cursor)?; + gateio::parse_futures_trades(&value, route.max_rows(), cursor) + } + TradeRoute::BitgetSpotFills | TradeRoute::BitgetMixFills => { + let value = bitget::fetch_trades(agent, route, market, from_ms, to_ms, cursor)?; + bitget::parse_fills(&value, route.max_rows(), from_ms) + } + TradeRoute::OkxHistoryTrades => { + let value = okx::fetch_trades(agent, route, market, to_ms, route.max_rows(), cursor)?; + okx::parse_history_trades(&value, route.max_rows(), from_ms) + } + } +} + /// Decode one response body and put it through that venue's classifier. /// /// The five GET venues repeat this exact sequence, differing only in which classifier runs and in diff --git a/crates/moon-core/src/market/trade_replay/rest/okx.rs b/crates/moon-core/src/market/trade_replay/rest/okx.rs index fa683384..3f3609b8 100644 --- a/crates/moon-core/src/market/trade_replay/rest/okx.rs +++ b/crates/moon-core/src/market/trade_replay/rest/okx.rs @@ -5,9 +5,10 @@ use serde_json::Value; -use super::{FetchError, cell_f32, cell_i64}; +use super::{FetchError, TradeCursor, TradePage, cell_f32, cell_i64}; +use crate::feed::types::{Side, Tick}; use crate::market::candles::ChartCandle; -use crate::market::trade_replay::venue_caps::KlineRoute; +use crate::market::trade_replay::venue_caps::{KlineRoute, TradeRoute}; /// Positional index of the cell holding BASE-asset volume on a SPOT row. pub(super) const SPOT_VOLUME_CELL: usize = 5; @@ -168,5 +169,155 @@ fn parse_row(row: &Value, volume_cell: usize) -> Option { }) } +/// Fetch one page of public trades and return the decoded body. +/// +/// # No time range at all — the FIRST page anchors on a timestamp, every later one on a trade id +/// +/// `history-trades` has no `startTime`/`endTime` pair whatsoever, unlike the candle endpoint. The +/// FIRST page of a slice has nothing else to anchor to, so it uses `type=2&after=`, walking strictly backward from the slice's own right edge. +/// +/// Every LATER page switches to `type=1&after=` instead of continuing with +/// `type=2` at the previous page's oldest timestamp. `type=2`'s `after` is EXCLUSIVE of the given +/// millisecond, and a full 100-row page routinely ends in the middle of several trades sharing +/// one millisecond on a liquid pair; continuing from that same timestamp would silently drop +/// every sibling trade at the boundary, a hole `full`/`covered` never notice and that ships as a +/// complete window. A trade id has no such collision. +/// +/// Args: +/// agent: Shared client. +/// route: [`TradeRoute::OkxHistoryTrades`]. +/// market: Exchange-native instrument id, e.g. `BTC-USDT` or `BTC-USDT-SWAP`. +/// to_ms: Right edge of this slice, inclusive; the first page's `after` bound. +/// max_rows: Row cap for this request. +/// cursor: Continuation from a previous page, or `None` for the first page. +/// +/// Returns: +/// The decoded response, or a classified failure. +pub(super) fn fetch_trades( + agent: &ureq::Agent, + route: TradeRoute, + market: &str, + to_ms: i64, + max_rows: usize, + cursor: Option, +) -> Result { + let request = agent + .get(route.url()) + .query("instId", market) + .query("limit", max_rows.to_string()); + let request = match cursor { + Some(TradeCursor::LessThanId(id)) => { + request.query("type", "1").query("after", id.to_string()) + } + None => request.query("type", "2").query("after", to_ms.to_string()), + Some(_) => { + debug_assert!( + false, + "okx trade route hands back only LessThanId after the first page" + ); + request.query("type", "2").query("after", to_ms.to_string()) + } + }; + let response = request + .call() + .map_err(|error| FetchError::Transient(error.to_string()))?; + super::decode_and_classify(response, "okx", classify) +} + +/// Parse an OKX `history-trades` envelope into a page of ticks. +/// +/// Rows arrive DESCENDING (newest first, walking backward), so the oldest row in this page is +/// the LAST one — that is what both the next cursor and the window-covered check key on. +/// +/// Args: +/// body: Decoded response. +/// max_rows: Row cap that was sent, so a FULL page can be told from a short, final one. +/// from_ms: Left edge of the slice, so completeness is judged from the oldest row's own +/// timestamp rather than assumed from the row count alone. +/// +/// Returns: +/// The page, or a failure when the envelope is missing. +pub(super) fn parse_history_trades( + body: &Value, + max_rows: usize, + from_ms: i64, +) -> Result { + let rows = body + .get("data") + .and_then(Value::as_array) + .ok_or_else(|| FetchError::Transient("okx: missing data".to_string()))?; + let ticks: Vec = rows.iter().filter_map(parse_trade_row).collect(); + // A page holding a malformed row alongside valid ones can still finish pagination with a + // non-empty tick vector that is silently missing rows — a hole must send the window to + // candles instead of drawing a partial tape as if it were whole. + if ticks.len() < rows.len() { + return Err(FetchError::Transient(format!( + "okx: page held {} unparseable row(s) of {} (parsed {})", + rows.len() - ticks.len(), + rows.len(), + ticks.len() + ))); + } + let oldest_ms = rows.last().and_then(|r| r.get("ts")).and_then(cell_i64); + let oldest_id = rows + .last() + .and_then(|r| r.get("tradeId")) + .and_then(cell_i64) + .map(|v| v as u64); + let full = rows.len() >= max_rows; + let covered = oldest_ms.is_some_and(|t| t < from_ms); + let next = match (full, covered, oldest_id) { + // Continuations paginate by TRADE ID, never by timestamp — see `fetch_trades`'s doc for + // why the boundary millisecond is unsafe to resume from. + (true, false, Some(id)) => Some(TradeCursor::LessThanId(id)), + // Full page, window not covered, but the oldest row's own `tradeId` did not parse: the + // cursor to continue from is unknowable. Treating this as completion would silently ship + // a truncated window as a whole one. + (true, false, None) => { + return Err(FetchError::Transient( + "okx: full page, window not covered, but the oldest row's tradeId did not parse" + .to_string(), + )); + } + _ => None, + }; + Ok(TradePage { ticks, next }) +} + +/// Parse one OKX `history-trades` row. +/// +/// **Unit note**: `sz` is a base-asset amount for SPOT but a CONTRACT COUNT for swap/futures — +/// [`TradeRoute::OkxHistoryTrades`] serves both markets through this one parser, unlike the +/// kline sibling, which is deliberately split into `OkxSpot`/`OkxSwap` for exactly this reason +/// (see `venue_caps.rs`'s doc on `OkxSwap`). No base-asset amount is available from this endpoint +/// for a swap instrument, so `Tick::qty` holds contracts there. The chart's volume bars stay +/// shape-correct regardless: the chart normalises the visible window against its own maximum +/// `qty`, and a per-instrument contract multiplier is a constant that cancels out of that +/// normalisation — only an ABSOLUTE volume figure would be wrong, and a tick series' aggregated +/// candles never reach the shared SQLite kline cache where one could be read. See +/// `venue_caps.rs`'s trade-route table for the same note. +/// +/// Args: +/// row: One element of `data`. +/// +/// Returns: +/// The tick, or `None` when the row is malformed. +fn parse_trade_row(row: &Value) -> Option { + let price = cell_f32(row.get("px")?)?; + let qty = cell_f32(row.get("sz")?)?; + let time_ms = cell_i64(row.get("ts")?)? as f64; + let side = match row.get("side").and_then(Value::as_str)? { + "sell" => Side::Sell, + _ => Side::Buy, + }; + Some(Tick { + time_ms, + price, + qty, + side, + }) +} + #[cfg(test)] mod tests; diff --git a/crates/moon-core/src/market/trade_replay/tests.rs b/crates/moon-core/src/market/trade_replay/tests.rs index 0dd6033c..c1bbbd72 100644 --- a/crates/moon-core/src/market/trade_replay/tests.rs +++ b/crates/moon-core/src/market/trade_replay/tests.rs @@ -21,6 +21,8 @@ fn bars_only_series() -> TradeReplaySeries { window: ReplayWindow { from_ms: 0, to_ms: 2 * MINUTE_MS, + open_ms: 0, + close_ms: 2 * MINUTE_MS, over_budget: false, }, tf_ms: MINUTE_MS, @@ -31,6 +33,9 @@ fn bars_only_series() -> TradeReplaySeries { ], ticks: Vec::new(), identity: 42, + tick_status: TickStatus::Pending, + bucket_ms: 0, + partial: false, } } @@ -158,6 +163,8 @@ fn cache_coverage_rejects_prefixes_and_oversized_holes() { let window = ReplayWindow { from_ms: 0, to_ms: 5 * MINUTE_MS, + open_ms: 0, + close_ms: 5 * MINUTE_MS, over_budget: false, }; let exact = [0, 1, 2, 3, 4, 5] @@ -348,3 +355,212 @@ fn replay_window_keeps_trade_and_floors_when_trimming_the_budget() { "replay_window discarding floors above MAX_SPAN_MS would hide that the request exceeds its budget" ); } + +/// `market/trade_replay/mod.rs:TradeReplaySeries::read_into` dropping the source identity salt, +/// or salting `Klines1m`, makes a tick upgrade leave exchange candles on screen or makes every +/// existing replay look changed to the chart. +#[test] +fn tick_and_kline_replays_keep_distinct_revisions_without_changing_kline_revision() { + let kline = bars_only_series(); + let mut ticks = kline.clone(); + ticks.source = TradeReplaySource::Ticks; + ticks.ticks = vec![crate::feed::types::Tick { + time_ms: MINUTE_MS as f64, + price: 101.0, + qty: 2.0, + side: crate::feed::types::Side::Buy, + }]; + + let mut kline_out = ChartHistoryBuffers::default(); + let kline_read = kline.read_into( + 0.0, + 0.0, + (2 * MINUTE_MS) as f32, + Some(&candle_params(0)), + &mut kline_out, + ); + let mut tick_out = ChartHistoryBuffers::default(); + let tick_read = ticks.read_into( + 0.0, + 0.0, + (2 * MINUTE_MS) as f32, + Some(&candle_params(0)), + &mut tick_out, + ); + + let unchanged_kline_revision = replay_revision(kline.identity, MINUTE_MS, 0, 2); + assert_eq!( + kline_read.revision, unchanged_kline_revision, + "Klines1m keeps the established replay revision for the same identity and window" + ); + assert_ne!( + tick_read.revision, kline_read.revision, + "a tick upgrade must force its aggregated candles to replace already shipped klines" + ); +} + +/// `market/trade_replay/mod.rs:time_slices` representing an unlimited query span as a saturating +/// integer can make pagination step backwards forever, hanging the sole replay worker and every +/// later trade-detail window. +#[test] +fn time_slices_keeps_unbounded_windows_whole_and_bounded_windows_gap_free() { + let window = ReplayWindow { + from_ms: 1_000, + to_ms: 7_200_999, + open_ms: 1_000, + close_ms: 7_200_999, + over_budget: false, + }; + + assert_eq!( + time_slices(window, None), + vec![(window.from_ms, window.to_ms)], + "an unlimited route issues one request for precisely its requested window" + ); + + let span_ms = 3_600_000; + let slices = time_slices(window, Some(span_ms)); + assert_eq!( + slices.first().copied(), + Some((window.from_ms, window.from_ms + span_ms - 1)), + "the first bounded request starts at the requested left edge and consumes one legal span" + ); + assert_eq!( + slices.last().copied().map(|(_, end)| end), + Some(window.to_ms), + "the final bounded request reaches the requested right edge" + ); + assert!( + slices + .iter() + .all(|(start, end)| end >= start && end - start < span_ms), + "each slice stays strictly within the documented exclusive maximum span" + ); + assert!( + slices.windows(2).all(|pair| pair[0].1 + 1 == pair[1].0), + "adjacent requests neither leave a market-data gap nor re-fetch a boundary millisecond" + ); +} + +/// `market/trade_replay/mod.rs:tick_plan` sorting tiles by clock time instead of focus-first +/// spends the budget on lead context and makes a partial replay omit the trade itself. +#[test] +fn tick_plan_prioritizes_focus_and_keeps_every_prefix_contiguous_after_clipping() { + let window = replay_window(100_000, 100_000).expect("a same-second scalp has floor context"); + let earliest_ms = window.from_ms + 20 * MINUTE_MS; + let plan = tick_plan(window, Some(60 * MINUTE_MS), Some(earliest_ms)); + let focus = window.focus(); + + assert!( + plan.focus_len > 0, + "the retained focus must have at least one slice" + ); + assert!( + plan.slices[0].0 <= window.open_ms && window.open_ms <= plan.slices[0].1, + "the 80-minute scalp's entry belongs to the very first fetched slice" + ); + let focus_slices = &plan.slices[..plan.focus_len]; + assert_eq!( + focus_slices.first().map(|slice| slice.0), + Some(focus.0), + "the focus prefix begins at the independently derived focus edge" + ); + assert_eq!( + focus_slices.last().map(|slice| slice.1), + Some(focus.1), + "the focus prefix reaches the independently derived focus edge" + ); + assert!( + plan.slices.iter().all(|(from, _)| *from >= earliest_ms), + "retention clipping must exclude tiles older than the route can answer" + ); + for prefix_len in 1..=plan.slices.len() { + let mut prefix = plan.slices[..prefix_len].to_vec(); + prefix.sort_unstable(); + assert!( + prefix.windows(2).all(|pair| pair[0].1 + 1 == pair[1].0), + "prefix {prefix_len} must form one gap-free covered interval rather than a comb" + ); + } +} + +/// `market/trade_replay/mod.rs:fit_ticks` dropping its terminal stride or thinning an already +/// fitting input can overflow the GPU ring or alter raw trade points without need. +#[test] +fn fit_ticks_obeys_every_budget_boundary_and_preserves_raw_inputs_that_fit() { + let raw = (0..10) + .map(|index| crate::feed::types::Tick { + time_ms: (index * 100) as f64, + price: 100.0 + index as f32, + qty: 1.0, + side: crate::feed::types::Side::Buy, + }) + .collect::>(); + + for budget in [0, 1, 2, 3] { + let (result, _) = fit_ticks(raw.clone(), budget); + assert!( + result.len() <= budget, + "a ten-row input must never exceed requested budget {budget}" + ); + } + for budget in [10, 11] { + let (result, bucket_ms) = fit_ticks(raw.clone(), budget); + assert_eq!( + result + .iter() + .map(|tick| (tick.time_ms, tick.price, tick.qty)) + .collect::>(), + raw.iter() + .map(|tick| (tick.time_ms, tick.price, tick.qty)) + .collect::>(), + "a raw vector that fits budget {budget} stays unchanged" + ); + assert_eq!(bucket_ms, 0, "an already fitting vector reports raw ticks"); + } + let (empty, bucket_ms) = fit_ticks(Vec::new(), 0); + assert!(empty.is_empty(), "empty input stays empty at zero budget"); + assert_eq!(bucket_ms, 0, "empty input already fits and remains raw"); +} + +/// `market/trade_replay/mod.rs:tick_identity_salt` including tick-status payload makes identical +/// candle rows re-upload merely because a tick attempt changed from pending to failed. +#[test] +fn kline_tick_statuses_keep_the_same_chart_revision_while_ticks_change_it() { + let pending = bars_only_series(); + let mut failed = pending.clone(); + failed.tick_status = TickStatus::Failed; + let mut tick_upgrade = pending.clone(); + tick_upgrade.source = TradeReplaySource::Ticks; + tick_upgrade.tick_status = TickStatus::Served; + tick_upgrade.ticks = vec![crate::feed::types::Tick { + time_ms: MINUTE_MS as f64, + price: 101.0, + qty: 1.0, + side: crate::feed::types::Side::Buy, + }]; + + let read_revision = |series: &TradeReplaySeries| { + let mut out = ChartHistoryBuffers::default(); + series + .read_into( + 0.0, + 0.0, + (2 * MINUTE_MS) as f32, + Some(&candle_params(0)), + &mut out, + ) + .revision + }; + + assert_eq!( + read_revision(&pending), + read_revision(&failed), + "pending and failed candle fallbacks carry identical rows and therefore one revision" + ); + assert_ne!( + read_revision(&pending), + read_revision(&tick_upgrade), + "a tick upgrade must have its own revision so the pane uploads its new points" + ); +} diff --git a/crates/moon-core/src/market/trade_replay/venue_caps.rs b/crates/moon-core/src/market/trade_replay/venue_caps.rs index 4c174aa1..3bac6332 100644 --- a/crates/moon-core/src/market/trade_replay/venue_caps.rs +++ b/crates/moon-core/src/market/trade_replay/venue_caps.rs @@ -64,6 +64,40 @@ //! distinguish an unknown coin from an outage**: both answer HTTP 500 with a body of literally //! `null`, so its classifier calls every failure transient, which is the direction that caches //! nothing. +//! +//! # The trade-route directory, in one table +//! +//! [`TradeRoute`] is the same idea one level down: which venues can answer a public "trades +//! between T1 and T2" question, for the trade-detail window's tick replay. Every row was read off +//! the vendor's own documentation, same discipline as the kline table above. Bybit, Hyperliquid +//! and HTX document no such endpoint reachable from this build, so [`trade_route`] answers +//! [`None`] for all three — the same honest degradation [`kline_route`] already gives HTX. +//! +//! | route | endpoint | gate key | rows/page | max query window | retention | cursor | order | unknown-symbol | doc | +//! |---|---|---|---|---|---|---|---|---|---| +//! | `BinanceSpotAggTrades` | `/api/v3/aggTrades` | `data-api.binance.vision` | 1000 | none documented | none documented | `fromId` | undocumented | HTTP 4xx `-1121` | developers.binance.com/docs/binance-spot-api-docs/rest-api/market-data-endpoints | +//! | `BinanceUsdMAggTrades` | `/fapi/v1/aggTrades` | `fapi.binance.com` | 1000 | **< 1 h** | **48 h** | `fromId` | undocumented | HTTP 4xx `-1121` | developers.binance.com/.../derivatives/usds-margined-futures/market-data/rest-api/Compressed-Aggregate-Trades-List | +//! | `BinanceCoinMAggTrades` | `/dapi/v1/aggTrades` | `dapi.binance.com` | 1000 | **< 1 h** | **48 h** | `fromId` | undocumented | HTTP 4xx `-1121` | developers.binance.com/.../derivatives/coin-margined-futures/market-data/rest-api/Compressed-Aggregate-Trades-List | +//! | `GateSpotTrades` | `/api/v4/spot/trades` | `api.gateio.ws` | 1000 | none (page cap `limit*(page-1) <= 100000`) | ~30 d | `from`/`to` in **SECONDS** + `page` | undocumented | label `INVALID_CURRENCY_PAIR` | gateio/gateapi-python docs/SpotApi.md | +//! | `GateFuturesTrades` | `/api/v4/futures/usdt/trades` | `api.gateio.ws` | **undocumented, default 100** | none | none documented | `from`/`to` in **SECONDS** + `offset` | undocumented | label `CONTRACT_NOT_FOUND` | gateio/gateapi-python docs/FuturesApi.md | +//! | `BitgetSpotFills` | `/api/v2/spot/market/fills-history` | `api.bitget.com` | 1000 | **7 d** | **90 d** | `idLessThan` | **desc** | envelope `code != "00000"` | bitget.com/api-doc/classic/spot/market/Get-Market-Trades | +//! | `BitgetMixFills` | `/api/v2/mix/market/fills-history` | `api.bitget.com` | 1000 | **7 d** | **90 d** | `idLessThan` (+`productType` required) | **desc** | envelope `code != "00000"` | bitget.com/api-doc/classic/contract/market/Get-Fills-History | +//! | `OkxHistoryTrades` | `/api/v5/market/history-trades` | `www.okx.com` | **100** | none (no `startTime`/`endTime` at all) | **3 months** | `type=2&after=` for the FIRST page, `type=1&after=` for every later one | **desc** | HTTP 200 + `code 51001` | okx.com/docs-v5/en/#order-book-trading-market-data-get-trades-history | +//! +//! Two venue facts that are silently wrong when mistaken: **Binance futures (both arms) retain +//! only 48 hours of aggTrades**, so the retention check must run before any request is spent; and +//! **Gate's `from`/`to` are in SECONDS**, while every other timestamp in this module is +//! milliseconds. +//! +//! **Three trade routes report a CONTRACT count where `Tick::qty` is documented as base-currency +//! quantity**, and this is deliberate rather than an oversight: `OkxHistoryTrades` +//! (SWAP instruments only — `sz` is base currency for SPOT), `BinanceCoinMAggTrades` (`q` on a +//! dapi row, with no `baseQty` alternative), and `GateFuturesTrades` (`size`, whose base amount +//! depends on the contract's `quanto_multiplier`). No code compensates for this: the chart's +//! volume bars stay shape-correct because the drawn scale is window-relative and a per-instrument +//! multiplier is a constant that cancels out of it, and a tick series' aggregated candles never +//! reach the shared kline cache where an absolute figure could be read as genuine history. See +//! each route's own parser for the fact restated in the vendor's own terms. use crate::venue::{Brand, MarketKind, Venue}; @@ -215,6 +249,178 @@ impl KlineRoute { } } +/// Route serving PUBLIC individual trades over a bounded past window. +/// +/// A variant is a REQUEST SHAPE, exactly as [`KlineRoute`] is one level up: the host, the query +/// grammar, the pagination cursor and the response envelope differ per family, and [`super::rest`] +/// matches on this to build and parse the call. Coverage is narrower than [`KlineRoute`]'s on +/// purpose — Bybit and Hyperliquid document no public trade-history endpoint this build can use, +/// and HTX inherits [`kline_route`]'s own reason for answering [`None`]. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub enum TradeRoute { + /// `GET https://data-api.binance.vision/api/v3/aggTrades`. + BinanceSpotAggTrades, + /// `GET https://fapi.binance.com/fapi/v1/aggTrades` — retains only 48h, `< 1h` per request. + BinanceUsdMAggTrades, + /// `GET https://dapi.binance.com/dapi/v1/aggTrades` — retains only 48h, `< 1h` per request. + BinanceCoinMAggTrades, + /// `GET https://api.gateio.ws/api/v4/spot/trades` — `from`/`to` in SECONDS. + GateSpotTrades, + /// `GET https://api.gateio.ws/api/v4/futures/usdt/trades` — `from`/`to` in SECONDS, truncates + /// silently at its page size with no error. + GateFuturesTrades, + /// `GET https://api.bitget.com/api/v2/spot/market/fills-history` — answers descending. + BitgetSpotFills, + /// `GET https://api.bitget.com/api/v2/mix/market/fills-history` — answers descending, needs + /// `productType`. + BitgetMixFills, + /// `GET https://www.okx.com/api/v5/market/history-trades` — spot and swap alike, no + /// `startTime`/`endTime` at all; paginates backward with `after` only. + OkxHistoryTrades, +} + +impl TradeRoute { + /// Return the fully qualified request URL for this route. + /// + /// Returns: + /// Absolute HTTPS endpoint, without any query string. + pub const fn url(self) -> &'static str { + match self { + Self::BinanceSpotAggTrades => "https://data-api.binance.vision/api/v3/aggTrades", + Self::BinanceUsdMAggTrades => "https://fapi.binance.com/fapi/v1/aggTrades", + Self::BinanceCoinMAggTrades => "https://dapi.binance.com/dapi/v1/aggTrades", + Self::GateSpotTrades => "https://api.gateio.ws/api/v4/spot/trades", + Self::GateFuturesTrades => "https://api.gateio.ws/api/v4/futures/usdt/trades", + Self::BitgetSpotFills => "https://api.bitget.com/api/v2/spot/market/fills-history", + Self::BitgetMixFills => "https://api.bitget.com/api/v2/mix/market/fills-history", + Self::OkxHistoryTrades => "https://www.okx.com/api/v5/market/history-trades", + } + } + + /// Rate-limit key, and it is DERIVED, never re-typed: every arm returns the corresponding + /// [`KlineRoute`]'s own [`KlineRoute::host`]. + /// + /// [`super::gate::ReplayGate`] keys pacing and refusal history by this literal, so a second + /// hand-typed copy of the same string is two production authorities that can drift on a later + /// endpoint change and split ONE real IP budget into two independent permits. Delegation makes + /// that drift unrepresentable rather than merely tested. + /// + /// Returns: + /// Bare host name, without scheme or path. + pub const fn host(self) -> &'static str { + match self { + Self::BinanceSpotAggTrades => KlineRoute::BinanceSpot.host(), + Self::BinanceUsdMAggTrades => KlineRoute::BinanceUsdM.host(), + Self::BinanceCoinMAggTrades => KlineRoute::BinanceCoinM.host(), + Self::GateSpotTrades => KlineRoute::GateSpot.host(), + Self::GateFuturesTrades => KlineRoute::GateFutures.host(), + Self::BitgetSpotFills => KlineRoute::BitgetSpot.host(), + Self::BitgetMixFills => KlineRoute::BitgetFutures.host(), + Self::OkxHistoryTrades => KlineRoute::OkxSpot.host(), + } + } + + /// Largest number of rows one request may ask for. + /// + /// Returns: + /// Maximum rows per request. + pub const fn max_rows(self) -> usize { + match self { + Self::BinanceSpotAggTrades + | Self::BinanceUsdMAggTrades + | Self::BinanceCoinMAggTrades => 1_000, + Self::GateSpotTrades => 1_000, + // UNDOCUMENTED: this is the MEASURED default page size, not a vendor-stated cap. + Self::GateFuturesTrades => 100, + Self::BitgetSpotFills | Self::BitgetMixFills => 1_000, + Self::OkxHistoryTrades => 100, + } + } + + /// Largest span one REQUEST may cover, or [`None`] when the vendor documents no cap. + /// + /// NEVER expressed as `i64::MAX`: [`None`] preserves the distinction between no documented + /// cap and a finite maximum span. + /// + /// Returns: + /// Widest request window in milliseconds, or `None` when unbounded. + pub const fn max_query_ms(self) -> Option { + const HOUR_MS: i64 = 3_600_000; + const DAY_MS: i64 = 24 * HOUR_MS; + match self { + Self::BinanceUsdMAggTrades | Self::BinanceCoinMAggTrades => Some(HOUR_MS), + Self::BitgetSpotFills | Self::BitgetMixFills => Some(7 * DAY_MS), + Self::BinanceSpotAggTrades + | Self::GateSpotTrades + | Self::GateFuturesTrades + | Self::OkxHistoryTrades => None, + } + } + + /// How far back the vendor documents that this endpoint answers, or [`None`] when it + /// documents no limit. + /// + /// Evaluated by [`super::worker::inside_retention`] BEFORE any request is spent, so an + /// out-of-retention window costs zero. + /// + /// Returns: + /// Retention window in milliseconds, or `None` when unbounded. + pub const fn retention_ms(self) -> Option { + const HOUR_MS: i64 = 3_600_000; + const DAY_MS: i64 = 24 * HOUR_MS; + match self { + Self::BinanceUsdMAggTrades | Self::BinanceCoinMAggTrades => Some(48 * HOUR_MS), + Self::GateSpotTrades => Some(30 * DAY_MS), + Self::BitgetSpotFills | Self::BitgetMixFills => Some(90 * DAY_MS), + Self::OkxHistoryTrades => Some(90 * DAY_MS), + Self::BinanceSpotAggTrades | Self::GateFuturesTrades => None, + } + } +} + +/// Return the public-trade route this venue is served by, if this build knows one. +/// +/// EVERY arm spelled out; no `_` catch-all, same discipline as [`kline_route`]. +/// +/// `(Binance, Quarterly)` maps to [`TradeRoute::BinanceCoinMAggTrades`], NOT to `None`: +/// `venue.rs:260-265` defines [`MarketKind::Quarterly`] as Binance COIN-M and `venue.rs:323-326` +/// maps the reachable `QBinance` code to it, so a blanket "Quarterly -> None" rule would make the +/// COIN-M route unreachable here exactly as it would in [`kline_route`]. +/// +/// [`None`] for Bybit, Hyperliquid, HTX — none document a public trade-history endpoint this +/// build can reach — and for the Gate / BitGet / OKX `Quarterly` arms, which name no product +/// those brands actually have, the same reason [`kline_route`] already gives them. +/// +/// Args: +/// venue: Venue resolved from the core's reported platform ordinal. +/// +/// Returns: +/// The route, or `None` when no verified public trade endpoint exists for it in this build. +pub const fn trade_route(venue: Venue) -> Option { + match (venue.brand, venue.kind) { + (Brand::Binance, MarketKind::Spot) => Some(TradeRoute::BinanceSpotAggTrades), + (Brand::Binance, MarketKind::Futures) => Some(TradeRoute::BinanceUsdMAggTrades), + (Brand::Binance, MarketKind::Quarterly) => Some(TradeRoute::BinanceCoinMAggTrades), + (Brand::Gate, MarketKind::Spot) => Some(TradeRoute::GateSpotTrades), + (Brand::Gate, MarketKind::Futures) => Some(TradeRoute::GateFuturesTrades), + (Brand::BitGet, MarketKind::Spot) => Some(TradeRoute::BitgetSpotFills), + (Brand::BitGet, MarketKind::Futures) => Some(TradeRoute::BitgetMixFills), + (Brand::Okx, MarketKind::Spot) | (Brand::Okx, MarketKind::Futures) => { + Some(TradeRoute::OkxHistoryTrades) + } + // Neither venue documents a public trade-history endpoint reachable from this build. + (Brand::Bybit, _) => None, + (Brand::Hyperliquid, _) => None, + // HTX has no trade route either, for the same reason `kline_route` gives it none. + (Brand::Htx, _) => None, + // `venue` yields no quarterly market for these three brands today; spelled out rather + // than folded into a catch-all for the same reason `kline_route` spells them out. + (Brand::Gate, MarketKind::Quarterly) + | (Brand::BitGet, MarketKind::Quarterly) + | (Brand::Okx, MarketKind::Quarterly) => None, + } +} + /// Return the one-minute-bar route this venue is served by, if this build knows one. /// /// Args: diff --git a/crates/moon-core/src/market/trade_replay/worker.rs b/crates/moon-core/src/market/trade_replay/worker.rs index 10ca68a7..047be1f9 100644 --- a/crates/moon-core/src/market/trade_replay/worker.rs +++ b/crates/moon-core/src/market/trade_replay/worker.rs @@ -17,6 +17,18 @@ //! worker, keyed by the exact question asked. That second cache is what actually satisfies "the //! second open of the same trade costs nothing": the SQLite cache cannot hold ticks at all, and //! nothing else in the process remembers that a given window was already answered. +//! +//! # The degrade ladder +//! +//! A tick stage never throws away what it already paid for. Cancellation is the one thing that +//! discards everything collected so far, because the window itself is gone; every other stop — +//! the page or tick budget, the job deadline, a venue's own refusal — instead SERVES what was +//! already fetched and names the reason in [`TradeReplaySeries::tick_status`], rather than +//! abandoning the whole stage and falling back to bars with no explanation. Only a harvest that +//! ends up genuinely empty reaches the candles-only outcome, and even then the bar layer drawn is +//! never blank: [`TickStage::candles`] carries the exchange's own one-minute klines forward from +//! the candle stage that ran first, so the window always has SOMETHING to show while the reasoned +//! caption explains what is missing and why. use std::collections::VecDeque; use std::sync::atomic::{AtomicBool, Ordering}; @@ -25,11 +37,12 @@ use std::sync::{Arc, Mutex, OnceLock}; use std::time::{Duration, Instant}; use super::gate::ReplayGate; -use super::venue_caps::{bybit_category, kline_route}; +use super::venue_caps::{TradeRoute, bybit_category, kline_route, trade_route}; use super::{ - ReplayWindow, TradeReplayEmpty, TradeReplayFailure, TradeReplayOutcome, TradeReplaySeries, - TradeReplaySource, pages, rest, + ReplayWindow, TickPlan, TickStatus, TradeReplayEmpty, TradeReplayFailure, TradeReplayOutcome, + TradeReplaySeries, TradeReplaySource, fit_ticks, pages, rest, tick_plan, }; +use crate::feed::types::Tick; use crate::market::candles::ChartCandle; use crate::market::kline_cache::{KlineCache, MergeItem}; use crate::market::source::ReplayAddress; @@ -58,6 +71,35 @@ const JOB_DEADLINE: Duration = Duration::from_secs(45); /// holds one bounded window's rows. const OUTCOME_CACHE_LEN: usize = 8; +/// Ceiling on the total number of ticks held across every remembered entry. +/// +/// A single entry can carry up to [`TICK_BUDGET`] ticks, and [`OUTCOME_CACHE_LEN`] entries of +/// that size would let the ring's own memory dwarf the point ring it feeds. This bounds the ring +/// independently of its entry count: eviction runs oldest-first, exactly as the entry-count +/// eviction does, and never touches the entry that was just inserted, so one huge series is held +/// rather than immediately discarded and re-fetched. +const OUTCOME_CACHE_MAX_TICKS: usize = 2 * TICK_BUDGET; + +/// Bounds the COMPOSED series and the outcome ring for one tick series — never the in-flight +/// fetch, which is bounded instead by [`TICK_PAGE_BUDGET`] times a route's own page size. A +/// budget crossed while paginating STOPS the walk and serves what is already held rather than +/// discarding it (see the module header's degrade ladder), so this constant ceilings what gets +/// drawn and remembered, not what a stage may fetch before giving up. +/// +/// Sits under the live chart's default `trades_limit` of 50 000 (`candles.rs:93`), so a tick +/// replay never asks the point ring for more than the main chart already draws. +pub(crate) const TICK_BUDGET: usize = 40_000; + +/// Bounds WALL TIME on the single worker thread for one tick stage. +/// +/// 60 pages at [`super::gate::ReplayGate::pace`]'s 100 ms floor plus a ~250 ms round trip is an +/// ORDER-OF-MAGNITUDE bound of a few tens of seconds, inside [`JOB_DEADLINE`] with room for a slow +/// venue. Not a precise figure: [`tick_plan`] now tiles the window into many small slices rather +/// than the one-or-two wide ones this constant was first sized against, and a quiet-market tile +/// still costs one round trip apiece, so the true page count for a given window depends on how +/// many tiles it takes as much as on how much data each holds. +const TICK_PAGE_BUDGET: usize = 60; + /// What one answered question is remembered as. /// /// An authoritative EMPTY is an answer too, and a valuable one: a delisted or halted market @@ -66,7 +108,13 @@ const OUTCOME_CACHE_LEN: usize = 8; #[derive(Clone, Debug)] enum Remembered { /// Rows to draw. - Ready(TradeReplaySeries), + Ready { + series: TradeReplaySeries, + /// Whether these rows are already a SETTLED tick series, so a reopen never re-asks for + /// ticks it already has, and a fresh entry (candles only, no tick attempt made yet) still + /// earns one. + ticks_settled: bool, + }, /// The venue answered and its answer held nothing in this window. Empty, } @@ -93,6 +141,135 @@ struct OutcomeKey { to_ms: i64, } +/// Which route, cache key and bar layer a queued tick stage answers. +/// +/// Carried on [`Job::Ticks`] alongside the original [`TradeReplayRequest`] rather than re-derived +/// when the stage finally runs, so a stage queued behind a long line of candle jobs answers the +/// same question `serve` decided it should — never a re-lookup that could disagree. `candles` is +/// the exchange klines `serve` already composed for this window: carrying them here removes the +/// dependence on the outcome ring not having evicted this key's entry between the two jobs, and is +/// what lets a tick outcome keep the bar layer whole even where its own points, per +/// [`TradeReplaySeries::partial`], cover only part of the window. +#[derive(Clone, Debug)] +pub(crate) struct TickStage { + /// Which venue endpoint to ask. + route: TradeRoute, + /// The ring key this stage's answer replaces on success. + key: OutcomeKey, + /// The exchange klines to carry forward as the bar layer of the eventual tick series. + candles: Vec, +} + +/// One unit of the worker's internal priority queue. +/// +/// A candle job and its own tick upgrade are two separate units on purpose: queuing the tick +/// stage inline would make a second report-row double-click wait behind it for its OWN candles — +/// see [`next_job`], which is what keeps candle jobs strictly ahead. +pub(crate) enum Job { + Candles(TradeReplayRequest), + Ticks(TradeReplayRequest, TickStage), +} + +/// Pop the next unit of work: any pending [`Job::Candles`] strictly ahead of every +/// [`Job::Ticks`], oldest first within each kind. +/// +/// Args: +/// queue: The worker's own pending-work deque. +/// +/// Returns: +/// The next job to run, or `None` when the queue is empty. +fn next_job(queue: &mut VecDeque) -> Option { + match queue.iter().position(|job| matches!(job, Job::Candles(_))) { + Some(index) => queue.remove(index), + None => queue.pop_front(), + } +} + +/// Why a tick stage's walk stopped without a usable harvest, or was cancelled outright. +/// +/// `Cancelled` throws away whatever was collected because the window itself closed. Every other +/// arm here is reached only when the harvest that stopped for that reason turned out EMPTY — +/// a non-empty one is served instead, whatever the stop reason was; see [`paginate_ticks`]. Each +/// arm is a DIFFERENT log line and a different test. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum TickAbandon { + Cancelled, + Deadline, + Transient, + Empty, + UnknownSymbol, + OverPageBudget, + /// The stage's own [`TickObserver::claim`] was refused: an active refusal is already + /// recorded for this host by some other request, and the tick stage must respect it rather + /// than send anyway on the strength of a candle stage's claim that already cleared. + RateLimited, +} + +/// What one tick stage's walk produced when it did produce something. +#[derive(Debug)] +pub(crate) struct TickHarvest { + /// Ticks collected, in the vendor's own per-page order within each slice — the global sort + /// and the clip to [`Self::covered`] both happen in [`serve_ticks`] after this returns, so a + /// test can hand in DESCENDING pages and observe that the SORT, not the pagination, is what + /// fixes them. + pub ticks: Vec, + /// The inclusive time range [`Self::ticks`] is guaranteed exhaustive over — [`serve_ticks`] + /// clips to this rather than to the request window, since a walk cut short still holds a + /// complete answer for the slices it actually finished. + pub covered: (i64, i64), + /// Whether every slice of the plan was walked to completion. + pub complete: bool, + /// Whether the walk stopped because the venue itself refused (`Transient`/`UnknownSymbol`), + /// as opposed to our own budget or the caller cancelling — see [`serve_ticks`]'s gate-clear. + pub venue_refused: bool, +} + +/// What one tick stage's walk produced. +/// +/// `Ready` carries the harvest exactly as walked; `Abandoned` carries the reason nothing usable +/// resulted. See [`TickHarvest`] and [`TickAbandon`]. +#[derive(Debug)] +pub(crate) enum TickVerdict { + Ready(TickHarvest), + Abandoned(TickAbandon), +} + +/// Records the gate calls one tick stage makes, so a test can assert exactly one claim per stage +/// and exactly one pace per fetched page, without a network or a real gate. +/// +/// `claim` takes a REAL send permit rather than merely observing one: the candle stage that ran +/// immediately before this one claimed and cleared its OWN permit already, and neither a +/// cache-answered candle stage nor a memory-ring reopen ever reaches `gate.claim` at all, so a +/// tick stage that trusted that prior claim would send its up-to-60 requests blind to an active +/// refusal recorded for this host by any other request — the exact escalation-to-ban path +/// [`ReplayGate`] exists to prevent. +pub(crate) trait TickObserver { + fn claim(&mut self, host: &str) -> Result<(), u32>; + fn pace(&mut self, host: &str); +} + +/// Bridges the pure [`TickObserver`] seam to the real [`ReplayGate`] for production use. +/// +/// Holds its own `host` rather than trusting the one handed to each call: [`TickObserver`]'s +/// methods take `&str` so the pure seam stays free of a lifetime a test double has no reason to +/// carry, while [`ReplayGate::claim`] and [`ReplayGate::pace`] need the `'static` the route +/// itself already guarantees. Pinning it at construction resolves that without widening the +/// trait's parameter type. +struct GateObserver<'a> { + gate: &'a ReplayGate, + host: &'static str, +} + +impl TickObserver for GateObserver<'_> { + fn claim(&mut self, _host: &str) -> Result<(), u32> { + self.gate.claim(self.host, Instant::now()) + } + + fn pace(&mut self, _host: &str) { + self.gate.pace(self.host); + } +} + /// One replay request. pub struct TradeReplayRequest { /// Exchange addressing resolved from the live source before the request was queued. @@ -143,7 +320,13 @@ pub fn request(request: TradeReplayRequest) { } } -/// Worker loop: one request at a time, forever. +/// Worker loop: an internal priority queue, forever. +/// +/// One request produces up to two jobs, run at different priorities rather than back to back — +/// see [`next_job`] for why an inline tick stage would break the first outcome's own promise. +/// Every iteration blocks on [`Receiver::recv`] only when the queue is empty; otherwise every +/// already-queued request is drained non-blockingly first, so a burst of report-row clicks is +/// batched into the queue before priority is applied rather than served one at a time. /// /// Args: /// rx: Queue of pending requests. @@ -151,18 +334,100 @@ fn run(rx: &Receiver) { let agent = rest::agent(); let gate = ReplayGate::new(); let cache: Mutex> = Mutex::new(VecDeque::new()); - while let Ok(request) = rx.recv() { - // A window that closed while its request sat in the queue costs nothing at all: this is - // the cheapest of the three cancellation guards and the only one that prevents the work. - if request.cancel.load(Ordering::Relaxed) { + let mut queue: VecDeque = VecDeque::new(); + loop { + if queue.is_empty() { + match rx.recv() { + Ok(request) => queue.push_back(Job::Candles(request)), + // Every sender lives inside `WORKER`, which is never dropped, so this is + // unreachable in practice; exiting is the honest answer if it ever happens. + Err(_) => return, + } + } + while let Ok(request) = rx.try_recv() { + queue.push_back(Job::Candles(request)); + } + let Some(job) = next_job(&mut queue) else { continue; + }; + match job { + Job::Candles(request) => { + // A window that closed while its request sat in the queue costs nothing at all: + // this is the cheapest of the three cancellation guards and the only one that + // prevents the work. + if request.cancel.load(Ordering::Relaxed) { + continue; + } + let served = serve(&agent, &gate, &cache, &request); + // The receiver is gone whenever the window closed mid-fetch. Normal, not an + // error — and exactly the signal that a queued tick stage would now answer no + // one, so it is never queued on a failed send. + let sent = request.reply.send(served.outcome).is_ok(); + if sent { + if let Some(stage) = served.tick_stage { + queue.push_back(Job::Ticks(request, stage)); + } + } + } + Job::Ticks(request, stage) => { + if request.cancel.load(Ordering::Relaxed) { + continue; + } + match serve_ticks(&agent, &gate, &request, &stage) { + Ok((series, venue_refused)) => { + // A PARTIAL harvest is still a COMPLETE run of the stage: the walk is + // done deciding what it can serve, so a reopen must not re-ask for ticks + // it already answered, whether or not `series.partial` is set — UNLESS + // the venue's own account of why it stopped was `Transient`: serving the + // partial rows is right (throwing away paid-for data is what this goal + // removes), but calling that answer SETTLED is not, because the venue + // called its own refusal transient. An exact-key reopen must retry it + // rather than pin the same partial answer until unrelated cache eviction. + remember_store( + &cache, + stage.key, + Remembered::Ready { + series: series.clone(), + ticks_settled: !venue_refused, + }, + ); + // Normal, not an error, for the same reason as the candle send above. + let _ = request.reply.send(TradeReplayOutcome::Ready(series)); + } + Err(Some(status)) => { + let mut series = compose(&request, request.address.venue, stage.candles); + series.tick_status = status; + // `NoTrades` is authoritative — the venue answered and held nothing — and + // is remembered settled exactly like a `Ready` harvest. `Failed` is not: + // the fetch itself did not produce an answer, so a reopen must retry it. + if status == TickStatus::NoTrades { + remember_store( + &cache, + stage.key, + Remembered::Ready { + series: series.clone(), + ticks_settled: true, + }, + ); + } + let _ = request.reply.send(TradeReplayOutcome::Ready(series)); + } + // The window closed; there is no one left to send a second outcome to. + Err(None) => {} + } + } } - let outcome = serve(&agent, &gate, &cache, &request); - // The receiver is gone whenever the window closed mid-fetch. Normal, not an error. - let _ = request.reply.send(outcome); } } +/// What one candle job resolves to: the outcome to send, and whether it earned a tick upgrade. +struct Served { + outcome: TradeReplayOutcome, + /// `Some` only when the CANDLE outcome above was `Ready`, so `run` may queue it onto the + /// BACK of the deque; see [`tick_stage_for`] for the four conditions that gate it. + tick_stage: Option, +} + /// Answer one request: memory cache, then SQLite cache, then the network. /// /// The order is fixed and each step earns its place. The memory cache answers a reopen with no @@ -170,6 +435,10 @@ fn run(rx: &Receiver) { /// gate is refusing — a user in backoff still sees the real chart rather than a countdown. Only /// then is a permit taken. /// +/// Each of the three points that produces a fresh candle answer (a non-settled ring hit, a +/// SQLite hit, a completed network fetch) also decides the tick stage for it and stamps the +/// outgoing series' [`TradeReplaySeries::tick_status`] to match, via [`stage_and_stamp`]. +/// /// Args: /// agent: Shared HTTP client. /// gate: Per-host pacing and backoff. @@ -177,16 +446,19 @@ fn run(rx: &Receiver) { /// request: The request being served. /// /// Returns: -/// The outcome to send back. +/// The outcome to send back, and the tick stage to queue behind it, if any. fn serve( agent: &ureq::Agent, gate: &ReplayGate, cache: &Mutex>, request: &TradeReplayRequest, -) -> TradeReplayOutcome { +) -> Served { let venue = request.address.venue; let Some(route) = kline_route(venue) else { - return TradeReplayOutcome::Empty(TradeReplayEmpty::NoEndpoint { brand: venue.brand }); + return Served { + outcome: TradeReplayOutcome::Empty(TradeReplayEmpty::NoEndpoint { brand: venue.brand }), + tick_stage: None, + }; }; let key = OutcomeKey { venue, @@ -196,41 +468,92 @@ fn serve( to_ms: request.window.to_ms, }; match remember_lookup(cache, &key, request.identity) { - Some(Remembered::Ready(series)) => return TradeReplayOutcome::Ready(series), + Some(Remembered::Ready { + series, + ticks_settled: true, + }) => { + // Sent exactly as stored: its own fields already carry the final answer, so no + // stage is re-decided and none is queued. + return Served { + outcome: TradeReplayOutcome::Ready(series), + tick_stage: None, + }; + } + Some(Remembered::Ready { + mut series, + ticks_settled: false, + }) => { + let tick_stage = stage_and_stamp(venue, request.window, &key, &mut series); + return Served { + outcome: TradeReplayOutcome::Ready(series), + tick_stage, + }; + } Some(Remembered::Empty) => { - return TradeReplayOutcome::Empty(TradeReplayEmpty::NoDataInWindow); + return Served { + outcome: TradeReplayOutcome::Empty(TradeReplayEmpty::NoDataInWindow), + tick_stage: None, + }; } None => {} } // The SQLite cache is read first and unconditionally: it costs no request and is not gated. if let Some(rows) = read_cached_bars(request.address.cache.as_ref(), request) { - let series = compose(request, venue, rows); - remember_store(cache, key, Remembered::Ready(series.clone())); - return TradeReplayOutcome::Ready(series); + let mut series = compose(request, venue, rows); + let tick_stage = stage_and_stamp(venue, request.window, &key, &mut series); + // Settled exactly when NO stage was queued: `stage_and_stamp` already stamped a TERMINAL + // status (`NoRoute`/`OutOfRetention`) in that case, and both are stable facts a reopen + // would only re-derive identically — a queued stage, by contrast, is still `Pending` and + // must be re-decided (or answered) on the next open. + remember_store( + cache, + key.clone(), + Remembered::Ready { + series: series.clone(), + ticks_settled: tick_stage.is_none(), + }, + ); + return Served { + outcome: TradeReplayOutcome::Ready(series), + tick_stage, + }; } if let Err(retry_in_s) = gate.claim(route.host(), Instant::now()) { - return TradeReplayOutcome::Failed(TradeReplayFailure::RateLimited { retry_in_s }); + return Served { + outcome: TradeReplayOutcome::Failed(TradeReplayFailure::RateLimited { retry_in_s }), + tick_stage: None, + }; } let category = bybit_category(venue, &request.market); let deadline = Instant::now() + JOB_DEADLINE; let mut rows: Vec = Vec::new(); - // Whether every page of the window was actually fetched. A cancelled run keeps its rows — they - // were paid for — but must NOT be remembered as this window's answer. + // Whether every page of the window was actually fetched. Two independent things can make this + // false, and only `cancelled` below may still be true when this is — see the tick-stage + // decision after the forming-bar drop for why the two must not be read as one fact. A + // cancelled run keeps its rows — they were paid for — but must NOT be remembered as this + // window's answer. let mut complete = true; + // Whether the WINDOW ITSELF closed mid-fetch, as opposed to `complete` going false for the + // forming-bar drop below: only this one discards the tick upgrade outright. + let mut cancelled = false; for (from_ms, to_ms) in pages(request.window, BAR_MS, route.max_rows()) { if request.cancel.load(Ordering::Relaxed) { // The window is gone, or a Retry superseded this request. Whatever was fetched is // still worth merging into the shared cache, so fall through rather than discarding a // page already paid for. complete = false; + cancelled = true; break; } if Instant::now() >= deadline { - return TradeReplayOutcome::Failed(TradeReplayFailure::Transient { - diagnostic: format!("trade replay exceeded {}s", JOB_DEADLINE.as_secs()), - }); + return Served { + outcome: TradeReplayOutcome::Failed(TradeReplayFailure::Transient { + diagnostic: format!("trade replay exceeded {}s", JOB_DEADLINE.as_secs()), + }), + tick_stage: None, + }; } gate.pace(route.host()); match rest::fetch_klines( @@ -248,10 +571,18 @@ fn serve( // claim here would make one bad market throttle every other market on that host, // and five of them would push it to the backoff ceiling for nothing. gate.clear(route.host()); - return TradeReplayOutcome::Failed(TradeReplayFailure::UnknownSymbol); + return Served { + outcome: TradeReplayOutcome::Failed(TradeReplayFailure::UnknownSymbol), + tick_stage: None, + }; } Err(rest::FetchError::Transient(diagnostic)) => { - return TradeReplayOutcome::Failed(TradeReplayFailure::Transient { diagnostic }); + return Served { + outcome: TradeReplayOutcome::Failed(TradeReplayFailure::Transient { + diagnostic, + }), + tick_stage: None, + }; } } } @@ -284,25 +615,587 @@ fn serve( // window has not been fully answered yet. Without this, a window whose only bar is the forming // one empties out and is remembered as an authoritative "this market did not trade". complete = complete && rows.len() == before_drop; - write_cached_bars(request.address.cache.as_ref(), request, &rows); + write_cached_bars( + request.address.cache.as_ref(), + request, + rows_for_cache(TradeReplaySource::Klines1m, &rows), + ); if rows.is_empty() { // Only a COMPLETE run may be remembered, empty or not: a cancelled one proves nothing // about the window it never finished reading. if complete { remember_store(cache, key, Remembered::Empty); } - return TradeReplayOutcome::Empty(TradeReplayEmpty::NoDataInWindow); + return Served { + outcome: TradeReplayOutcome::Empty(TradeReplayEmpty::NoDataInWindow), + tick_stage: None, + }; } - let series = compose(request, venue, rows); + let mut series = compose(request, venue, rows); // Only a COMPLETE run may be remembered. Pages are issued left to right, so a cancelled run // holds the window's left-hand prefix — typically missing exactly the bars around the exit — // and the in-memory ring, unlike the SQLite path, has no coverage re-check to catch that on // read. Storing it would serve a silently truncated chart as `Ready` for the life of the // entry. The SQLite merge above is unaffected: `cache_covers` re-checks it on every read. + // + // The tick stage is gated on `cancelled` alone, NOT on `complete`: a forming-bar drop leaves + // `complete` false too, but the window is fine and its ticks are fetched independently of the + // bar layer — queuing the stage is the whole point of this feature, and skipping it here is + // exactly what used to leave a freshly closed trade stuck on "tics ещё грузятся" forever. + // CANCELLED is the one reason to skip it outright: the window itself is gone. + let tick_stage = if cancelled { + // No stage is queued, so the status must be TERMINAL: `Pending` (compose()'s default) + // asserts a stage is in flight, and none is. `Failed` reads honestly — whatever a retry + // would have answered, it never ran. + series.tick_status = TickStatus::Failed; + None + } else { + stage_and_stamp(venue, request.window, &key, &mut series) + }; if complete { - remember_store(cache, key, Remembered::Ready(series.clone())); + // Settled exactly when no stage was queued — see the SQLite-hit branch above for why a + // terminal `stage_and_stamp` result never needs re-deciding, while a queued stage's + // `Pending` must be. + remember_store( + cache, + key, + Remembered::Ready { + series: series.clone(), + ticks_settled: tick_stage.is_none(), + }, + ); + } + Served { + outcome: TradeReplayOutcome::Ready(series), + tick_stage, + } +} + +/// Decide the tick stage for a just-built candle series, and stamp its own `tick_status` in place +/// to match — `Pending` when a stage is queued, or the reason it is not, via [`tick_stage_for`]. +/// +/// One helper for the three sites in [`serve`] that each produce a fresh candle answer: the +/// ring-hit-but-not-settled branch, the SQLite-cache-hit branch, and the completed-network-fetch +/// branch — the last of these calls it only on its NON-CANCELLED path; the cancelled sub-branch +/// skips it entirely and stamps [`TickStatus::Failed`] directly, since there is no fresh route +/// decision to make for a window that is already gone. `series.tick_status` already reads +/// `Pending` from [`compose`], so this only overwrites it when a stage is NOT queued. +/// +/// Args: +/// venue: Venue the candles came from. +/// window: The window the candles cover. +/// key: The ring key this stage would replace on success. +/// series: The just-built series; its `tick_status` is overwritten in place when no stage is +/// queued for it. +/// +/// Returns: +/// The stage to queue, or `None`. +fn stage_and_stamp( + venue: crate::venue::Venue, + window: ReplayWindow, + key: &OutcomeKey, + series: &mut TradeReplaySeries, +) -> Option { + match tick_stage_for(venue, window, key, &series.candles) { + Ok(stage) => Some(stage), + Err(status) => { + series.tick_status = status; + None + } + } +} + +/// Decide whether a just-built CANDLE series earns a queued tick upgrade, or the reason it does +/// not. +/// +/// The "already settled" short-circuit this used to take as a parameter no longer lives here: it +/// is checked once, in [`serve`]'s ring-hit branch, before this is ever called — a settled entry +/// is sent exactly as stored, with no stage queued and nothing here re-decided. +/// +/// A clock that cannot be read (`now_unix_ms_i64` answering `0`) is treated as INSIDE retention +/// rather than refused, the same permissive default [`serve`] already applies to the closed-bar +/// drop above: nothing here can prove the window is too old, so nothing here refuses it. +/// +/// Args: +/// venue: Venue the candles came from. +/// window: The window the candles cover. +/// key: The ring key this stage would replace on success. +/// candles: The exchange klines just composed, carried forward as the eventual tick series' +/// bar layer — see [`TickStage::candles`]. +/// +/// Returns: +/// The stage to queue, or the reason it is not queued. +fn tick_stage_for( + venue: crate::venue::Venue, + window: ReplayWindow, + key: &OutcomeKey, + candles: &[ChartCandle], +) -> Result { + let route = trade_route(venue).ok_or(TickStatus::NoRoute)?; + let now_ms = crate::util::time::now_unix_ms_i64(); + if now_ms > 0 && !inside_retention(route, window, now_ms) { + // `inside_retention` is false here only when the route documents a retention: it is + // unconditionally true otherwise, so this default is never actually reached — see its own + // doc comment. + let retention_ms = route.retention_ms().unwrap_or(0); + return Err(TickStatus::OutOfRetention { retention_ms }); + } + Ok(TickStage { + route, + key: key.clone(), + candles: candles.to_vec(), + }) +} + +/// Whether a window is within a trade route's own documented retention. +/// +/// Judges the FOCUS's own right edge — the trade's EXIT ([`ReplayWindow::focus`]) — never the +/// window's padded `from_ms` (D2-3), and never the focus's left edge either: the lead context is +/// optional padding, but the trade itself is not, and [`tick_plan`]'s own retention clip already +/// asks only that the exit be inside retention, clipping everything older. Judging the entry +/// instead is a STRICTLY STRONGER check that runs first, in [`tick_stage_for`], and made +/// `tick_plan`'s whole retention-clipping recovery path unreachable for exactly the windows it was +/// written to rescue: a Binance futures trade held ~10 h and closed 40 h ago (retention 48 h) was +/// refused outright although its exit's ticks were comfortably inside retention. +/// +/// Free, and evaluated BEFORE any request is spent — see [`tick_stage_for`], the only caller. +/// +/// Args: +/// route: The trade route in question. +/// window: The window to check. +/// now_ms: Current Unix time in milliseconds. +/// +/// Returns: +/// `true` when the route documents no retention limit, or when the focus's own right edge +/// falls inside the one it does document. +pub(crate) fn inside_retention(route: TradeRoute, window: ReplayWindow, now_ms: i64) -> bool { + route + .retention_ms() + .is_none_or(|r| window.focus().1 >= now_ms - r) +} + +/// Run one queued tick stage to completion. +/// +/// Args: +/// agent: Shared HTTP client. +/// gate: Per-host pacing. +/// request: The original request this stage upgrades. +/// stage: Which route, cache key and bar layer this stage answers. +/// +/// Returns: +/// `Ok((series, venue_refused))` with the tick series to send as the SECOND outcome — +/// `venue_refused` is [`TickHarvest::venue_refused`], carried out here so the caller can +/// decide whether this harvest is safe to remember settled (see [`run`]'s `Job::Ticks` arm). +/// `Err(Some(status))` when the stage ended for a reason the window must print instead — the +/// caller composes that second outcome from [`TickStage::candles`] carrying `status`. +/// `Err(None)` only for a cancelled window, where the requester is already gone and nothing +/// more is sent. +fn serve_ticks( + agent: &ureq::Agent, + gate: &ReplayGate, + request: &TradeReplayRequest, + stage: &TickStage, +) -> Result<(TradeReplaySeries, bool), Option> { + let route = stage.route; + let deadline = Instant::now() + JOB_DEADLINE; + // Re-derived rather than trusted from `tick_stage_for`'s own permissive pass: that check ran + // BEFORE this stage was even queued, and a clock that could not be read then still cannot + // prove the window is too old now, so the same `now_ms > 0` guard applies here. + let now_ms = crate::util::time::now_unix_ms_i64(); + let earliest_ms = match now_ms > 0 { + true => route.retention_ms().map(|r| now_ms - r), + false => None, + }; + let plan = tick_plan(request.window, route.max_query_ms(), earliest_ms); + if plan.slices.is_empty() { + // The FOCUS itself — the trade, not its optional context — lies entirely before + // `earliest_ms`: nothing worth fetching remains, so this is reported as retention rather + // than as an empty venue answer. + return Err(Some(TickStatus::OutOfRetention { + retention_ms: route.retention_ms().unwrap_or(0), + })); + } + let mut observer = GateObserver { + gate, + host: route.host(), + }; + let verdict = paginate_ticks( + route, + &plan, + TICK_BUDGET, + TICK_PAGE_BUDGET, + || request.cancel.load(Ordering::Relaxed), + || Instant::now() >= deadline, + &mut observer, + |from_ms, to_ms, cursor| { + rest::fetch_trades(agent, route, &request.market, from_ms, to_ms, cursor) + }, + ); + let harvest = match verdict { + TickVerdict::Ready(harvest) => harvest, + TickVerdict::Abandoned(reason) => { + // RELEASE THE PERMIT THIS STAGE TOOK, unless the venue is the reason we stopped. + // + // `TickObserver::claim` records a real attempt on the host's shared claim map, and + // only an explicit `clear` erases it. So an abandonment that is OUR OWN doing — the + // user closed the window, the job deadline expired, either budget was crossed, the + // market was simply quiet, or the venue answered that it does not list this symbol — + // would otherwise leave that attempt standing and put the host into 30-600 s of + // backoff. The next request to the SAME host is then refused, and because one host + // serves several venues that request belongs to an unrelated trade, and is usually a + // CANDLE stage that would have worked. The candle path one function up makes exactly + // this distinction already: it clears unconditionally after its own loop, its own + // cancellation break included, and clears on `UnknownSymbol` for the stated reason + // that "one bad market would throttle every other market on that host". + // + // TWO reasons keep the record, and both are the venue's own word rather than ours: + // `Transient` is a refusal or failure it just gave us, and `RateLimited` means our + // claim was REFUSED — we recorded nothing, so clearing would erase somebody else's + // legitimate backoff. + match reason { + TickAbandon::Transient | TickAbandon::RateLimited => {} + TickAbandon::Cancelled + | TickAbandon::Deadline + | TickAbandon::Empty + | TickAbandon::UnknownSymbol + | TickAbandon::OverPageBudget => gate.clear(route.host()), + } + log::info!( + "[x] trade-replay tick stage abandoned on {}: {reason:?}", + route.host() + ); + return Err(match reason { + TickAbandon::Cancelled => None, + TickAbandon::Empty => Some(TickStatus::NoTrades), + _ => Some(TickStatus::Failed), + }); + } + }; + let TickHarvest { + mut ticks, + covered, + complete, + venue_refused, + } = harvest; + // The venue answered without refusing anywhere along the walk, so its refusal history is + // stale — exactly the candle stage's own `gate.clear` above. A refusal it gave us mid-walk + // (`venue_refused`) must stand, or the next request to this host sends blind into a burst it + // just declined. + if !venue_refused { + gate.clear(route.host()); + } + // Stably sorted ascending, BEFORE anything else: per-page sorting is NOT enough for the two + // BACKWARD-paginating venues (Bitget, OKX), whose concatenated pages walk backwards across + // every page boundary. + ticks.sort_by(|a, b| { + a.time_ms + .partial_cmp(&b.time_ms) + .unwrap_or(std::cmp::Ordering::Equal) + }); + // Clipped to what the walk actually finished (`covered`), not to the request window: a walk + // cut short still holds a complete answer for the slices it actually walked, and clipping to + // the wider window would let a stray page-overshoot outside `covered` back in. + ticks.retain(|t| { + t.time_ms.is_finite() && (t.time_ms as i64) >= covered.0 && (t.time_ms as i64) <= covered.1 + }); + // `partial` must reflect what `covered` actually spans, not merely whether the walk finished. + // `tick_plan`'s own `earliest_ms` clip can make the PLAN narrower than `request.window` before + // the walk even starts, so a retention-clipped plan that completes still leaves the served + // ticks short of the requested window on one or both edges. + let partial = + !complete || covered.0 > request.window.from_ms || covered.1 < request.window.to_ms; + let (ticks, bucket_ms) = fit_ticks(ticks, TICK_BUDGET); + // An empty harvest after the coverage clip is not a success: some slice genuinely produced + // rows (`paginate_ticks` already refuses an empty one, above), so a caption of "Served" with + // zero points would be the lying-chart failure this module exists to remove. `Failed`, not + // `NoTrades` — a retry is honest here, while `NoTrades` claims an authoritative empty. + if ticks.is_empty() { + return Err(Some(TickStatus::Failed)); + } + Ok(( + compose_ticks( + request, + request.address.venue, + ticks, + bucket_ms, + partial, + stage.candles.clone(), + ), + venue_refused, + )) +} + +/// Walk every tile of one tick stage's [`TickPlan`], paginating each with the venue's own cursor, +/// until the plan is exhausted or a stop condition is reached — and, unlike a candle job, a stop +/// never discards what was already collected except when the window itself closed. +/// +/// `fetch` is the injected seam — no network, no clock, no gate inside this function, which is +/// what makes it testable with a fake fetcher and a fake [`TickObserver`]. +/// +/// The loop rule, in order of precedence (D2-2, replacing this function's earlier design in +/// full): +/// - [`cancelled`] stops EVERYTHING, always, and discards whatever was collected — the window is +/// gone and there is no one left to serve it to. +/// - The job deadline and the page budget each stop the WALK without abandoning it, at any point: +/// the harvest collected so far is kept, and the loop moves straight to the verdict. This is +/// the whole point of this function's redesign — a budget or a deadline crossed an hour of lead +/// context away from the trade must never throw away the tiles around the trade that were +/// already paid for. +/// - A venue's own answer — `Transient`/`UnknownSymbol` — also stops the walk rather than the +/// whole stage, and marks the harvest [`TickHarvest::venue_refused`], so [`serve_ticks`] knows +/// not to clear a refusal the venue just gave it. +/// - The FOCUS tiles (the leading [`TickPlan::focus_len`] entries) are never truncated by the +/// tick budget: it is checked only around a NON-focus tile, before it starts and again once it +/// finishes, so a tile is walked whole or not at all — never cut mid-body. +/// - Every page is clipped to the SLICE it was fetched for before it is counted toward any budget +/// (D2-1): Binance's forward pager and OKX's backward one both routinely return a page that +/// overshoots its own slice edge, and [`Tick`] carries no exchange trade id, so an unclipped +/// overlap between two adjacent slices is undetectable once concatenated, not merely unnoticed. +/// The aggregate clip to [`TickHarvest::covered`] in [`serve_ticks`] is the OUTER bound and does +/// not replace this inner one. +/// - The verdict is one question: is the harvest empty? A non-empty one is always `Ready`, +/// whatever stopped the walk; only an empty one reaches [`TickVerdict::Abandoned`], carrying +/// whichever reason actually stopped it. +/// +/// Args: +/// route: Which venue endpoint this stage answers. +/// plan: The window's own [`tick_plan`] output — tiles in fetch-priority order, with the +/// first [`TickPlan::focus_len`] of them being the trade's own focus. +/// tick_budget: Ceiling on the total ticks collected before a non-focus tile is skipped. +/// page_budget: Ceiling on the total pages fetched across every tile. +/// cancelled: Answers whether the requester's window has closed. +/// expired: Answers whether this stage's own deadline has passed. +/// observer: Records the `claim`/`pace` calls this stage makes. +/// fetch: Fetches one page for a given slice and cursor. +/// +/// Returns: +/// The harvest, or the reason nothing was collected. +pub(crate) fn paginate_ticks( + route: TradeRoute, + plan: &TickPlan, + tick_budget: usize, + page_budget: usize, + cancelled: impl Fn() -> bool, + expired: impl Fn() -> bool, + observer: &mut O, + mut fetch: F, +) -> TickVerdict +where + F: FnMut(i64, i64, Option) -> Result, + O: TickObserver, +{ + if plan.slices.is_empty() { + return TickVerdict::Abandoned(TickAbandon::Empty); + } + if observer.claim(route.host()).is_err() { + return TickVerdict::Abandoned(TickAbandon::RateLimited); + } + let mut ticks: Vec = Vec::new(); + let mut pages_fetched = 0usize; + let mut covered: Option<(i64, i64)> = None; + let mut complete = true; + let mut venue_refused = false; + let mut stop_reason: Option = None; + // The tile still being walked when a `break 'walk` fired mid-body — its own bounds, where its + // rows begin in `ticks`, and the cursor most recently used for it — so its paid-for rows can + // extend `covered` afterward rather than being reclaimed by the clip in `serve_ticks` (F2). + // `None` whenever every stop happened BETWEEN tiles (or the walk was cancelled outright, which + // returns before this is ever read). + let mut interrupted: Option<(i64, i64, usize, Option)> = None; + + 'walk: for (index, &(slice_from, slice_to)) in plan.slices.iter().enumerate() { + let is_focus = index < plan.focus_len; + // The tick budget never truncates a focus slice — checked only around a NON-focus one, so + // a slice is whole or absent rather than cut mid-body. See the after-check below for the + // other half of this rule. + if !is_focus && ticks.len() >= tick_budget { + complete = false; + break; + } + let start_len = ticks.len(); + let mut cursor: Option = None; + loop { + if cancelled() { + // The window is gone; nothing collected so far is worth keeping. + return TickVerdict::Abandoned(TickAbandon::Cancelled); + } + if expired() { + complete = false; + stop_reason = Some(TickAbandon::Deadline); + interrupted = Some((slice_from, slice_to, start_len, cursor)); + break 'walk; + } + if pages_fetched >= page_budget { + complete = false; + stop_reason = Some(TickAbandon::OverPageBudget); + interrupted = Some((slice_from, slice_to, start_len, cursor)); + break 'walk; + } + observer.pace(route.host()); + let page = match fetch(slice_from, slice_to, cursor) { + Ok(page) => page, + Err(rest::FetchError::UnknownSymbol) => { + complete = false; + venue_refused = true; + stop_reason = Some(TickAbandon::UnknownSymbol); + interrupted = Some((slice_from, slice_to, start_len, cursor)); + break 'walk; + } + Err(rest::FetchError::Transient(_)) => { + complete = false; + venue_refused = true; + stop_reason = Some(TickAbandon::Transient); + interrupted = Some((slice_from, slice_to, start_len, cursor)); + break 'walk; + } + }; + pages_fetched += 1; + let mut rows = page.ticks; + // D2-1: clip THIS page to the slice it was fetched for, before extending or counting + // toward the budget — see this function's own doc comment for the vendor evidence. + rows.retain(|t| { + t.time_ms.is_finite() + && (t.time_ms as i64) >= slice_from + && (t.time_ms as i64) <= slice_to + }); + ticks.extend(rows); + match page.next { + Some(next_cursor) => cursor = Some(next_cursor), + None => break, + } + } + // The slice's own pagination completed. For a non-focus slice only, a tick budget crossed + // during it removes the WHOLE slice rather than leaving it half-drawn. + if !is_focus && ticks.len() > tick_budget { + ticks.truncate(start_len); + complete = false; + break; + } + covered = Some(match covered { + None => (slice_from, slice_to), + Some((c_from, c_to)) => (c_from.min(slice_from), c_to.max(slice_to)), + }); + } + + if ticks.is_empty() { + return TickVerdict::Abandoned(stop_reason.unwrap_or(TickAbandon::Empty)); + } + // Extend `covered` by the interrupted tile's own paid-for rows — but ONLY when its pagination + // direction actually reached the edge touching `covered`, never unconditionally. Within one + // slice a paginated run is contiguous, but its direction is per-venue: Binance's `FromId` + // cursor walks FORWARD from the tile's own `slice_from` (a prefix of the tile); Bitget/OKX's + // `LessThanId` walks BACKWARD from `slice_to` (a suffix). A tile to the RIGHT of `covered` only + // touches the shared edge under a FORWARD cursor (it starts at `slice_from`, which sits right + // beside `covered`); a tile to the LEFT only under a BACKWARD one (it starts at `slice_to`, + // beside `covered` on that side). Gate's `Page`/`Offset` cursors carry an UNDOCUMENTED order + // (`venue_caps.rs`), so neither side ever trusts them. Getting this wrong would union in a + // stretch of the tile that was never actually fetched — the exact false "the market was quiet + // here" gap this whole design exists to prevent. + if let (Some((c_from, c_to)), Some((slice_from, slice_to, start, cursor))) = + (covered, interrupted) + { + if start < ticks.len() { + let (lo, hi) = ticks[start..] + .iter() + .fold((i64::MAX, i64::MIN), |(lo, hi), t| { + let time_ms = t.time_ms as i64; + (lo.min(time_ms), hi.max(time_ms)) + }); + // `AfterMs` is excluded from both: its own doc says no current route ever emits it, so + // there is no evidence for which edge it would touch. + let forward = matches!(cursor, Some(rest::TradeCursor::FromId(_))); + let backward = matches!(cursor, Some(rest::TradeCursor::LessThanId(_))); + covered = Some(if slice_from > c_to && forward { + (c_from, c_to.max(hi)) + } else if slice_to < c_from && backward { + (c_from.min(lo), c_to) + } else { + (c_from, c_to) + }); + } + } + let covered = covered.unwrap_or_else(|| { + // No slice ever reached natural completion, yet a stop mid-walk still left partial pages + // in `ticks` — this function serves what is held rather than discarding it (D2-2). The + // observed extremes of what was actually fetched can only UNDER-state true coverage, + // never claim more than was really walked, which the interrupted slice's own nominal + // bounds could. + let (lo, hi) = ticks.iter().fold((i64::MAX, i64::MIN), |(lo, hi), t| { + let time_ms = t.time_ms as i64; + (lo.min(time_ms), hi.max(time_ms)) + }); + (lo, hi) + }); + TickVerdict::Ready(TickHarvest { + ticks, + covered, + complete, + venue_refused, + }) +} + +/// Build the frozen TICK series one tick stage answers with. +/// +/// `ticks` must already be globally sorted ascending and clipped to the harvest's own +/// [`TickHarvest::covered`] range — this function does neither; [`serve_ticks`] does both before +/// calling it. `candles` is the EXCHANGE'S OWN klines carried forward from the candle stage that +/// ran first ([`TickStage::candles`]), never aggregated from `ticks`: the bar layer covers the +/// whole window even where the points, per `partial`, cover only part of it. +/// +/// Args: +/// request: The request being served. +/// venue: Venue the ticks came from. +/// ticks: Trade points, ascending, already clipped to the harvest's covered range. +/// bucket_ms: The bucket [`fit_ticks`] thinned the points to; `0` means raw. +/// partial: Whether `ticks` covers only part of `request.window`. +/// candles: The exchange klines to carry as the bar layer. +/// +/// Returns: +/// The series to hand the chart. +fn compose_ticks( + request: &TradeReplayRequest, + venue: crate::venue::Venue, + ticks: Vec, + bucket_ms: i64, + partial: bool, + candles: Vec, +) -> TradeReplaySeries { + TradeReplaySeries { + source: TradeReplaySource::Ticks, + venue, + window: request.window, + tf_ms: BAR_MS, + candles, + ticks, + identity: request.identity, + tick_status: TickStatus::Served, + bucket_ms, + partial, + } +} + +/// Answer the rows one cache write may actually carry, keyed on the series it came from. +/// +/// The SQLite isolation seam (acceptance criterion 7): a [`TradeReplaySource::Ticks`] series must +/// NEVER reach [`write_cached_bars`], because that table is the SHARED kline cache the live +/// recorder writes too. In practice no call site ever offers one this way — [`serve`] is the only +/// caller and always passes [`TradeReplaySource::Klines1m`], since [`serve_ticks`] writes nothing +/// back to SQLite at all — but the guard is keyed on the TYPE rather than on that fact, so the +/// invariant survives a future call site instead of depending on every one of them getting it +/// right by omission. +/// +/// Args: +/// source: Which kind of series `rows` was built for. +/// rows: The candidate rows. +/// +/// Returns: +/// `rows` unchanged for [`TradeReplaySource::Klines1m`]; an empty slice for +/// [`TradeReplaySource::Ticks`]. +pub(crate) fn rows_for_cache(source: TradeReplaySource, rows: &[ChartCandle]) -> &[ChartCandle] { + match source { + TradeReplaySource::Klines1m => rows, + TradeReplaySource::Ticks => &[], } - TradeReplayOutcome::Ready(series) } /// Build the frozen series one request answers with. @@ -327,6 +1220,9 @@ fn compose( candles: rows, ticks: Vec::new(), identity: request.identity, + tick_status: TickStatus::Pending, + bucket_ms: 0, + partial: false, } } @@ -349,12 +1245,18 @@ fn remember_lookup( .unwrap_or_else(std::sync::PoisonError::into_inner); let hit = cache.iter().find(|(k, _)| k == key)?; Some(match hit.1.clone() { - Remembered::Ready(mut series) => { + Remembered::Ready { + mut series, + ticks_settled, + } => { // The identity belongs to the WINDOW that asked, not to the cached rows: two windows // on the same trade must not share a chart revision, or the second would be told // nothing changed and would draw nothing. series.identity = identity; - Remembered::Ready(series) + Remembered::Ready { + series, + ticks_settled, + } } Remembered::Empty => Remembered::Empty, }) @@ -362,6 +1264,11 @@ fn remember_lookup( /// Remember one answered window, evicting the oldest when full. /// +/// Two independent ceilings, both enforced oldest-first: [`OUTCOME_CACHE_LEN`] bounds the number +/// of entries, [`OUTCOME_CACHE_MAX_TICKS`] bounds their combined tick count. Neither ever evicts +/// the entry this call just inserted, so a single series alone can outrun the tick ceiling +/// without being immediately discarded. +/// /// Args: /// cache: The ring. /// key: The question that was answered. @@ -379,6 +1286,26 @@ fn remember_store( while cache.len() > OUTCOME_CACHE_LEN { cache.pop_front(); } + while cache.len() > 1 && total_ticks(&cache) > OUTCOME_CACHE_MAX_TICKS { + cache.pop_front(); + } +} + +/// Sum the ticks carried by every remembered entry. +/// +/// Args: +/// cache: The ring. +/// +/// Returns: +/// Combined tick count across every entry. +fn total_ticks(cache: &VecDeque<(OutcomeKey, Remembered)>) -> usize { + cache + .iter() + .map(|(_, answer)| match answer { + Remembered::Ready { series, .. } => series.ticks.len(), + Remembered::Empty => 0, + }) + .sum() } /// Read the window's bars from the shared kline cache, when it covers the window. @@ -447,3 +1374,6 @@ fn write_cached_bars( rows: rows.to_vec(), }]); } + +#[cfg(test)] +mod tests; diff --git a/crates/moon-core/src/market/trade_replay/worker/tests.rs b/crates/moon-core/src/market/trade_replay/worker/tests.rs new file mode 100644 index 00000000..1891d0c4 --- /dev/null +++ b/crates/moon-core/src/market/trade_replay/worker/tests.rs @@ -0,0 +1,222 @@ +use std::cell::Cell; + +use super::*; +use crate::feed::Side; +use crate::market::trade_replay::rest::{FetchError, TradePage}; + +/// Records the pagination seam without touching a real host gate. +#[derive(Default)] +struct FakeObserver { + claims: usize, + paces: usize, +} + +impl TickObserver for FakeObserver { + fn claim(&mut self, _host: &str) -> Result<(), u32> { + self.claims += 1; + Ok(()) + } + + fn pace(&mut self, _host: &str) { + self.paces += 1; + } +} + +/// Builds one real-looking trade row for a deterministic fake page. +fn tick(time_ms: i64, price: f32) -> Tick { + Tick { + time_ms: time_ms as f64, + price, + qty: 1.0, + side: Side::Buy, + } +} + +/// Returns a completed fake page with the supplied rows. +fn page(ticks: Vec) -> Result { + Ok(TradePage { ticks, next: None }) +} + +/// `market/trade_replay/worker.rs:rows_for_cache` returning rows for both arms would file +/// tick-derived data as shared settled klines for every core. +#[test] +fn ticks_are_never_eligible_for_the_shared_kline_cache() { + let rows = [ChartCandle { + t_open_ms: 10_000.0, + open: 10.0, + high: 12.0, + low: 9.0, + close: 11.0, + volume: 5.0, + quote_volume: 55.0, + }]; + + assert!( + rows_for_cache(TradeReplaySource::Ticks, &rows).is_empty(), + "a tick series must never write a candle-shaped row into shared klines.sqlite" + ); + assert_eq!( + rows_for_cache(TradeReplaySource::Klines1m, &rows), + &rows, + "genuine exchange klines remain cacheable" + ); +} + +/// `market/trade_replay/worker.rs:inside_retention` judging `window.from_ms` instead of the +/// focus rejects a still-retained trade merely because optional lead context is older. +#[test] +fn retention_is_decided_from_the_trade_focus_not_padded_context() { + const HOUR_MS: i64 = 3_600_000; + let now_ms = 1_000 * HOUR_MS; + let window = ReplayWindow { + from_ms: now_ms - 60 * HOUR_MS, + to_ms: now_ms - 10 * HOUR_MS, + open_ms: now_ms - 47 * HOUR_MS, + close_ms: now_ms - 46 * HOUR_MS, + over_budget: false, + }; + + assert!( + inside_retention(TradeRoute::BinanceUsdMAggTrades, window, now_ms), + "the focus begins inside Binance USD-M's 48-hour retention despite older optional lead" + ); +} + +/// `market/trade_replay/worker.rs:paginate_ticks` restoring an over-budget abandonment would +/// throw away a non-empty focus harvest and fall back to candles. +#[test] +fn budget_stops_after_whole_focus_slices_and_serves_their_harvest() { + let plan = TickPlan { + slices: vec![(100, 199), (200, 299), (0, 99)], + focus_len: 2, + }; + let mut observer = FakeObserver::default(); + + let verdict = paginate_ticks( + TradeRoute::BinanceUsdMAggTrades, + &plan, + 40_000, + 10, + || false, + || false, + &mut observer, + |from_ms, _, _| { + page( + (0..30_000) + .map(|n| tick(from_ms + (n % 100), n as f32)) + .collect(), + ) + }, + ); + + let TickVerdict::Ready(harvest) = verdict else { + panic!("a non-empty harvest stopped by the tick budget must be served") + }; + assert_eq!( + harvest.ticks.len(), + 60_000, + "the 40,000 budget must not truncate either 30,000-row focus slice" + ); + assert_eq!( + harvest.covered, + (100, 299), + "only the two completed focus slices are covered after the budget stops the walk" + ); + assert!( + !harvest.complete, + "skipping the non-focus slice is a partial, not a complete, harvest" + ); + assert_eq!(observer.claims, 1, "a stage takes one host permit"); +} + +/// `market/trade_replay/worker.rs:paginate_ticks` returning `Abandoned(Deadline)` after any +/// fetched page discards usable ticks and replaces the user's trade with candles. +#[test] +fn deadline_with_a_non_empty_harvest_is_ready_not_abandoned() { + let plan = TickPlan { + slices: vec![(100, 199), (0, 99)], + focus_len: 1, + }; + let fetched = Cell::new(false); + let mut observer = FakeObserver::default(); + + let verdict = paginate_ticks( + TradeRoute::BinanceUsdMAggTrades, + &plan, + 40_000, + 10, + || false, + || fetched.get(), + &mut observer, + |from_ms, _, _| { + fetched.set(true); + page(vec![tick(from_ms + 5, 10.0)]) + }, + ); + + let TickVerdict::Ready(harvest) = verdict else { + panic!("a deadline after a fetched focus page must preserve the non-empty harvest") + }; + assert_eq!( + harvest + .ticks + .iter() + .map(|tick| (tick.time_ms as i64, tick.price, tick.qty)) + .collect::>(), + vec![(105, 10.0, 1.0)], + "the deadline preserves the fetched tick's time, price, and quantity" + ); + assert_eq!(harvest.covered, (100, 199)); + assert!( + !harvest.complete, + "the deadline leaves remaining slices unwalked" + ); +} + +/// `market/trade_replay/worker.rs:paginate_ticks` dropping its per-page retain lets adjacent +/// slices duplicate overshot exchange trades and spend the tick budget twice. +#[test] +fn each_fetched_page_is_clipped_to_its_own_slice_before_collection() { + let plan = TickPlan { + slices: vec![(100, 199), (200, 299)], + focus_len: 1, + }; + let mut observer = FakeObserver::default(); + let verdict = paginate_ticks( + TradeRoute::BinanceUsdMAggTrades, + &plan, + 40_000, + 10, + || false, + || false, + &mut observer, + |from_ms, _, _| match from_ms { + 100 => page(vec![ + tick(99, 1.0), + tick(100, 2.0), + tick(199, 3.0), + tick(200, 4.0), + ]), + 200 => page(vec![ + tick(199, 5.0), + tick(200, 6.0), + tick(299, 7.0), + tick(300, 8.0), + ]), + _ => unreachable!("the plan contains only two slices"), + }, + ); + + let TickVerdict::Ready(harvest) = verdict else { + panic!("the fake pages contain in-slice ticks") + }; + assert_eq!( + harvest + .ticks + .iter() + .map(|tick| tick.time_ms as i64) + .collect::>(), + vec![100, 199, 200, 299], + "only rows inside their own requested slice may enter the aggregate harvest" + ); +} diff --git a/crates/moon-core/src/strat_db/mod.rs b/crates/moon-core/src/strat_db/mod.rs index 4ab51606..1508757f 100644 --- a/crates/moon-core/src/strat_db/mod.rs +++ b/crates/moon-core/src/strat_db/mod.rs @@ -278,6 +278,7 @@ fn open_ro(path: &std::path::Path) -> rusqlite::Result { rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY | rusqlite::OpenFlags::SQLITE_OPEN_NO_MUTEX, )?; let _ = conn.busy_timeout(std::time::Duration::from_secs(3)); + crate::db::trace::install_on(&conn); Ok(conn) } diff --git a/crates/moon-core/src/strat_db/stats.rs b/crates/moon-core/src/strat_db/stats.rs index 59284601..19be6302 100644 --- a/crates/moon-core/src/strat_db/stats.rs +++ b/crates/moon-core/src/strat_db/stats.rs @@ -41,6 +41,7 @@ fn open_rw() -> Option { } let conn = Connection::open(&path).ok()?; let _ = conn.busy_timeout(std::time::Duration::from_secs(3)); + crate::db::trace::install_on(&conn); Some(conn) } diff --git a/crates/moon-ui-gpui/src/analytics/calendar/mod.rs b/crates/moon-ui-gpui/src/analytics/calendar/mod.rs index 522bc687..cbe317b0 100644 --- a/crates/moon-ui-gpui/src/analytics/calendar/mod.rs +++ b/crates/moon-ui-gpui/src/analytics/calendar/mod.rs @@ -38,14 +38,20 @@ use moon_core::db::{ProfitScope, ProfitUnit, ReadResult}; /// previous: Previous-month aggregate UI load state. /// period_result: Current cells and optional comparison from one SQLite snapshot. /// has_previous: Whether the active mode requested a comparison period. +/// preserve_transient: Whether a same-scope catch-up should keep the visible days and +/// previous-period cells instead of publishing the classified result — decided by the +/// caller via `refresh::preserve_on_catch_up` (a transient outcome with a scheduled +/// correction left; every other outcome is still published). /// /// Returns: -/// `true` when the shared period read failed. +/// `true` when the shared period read did not settle into a real replacement (a failure or a +/// split totals result, published or preserved alike), so `cal_dirty` stays set. fn apply_calendar_results( days: &mut ProfitLoadState>, previous: &mut LoadState>, period_result: ReadResult>, has_previous: bool, + preserve_transient: bool, ) -> bool { match period_result { Ok(ProfitScope::Comparable { unit, data: period }) => { @@ -62,13 +68,19 @@ fn apply_calendar_results( false } Ok(ProfitScope::Split(totals)) => { - days.apply(Ok(ProfitScope::Split(totals))); - previous.apply(Ok(None)); - false + // Mirrors the `Err` arm: a Split result is never a real replacement, so `cal_dirty` + // stays set regardless of whether this pass published it or kept the prior snapshot. + if !preserve_transient { + days.apply(Ok(ProfitScope::Split(totals))); + previous.apply(Ok(None)); + } + true } Err(error) => { - days.apply(Err(error.clone())); - previous.apply(if has_previous { Err(error) } else { Ok(None) }); + if !preserve_transient { + days.apply(Err(error.clone())); + previous.apply(if has_previous { Err(error) } else { Ok(None) }); + } true } } @@ -669,7 +681,7 @@ impl AnalyticsView { let (from, to) = match self.cal_mode { CalMode::Month => month_range(self.cal_ym, self.bound_zone()), CalMode::Year => all_history_range(self.bound_zone()), - // "Day" loads a 7-day window (selected day centered/at the bottom). + // "Day" loads the 30-day window, with the selected day centered or at the bottom. CalMode::Day => { let (top, bottom) = day_window(self.cal_day, self.bound_zone()); let to = @@ -741,7 +753,10 @@ impl AnalyticsView { /// Start the Calendar query with optional post-commit metadata chaining. /// /// Args: - /// after_report: Whether to preserve report-style catch-up and retry semantics. + /// after_report: Whether to preserve report-style catch-up and retry semantics, including + /// keeping the visible cells over a same-scope outcome that is transient with a + /// scheduled correction left (retry allowance or a newer generation, + /// `refresh::preserve_on_catch_up`) — every other outcome still publishes. /// show_overlay: Whether this refresh must block interaction with visible progress feedback. /// cx: GPUI context used to execute and publish the reads. fn reload_calendar_inner( @@ -783,44 +798,41 @@ impl AnalyticsView { if this.cal_seq != req { return; // mode/filters already changed } - let retry = data - .period + // Computed before `data.period` moves into the call below. + let period_outcome = super::refresh::CatchUpOutcome::of_scope(&data.period); + let cores_outcome = data + .cores .as_ref() - .err() - .filter(|error| error.kind() == Some(moon_core::db::FailKind::Busy)) - .or_else(|| { - data.cores - .as_ref() - .and_then(|cores| cores.as_ref().err()) - .filter(|error| error.kind() == Some(moon_core::db::FailKind::Busy)) - }) - .or_else(|| { - data.undated - .as_ref() - .err() - .filter(|error| error.kind() == Some(moon_core::db::FailKind::Busy)) - }) - .cloned(); + .map(super::refresh::CatchUpOutcome::of_read); + let undated_outcome = super::refresh::CatchUpOutcome::of_read(&data.undated); + let transient = period_outcome.is_transient() + || cores_outcome.is_some_and(super::refresh::CatchUpOutcome::is_transient) + || undated_outcome.is_transient(); let metadata_failed = data.cores.as_ref().is_some_and(|cores| cores.is_err()) || data.undated.is_err(); + // Peek at `data.period`'s outcome before it moves into the call below: only a + // transient outcome with a scheduled correction left may keep the visible cells. + let preserve_transient = !matches!(this.cal_days, ProfitLoadState::Loading) + && this.keep_on_catch_up(after_report, period_outcome, report_req); let read_failed = apply_calendar_results( &mut this.cal_days, &mut this.cal_prev, data.period, has_previous, + preserve_transient, ); if let Some(Ok(cores)) = data.cores { this.cores = cores; this.last_cores_at = Some(std::time::Instant::now()); this.core_refresh_needed = false; } - this.apply_undated_result(data.undated, true); + this.apply_undated_result(data.undated, after_report, report_req); this.cal_dirty = super::refresh::report_result_is_stale( report_req, this.current_report_generation(), read_failed || metadata_failed, ); - this.settle_report_refresh_retry(retry.as_ref(), cx); + this.settle_report_refresh_retry(transient, cx); cx.notify(); }, ); diff --git a/crates/moon-ui-gpui/src/analytics/calendar/tests.rs b/crates/moon-ui-gpui/src/analytics/calendar/tests.rs index 297fa3c8..dfa0ee73 100644 --- a/crates/moon-ui-gpui/src/analytics/calendar/tests.rs +++ b/crates/moon-ui-gpui/src/analytics/calendar/tests.rs @@ -23,7 +23,7 @@ fn current_and_previous_failures_remain_visible() { let mut days = ProfitLoadState::>::default(); let mut previous = LoadState::>::default(); - let failed = apply_calendar_results(&mut days, &mut previous, Err(failure()), true); + let failed = apply_calendar_results(&mut days, &mut previous, Err(failure()), true, false); assert!(failed); assert!(matches!( @@ -42,6 +42,94 @@ fn current_and_previous_failures_remain_visible() { )); } +/// `analytics/calendar/mod.rs:apply_calendar_results` must leave both settled Calendar snapshots +/// intact for a report-driven failure. Returning `false` or applying that failure clears the cells, +/// so `cal_dirty` can stop the catch-up and the visible calendar stays stale or flashes blank. +#[test] +fn report_catch_up_failure_preserves_current_and_previous_calendar_snapshots() { + let mut days = ProfitLoadState::Ready { + unit: None, + data: Arc::new(vec![DayCell { + start: 123, + ..Default::default() + }]), + }; + let mut previous = LoadState::Ready(Arc::new(Some(Default::default()))); + let original_days = days.data().expect("settled Calendar cells").clone(); + let original_previous = previous + .data() + .expect("settled previous-period total") + .clone(); + let failure = ReadFail::Failed { + kind: FailKind::Busy, + msg: Arc::from("busy calendar"), + }; + + assert!( + apply_calendar_results(&mut days, &mut previous, Err(failure), true, true), + "a failed period read must keep Calendar dirty for the bounded retry" + ); + assert!( + Arc::ptr_eq( + &original_days, + days.data().expect("preserved Calendar cells") + ), + "the catch-up failure must retain the exact current-cell snapshot" + ); + assert!( + Arc::ptr_eq( + &original_previous, + previous.data().expect("preserved previous-period total") + ), + "the catch-up failure must retain the exact comparison snapshot" + ); +} + +/// `analytics/calendar/mod.rs:apply_calendar_results` must preserve both settled Calendar +/// snapshots for a transient Split result and keep Calendar dirty. Publishing Split unconditionally +/// flashes incomplete cells, while returning false stops the catch-up that would replace them. +#[test] +fn transient_split_preserves_calendar_snapshots_and_requests_a_catch_up() { + let mut days = ProfitLoadState::Ready { + unit: None, + data: Arc::new(vec![DayCell { + start: 456, + ..Default::default() + }]), + }; + let mut previous = LoadState::Ready(Arc::new(Some(Default::default()))); + let original_days = days.data().expect("settled Calendar cells").clone(); + let original_previous = previous + .data() + .expect("settled previous-period total") + .clone(); + + assert!( + apply_calendar_results( + &mut days, + &mut previous, + Ok(moon_core::db::ProfitScope::Split(Default::default())), + true, + true, + ), + "a transient Split result must keep Calendar dirty for its scheduled correction" + ); + assert!( + Arc::ptr_eq( + &original_days, + days.data().expect("preserved Calendar cells") + ), + "a transient Split must retain the exact current-cell snapshot" + ); + assert!( + Arc::ptr_eq( + &original_previous, + previous.data().expect("preserved previous-period total") + ), + "a transient Split must retain the exact comparison snapshot" + ); +} + /// Replacing `calendar::hour_start`'s gap rejection with the shared picker clamp would map both /// Warsaw 02:00 and 03:00 to the same bucket and duplicate that hour's profit in the Day grid. #[test] diff --git a/crates/moon-ui-gpui/src/analytics/mod.rs b/crates/moon-ui-gpui/src/analytics/mod.rs index 1cd7f75a..eb57a00f 100644 --- a/crates/moon-ui-gpui/src/analytics/mod.rs +++ b/crates/moon-ui-gpui/src/analytics/mod.rs @@ -9,6 +9,10 @@ //! SQLite query never runs on the UI thread). Direct scope edits reload immediately; stale //! tab/mode entry and committed report generations use the shared quiet-period and maximum-wait //! gate. Hidden surfaces remain marked stale until entry, and automatic scans never overlap. +//! +//! A writer-driven catch-up preserves a settled snapshot only for a transient outcome that a +//! scheduled correction — a bounded retry or a newer generation — will resolve. Any durable +//! outcome still publishes, so stale data never looks current with no correction path left. /// The shared spawn+overlay envelope of every background DB read. mod bg; @@ -54,11 +58,12 @@ use crate::controls::date_range::{self, Bound}; use moon_core::db::ReportAxis; use moon_core::db::analytics::{DayCell, PreviousPeriodBasis, Query, StrategyBase, Summary}; use moon_core::db::valuation::{ValuationMode, ValuationStatus}; -use moon_core::db::{FailKind, ProfitMetric, ProfitUnit, ReadFail, SideFilter}; +use moon_core::db::{ProfitMetric, ProfitUnit, ReadFail, SideFilter}; use crate::load_state::{LoadState, note_el}; use refresh::{ - BusyRetryBudget, RefreshGate, RefreshPlan, RefreshUrgency, VisibleRefresh, visible_refresh, + BusyRetryBudget, CatchUpOutcome, RefreshGate, RefreshPlan, RefreshUrgency, VisibleRefresh, + visible_refresh, }; const ANALYTICS_HEADER_H: f32 = 32.0; @@ -398,7 +403,7 @@ pub struct AnalyticsView { last_valuation_status_rev: u64, /// Debounce/max-wait state for automatic report-driven refreshes. report_refresh: RefreshGate, - /// Bounded automatic Busy retries for the active contention episode. + /// Bounded automatic retries for the active transient-outcome episode. report_busy_retries: BusyRetryBudget, tab: Tab, /// Period of the Summary tab (presets or the from/to range). @@ -409,6 +414,9 @@ pub struct AnalyticsView { /// Period currently represented by `data` (summary/strategy list). Entering a tab /// with a different time window triggers a reload. data_period: Period, + /// Resolved `[from, to)` bounds `data` was computed for. Presets survive civil rollovers, so + /// the enum alone cannot tell when their re-resolved bounds make retained hovers stale. + data_range: (i64, i64), /// Whether `data` predates the latest committed report generation. data_dirty: bool, /// Cores from the replica (for the combo box) plus multi-selection (empty = all), using @@ -552,8 +560,8 @@ pub struct AnalyticsView { /// Content-measured preferred width of the strategy list's core column as /// `(font_scale_it_was_measured_under, width_in_base_px)` (`tuner::list::table::core_col_w`). /// Filled lazily on render; measuring lays out a glyph per character for every distinct core - /// name, too much to repay on an idle repaint. Invalidated two ways: cleared to `None` where - /// `strategy_data` is replaced (the names may have changed), and + /// name, too much to repay on an idle repaint. It is cleared only when a published base + /// changes the single-core names (`tuner::core_names_changed`), and /// recomputed when the stored font scale no longer matches the current one, so a Font-slider /// move OR a theme whose mode carries a different base mono size re-measures instead of /// scaling a width that assumed the old base. @@ -967,6 +975,7 @@ impl AnalyticsView { period: saved_period.unwrap_or(Period::CurMonth), strat_period: saved_strat_period.unwrap_or(Period::CurMonth), data_period: saved_period.unwrap_or(Period::CurMonth), + data_range: saved_period.unwrap_or(Period::CurMonth).range(bound_zone), data_dirty: false, cores: Vec::new(), last_cores_at: None, @@ -1107,26 +1116,44 @@ impl AnalyticsView { cx.notify(); } - /// Adopt a newly measured core offset, reloading every surface that was computed on the old - /// one. + /// Adopt a newly measured core offset, catching up every surface that was computed on the + /// old one. /// /// Polled beside the valuation health rather than pushed, following this window's own idiom. /// Unlike health, an adoption DOES change rows — every date bucket, every calendar cell and - /// every period bound moves — so it reloads rather than merely repainting. + /// every period bound moves. But the SCOPE (period, filters) does not, so this is a + /// writer-driven catch-up, not a user reload: the visible snapshot stays on screen, with no + /// blocking overlay, until the replacement lands. The observer retires EVERY in-flight read + /// identity for the old axis — `seq`, `cal_seq`, `cancel_latest_reads`, plus `tuner`, + /// `time_tuner`, `coins` and `coin_lists` `invalidate()` for the axes that keep their own + /// request generations — because a cancelled read is not silently dropped: the DB layer + /// raises a real SQLite interrupt that gets classified as a durable `Settled` failure, so a + /// read whose identity was not retired would pass its own `seq != req` guard and publish that + /// failure as if it were a real result. Retiring the tuner's identity here also clears any + /// unsaved filter draft, matching this observer's behavior before it stopped calling + /// `reload()` for axis changes. /// /// Args: - /// cx: Analytics window context, reloaded only when the axis actually moved. + /// cx: Analytics window context used to schedule a catch-up only when the axis moved. /// /// Returns: - /// Nothing; an axis change marks calendar state dirty and reloads Analytics. + /// Nothing; an axis change retires every in-flight read identity and schedules a + /// writer-driven catch-up. fn observe_report_axis(&mut self, cx: &mut Context) { let axis = self.backend.read(cx).report_axis(self.display_zone); if axis == self.axis { return; } self.axis = axis; - self.cal_dirty = true; - self.reload(cx); + self.seq = self.seq.wrapping_add(1); + self.cal_seq = self.cal_seq.wrapping_add(1); + self.cancel_latest_reads(); + self.tuner.invalidate(); + self.time_tuner.invalidate(); + self.coins.invalidate(); + self.coin_lists.invalidate(); + self.mark_report_data_stale(); + self.request_report_refresh(RefreshUrgency::Writer, false, cx); cx.notify(); } @@ -1191,16 +1218,16 @@ impl AnalyticsView { .refresh_started(generation, std::time::Instant::now()); } - /// Settle the Busy retry episode and optionally schedule its next bounded attempt. + /// Settle the transient retry episode and optionally schedule its next bounded attempt. /// /// Permanent corruption and unclassified I/O failures remain visible instead of creating an /// endless full-history retry loop. /// /// Args: - /// error: Transient read failure, or `None` when the read escaped SQLite contention. + /// transient: Whether any part of the completed read classified as a transient outcome. /// cx: GPUI context used to arm the quiet-period retry. - fn settle_report_refresh_retry(&mut self, error: Option<&ReadFail>, cx: &mut Context) { - if error.and_then(ReadFail::kind) != Some(FailKind::Busy) { + fn settle_report_refresh_retry(&mut self, transient: bool, cx: &mut Context) { + if !transient { self.report_busy_retries.resolve(); return; } @@ -1208,8 +1235,8 @@ impl AnalyticsView { log::warn!("analytics: automatic database retry budget exhausted"); return; } - // WRITER urgency on purpose: a bounded Busy retry that fired immediately would hammer a - // database already under contention, which is the one thing the quiet period is for. + // WRITER urgency lets this follow-up share the quiet period with a fresh report + // generation instead of immediately starting another full-period read. self.report_refresh.request_refresh( std::time::Instant::now(), false, @@ -1218,6 +1245,35 @@ impl AnalyticsView { self.schedule_report_refresh(cx); } + /// Decide whether a writer-driven catch-up outcome may keep a visible snapshot instead of + /// publishing it. + /// + /// Centralizing the predicate keeps every load surface aligned as their call sites evolve. A + /// preserved snapshot is always covered by a scheduled correction: a transient outcome with a + /// scheduled correction (retry allowance or a newer generation) left, never a bare exhausted + /// budget. + /// + /// Args: + /// after_report: Whether this is a writer-driven catch-up rather than a manual reload. + /// outcome: The classified outcome this read completed with. + /// started_generation: Report generation captured immediately before the read started. + /// + /// Returns: + /// `true` only when the outcome is transient AND a correction is already scheduled. + fn keep_on_catch_up( + &self, + after_report: bool, + outcome: CatchUpOutcome, + started_generation: u64, + ) -> bool { + refresh::preserve_on_catch_up( + after_report, + outcome, + self.report_busy_retries.has_allowance(), + started_generation != self.current_report_generation(), + ) + } + /// Queue a visible catch-up behind any Analytics database work already in flight. /// /// Args: @@ -1621,8 +1677,8 @@ impl AnalyticsView { /// Reload the full Summary without resetting tuner drafts. /// /// Args: - /// after_report: Whether report-style catch-up may preserve the previous undated value - /// while exposing a classified read error. + /// after_report: Whether this writer-driven catch-up may keep a settled snapshot while a + /// transient outcome with a scheduled correction remains. /// show_overlay: Whether this refresh must block interaction with visible progress feedback. /// cx: GPUI context used to run and publish the shared background query. fn reload_summary(&mut self, after_report: bool, show_overlay: bool, cx: &mut Context) { @@ -1639,18 +1695,25 @@ impl AnalyticsView { if !after_report { self.data = ProfitLoadState::default(); } - // Drop every chart hover: the bars/columns under the cursor are about to be - // replaced (and on a single-day period the right card swaps its whole element - // tree), so they never fire `hovered = false` and a stale index would re-open a - // popup with no cursor on it — pointing at another period's bucket. - self.hover_daily_bucket = None; - self.hover_cum_bucket = None; + // Presets keep their enum across civil rollovers while their resolved bounds slide, so + // only the range can tell whether retained bucket hovers still name the same data. + let active_range = self.active_period().range(self.bound_zone()); + let range_moved = active_range != self.data_range; + if !after_report || range_moved { + // Bucket indices are time-ordered and survive a same-scope catch-up, but a moved + // range shifts their meaning without changing the preset enum. + self.hover_daily_bucket = None; + self.hover_cum_bucket = None; + } + // Kinds are profit-sorted on every read, so an existing index can name another kind even + // during a same-scope catch-up; a closed popup is safer than a silently wrong one. self.hover_kind = None; self.seq = self.seq.wrapping_add(1); let req = self.seq; let report_req = self.current_report_generation(); // Record the ACTIVE tab's time window that `data` is being computed for. self.data_period = self.active_period(); + self.data_range = active_range; let q = self.query(); let read_cores = self.core_metadata_due(cx); self.spawn_latest_db( @@ -1665,40 +1728,39 @@ impl AnalyticsView { let data = result.data; let undated = result.undated; let cores = result.cores; - let data_error = data.as_ref().err().cloned(); + // Computed before `data` moves into `apply` below. + let data_outcome = CatchUpOutcome::of_scope(&data); let undated_error = undated.as_ref().err().cloned(); let cores_error = cores .as_ref() .and_then(|cores| cores.as_ref().err()) .cloned(); - let retry_error = data_error - .as_ref() - .filter(|error| error.kind() == Some(FailKind::Busy)) - .or_else(|| { - undated - .as_ref() - .err() - .filter(|error| error.kind() == Some(FailKind::Busy)) - }) - .or_else(|| { - cores_error - .as_ref() - .filter(|error| error.kind() == Some(FailKind::Busy)) - }) - .cloned(); + let transient = data_outcome.is_transient() + || CatchUpOutcome::of_read(&undated).is_transient() + || cores + .as_ref() + .is_some_and(|cores| CatchUpOutcome::of_read(cores).is_transient()); if let Some(Ok(cores)) = cores { this.cores = cores; this.last_cores_at = Some(std::time::Instant::now()); this.core_refresh_needed = false; } - this.data.apply(data); + // A snapshot survives only while a scheduled correction can replace it; otherwise + // stale values would look current with no remaining correction path. + let preserve_snapshot = !matches!(this.data, ProfitLoadState::Loading) + && this.keep_on_catch_up(after_report, data_outcome, report_req); + if !preserve_snapshot { + this.data.apply(data); + } this.data_dirty = refresh::report_result_is_stale( report_req, this.current_report_generation(), - data_error.is_some() || undated_error.is_some() || cores_error.is_some(), + !matches!(data_outcome, CatchUpOutcome::Replacement) + || undated_error.is_some() + || cores_error.is_some(), ); - this.apply_undated_result(undated, after_report); - this.settle_report_refresh_retry(retry_error.as_ref(), cx); + this.apply_undated_result(undated, after_report, report_req); + this.settle_report_refresh_retry(transient, cx); cx.notify(); }, ); @@ -1707,8 +1769,8 @@ impl AnalyticsView { /// Reload the compact Strategies base and optionally continue with its visible axis. /// /// Args: - /// after_report: Whether to preserve the visible snapshot while loading and on read - /// failure, and to use report-style catch-up and retry semantics. + /// after_report: Whether a transient outcome with a scheduled correction may preserve the + /// visible snapshot under report-style catch-up semantics. /// chain_visible_axis: Whether a successful base read should continue into the active axis. /// show_overlay: Whether this refresh must block interaction with visible progress feedback. /// cx: GPUI context used to run and publish the shared background query. @@ -1743,50 +1805,43 @@ impl AnalyticsView { let data = result.data; let undated = result.undated; let cores = result.cores; + // Computed before `data` moves into `apply` below. + let data_outcome = CatchUpOutcome::of_scope(&data); let data_error = data.as_ref().err().cloned(); let undated_error = undated.as_ref().err().cloned(); let cores_error = cores .as_ref() .and_then(|cores| cores.as_ref().err()) .cloned(); - let retry_error = data_error - .as_ref() - .filter(|error| error.kind() == Some(FailKind::Busy)) - .or_else(|| { - undated_error - .as_ref() - .filter(|error| error.kind() == Some(FailKind::Busy)) - }) - .or_else(|| { - cores_error - .as_ref() - .filter(|error| error.kind() == Some(FailKind::Busy)) - }) - .cloned(); + let transient = data_outcome.is_transient() + || CatchUpOutcome::of_read(&undated).is_transient() + || cores + .as_ref() + .is_some_and(|cores| CatchUpOutcome::of_read(cores).is_transient()); if let Some(Ok(cores)) = cores { this.cores = cores; this.last_cores_at = Some(std::time::Instant::now()); this.core_refresh_needed = false; } - // A same-scope automatic failure must leave the last ready snapshot visible while - // the retry gate settles. Manual scope changes have already retired the old data, - // so they publish the classified failure instead of showing stale values. - // - // "Keep what is visible" needs something visible to keep. An automatic catch-up - // can reach here over a NEVER-SETTLED state, and there the rule would leave the - // tuner reading "Loading" forever with nothing in flight — a pending state that - // was really a classified failure, which is the one thing this window must never - // show. - // - // The test is `Loading`, deliberately NOT "has scalar data": `Split` carries a - // real per-quote breakdown the toolbar renders, and `NotReady` / `Failed` are - // settled answers too. All three stay preserved exactly as before, so no path - // that existed before this change behaves differently. - let preserve_snapshot = - after_report && !matches!(this.strategy_data, ProfitLoadState::Loading); - if !preserve_snapshot || data_error.is_none() { + // Read `data` before `apply` moves it: the comparison needs both the currently + // shown group set and the one about to be published. + let core_names_changed = tuner::core_names_changed( + this.strategy_data.data().map(|d| d.strategies.as_slice()), + tuner::published_groups(&data), + ); + // Keep a settled same-scope snapshot only until a scheduled correction replaces + // it. An initial `Loading` state must publish its failure rather than remain + // pending forever; `Split`, `NotReady`, and `Failed` are already settled + // snapshots unless still transient (see `CatchUpOutcome`). + let preserve_snapshot = !matches!(this.strategy_data, ProfitLoadState::Loading) + && this.keep_on_catch_up(after_report, data_outcome, report_req); + if !preserve_snapshot { this.strategy_data.apply(data); - this.strat_core_w = None; + // Measuring every core name is expensive, so invalidate only when its input + // text changed. + if core_names_changed { + this.strat_core_w = None; + } // Both caches describe the group set that was just replaced. The memo's key // also carries that set's address, but an address is only unique among LIVE // allocations: a failed load drops the old buffer and a later successful one @@ -1797,9 +1852,11 @@ impl AnalyticsView { this.strategy_dirty = refresh::report_result_is_stale( report_req, this.current_report_generation(), - data_error.is_some() || undated_error.is_some() || cores_error.is_some(), + !matches!(data_outcome, CatchUpOutcome::Replacement) + || undated_error.is_some() + || cores_error.is_some(), ); - this.apply_undated_result(undated, after_report); + this.apply_undated_result(undated, after_report, report_req); let probe_took_over = probe_selects_strategy() && this.probe_select_first(cx); if refresh::strategy_base_allows_axis( data_error.is_some(), @@ -1812,27 +1869,39 @@ impl AnalyticsView { { this.reload_axis_after_report(this.strat_mode, show_overlay, cx); } - this.settle_report_refresh_retry(retry_error.as_ref(), cx); + this.settle_report_refresh_retry(transient, cx); cx.notify(); }, ); } - /// Apply an undated-close result without erasing a same-scope value on automatic failure. + /// Apply an undated-close result without replacing a settled strip with a transient alert + /// that a scheduled correction will immediately remove. + /// + /// Every other failure still publishes: otherwise stale counts would look current without a + /// correction path. + /// + /// Args: + /// result: Current undated-close read outcome. + /// after_report: Whether this is a writer-driven catch-up rather than a manual reload. + /// started_generation: Report generation captured immediately before the read started. fn apply_undated_result( &mut self, result: moon_core::db::ReadResult, - preserve_previous: bool, + after_report: bool, + started_generation: u64, ) { + let outcome = CatchUpOutcome::of_read(&result); match result { Ok(undated) => { self.undated = Some(undated); self.undated_error = None; } Err(error) => { - if !preserve_previous { - self.undated = None; + if self.keep_on_catch_up(after_report, outcome, started_generation) { + return; } + self.undated = None; self.undated_error = Some(error); } } diff --git a/crates/moon-ui-gpui/src/analytics/refresh.rs b/crates/moon-ui-gpui/src/analytics/refresh.rs index 06225cfc..2b94e76e 100644 --- a/crates/moon-ui-gpui/src/analytics/refresh.rs +++ b/crates/moon-ui-gpui/src/analytics/refresh.rs @@ -9,9 +9,19 @@ //! runs immediately: [`RefreshUrgency`] is what separates the two, and it is the difference //! between a tuner tab that opens now and one that opens a second from now. The `db_active` //! interlock is unaffected: nothing here ever overlaps work already in flight. +//! +//! [`preserve_on_catch_up`] is the rule every writer-driven completion handler in the Analytics +//! window's own surfaces -- Summary, Strategies, Calendar and the three tuner axes -- consults +//! before publishing a failed or split read: only a transient outcome with a scheduled +//! correction (retry allowance or a newer generation) left may keep a settled surface on screen +//! instead of the classified result, because that is the only kind of outcome the bounded +//! [`BusyRetryBudget`] retry -- or the writer's own next generation -- will actually clear. The +//! Profit Monitor keeps its own separate Busy-retry flow and does not consult this rule. use std::time::{Duration, Instant}; +use moon_core::db::{FailKind, ProfitScope, ReadFail, ReadResult}; + use super::Tab; /// Quiet time that coalesces adjacent report commits into one analytical scan. @@ -26,7 +36,7 @@ const CORE_METADATA_INTERVAL: Duration = Duration::from_secs(60); /// Maximum automatic retries for one report generation under persistent SQLite contention. const MAX_BUSY_RETRIES: u8 = 3; -/// Bounded retry allowance for transient database contention. +/// Bounded retry allowance for a transient database outcome. #[derive(Default)] pub(super) struct BusyRetryBudget { attempts: u8, @@ -58,7 +68,14 @@ impl BusyRetryBudget { true } - /// Close a retry episode after a read completes without SQLite contention. + /// Whether a transient outcome can still be corrected automatically. + /// + /// This is read-only because only `claim` may spend an attempt. + pub(super) fn has_allowance(&self) -> bool { + self.attempts < MAX_BUSY_RETRIES + } + + /// Close a retry episode after a read completes without a transient outcome. pub(super) fn resolve(&mut self) { self.attempts = 0; self.active = false; @@ -133,10 +150,117 @@ pub(super) fn report_result_is_stale( read_failed || started_generation != current_generation } +/// Classification of a completed report-derived read for catch-up preservation. +/// +/// `Transient` is deliberately wider than a bare `Busy` failure only in that a `Split` scope still +/// short of full valuation coverage joins it, correctable by the writer's own next generation. A +/// `NotReady` schema gap during a replica reset is a COMPLETED absence, not a momentary one: it +/// stays `Settled` and publishes now. See the module docs and `CatchUpOutcome::of_scope` for the +/// narrowed Split rule. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(super) enum CatchUpOutcome { + /// A real result: publish it. + Replacement, + /// Correctable by a scheduled retry or a newer generation; may be hidden behind the last + /// settled snapshot instead of publishing. + Transient, + /// Durable: no correction is coming, so it must publish now. + Settled, +} + +/// Classify a failed read into [`CatchUpOutcome::Transient`] or [`CatchUpOutcome::Settled`]. +/// +/// Only `Busy` is transient: it is the sole failure kind the bounded retry actually clears. +/// `NotReady` is a completed absence rather than proof of momentariness (see the module docs). +fn classify_fail(error: &ReadFail) -> CatchUpOutcome { + if error.kind() == Some(FailKind::Busy) { + CatchUpOutcome::Transient + } else { + CatchUpOutcome::Settled + } +} + +impl CatchUpOutcome { + /// Classify a plain fallible report read. + /// + /// Args: + /// result: The completed read. + /// + /// Returns: + /// `Replacement` for `Ok`, otherwise the failure's transient/settled classification. + pub(super) fn of_read(result: &Result) -> Self { + match result { + Ok(_) => CatchUpOutcome::Replacement, + Err(error) => classify_fail(error), + } + } + + /// Classify a scope-shaped report read. + /// + /// `Split` is transient only while the valuation worker demonstrably still has eligible rows + /// left to value; an unknown-quote identity or a worker that finished unroutable rows is a + /// durable safety verdict and must publish (see the module's CORRECTION A). + /// + /// Args: + /// result: The completed scope read. + /// + /// Returns: + /// `Replacement` for `Comparable`/`Empty`, the narrowed Split classification for `Split`, + /// otherwise the failure's transient/settled classification. + pub(super) fn of_scope(result: &ReadResult>) -> Self { + match result { + Ok(ProfitScope::Split(totals)) => { + if totals.unknown_orders == 0 + && totals.valuation.is_some_and(|coverage| { + coverage.valued_orders + coverage.unavailable_orders + < coverage.eligible_orders + }) + { + CatchUpOutcome::Transient + } else { + CatchUpOutcome::Settled + } + } + Ok(ProfitScope::Comparable { .. } | ProfitScope::Empty(_)) => { + CatchUpOutcome::Replacement + } + Err(error) => classify_fail(error), + } + } + + /// Whether this outcome is a scheduled-correction candidate. + pub(super) fn is_transient(self) -> bool { + matches!(self, CatchUpOutcome::Transient) + } +} + +/// Decide whether a writer-driven catch-up may keep a settled snapshot instead of publishing. +/// +/// Only a transient outcome with a scheduled correction still open may be hidden: any other +/// result, including an exhausted retry budget with no newer generation pending, must publish so +/// stale values never look current without a correction path. +/// +/// Args: +/// after_report: Whether this is a writer-driven catch-up rather than a manual reload. +/// outcome: The classified outcome this read completed with. +/// retry_allowance: Whether another automatic transient retry remains for this episode. +/// superseded: Whether a newer generation is already pending behind this read. +/// +/// Returns: +/// `true` only when the outcome is transient AND a correction is already scheduled. +pub(super) fn preserve_on_catch_up( + after_report: bool, + outcome: CatchUpOutcome, + retry_allowance: bool, + superseded: bool, +) -> bool { + after_report && outcome == CatchUpOutcome::Transient && (retry_allowance || superseded) +} + /// Decide whether the compact Strategies base may continue into its visible axis. /// /// The axis is a child of the compound base refresh. Starting it after a partial base failure -/// wastes another scan and lets its success resolve the parent's active Busy retry episode. +/// wastes another scan and lets its success resolve the parent's active transient retry episode. /// /// Args: /// data_failed: Whether the strategy-list aggregate failed. diff --git a/crates/moon-ui-gpui/src/analytics/refresh/tests.rs b/crates/moon-ui-gpui/src/analytics/refresh/tests.rs index 6067162f..7daee7fb 100644 --- a/crates/moon-ui-gpui/src/analytics/refresh/tests.rs +++ b/crates/moon-ui-gpui/src/analytics/refresh/tests.rs @@ -3,10 +3,138 @@ use std::time::{Duration, Instant}; use super::{ - BusyRetryBudget, RefreshGate, RefreshPlan, RefreshUrgency, VisibleRefresh, core_metadata_wait, - report_result_is_stale, strategy_base_allows_axis, visible_refresh, + BusyRetryBudget, CatchUpOutcome, RefreshGate, RefreshPlan, RefreshUrgency, VisibleRefresh, + core_metadata_wait, preserve_on_catch_up, report_result_is_stale, strategy_base_allows_axis, + visible_refresh, }; use crate::analytics::Tab; +use moon_core::db::{ + FailKind, ProfitScope, ProfitUnit, QuoteBreakdown, ReadFail, ValuationCoverage, +}; +use std::sync::Arc; + +/// Construct a classified database failure without depending on display text. +fn failure(kind: FailKind) -> ReadFail { + ReadFail::Failed { + kind, + msg: Arc::from("test failure"), + } +} + +/// `analytics/refresh.rs:preserve_on_catch_up` must preserve only a transient result covered by a +/// retry allowance or a newer generation. Dropping that correction guard, or letting a settled +/// result through, leaves stale figures under a current period label with nothing scheduled to +/// correct them. +#[test] +fn catch_up_preservation_requires_a_transient_outcome_and_scheduled_correction() { + for (label, outcome, is_transient) in [ + ("Replacement", CatchUpOutcome::Replacement, false), + ("Transient", CatchUpOutcome::Transient, true), + ("Settled", CatchUpOutcome::Settled, false), + ] { + for after_report in [false, true] { + for retry_allowance in [false, true] { + for superseded in [false, true] { + assert_eq!( + preserve_on_catch_up(after_report, outcome, retry_allowance, superseded,), + after_report && is_transient && (retry_allowance || superseded), + "{label}: after_report={after_report}, retry_allowance={retry_allowance}, \ + superseded={superseded}" + ); + } + } + } + } +} + +/// `analytics/refresh.rs:CatchUpOutcome::{of_scope,of_read}` must distinguish a valuation gap +/// from unknown-quote or settled results. Removing the unknown-quote guard hides money behind a +/// stale comparable scalar in the wrong unit, while moving Empty or Corrupt to transient hides a +/// legitimate purge result or permanent database failure. +#[test] +fn catch_up_outcome_classifies_scope_and_read_results() { + let incomplete_split = Ok::, ReadFail>(ProfitScope::Split(QuoteBreakdown { + unknown_orders: 0, + valuation: Some(ValuationCoverage { + eligible_orders: 5, + valued_orders: 3, + unavailable_orders: 1, + ..Default::default() + }), + ..Default::default() + })); + let unknown_quote_split = Ok::, ReadFail>(ProfitScope::Split(QuoteBreakdown { + unknown_orders: 1, + valuation: Some(ValuationCoverage { + eligible_orders: 5, + valued_orders: 3, + unavailable_orders: 1, + ..Default::default() + }), + ..Default::default() + })); + let settled_split = Ok::, ReadFail>(ProfitScope::Split(Default::default())); + let comparable = Ok(ProfitScope::Comparable { + unit: ProfitUnit::Percent, + data: (), + }); + let empty = Ok(ProfitScope::Empty(())); + + assert!(matches!( + CatchUpOutcome::of_scope(&incomplete_split), + CatchUpOutcome::Transient + )); + assert!(matches!( + CatchUpOutcome::of_scope(&unknown_quote_split), + CatchUpOutcome::Settled + )); + assert!(matches!( + CatchUpOutcome::of_scope(&settled_split), + CatchUpOutcome::Settled + )); + assert!(matches!( + CatchUpOutcome::of_scope(&comparable), + CatchUpOutcome::Replacement + )); + assert!(matches!( + CatchUpOutcome::of_scope(&empty), + CatchUpOutcome::Replacement + )); + + let cases: [(&str, Result<(), ReadFail>, bool); 6] = [ + ("Busy", Err(failure(FailKind::Busy)), true), + ("NotReady", Err(ReadFail::NotReady), false), + ("Corrupt", Err(failure(FailKind::Corrupt)), false), + ("Other", Err(failure(FailKind::Other)), false), + ("IncomparableQuote", Err(ReadFail::IncomparableQuote), false), + ("PeriodOutOfRange", Err(ReadFail::PeriodOutOfRange), false), + ]; + for (label, result, expected_transient) in cases { + assert_eq!( + matches!(CatchUpOutcome::of_read(&result), CatchUpOutcome::Transient), + expected_transient, + "{label} must be transient exactly when a scheduled correction can resolve it" + ); + } +} + +/// `analytics/refresh.rs:BusyRetryBudget::has_allowance` must read without spending a retry. +/// Letting this publication query increment attempts exhausts a transient Busy recovery budget +/// before `claim` schedules the retry, leaving stale data or a false error. +#[test] +fn checking_the_busy_retry_allowance_does_not_consume_an_attempt() { + let mut budget = BusyRetryBudget::default(); + + assert!(budget.has_allowance()); + assert!(budget.has_allowance()); + assert!(budget.claim()); + assert!(budget.has_allowance()); + assert!(budget.claim()); + assert!(budget.has_allowance()); + assert!(budget.claim()); + assert!(!budget.has_allowance()); + assert!(!budget.claim()); +} /// `analytics/refresh.rs:report_result_is_stale` must compare the start and completion /// generations; removing that comparison lets a scan started before a new trade clear the diff --git a/crates/moon-ui-gpui/src/analytics/tuner/coins/load.rs b/crates/moon-ui-gpui/src/analytics/tuner/coins/load.rs index e9a0fda6..4b54c1f2 100644 --- a/crates/moon-ui-gpui/src/analytics/tuner/coins/load.rs +++ b/crates/moon-ui-gpui/src/analytics/tuner/coins/load.rs @@ -10,7 +10,7 @@ use gpui::*; use super::super::super::AnalyticsView; use super::state::{CoinLists, CoinPlan}; -use crate::analytics::refresh::report_result_is_stale; +use crate::analytics::refresh::{CatchUpOutcome, report_result_is_stale}; use moon_core::db::analytics::GroupStat; /// How long a burst of ticks is allowed to keep coalescing before the KPI is rescanned. @@ -28,7 +28,7 @@ impl AnalyticsView { self.reload_coins_inner(false, true, cx); } - /// Recompute report-stale coin data and retry transient database contention. + /// Recompute report-stale coin data through the writer-driven catch-up path. /// /// Args: /// show_overlay: Whether queued user work requires blocking progress feedback. @@ -44,7 +44,7 @@ impl AnalyticsView { /// Start one coin-axis snapshot with behavior selected by its reload cause. /// /// Args: - /// after_report: Whether transient database contention should re-arm automatic refresh. + /// after_report: Whether this is a writer-driven catch-up that may arm a follow-up refresh. /// show_overlay: Whether this refresh must block interaction with visible progress feedback. /// cx: GPUI context used to run and publish the combined background query. fn reload_coins_inner( @@ -130,7 +130,8 @@ impl AnalyticsView { ) }, move |this, (stats, kpi, lists, bl_n, wl_n, entries, picked_strategies), cx| { - let entries_error = entries.as_ref().err().cloned(); + let entries_outcome = CatchUpOutcome::of_read(&entries); + let picked_outcome = CatchUpOutcome::of_read(&picked_strategies); let picked_error = picked_strategies.as_ref().err().cloned(); // Guarded separately from the table below: the panels have their own // generation, so a scope change that retired only them still lands here @@ -165,34 +166,12 @@ impl AnalyticsView { // what says "the user has touched the working lists since this request // started" — so it must not be overwritten by a baseline read before it. let edited = this.coins.kpi_seq != kpi_req; - let retry = stats - .as_ref() - .err() - .filter(|error| error.kind() == Some(moon_core::db::FailKind::Busy)) - .or_else(|| { - if edited { - None - } else { - kpi.as_ref() - .err() - .filter(|error| error.kind() == Some(moon_core::db::FailKind::Busy)) - } - }) - .or_else(|| { - entries_error - .as_ref() - .filter(|error| error.kind() == Some(moon_core::db::FailKind::Busy)) - }) - .or_else(|| { - if picked_current { - picked_error - .as_ref() - .filter(|error| error.kind() == Some(moon_core::db::FailKind::Busy)) - } else { - None - } - }) - .cloned(); + let stats_outcome = CatchUpOutcome::of_read(&stats); + let kpi_outcome = CatchUpOutcome::of_read(&kpi); + let transient = stats_outcome.is_transient() + || (!edited && kpi_outcome.is_transient()) + || entries_outcome.is_transient() + || (picked_current && picked_outcome.is_transient()); // Clear "recompute me" only when everything this pass owed came back. The // KPI counts only when it is actually applied — a discarded request's // error says nothing about the table. An empty universe means the plan @@ -207,7 +186,8 @@ impl AnalyticsView { this.current_report_generation(), read_failed, ); - this.coins.stats.apply(stats); + let keep_stats = this.keep_on_catch_up(after_report, stats_outcome, report_req); + this.coins.stats.apply_or_keep(stats, keep_stats); // Only a confirmed strategies snapshot may replace the baseline. On a failed // read, preserving both sets is safer than replaying the draft against a // fabricated empty list and silently changing the next Save. @@ -216,7 +196,8 @@ impl AnalyticsView { // left to show; fall back before the table renders "no matches". this.coins.settle_filter(); if !edited { - this.coins.kpi.apply(kpi); + let keep_kpi = this.keep_on_catch_up(after_report, kpi_outcome, report_req); + this.coins.kpi.apply_or_keep(kpi, keep_kpi); this.coins.kpi_bl = bl_n; this.coins.kpi_wl = wl_n; } else { @@ -225,7 +206,7 @@ impl AnalyticsView { this.arm_coin_kpi(cx); } if after_report { - this.settle_report_refresh_retry(retry.as_ref(), cx); + this.settle_report_refresh_retry(transient, cx); } cx.notify(); }, diff --git a/crates/moon-ui-gpui/src/analytics/tuner/filter/mod.rs b/crates/moon-ui-gpui/src/analytics/tuner/filter/mod.rs index 0a21a502..e77ee6a4 100644 --- a/crates/moon-ui-gpui/src/analytics/tuner/filter/mod.rs +++ b/crates/moon-ui-gpui/src/analytics/tuner/filter/mod.rs @@ -25,6 +25,7 @@ use moon_ui::{ }; use rust_i18n::t; +use super::super::refresh::CatchUpOutcome; use super::super::{AnalyticsView, LoadState}; pub(in crate::analytics::tuner) use super::shared::{N_VAR, TunerKind, card, glyph_btn}; use crate::design; @@ -216,10 +217,13 @@ impl AnalyticsView { }, move |this, (stats, histogram, strat), cx| { let mut hist_error = None; + let mut hist_outcome = CatchUpOutcome::Replacement; if this.tuner.hist_seq == hist_req { this.tuner.hist_loading = false; + hist_outcome = CatchUpOutcome::of_read(&histogram); hist_error = histogram.as_ref().err().cloned(); - this.tuner.hist.apply(histogram); + let keep_hist = this.keep_on_catch_up(after_report, hist_outcome, report_req); + this.tuner.hist.apply_or_keep(histogram, keep_hist); this.tuner.hist_dirty = super::super::refresh::report_result_is_stale( report_req, this.current_report_generation(), @@ -227,16 +231,20 @@ impl AnalyticsView { ); } if this.tuner.seq != req { - if let Some(error) = hist_error.as_ref() { - this.settle_report_refresh_retry(Some(error), cx); + if hist_error.is_some() { + this.settle_report_refresh_retry(hist_outcome.is_transient(), cx); } cx.notify(); return; } + let stats_outcome = CatchUpOutcome::of_read(&stats); let error = stats.as_ref().err().cloned(); - // A completed non-data result clears stale numbers because - // values under a changed period label must belong to it. - this.tuner.stats.apply(stats); + // A completed non-data result clears stale numbers, unless it is a transient + // outcome with a scheduled correction left (`keep_on_catch_up`) — values + // under a changed period label must belong to it, but a settled snapshot with a + // scheduled correction must not blink through the intermediate result. + let keep_stats = this.keep_on_catch_up(after_report, stats_outcome, report_req); + this.tuner.stats.apply_or_keep(stats, keep_stats); // `strategy_filters` is intentionally lossy and reports an unreadable row as // `found=false`. An automatic report refresh must not turn that ambiguity into // an empty Save baseline; explicit scope changes may clear the old strategy. @@ -246,15 +254,8 @@ impl AnalyticsView { this.current_report_generation(), error.is_some(), ); - let retry_error = error - .as_ref() - .filter(|error| error.kind() == Some(moon_core::db::FailKind::Busy)) - .or_else(|| { - hist_error - .as_ref() - .filter(|error| error.kind() == Some(moon_core::db::FailKind::Busy)) - }); - this.settle_report_refresh_retry(retry_error, cx); + let transient = stats_outcome.is_transient() || hist_outcome.is_transient(); + this.settle_report_refresh_retry(transient, cx); cx.notify(); }, ); @@ -311,8 +312,10 @@ impl AnalyticsView { if this.tuner.seq != req { return; } + let stats_outcome = CatchUpOutcome::of_read(&stats); let error = stats.as_ref().err().cloned(); - this.tuner.stats.apply(stats); + let keep_stats = this.keep_on_catch_up(after_report, stats_outcome, report_req); + this.tuner.stats.apply_or_keep(stats, keep_stats); this.tuner.apply_strategy_read(filters, after_report); this.tuner.dirty = super::super::refresh::report_result_is_stale( report_req, @@ -320,7 +323,7 @@ impl AnalyticsView { error.is_some(), ); if after_report { - this.settle_report_refresh_retry(error.as_ref(), cx); + this.settle_report_refresh_retry(stats_outcome.is_transient(), cx); } cx.notify(); }, diff --git a/crates/moon-ui-gpui/src/analytics/tuner/list/mod.rs b/crates/moon-ui-gpui/src/analytics/tuner/list/mod.rs index 5f7f06f1..f014400e 100644 --- a/crates/moon-ui-gpui/src/analytics/tuner/list/mod.rs +++ b/crates/moon-ui-gpui/src/analytics/tuner/list/mod.rs @@ -33,7 +33,8 @@ use super::{ }; use crate::design; use crate::design::moon; -use moon_core::db::analytics::GroupStat; +use moon_core::db::ProfitScope; +use moon_core::db::analytics::{GroupStat, StrategyBase}; /// "Which strategies name coins in a list?" — the list filter of the strategy table. /// @@ -244,6 +245,45 @@ fn memo_is_fresh(cached: Option<&VisibleRows>, key: &VisibleKey) -> bool { cached.is_some_and(|c| &c.key == key) } +/// Whether a published group set changes the names measured for the core-column width. +/// +/// This matches `core_col_w`'s `cores_n <= 1` filter so the cache invalidates only when its +/// measured text changes; multi-core rows render an aggregate instead of a name. No new group +/// set means no replacement was published, while a missing old set must measure the first one. +pub(in crate::analytics) fn core_names_changed( + old: Option<&[GroupStat]>, + new: Option<&[GroupStat]>, +) -> bool { + let Some(new) = new else { + return false; + }; + let Some(old) = old else { + return true; + }; + fn names(rows: &[GroupStat]) -> std::collections::HashSet<&str> { + rows.iter() + .filter(|g| g.cores_n <= 1) + .map(|g| g.core.as_str()) + .collect() + } + names(old) != names(new) +} + +/// Borrow the strategy rows a just-completed base read published, when it published any. +/// +/// `Split` carries no comparable group set (only raw per-quote totals), and a failed read +/// carries none at all — both read as "nothing new to measure", not as "the set became empty". +pub(in crate::analytics) fn published_groups( + result: &moon_core::db::ReadResult>, +) -> Option<&[GroupStat]> { + match result { + Ok(ProfitScope::Comparable { data, .. }) | Ok(ProfitScope::Empty(data)) => { + Some(&data.strategies) + } + Ok(ProfitScope::Split(_)) | Err(_) => None, + } +} + impl AnalyticsView { /// Rebuild the row order unless the cache already holds the same inputs. /// diff --git a/crates/moon-ui-gpui/src/analytics/tuner/list/tests.rs b/crates/moon-ui-gpui/src/analytics/tuner/list/tests.rs index 0a8e34cb..3259b76f 100644 --- a/crates/moon-ui-gpui/src/analytics/tuner/list/tests.rs +++ b/crates/moon-ui-gpui/src/analytics/tuner/list/tests.rs @@ -6,8 +6,8 @@ use moon_core::db::analytics::GroupStat; use super::{ - SORT_NAME, StratListFilter, VisibleKey, VisibleRows, filter_sort_indices, memo_is_fresh, - restore_strat_sort, + SORT_NAME, StratListFilter, VisibleKey, VisibleRows, core_names_changed, filter_sort_indices, + memo_is_fresh, restore_strat_sort, }; /// A group with just the fields these tests filter and sort on. @@ -23,6 +23,67 @@ fn g(name: &str, kind: &str, alive: Option, bl: i64, profit: f64) -> GroupS } } +/// A group whose Core column either names one core or summarizes several. +fn core_group(core: &str, cores_n: i64) -> GroupStat { + let mut group = g(core, "K", Some(2), 0, 0.0); + group.core = core.to_string(); + group.cores_n = cores_n; + group +} + +/// `analytics/tuner/list/mod.rs:core_names_changed` must compare the set of names measured by +/// the Core column. Replacing its set comparison with row order would remeasure every glyph after +/// an otherwise identical report refresh, restoring the writer-path hitch this cache avoids. +#[test] +fn core_name_changes_ignore_row_order_and_multi_core_summaries() { + let old = vec![core_group("Alpha", 1), core_group("Beta", 1)]; + let reordered = vec![core_group("Beta", 1), core_group("Alpha", 1)]; + let with_summary_only = vec![ + core_group("Beta", 1), + core_group("Alpha", 1), + core_group("Many cores", 2), + ]; + let with_new_single_core = vec![ + core_group("Alpha", 1), + core_group("Beta", 1), + core_group("Gamma", 1), + ]; + + assert!( + !core_names_changed(Some(&old), Some(&reordered)), + "the same measured names in another row order retain the width cache" + ); + assert!( + !core_names_changed(Some(&old), Some(&with_summary_only)), + "a multi-core summary is not a separately measured Core-column name" + ); + assert!( + core_names_changed(Some(&old), Some(&with_new_single_core)), + "a newly measured single-core name needs a fresh width" + ); + assert!( + core_names_changed(Some(&old), Some(&[core_group("Alpha", 1)])), + "a disappearing measured name also changes the rendered column" + ); +} + +/// `analytics/tuner/list/mod.rs:core_names_changed` must treat an unpublished replacement as no +/// change. Treating `new == None` as changed clears the settled width cache for every failed or +/// split read, bringing the visible writer-path hitch back without new names to measure. +#[test] +fn unpublished_core_names_do_not_invalidate_a_settled_width() { + let old = vec![core_group("Alpha", 1)]; + + assert!( + core_names_changed(None, Some(&old)), + "the first published Core-column set needs an initial measurement" + ); + assert!( + !core_names_changed(Some(&old), None), + "a failed or split replacement publishes no names and keeps the old width" + ); +} + /// The default key over group set `base`: no search, no kind, no list filter, active-only off, /// unsorted. fn key_over(base: usize) -> VisibleKey { diff --git a/crates/moon-ui-gpui/src/analytics/tuner/mod.rs b/crates/moon-ui-gpui/src/analytics/tuner/mod.rs index ab8eeb47..c7b06a21 100644 --- a/crates/moon-ui-gpui/src/analytics/tuner/mod.rs +++ b/crates/moon-ui-gpui/src/analytics/tuner/mod.rs @@ -47,7 +47,9 @@ mod time; pub(super) use coins::picker::CoinListsState; pub(super) use coins::state::CoinsState; pub(super) use filter::state::TunerState; -pub(super) use list::{StratListFilter, VisibleRows, restore_strat_sort}; +pub(super) use list::{ + StratListFilter, VisibleRows, core_names_changed, published_groups, restore_strat_sort, +}; pub(super) use time::state::TimeTunerState; // Column descriptors of the comparison tables — re-exported so the submodules (`list`) take diff --git a/crates/moon-ui-gpui/src/analytics/tuner/time/mod.rs b/crates/moon-ui-gpui/src/analytics/tuner/time/mod.rs index 1ce0a742..05285de3 100644 --- a/crates/moon-ui-gpui/src/analytics/tuner/time/mod.rs +++ b/crates/moon-ui-gpui/src/analytics/tuner/time/mod.rs @@ -21,6 +21,7 @@ use moon_ui::{MoonPalette, h_flex, v_flex}; use rust_i18n::t; use super::super::AnalyticsView; +use super::super::refresh::CatchUpOutcome; use super::super::summary::{fmt_signed, sign_color}; use super::kpi::kpi_matrix_card; use crate::design; @@ -36,7 +37,7 @@ impl AnalyticsView { self.reload_time_inner(false, true, cx); } - /// Recompute report-stale time data and retry transient database contention. + /// Recompute report-stale time data through the writer-driven catch-up path. /// /// Args: /// show_overlay: Whether queued user work requires blocking progress feedback. @@ -127,36 +128,34 @@ impl AnalyticsView { if this.time_tuner.seq != req { return; // the period/filters/strategy/grid have already changed } - let retry = profiles - .as_ref() - .err() - .filter(|error| error.kind() == Some(moon_core::db::FailKind::Busy)) - .or_else(|| { - stats - .as_ref() - .err() - .filter(|error| error.kind() == Some(moon_core::db::FailKind::Busy)) - }) - .or_else(|| { - slider - .as_ref() - .err() - .filter(|error| error.kind() == Some(moon_core::db::FailKind::Busy)) - }) - .cloned(); + let profiles_outcome = CatchUpOutcome::of_read(&profiles); + let stats_outcome = CatchUpOutcome::of_read(&stats); + let slider_outcome = CatchUpOutcome::of_read(&slider); this.time_tuner.dirty = super::super::refresh::report_result_is_stale( report_req, this.current_report_generation(), profiles.is_err() || stats.is_err() || slider.is_err() || current.is_none(), ); // The profile, slider colors, and KPI are independent load surfaces, so each - // retains its own classified error instead of collapsing it to "no data". - this.time_tuner.profiles.apply(profiles); - this.time_tuner.slider.apply(slider); - this.time_tuner.stats.apply(stats); + // retains its own classified error instead of collapsing it to "no data" — unless + // that outcome is transient with a scheduled correction left + // (`keep_on_catch_up`), in which case its own settled snapshot survives the + // catch-up instead of blinking through the intermediate result. + let keep_profiles = + this.keep_on_catch_up(after_report, profiles_outcome, report_req); + let keep_slider = this.keep_on_catch_up(after_report, slider_outcome, report_req); + let keep_stats = this.keep_on_catch_up(after_report, stats_outcome, report_req); + this.time_tuner + .profiles + .apply_or_keep(profiles, keep_profiles); + this.time_tuner.slider.apply_or_keep(slider, keep_slider); + this.time_tuner.stats.apply_or_keep(stats, keep_stats); this.time_tuner.apply_current_read(current, after_report); if after_report { - this.settle_report_refresh_retry(retry.as_ref(), cx); + let transient = profiles_outcome.is_transient() + || stats_outcome.is_transient() + || slider_outcome.is_transient(); + this.settle_report_refresh_retry(transient, cx); } cx.notify(); }, diff --git a/crates/moon-ui-gpui/src/load_state.rs b/crates/moon-ui-gpui/src/load_state.rs index 185ec6d8..670644fa 100644 --- a/crates/moon-ui-gpui/src/load_state.rs +++ b/crates/moon-ui-gpui/src/load_state.rs @@ -74,6 +74,23 @@ impl LoadState { }; } + /// Keep revalidation stale data only for a caller-approved transient failure. + /// + /// `NotReady` never qualifies: it is a completed absence, so preserving data would promise a + /// recovery that no retry can provide. + pub(crate) fn apply_or_keep(&mut self, r: Result, keep_on_failure: bool) { + let keeps = keep_on_failure + && matches!(r, Err(ref e) if !matches!(e, ReadFail::NotReady)) + && matches!(self, LoadState::Loading { stale: Some(_) }); + if keeps { + if let LoadState::Loading { stale: Some(v) } = self { + *self = LoadState::Ready(Arc::clone(v)); + } + return; + } + self.apply(r); + } + /// What this surface should render: either the data, or the placeholder /// standing in for it. `empty` decides whether loaded data counts as empty. /// diff --git a/crates/moon-ui-gpui/src/load_state/tests.rs b/crates/moon-ui-gpui/src/load_state/tests.rs index b67272ca..6ace1298 100644 --- a/crates/moon-ui-gpui/src/load_state/tests.rs +++ b/crates/moon-ui-gpui/src/load_state/tests.rs @@ -1,7 +1,65 @@ //! Classified load-state presentation regression tests. use super::{LoadState, Note}; -use moon_core::db::ReadFail; +use moon_core::db::{FailKind, ReadFail}; +use std::sync::Arc; + +/// Construct a classified read failure without coupling tests to a rendered error message. +fn failure(kind: FailKind) -> ReadFail { + ReadFail::Failed { + kind, + msg: Arc::from("test failure"), + } +} + +/// `load_state.rs:LoadState::apply_or_keep` must retain only a settled stale snapshot when a +/// report catch-up gets Busy. Keeping a NotReady result would show numbers from an unavailable +/// replica, while dropping a valid stale snapshot makes the Analytics surface flash to an error. +#[test] +fn catch_up_failure_keeps_only_a_real_stale_snapshot() { + let mut stale = LoadState::Ready(Arc::new(vec![42])); + stale.begin(); + stale.apply_or_keep(Err(failure(FailKind::Busy)), true); + assert!( + matches!(&stale, LoadState::Ready(_)), + "a Busy revalidation restores the settled stale snapshot" + ); + + let mut no_stale = LoadState::>::default(); + no_stale.apply_or_keep(Err(failure(FailKind::Busy)), true); + assert!( + matches!(&no_stale, LoadState::Failed(_)), + "an initial read failure has no snapshot to preserve" + ); + + let mut unavailable = LoadState::Ready(Arc::new(vec![7])); + unavailable.begin(); + unavailable.apply_or_keep(Err(ReadFail::NotReady), true); + assert!( + matches!(&unavailable, LoadState::NotReady), + "NotReady is a completed unavailable-replica answer, never stale data" + ); +} + +/// `load_state.rs:LoadState::apply_or_keep` must publish a failure when preservation is disabled. +/// Keeping the old snapshot after a user scope change would put the prior scope's numbers under +/// the new label, which is worse than a visible classified read failure. +#[test] +fn scope_changes_do_not_keep_the_previous_snapshot_after_a_failure() { + let mut state = LoadState::Ready(Arc::new(vec![42])); + state.begin(); + state.apply_or_keep(Err(failure(FailKind::Busy)), false); + assert!( + matches!(&state, LoadState::Failed(_)), + "a non-preserving scope change must surface the completed failure" + ); + + state.apply_or_keep(Ok(vec![9]), true); + assert!( + matches!(&state, LoadState::Ready(_)), + "a successful replacement still settles normally" + ); +} /// Incomparable quote scope is guidance, not a reports-database failure. /// diff --git a/crates/moon-ui-gpui/src/settings/connections/columns.rs b/crates/moon-ui-gpui/src/settings/connections/columns.rs index 211327ea..432b0fe2 100644 --- a/crates/moon-ui-gpui/src/settings/connections/columns.rs +++ b/crates/moon-ui-gpui/src/settings/connections/columns.rs @@ -16,6 +16,18 @@ //! the two kinds genuinely coexist in one row, and scaling all of them would be as wrong as //! scaling none. //! +//! The three columns holding user-typed text -- name, key, group -- all GROW, and two of them +//! carry a [`ConnCol::max`]. That is the second thing stated exactly once here, and it is a +//! response to the row having TWO regimes rather than one. `table.rs::cell` gives a non-growing +//! column `flex_shrink_0` as well, so `grow: false` means rigid in both directions; the rigid +//! columns plus the inset, the scrollbar gutter and the twelve gaps come to about 661px of the +//! 824px body the DEFAULT 860px Settings window leaves, against about 487px of text-column bases. +//! So the default window is a SHRINK regime -- the caps are inert there and the BASES decide who +//! keeps what -- while a wide window is a GROW regime, where the caps are what stop the key and +//! the group from spending width they have nothing readable to put in it. Both dials are needed, +//! and neither is the other's fallback. All three carry ONE width policy, which is what makes +//! their 150 > 140 > 85 ordering a property of the literals rather than of the Font slider. +//! //! Pure and GPUI-free on purpose -- its sibling test file can assert the header and the rows agree //! without a window. @@ -60,6 +72,24 @@ pub(super) enum ConnColWidth { Raw, /// A design reference the Micro dropdown trigger scales by `tokens.font(10) / 10`. MicroTrigger, + /// A design reference stated in CHARACTERS of a `MoonInput::small()`, scaled by the same + /// `tokens.font(10) / 10` ratio. Carried by all three text columns -- `h-name`, `h-key` and + /// `h-group`. + /// + /// The small input renders its text at `tokens.font(10)` (MoonUI + /// `MoonInputMetrics::base_for_size(Size::Small)`), so a width that means "this many readable + /// characters" is only true at the default Font setting unless it follows that ratio -- at the + /// shipped +3 delta a raw 140px cell holds about 18 characters where the design reference + /// promised 24. It shares [`MicroTriggerMetrics::scale`] because that IS the Font ratio, and + /// deliberately NOT the floor beside it: `min_width` is the dropdown TRIGGER's own minimum and + /// means nothing to an input. + /// + /// The three text columns share it for a second reason: their shrink order at a narrow window + /// is decided by their RESOLVED bases, so a mixed policy would let one Font setting overtake + /// another column. `repair_ui_font_delta` (`moon-core/src/config/schema.rs`) deliberately + /// preserves any finite hand-edited delta, well past the slider's +6, so "no reachable setting + /// reverses it" is only true when the comparison is scale-free. + TextScaled, } /// One column of the core table, as both the header and the row read it. @@ -74,10 +104,33 @@ pub(super) struct ConnCol { /// Flex basis in rendered pixels. Authoritative: every cell also carries `min_w_0()`, so a /// wider child paints over its neighbour instead of pushing it. pub(super) basis: f32, - /// Whether the column absorbs free space. Only columns holding user-typed text that - /// TRUNCATES do -- `h-name` and `h-group`. `h-key` deliberately does not: its content is - /// masked, so extra width buys more dots and nothing readable. + /// Whether the column absorbs free space. The three columns carrying user-typed text do -- + /// `h-name`, `h-key` and `h-group` -- and only `h-name` does so without a [`ConnCol::max`]. + /// + /// It is also what decides whether the column can SHRINK: `table.rs::cell` gives a + /// non-growing column `flex_shrink_0`, so `grow: false` means "this width is rigid in BOTH + /// directions". The default 860px Settings window does not fit this row, so every column that + /// can afford to give way there has to be ABLE to -- see [`ConnCol::max`] for the arithmetic. pub(super) grow: bool, + /// Upper bound on a GROWING column, in the same units as [`ConnCol::basis`] and resolved by + /// the same [`ConnCol::width`] policy. `None` means "grow without limit"; only `h-name` has it. + /// + /// GROW-WITH-A-CAP rather than `grow: false` is the load-bearing choice here, and the 860px + /// window is why. `settings/render.rs` leaves an 824px body inside its 18px padding there; the + /// rigid columns -- two checkboxes, the bundle field, the colour picker, the three Micro + /// dropdowns, the delete, reconnect and status glyphs -- plus the table inset, the scrollbar + /// gutter and twelve gaps take about 661px of it at the SHIPPED Font delta of +3, leaving + /// roughly 163px for three text columns whose bases resolve to about 487px. The default window + /// is therefore a SHRINK regime, and anything pinned `flex_shrink_0` in it is width the text + /// columns can never get back: a fixed 260px key would simply BE the widest column on a + /// default install while the name column collapsed to nothing. Growable, all three give way in + /// proportion to their RESOLVED bases instead -- which is why those bases are ordered, and why + /// all three carry the same [`ConnColWidth::TextScaled`] policy so that the ordering cannot + /// depend on the Font setting. + /// + /// The cap governs the other regime. Widen the window and Taffy freezes each capped item at + /// its cap, then hands the remaining free space to the only uncapped one -- `h-name`. + pub(super) max: Option, /// How [`ConnCol::basis`] becomes a rendered width. pub(super) width: ConnColWidth, /// Where the header label and the row control sit inside the cell. @@ -100,6 +153,7 @@ const CONN_COLS: [ConnCol; 13] = [ tip: Some("conn.tip.act"), basis: 34.0, grow: false, + max: None, width: ConnColWidth::Raw, align: ConnColAlign::Center, head_pad: 0.0, @@ -110,34 +164,52 @@ const CONN_COLS: [ConnCol; 13] = [ tip: Some("conn.tip.win"), basis: 34.0, grow: false, + max: None, width: ConnColWidth::Raw, align: ConnColAlign::Center, head_pad: 0.0, }, + // The ONLY uncapped column, which is what makes it the widest of the three on any window with + // free space to give: `h-key` and `h-group` freeze at their caps and Taffy hands what is left + // to whatever is still growable. Its 150 is also the largest of the three bases, and because + // all three share one width policy that ordering cannot be reversed by any Font setting -- so + // it stays the widest of them on a window too narrow for the table as well, which the default + // 860px one is. ConnCol { id: "h-name", label: Some("conn.col.name"), tip: Some("conn.tip.name"), basis: 150.0, grow: true, - width: ConnColWidth::Raw, + max: None, + width: ConnColWidth::TextScaled, align: ConnColAlign::Left, head_pad: 8.0, }, - // FIXED, not growing, at the basis it always had. The key field is MASKED and MoonUI draws - // one bullet per character of a variable-length key, so there is no "width of the masked - // value" to size to -- growth simply handed the column every spare pixel, ~480px of a 1791px - // window spent on identical dots while `Имя` and `Группа` truncated real text beside them. - // 200 keeps a usable text viewport next to the input's mask-toggle and clear affixes and the - // sibling Paste glyph (`table.rs::paste_key_affix`); a longer key scrolls inside the field, - // which is what it did before and what a masked field can afford. + // Capped rather than uncapped, because the key field is MASKED: MoonUI draws one bullet per + // character of a variable-length key, so there is no "width of the masked value" to size to, + // and uncapped growth simply handed the column every spare pixel -- ~480px of a 1791px window + // spent on identical dots while the name and group columns truncated real text beside them. + // + // 260 is what the field is actually asking for. At 200 the text viewport left over after the + // input's mask-toggle and clear affixes, its own padding and the sibling Paste glyph + // (`table.rs::paste_key_affix`) was still cramped, and that -- the VIEWPORT, not the key's + // length -- is what the user reported; a longer key scrolls inside the field and always did. + // + // The 140 basis is deliberately far BELOW the cap and below `h-name`'s. Shrinkage is + // proportional to the resolved basis, so on a window too narrow to hold the table -- and + // 860px, the default, is exactly such a window -- the bases alone decide the order the three + // text columns end up in, and 140 puts the key second. Pinning this column at a rigid 260 + // instead would have made the masked key the widest thing on screen there while the name + // column collapsed. ConnCol { id: "h-key", label: Some("conn.col.key"), tip: Some("conn.tip.key"), - basis: 200.0, - grow: false, - width: ConnColWidth::Raw, + basis: 140.0, + grow: true, + max: Some(260.0), + width: ConnColWidth::TextScaled, align: ConnColAlign::Left, head_pad: 8.0, }, @@ -150,6 +222,7 @@ const CONN_COLS: [ConnCol; 13] = [ tip: Some("conn.tip.proto"), basis: 52.0, grow: false, + max: None, width: ConnColWidth::MicroTrigger, align: ConnColAlign::Center, head_pad: 0.0, @@ -160,19 +233,30 @@ const CONN_COLS: [ConnCol; 13] = [ tip: Some("conn.tip.preset"), basis: 72.0, grow: false, + max: None, width: ConnColWidth::MicroTrigger, align: ConnColAlign::Center, head_pad: 0.0, }, - // Grows with `h-name`: both hold user-typed text that truncates, and the width the masked - // key gave up is exactly what they were missing. + // The lowest basis AND the lowest cap of the three, because a group name is one word. The 85 + // basis is what puts it last when the window is too narrow for the row; it is only ever + // compared against the other two, which share its policy, so the ordering is a property of the + // three literals and holds at every Font setting rather than only inside the slider's range. + // + // The cap: 140 is 126px of text viewport once the small input's 7px paddings are removed, + // about 24 lowercase glyphs at the design-reference font size -- comfortable for a name like + // "default" and for anything a user would actually type, and nothing like the ~420px this + // column was taking of a 1791px window while the name column truncated beside it. Uncapped + // growth here is what made the two growing columns split the free width evenly when only one + // of them holds a name long enough to need it. ConnCol { id: "h-group", label: Some("conn.col.group"), tip: Some("conn.tip.group"), - basis: 110.0, + basis: 85.0, grow: true, - width: ConnColWidth::Raw, + max: Some(140.0), + width: ConnColWidth::TextScaled, align: ConnColAlign::Left, head_pad: 8.0, }, @@ -182,6 +266,7 @@ const CONN_COLS: [ConnCol; 13] = [ tip: Some("conn.tip.bundle"), basis: 96.0, grow: false, + max: None, width: ConnColWidth::Raw, align: ConnColAlign::Left, head_pad: 8.0, @@ -192,6 +277,7 @@ const CONN_COLS: [ConnCol; 13] = [ tip: Some("conn.tip.flags"), basis: 52.0, grow: false, + max: None, width: ConnColWidth::MicroTrigger, align: ConnColAlign::Center, head_pad: 0.0, @@ -204,6 +290,7 @@ const CONN_COLS: [ConnCol; 13] = [ tip: Some("conn.tip.color"), basis: 128.0, grow: false, + max: None, width: ConnColWidth::Raw, align: ConnColAlign::Center, head_pad: 0.0, @@ -214,6 +301,7 @@ const CONN_COLS: [ConnCol; 13] = [ tip: None, basis: 24.0, grow: false, + max: None, width: ConnColWidth::Raw, align: ConnColAlign::Center, head_pad: 0.0, @@ -224,6 +312,7 @@ const CONN_COLS: [ConnCol; 13] = [ tip: None, basis: 24.0, grow: false, + max: None, width: ConnColWidth::Raw, align: ConnColAlign::Center, head_pad: 0.0, @@ -234,6 +323,7 @@ const CONN_COLS: [ConnCol; 13] = [ tip: None, basis: 16.0, grow: false, + max: None, width: ConnColWidth::Raw, align: ConnColAlign::Center, head_pad: 0.0, @@ -250,6 +340,10 @@ const CONN_COLS: [ConnCol; 13] = [ #[derive(Clone, Copy, Debug)] pub(super) struct MicroTriggerMetrics { /// Multiplier from a design-reference trigger width to its rendered width. + /// + /// It is `tokens.font(10) / 10`, the Font-slider ratio itself, which is why + /// [`ConnColWidth::TextScaled`] reads it too: a `MoonInput::small()` sizes its text at the + /// same `tokens.font(10)`. pub(super) scale: f32, /// Floor MoonUI clamps a scaled trigger up to. pub(super) min_width: f32, @@ -307,12 +401,46 @@ impl ConnColId { /// The width both the header cell and the row cell are laid out at. pub(super) fn width(self, micro: MicroTriggerMetrics) -> f32 { let spec = self.spec(); - match spec.width { - ConnColWidth::Raw => spec.basis, + Self::resolve(spec.width, spec.basis, micro) + } + + /// Upper bound this column may grow to, in pixels, or `None` for an uncapped one. + /// + /// Read by `table::SettingsView::cell` as `max_w`, which is what stops a growing column from + /// taking free width it has no readable content to spend. + /// + /// Args: + /// micro: The Micro dropdown trigger's rendered-width inputs, from + /// `table::micro_trigger_metrics`. + /// + /// Returns: + /// The cap both the header cell and the row cell are laid out against, if there is one. + pub(super) fn max_width(self, micro: MicroTriggerMetrics) -> Option { + let spec = self.spec(); + // Same policy as the basis on purpose: a cap in different units from the width it bounds + // is a cap that stops meaning what it says the moment a slider moves. + spec.max.map(|max| Self::resolve(spec.width, max, micro)) + } + + /// Turn one design reference into rendered pixels under a column's width policy. + /// + /// Args: + /// policy: The column's [`ConnColWidth`]. + /// reference: A basis or a cap, in that policy's units. + /// micro: The Micro dropdown trigger's rendered-width inputs. + /// + /// Returns: + /// The rendered pixel value. + fn resolve(policy: ConnColWidth, reference: f32, micro: MicroTriggerMetrics) -> f32 { + match policy { + ConnColWidth::Raw => reference, // MIRRORS MoonUI's own `max(scaled(basis), minimum_width)`. Dropping the floor let // a trigger outgrow its column at a large `ui_scale`; `min_w_0` would then have it // paint across its neighbour instead of moving it -- misaligned either way. - ConnColWidth::MicroTrigger => (spec.basis * micro.scale).max(micro.min_width), + ConnColWidth::MicroTrigger => (reference * micro.scale).max(micro.min_width), + // No floor: the floor beside `scale` is the dropdown trigger's own minimum, and an + // input has nothing to do with it. + ConnColWidth::TextScaled => reference * micro.scale, } } diff --git a/crates/moon-ui-gpui/src/settings/connections/columns/tests.rs b/crates/moon-ui-gpui/src/settings/connections/columns/tests.rs index efddf812..89a52158 100644 --- a/crates/moon-ui-gpui/src/settings/connections/columns/tests.rs +++ b/crates/moon-ui-gpui/src/settings/connections/columns/tests.rs @@ -46,24 +46,32 @@ fn indent_parts_match_the_header_inset() { ); } -/// `columns.rs:CONN_COLS` must preserve the three Micro-trigger columns. Replacing `h-preset`'s -/// 72px `MicroTrigger` policy with a 92px `Raw` width makes the control stop scaling with its -/// rendered trigger and lets the header and server row disagree at non-default font scales. +/// `columns.rs:CONN_COLS` must preserve each column's declared scaling policy. Replacing +/// any of `h-name`, `h-key` or `h-group`'s `TextScaled` policies with `Raw`, or `h-preset`'s +/// `MicroTrigger` policy with `Raw`, makes the header and server row disagree with their controls +/// at non-default font scales. #[test] fn widths_follow_the_frozen_per_column_policy() { const MICRO_COLUMNS: [ConnColId; 3] = [ConnColId::Proto, ConnColId::Preset, ConnColId::Data]; + const TEXT_SCALED_COLUMNS: [ConnColId; 3] = [ConnColId::Name, ConnColId::Key, ConnColId::Group]; const SCALES: [f32; 3] = [0.75, 1.0, 1.3]; for column in ConnColId::ALL { let basis = column.spec().basis; let is_micro = MICRO_COLUMNS.contains(&column); + let is_text_scaled = TEXT_SCALED_COLUMNS.contains(&column); let expected_policy = if is_micro { ConnColWidth::MicroTrigger + } else if is_text_scaled { + ConnColWidth::TextScaled } else { ConnColWidth::Raw }; for scale in SCALES { - let expected_width = if is_micro { basis * scale } else { basis }; + let expected_width = match expected_policy { + ConnColWidth::Raw => basis, + ConnColWidth::MicroTrigger | ConnColWidth::TextScaled => basis * scale, + }; assert_eq!( column.width(MicroTriggerMetrics { scale, @@ -82,9 +90,8 @@ fn widths_follow_the_frozen_per_column_policy() { } } -/// `columns.rs:CONN_COLS` must let only Name and Group absorb spare width: Key is excluded because -/// its masked content has no readable length to reward with width. Turning `h-key.grow` back on -/// wastes space on dots, while removing growth from Name or Group truncates user-entered text; +/// `columns.rs:CONN_COLS` must let Name, Key and Group absorb spare width without making any of +/// them rigid at the default window size. Removing growth from one truncates user-entered text; /// every visible header label also needs help text. #[test] fn growth_and_tooltips_match_the_text_column_contract() { @@ -92,7 +99,7 @@ fn growth_and_tooltips_match_the_text_column_contract() { .into_iter() .filter(|column| column.spec().grow) .collect(); - assert_eq!(growing, [ConnColId::Name, ConnColId::Group]); + assert_eq!(growing, [ConnColId::Name, ConnColId::Key, ConnColId::Group]); for column in ConnColId::ALL { let spec = column.spec(); @@ -104,3 +111,79 @@ fn growth_and_tooltips_match_the_text_column_contract() { } } } + +/// `columns.rs:CONN_COLS` must cap only Key and Group through their own policies. Removing either +/// cap lets that column consume wide-window space, while resolving either cap as raw pixels makes +/// its readable character count shrink when the user raises the Font setting. +#[test] +fn caps_match_the_text_column_contract_at_each_font_scale() { + const MICRO_COLUMNS: [ConnColId; 3] = [ConnColId::Proto, ConnColId::Preset, ConnColId::Data]; + const TEXT_SCALED_COLUMNS: [ConnColId; 3] = [ConnColId::Name, ConnColId::Key, ConnColId::Group]; + const SCALES: [f32; 3] = [0.75, 1.0, 1.3]; + + for column in ConnColId::ALL { + let expected_cap = match column { + ConnColId::Key => Some(260.0), + ConnColId::Group => Some(140.0), + _ => None, + }; + let expected_policy = if MICRO_COLUMNS.contains(&column) { + ConnColWidth::MicroTrigger + } else if TEXT_SCALED_COLUMNS.contains(&column) { + ConnColWidth::TextScaled + } else { + ConnColWidth::Raw + }; + + assert_eq!(column.spec().max, expected_cap, "{column:?} cap reference"); + for scale in SCALES { + let metrics = MicroTriggerMetrics { + scale, + min_width: 0.0, + }; + let expected_max = expected_cap.map(|cap| match expected_policy { + ConnColWidth::Raw => cap, + ConnColWidth::MicroTrigger | ConnColWidth::TextScaled => cap * scale, + }); + assert_eq!( + column.max_width(metrics), + expected_max, + "{column:?} cap at scale {scale}" + ); + + if let Some(expected_max) = expected_max { + let expected_basis = column.width(metrics); + assert!( + expected_max > expected_basis, + "{column:?} cap must exceed its basis at scale {scale}" + ); + } + } + } +} + +/// `columns.rs:CONN_COLS` must leave Name as the only uncapped growing column and resolve Name +/// wider than Key wider than Group at every finite Font scale. Changing `h-key` from `TextScaled` +/// to `Raw` makes Group outrank the masked Key at a hand-edited +10 Font delta, leaving less room +/// for the user-entered name. +#[test] +fn name_is_the_only_uncapped_growing_column_with_the_widest_narrow_width() { + let uncapped_growing: Vec<_> = ConnColId::ALL + .into_iter() + .filter(|column| column.spec().grow && column.spec().max.is_none()) + .collect(); + assert_eq!(uncapped_growing, [ConnColId::Name]); + + const SCALES: [f32; 5] = [0.75, 1.0, 1.3, 1.6, 2.0]; + for scale in SCALES { + let metrics = MicroTriggerMetrics { + scale, + min_width: 0.0, + }; + assert!( + ConnColId::Name.width(metrics) > ConnColId::Key.width(metrics) + && ConnColId::Key.width(metrics) > ConnColId::Group.width(metrics), + "Name, Key and Group must shrink in resolved-width order at scale {scale}" + ); + } +} diff --git a/crates/moon-ui-gpui/src/settings/connections/table.rs b/crates/moon-ui-gpui/src/settings/connections/table.rs index f0d113da..0df29f97 100644 --- a/crates/moon-ui-gpui/src/settings/connections/table.rs +++ b/crates/moon-ui-gpui/src/settings/connections/table.rs @@ -1039,6 +1039,13 @@ impl SettingsView { } else { d }; + // A cap, where a column has one, bounds growth WITHOUT freezing the column: it still + // shrinks when the window is too narrow for the row, which `flex_grow_0().flex_shrink_0()` + // below would not. + let d = match col.max_width(micro) { + Some(max) => d.max_w(px(max)), + None => d, + }; if spec.grow { d.flex_grow_1() } else { diff --git a/crates/moon-ui-gpui/src/trade_window/mod.rs b/crates/moon-ui-gpui/src/trade_window/mod.rs index faa7ba8e..a7814a88 100644 --- a/crates/moon-ui-gpui/src/trade_window/mod.rs +++ b/crates/moon-ui-gpui/src/trade_window/mod.rs @@ -50,10 +50,11 @@ use gpui::*; use moon_core::db::{ChartTradeRecord, TradeMeta}; use moon_core::market::trade_replay::worker::{self, TradeReplayRequest}; use moon_core::market::trade_replay::{ - TradeReplayEmpty, TradeReplayFailure, TradeReplayOutcome, TradeReplaySeries, TradeReplaySource, - replay_window, + TickStatus, TradeReplayEmpty, TradeReplayFailure, TradeReplayOutcome, TradeReplaySeries, + TradeReplaySource, replay_window, }; use moon_core::session::CoreId; +use moon_core::venue::Brand; pub(crate) use window::open_trade_window; @@ -73,9 +74,27 @@ pub(crate) enum TradeWindowState { /// five-minute bucket — a wrong label over real-money data, which is worse than an honest gap. /// A typed number rather than a formatted string, because this value is produced beside /// `moon-core` data and only the UI can localize the sentence around it. + /// + /// The next four fields exist for the same reason: the caption cannot ask the network again + /// for a fact only the fetch answer holds, so each is read off the series once, in `apply`, + /// before it is moved into the panel. Ready { source: TradeReplaySource, tf_min: u16, + /// How the tick attempt for this window ended — what a `Klines1m` caption NAMES as its + /// reason. `Served` only ever rides a `Ticks` source. + tick_status: TickStatus, + /// Bucket the `Ticks` points were thinned to, in ms; `0` means raw. Meaningless (and + /// always `0`) on `Klines1m`. + bucket_ms: i64, + /// Whether the `Ticks` points cover only part of the window. Always `false` on + /// `Klines1m`. + partial: bool, + /// Brand of the venue the rows came from, so a `NoRoute` caption can name it. Read off + /// `series.venue` rather than kept as a `TradeWindowView` field: `window.rs`'s + /// construction of that struct is outside this branch's file bounds, so nothing here may + /// add a field it would have to initialize. + brand: Brand, }, /// Nothing to draw, for a reason the window names. Empty(TradeReplayEmpty), @@ -227,6 +246,73 @@ pub(super) fn trade_labels( ) } +/// What one worker answer does to an already-drawn (or still-loading) window. +/// +/// One function rather than three predicates, so the whole state transition is pinnable by a +/// test without a GPUI harness — nothing in this repo renders GPUI in tests. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) struct Fold { + /// Whether `outcome` replaces `state` at all. A drawn window never regresses to an error or + /// empty message; once it is [`TradeWindowState::Ready`], only a tick upgrade or a terminal + /// answer that resolves its provisional `Pending` caption is let through. + pub accept: bool, + /// Whether the user's own candle mode should be restored before this outcome is drawn. + pub restore_candle_mode: bool, + /// Whether the trade's arrows and viewport should be (re)framed for this outcome. + pub frame: bool, +} + +/// Decide what one worker answer does to the window's state. +/// +/// Args: +/// state: What the window currently shows, i.e. what the LAST accepted outcome produced. +/// published_this_sequence: Whether a picture has already been framed for the fetch +/// `outcome` belongs to — `false` for a request's first outcome, `true` for its second. +/// outcome: The answer being folded in. +/// +/// Returns: +/// The decision. `accept == false` means `outcome` is dropped and `state` is left exactly as +/// it is. +pub(super) fn fold_outcome( + state: &TradeWindowState, + published_this_sequence: bool, + outcome: &TradeReplayOutcome, +) -> Fold { + let accept = match state { + // The rule is stated for what it means rather than for what the worker happens to send + // (the existing comment's own standard): a chart already showing something never + // regresses, but a caption that is still PROVISIONAL may be resolved. So a `Ready` state + // whose tick status has not answered yet accepts a second `Ready` outcome once that + // outcome's own status has moved past `Pending` and actually carries rows — the only + // thing that turns "пробуем тики…" into a stated reason. A state whose status is already + // terminal (a settled `Ticks` series, or a `Klines1m` reason already printed) accepts + // nothing. + TradeWindowState::Ready { tick_status, .. } => { + *tick_status == TickStatus::Pending + && matches!( + outcome, + TradeReplayOutcome::Ready(series) + if series.tick_status != TickStatus::Pending && !series.is_empty() + ) + } + // Loading, Empty or Failed has nothing on screen to protect. + TradeWindowState::Loading | TradeWindowState::Empty(_) | TradeWindowState::Failed(_) => { + true + } + }; + Fold { + accept, + // Keyed on the outcome's SOURCE, never on its position in the sequence: protocol item 2's + // reopen case delivers a settled `Ticks` series as the FIRST outcome, and a position-keyed + // rule would leave a reopened window stuck in the candle stage's forced mode. + restore_candle_mode: accept + && matches!(outcome, TradeReplayOutcome::Ready(series) if series.source == TradeReplaySource::Ticks), + // The upgrade re-uses the picture already framed rather than re-framing it, so a user who + // panned while the ticks loaded is not yanked back to the trade. + frame: accept && !published_this_sequence, + } +} + /// Name one strategy through the session's own store. /// /// `as u64` because the store keys strategies by the same bits the wire carries, which is the @@ -260,6 +346,15 @@ pub(crate) struct TradeWindowView { /// user clicked. stamps: (String, String), state: TradeWindowState, + /// Whether the current sequence's picture has already been framed (arrows + viewport), so a + /// tick upgrade landing on top of it does not yank back a user who panned while it loaded. + /// Reset to `false` every [`Self::fetch`], set once `fold_outcome`'s `frame` fires. + framed_this_sequence: bool, + /// The mode the user's own chart is actually on, captured in `window.rs` before the candle + /// stage's `CANDLE_MODE_OFF` force is applied. A tick outcome restores it — see + /// [`Self::restore_candle_mode`] — because a tick series has no reason to hide behind that + /// force: it draws points, not candles. + user_candle_mode: u8, /// Monotonic dispatch counter, so a Retry supersedes an in-flight fetch instead of racing it. sequence: u64, /// What the replica said this trade carried, kept so the strategy can be named LATER. @@ -473,6 +568,7 @@ impl TradeWindowView { self.sequence = self.sequence.wrapping_add(1); let sequence = self.sequence; self.state = TradeWindowState::Loading; + self.framed_this_sequence = false; cx.notify(); let (tx, rx) = mpsc::channel(); @@ -486,22 +582,47 @@ impl TradeWindowView { }); cx.spawn(async move |this, cx| { let executor = cx.update(|cx| cx.background_executor().clone()); - // The blocking receive sits on the background executor; the worker's own job deadline - // is what bounds it, so this task cannot outlive a stalled fetch indefinitely. - let Ok(outcome) = executor.spawn(async move { rx.recv() }).await else { - return; - }; - cx.update(|cx| { - let _ = this.update(cx, |this, cx| { - this.apply(sequence, outcome, buy_utc, close_utc, cx) + // One `TradeReplayRequest` now answers with ONE or TWO outcomes before the worker + // drops `reply` — the candle stage, then an optional tick upgrade — so this task + // receives in a loop rather than once, and the drop (a `RecvError`) is its exit + // signal, exactly like the view going away is (`this.update` failing below). The + // receiver travels into the background task and back out each iteration instead of + // living behind a lock across the await, so the blocking `recv` never holds anything + // this foreground task still needs between messages. + let mut rx = rx; + loop { + let (returned_rx, received) = executor + .spawn(async move { + let received = rx.recv(); + (rx, received) + }) + .await; + rx = returned_rx; + let Ok(outcome) = received else { + return; + }; + let applied = cx.update(|cx| { + this.update(cx, |this, cx| { + this.apply(sequence, outcome, buy_utc, close_utc, cx) + }) }); - }); + // The window closed while this outcome was in flight; nothing left to fold it + // into, and no later outcome for this sequence has anywhere to land either. + if applied.is_err() { + return; + } + } }) .detach(); } /// Fold one fetch answer into the window. /// + /// A `fetch` may now deliver up to two answers on the same `sequence` — the candle stage, + /// then an optional tick upgrade — so this runs once per outcome rather than once per fetch. + /// [`fold_outcome`] carries every decision that depends on what is already on screen; this + /// method only carries it out. + /// /// Args: /// sequence: Dispatch counter the answer belongs to. /// outcome: What the worker produced. @@ -522,6 +643,10 @@ impl TradeWindowView { if sequence != self.sequence { return; } + let fold = fold_outcome(&self.state, self.framed_this_sequence, &outcome); + if !fold.accept { + return; + } self.state = match outcome { TradeReplayOutcome::Ready(series) if series.is_empty() => { TradeWindowState::Empty(TradeReplayEmpty::NoDataInWindow) @@ -536,8 +661,27 @@ impl TradeWindowView { // range: a floor of one minute for anything finer, and no `as` wrap for anything // absurdly coarse. let tf_min = (series.tf_ms / 60_000).clamp(1, i64::from(u16::MAX)) as u16; - self.publish(series, buy_utc, close_utc, cx); - TradeWindowState::Ready { source, tf_min } + // Same reasoning, same moment: the caption's four remaining facts are the series' + // alone to give. + let tick_status = series.tick_status; + let bucket_ms = series.bucket_ms; + let partial = series.partial; + let brand = series.venue.brand; + if fold.restore_candle_mode { + self.restore_candle_mode(cx); + } + self.publish(series, buy_utc, close_utc, fold.frame, cx); + if fold.frame { + self.framed_this_sequence = true; + } + TradeWindowState::Ready { + source, + tf_min, + tick_status, + bucket_ms, + partial, + brand, + } } TradeReplayOutcome::Empty(empty) => TradeWindowState::Empty(empty), TradeReplayOutcome::Failed(failure) => { @@ -552,6 +696,23 @@ impl TradeWindowView { cx.notify(); } + /// Restore the user's own candle mode once a tick series is about to be drawn. + /// + /// `window.rs` forces candles on while no tick series has arrived because a candle replay has + /// no ticks; a tick series does, so that force's reason is gone and the user's real choice — + /// including Off, a pure tick chart — comes back. + /// + /// Args: + /// cx: View context. + fn restore_candle_mode(&mut self, cx: &mut Context) { + let mode = self.user_candle_mode; + self.panel.update(cx, |panel, pcx| { + let mut view = panel.effective_candle_view(pcx); + view.mode = mode; + panel.set_candle_view(Some(view), pcx); + }); + } + /// The price scale this window's chart is set to, for its own control to state. /// /// Read off the PANEL rather than back out of the layout, so the trigger reports what this @@ -589,7 +750,8 @@ impl TradeWindowView { cx.notify(); } - /// Hand a fetched series to this window's chart and focus it on the trade. + /// Hand a fetched series to this window's chart, and focus it on the trade on the FIRST + /// publish of a sequence. /// /// Args: /// series: The frozen rows. @@ -599,27 +761,40 @@ impl TradeWindowView { /// necessarily frames the exact pair `fetch` used to build the REST request whose /// `series` this now is. /// close_utc: True-UTC exit stamp, same provenance as `buy_utc`. + /// first_publish: Whether this is the first publish of the fetch's sequence. `false` is a + /// tick upgrade landing on a picture the candle stage already framed — re-running the + /// arrows and the viewport would yank back a user who panned while it loaded, so both + /// are skipped and only the chart's own rows are replaced. /// cx: View context. fn publish( &mut self, series: TradeReplaySeries, buy_utc: i64, close_utc: i64, + first_publish: bool, cx: &mut Context, ) { // Read off the series BEFORE it is moved into the panel, exactly as `source` and `tf_min` - // are in `apply`. - let frame = frame::trade_frame( - buy_utc.saturating_mul(1_000), - close_utc.saturating_mul(1_000), - series.tf_ms, - ); + // are in `apply`. Skipped entirely on an upgrade: the frame this trade opened on is not + // recomputed, only reused. + let frame = first_publish + .then(|| { + frame::trade_frame( + buy_utc.saturating_mul(1_000), + close_utc.saturating_mul(1_000), + series.tf_ms, + ) + }) + .flatten(); // The RAW record, deliberately: correcting it here would double-correct, since B.1 // (`chartdx/trade_history_sync.rs`) already applies the axis inside // `append_trade_history_geometry`. let record = self.record.clone(); self.panel.update(cx, |panel, pcx| { panel.attach_trade_replay(Some(std::rc::Rc::new(series)), pcx); + if !first_publish { + return; + } // THE ENTRY AND EXIT ARROWS. Owning a `ChartPanel` is not enough on its own: the // marker geometry is built during the userdata pass, and that pass only ever sees the // trades handed to this layer. The live Report path fills it from the open-request diff --git a/crates/moon-ui-gpui/src/trade_window/render.rs b/crates/moon-ui-gpui/src/trade_window/render.rs index 8fed9be8..bd443696 100644 --- a/crates/moon-ui-gpui/src/trade_window/render.rs +++ b/crates/moon-ui-gpui/src/trade_window/render.rs @@ -7,7 +7,9 @@ use gpui::prelude::FluentBuilder; use gpui::*; -use moon_core::market::trade_replay::{TradeReplayEmpty, TradeReplayFailure, TradeReplaySource}; +use moon_core::market::trade_replay::{ + TickStatus, TradeReplayEmpty, TradeReplayFailure, TradeReplaySource, +}; use moon_ui::{ MoonButton, MoonButtonSize, MoonPalette, MoonWindowFrame, MoonWindowFrameControls, h_flex, v_flex, @@ -183,11 +185,70 @@ impl TradeWindowView { let caption = match &self.state { TradeWindowState::Ready { source: TradeReplaySource::Ticks, + tf_min, + bucket_ms, + partial, .. - } => t!("trade_window.source.ticks").to_string(), - TradeWindowState::Ready { tf_min, .. } => { - t!("trade_window.source.candles_tf", min = tf_min).to_string() + } => { + let base = if *bucket_ms == 0 { + t!("trade_window.source.ticks").to_string() + } else { + t!( + "trade_window.source.ticks_bucketed", + secs = bucket_ms / 1_000 + ) + .to_string() + }; + if *partial { + // The join happens IN CODE, so no locale value carries a separator glyph. + // Every sibling Russian caption in `trade_window.yml` joins its two halves + // with an em dash, never the ASCII hyphen the other locales use — so the + // glyph itself must follow the active locale, not just the words either + // side of it. + let edges = t!("trade_window.source.ticks_edges", min = tf_min).to_string(); + let sep = match rust_i18n::locale().as_ref() { + "ru" => "—", + _ => "-", + }; + format!("{base} {sep} {edges}") + } else { + base + } } + TradeWindowState::Ready { + source: TradeReplaySource::Klines1m, + tf_min, + tick_status, + brand, + .. + } => match tick_status { + TickStatus::Pending => { + t!("trade_window.source.candles_ticks_pending", min = tf_min).to_string() + } + TickStatus::NoRoute => t!( + "trade_window.source.candles_no_route", + min = tf_min, + brand = brand.display() + ) + .to_string(), + TickStatus::OutOfRetention { retention_ms } => t!( + "trade_window.source.candles_retention", + min = tf_min, + hours = retention_ms / 3_600_000 + ) + .to_string(), + TickStatus::NoTrades => { + t!("trade_window.source.candles_no_trades", min = tf_min).to_string() + } + TickStatus::Failed => { + t!("trade_window.source.candles_failed", min = tf_min).to_string() + } + // Unreachable: `Served` only ever rides a `Ticks` source (see `TickStatus`'s + // own doc comment), never a `Klines1m` one. Answered rather than panicked, the + // way the `Loading | Empty | Failed` arm below answers its own unreachable + // case. + TickStatus::Served => String::new(), + }, // Unreachable: the caller checked `overlays_chart` first. Answered rather than // panicked, for the same reason `overlay_message` answers its own unreachable arm. TradeWindowState::Loading diff --git a/crates/moon-ui-gpui/src/trade_window/window.rs b/crates/moon-ui-gpui/src/trade_window/window.rs index 14685f2f..1724255a 100644 --- a/crates/moon-ui-gpui/src/trade_window/window.rs +++ b/crates/moon-ui-gpui/src/trade_window/window.rs @@ -195,20 +195,30 @@ pub(crate) fn open_trade_window( // minute rows into that coarser bucket // (`moon-core/src/market/trade_replay/mod.rs`: the caller's `tf_ms` wins, and a wider one // resamples), while the caption underneath claims minutes. That is the whole of the user's - // "нужны минутные свечи, а не 5минутные". + // "I need one-minute candles, not five-minute candles." // // Only `tf_min` is forced. Starting from the EFFECTIVE settings rather than // `CandleViewCfg::default()` keeps the user's candle mode, outline width, in-zone colours // and MoonShot corridor exactly as they are on their own charts. + // Captured immediately BEFORE the candle-stage force below, so `TradeWindowView::apply` + // can hand it back to `restore_candle_mode` once a tick outcome lands and that force's + // reason is gone. A window that never gets past the candle stage never reaches the + // restore path either, and keeps the forced mode for its whole life by construction — + // which is exactly right, since a candle replay genuinely has no ticks to fall back to. + let mut user_candle_mode = 0; panel.update(cx, |panel, pcx| { let mut view = panel.effective_candle_view(pcx); view.tf_min = 1; - // The SECOND thing the effective settings can carry that makes this window useless: - // candle mode Off is a pure TICK chart, and a candle replay has no ticks. Inherited - // unchanged it would draw an empty pane under a caption naming candles — the same - // dishonesty the timeframe pin exists to remove, one step further along. A user who - // turned candles off on their live chart still asked to SEE this trade, so the window - // falls back to the shipped drawing mode rather than to nothing. + user_candle_mode = view.mode; + // THE INITIAL CANDLE-MODE FORCE. It holds until a tick outcome arrives: a fresh + // request yields candles before its optional tick upgrade, while a settled tick-cache + // hit can yield ticks as its FIRST outcome. Candle mode Off is a pure TICK chart, and + // a candle replay has no ticks. Inherited unchanged it would draw an empty pane under + // a caption naming candles — the same dishonesty the timeframe pin exists to remove, + // one step further along. A user who turned candles off on their live chart still + // asked to SEE this trade, so the window falls back to the shipped drawing mode rather + // than to nothing, until `apply` restores the real choice once a tick series is on + // offer. if view.mode == moon_core::market::candles::CANDLE_MODE_OFF { view.mode = moon_core::market::CandleViewCfg::default().mode; } @@ -238,6 +248,8 @@ pub(crate) fn open_trade_window( market: market.clone(), stamps: stamps.clone(), state: TradeWindowState::Loading, + framed_this_sequence: false, + user_candle_mode, meta, strategy_pending: !named, // Nothing searched yet; the first notification does the walk. diff --git a/crates/moon-ui-gpui/tests/theme_contract/analytics.rs b/crates/moon-ui-gpui/tests/theme_contract/analytics.rs index 6764eab1..7ebf7279 100644 --- a/crates/moon-ui-gpui/tests/theme_contract/analytics.rs +++ b/crates/moon-ui-gpui/tests/theme_contract/analytics.rs @@ -3,6 +3,46 @@ use super::support::*; +/// `analytics/mod.rs:observe_report_axis` must send a machine axis observation through the Writer +/// refresh path instead of reload, while `observe_valuation_mode` remains a real scope reload. +/// Merging these paths blanks and dims Analytics on every feed reconnect; removing the valuation +/// reload leaves an actual mode change under stale values. +#[test] +fn report_axis_observation_uses_writer_refresh_while_valuation_mode_reloads() { + let analytics = read_src("analytics/mod.rs"); + let report_axis = code_only(braced_body(&analytics, "fn observe_report_axis(")); + let valuation_mode = code_only(braced_body(&analytics, "fn observe_valuation_mode(")); + + assert!( + !report_axis.contains("reload("), + "a report-axis observation must not blank the settled surface through reload" + ); + assert!( + report_axis.contains("self.request_report_refresh(") + && report_axis.contains("RefreshUrgency::Writer,") + && report_axis.contains("false,"), + "a report-axis observation must request a nonblocking Writer refresh" + ); + for required in [ + "self.seq = self.seq.wrapping_add(1);", + "self.cal_seq = self.cal_seq.wrapping_add(1);", + "self.cancel_latest_reads();", + "self.tuner.invalidate();", + "self.time_tuner.invalidate();", + "self.coins.invalidate();", + "self.coin_lists.invalidate();", + ] { + assert!( + report_axis.contains(required), + "a report-axis observation must retire every stale read identity: {required}" + ); + } + assert!( + valuation_mode.contains("self.reload("), + "a valuation-mode change remains a real scope change and must reload" + ); +} + /// Every retained view that caches civil-time presentation must observe the one shared display /// zone revision published by the header clock. /// @@ -1425,22 +1465,200 @@ fn automatic_strategy_refresh_keeps_the_visible_snapshot() { manual.contains("self.reload_strategy_base(false, true, true, cx)"), "manual scope refresh must retire values from the previous scope" ); - let preserve_snapshot = "let preserve_snapshot =\n after_report && !matches!(this.strategy_data, ProfitLoadState::Loading);"; + let preserve_snapshot = "let preserve_snapshot = !matches!(this.strategy_data, ProfitLoadState::Loading)\n && this.keep_on_catch_up(after_report, data_outcome, report_req);"; assert!( reload.contains(preserve_snapshot), "only a settled strategy snapshot may survive an automatic report refresh" ); let automatic_result = chain_between( reload, - "if !preserve_snapshot || data_error.is_none() {", + "let preserve_snapshot =", "this.strategy_dirty = refresh::report_result_is_stale(", "automatic strategy result publication", ); assert!( - automatic_result.contains("this.strategy_data.apply(data);") + automatic_result.contains("this.keep_on_catch_up(after_report, data_outcome, report_req)") + && !automatic_result.contains("if !preserve_snapshot || data_error.is_none() {") + && automatic_result.contains("this.strategy_data.apply(data)") && automatic_result.contains("this.strat_core_w = None;") && automatic_result.contains("this.strat_visible = None;"), - "an automatic read failure must preserve the complete visible strategy snapshot" + "only a retryable Busy failure may preserve the complete visible strategy snapshot" + ); +} + +/// `analytics/mod.rs:reload_summary` must retain only time-ordered chart hovers across a catch-up. +/// Moving a time-bucket clear back outside the scope-reset branch closes a valid popup on every +/// landed trade, while retaining profit-ordered `hover_kind` can show another kind's data. +#[test] +fn report_catch_up_keeps_time_bucket_hovers_but_clears_profit_ordered_kind_hover() { + let analytics = read_src("analytics/mod.rs"); + let reload = braced_body(&analytics, "fn reload_summary("); + let before_request = chain_between( + reload, + "fn reload_summary(", + "self.seq = self.seq.wrapping_add(1);", + "Summary reset path", + ); + assert!( + before_request.contains("if !after_report || range_moved {"), + "a resolved period rollover must clear time-bucket hovers even for an automatic catch-up" + ); + let scope_reset = code_only(braced_body( + before_request, + "if !after_report || range_moved", + )); + + for assignment in [ + "self.hover_daily_bucket = None;", + "self.hover_cum_bucket = None;", + ] { + assert!( + scope_reset.contains(assignment), + "{assignment} must belong only to the manual scope-reset branch" + ); + assert_eq!( + before_request.matches(assignment).count(), + 1, + "{assignment} must not also clear a report-driven catch-up" + ); + } + assert!( + !scope_reset.contains("self.hover_kind = None;"), + "kind hovers are ordered by profit and must clear during a same-scope catch-up" + ); + assert_eq!( + before_request.matches("self.hover_kind = None;").count(), + 1, + "kind hover must have one unconditional clear before the replacement request" + ); +} + +/// `analytics/mod.rs:reload_strategy_base` must invalidate the Core-column width only when the +/// measured name set changed. Unwrapping that assignment makes every published writer result +/// remeasure glyphs and visibly hitches the Strategies list despite unchanged core names. +#[test] +fn strategy_base_remeasures_core_width_only_for_changed_names() { + let analytics = read_src("analytics/mod.rs"); + let reload = braced_body(&analytics, "fn reload_strategy_base("); + let publication = chain_between( + reload, + "let preserve_snapshot =", + "this.strategy_dirty = refresh::report_result_is_stale(", + "strategy base publication", + ); + let name_change = code_only(braced_body(publication, "if core_names_changed")); + + assert_eq!( + publication.matches("this.strat_core_w = None;").count(), + 1, + "the width cache must have one invalidation site in result publication" + ); + assert!( + name_change.contains("this.strat_core_w = None;"), + "the sole width-cache invalidation must be guarded by changed measured names" + ); +} + +/// `analytics/mod.rs:reload_summary` must preserve a settled snapshot only across a retryable Busy +/// catch-up failure. Broadening that publication leaves stale Summary numbers current-looking after +/// NotReady, corruption, or an exhausted Busy budget with no retry left to correct them. +#[test] +fn summary_catch_up_preserves_only_a_retryable_busy_snapshot() { + let analytics = read_src("analytics/mod.rs"); + let reload = braced_body(&analytics, "fn reload_summary("); + let decision = code_only(chain_between( + reload, + "let preserve_snapshot =", + "this.data_dirty = refresh::report_result_is_stale(", + "Summary result publication", + )); + + assert!( + reload.contains("let preserve_snapshot ="), + "Summary completion must distinguish automatic settled snapshots from initial loading" + ); + assert!( + decision.contains("this.keep_on_catch_up(after_report, data_outcome, report_req)") + && !decision.contains("if !preserve_snapshot || data_error.is_none() {") + && decision.contains("this.data.apply(data);"), + "only a retryable Busy failure may retain the Summary snapshot" + ); +} + +/// The report-generation observer chain must not repaint Analytics before a result lands. +/// Adding `cx.notify()` to bookkeeping or scheduling turns each report generation into a whole +/// window rebuild rather than the one repaint owed by each completed visible result. +#[test] +fn report_generation_bookkeeping_has_no_repaint_and_each_completion_has_one() { + let analytics = read_src("analytics/mod.rs"); + for signature in [ + "fn observe_report_generation(", + "fn mark_report_data_stale(", + "fn schedule_report_refresh(", + "fn refresh_visible_report_data(", + ] { + let body = code_only(braced_body(&analytics, signature)); + assert!( + !body.contains("cx.notify()"), + "{signature} must only mark, schedule, or start work; completion owns repaint" + ); + } + + for (rel, signature, completion) in [ + ( + "analytics/mod.rs", + "fn reload_summary(", + "move |this, result, cx|", + ), + ( + "analytics/mod.rs", + "fn reload_strategy_base(", + "move |this, result, cx|", + ), + ( + "analytics/calendar/mod.rs", + "fn reload_calendar_inner(", + "move |this, data, cx|", + ), + ] { + let source = if rel == "analytics/mod.rs" { + analytics.clone() + } else { + read_src(rel) + }; + let reload = braced_body(&source, signature); + let callback = code_only(braced_body(reload, completion)); + assert_eq!( + reload.matches("cx.notify();").count(), + 1, + "{rel}: {signature} must repaint exactly once per landed result" + ); + assert_eq!( + callback.matches("cx.notify();").count(), + 1, + "{rel}: {signature} must put its one repaint in the database completion" + ); + } +} + +/// `analytics/mod.rs:apply_undated_result` must call `keep_on_catch_up` before it publishes an +/// error. Dropping that gate flashes a retryable Busy catch-up, while broadening it hides an +/// exhausted Busy failure and leaves the retained undated count falsely current forever. +#[test] +fn undated_catch_up_gate_runs_before_the_error_is_published() { + let analytics = read_src("analytics/mod.rs"); + let apply = braced_body(&analytics, "fn apply_undated_result("); + let failure_arm = code_only(braced_body(apply, "Err(error) =>")); + let before_error = chain_between( + &failure_arm, + "Err(error) =>", + "self.undated_error = Some(error);", + "undated failure publication", + ); + + assert!( + before_error.contains("self.keep_on_catch_up(after_report, outcome, started_generation)"), + "the retryable-Busy gate must run before the undated error can replace a retained count" ); } diff --git a/crates/moon-ui-gpui/tests/theme_contract/theme.rs b/crates/moon-ui-gpui/tests/theme_contract/theme.rs index 9244e0c2..b8c93807 100644 --- a/crates/moon-ui-gpui/tests/theme_contract/theme.rs +++ b/crates/moon-ui-gpui/tests/theme_contract/theme.rs @@ -383,3 +383,21 @@ fn popover_contents_do_not_paint_a_second_surface() { ); } } + +/// `settings/connections/table.rs:SettingsView::cell` must apply a column's resolved cap through +/// `max_w`. Deleting that match lets Group consume wide-window space past its 140px cap and +/// truncates the uncapped Name field. +#[test] +fn connections_cells_apply_their_resolved_growth_cap() { + let table = read_src("settings/connections/table.rs"); + let cell = code_only(braced_body( + &table, + "fn cell(col: ConnColId, micro: MicroTriggerMetrics) -> Div", + )); + + assert!( + cell.contains("let d = match col.max_width(micro) {") + && cell.contains("Some(max) => d.max_w(px(max)),"), + "SettingsView::cell must resolve ConnColId::max_width and apply it with Div::max_w" + ); +} diff --git a/locales/trade_window.yml b/locales/trade_window.yml index 9ddfd5a1..61912c12 100644 --- a/locales/trade_window.yml +++ b/locales/trade_window.yml @@ -43,12 +43,45 @@ trade_window.source.ticks: en: "exchange ticks" es: "ticks del exchange" +trade_window.source.ticks_bucketed: + ru: "тики с биржи, шаг %{secs} с" + en: "exchange ticks, %{secs} s step" + es: "ticks del exchange, paso de %{secs} s" + +# Присоединяется к одной из двух строк выше через " - " прямо в коде — глиф-разделитель в +# словаре не хранится. +trade_window.source.ticks_edges: + ru: "края закрыты свечами %{min} мин" + en: "edges filled by %{min}-minute candles" + es: "bordes cubiertos por velas de %{min} min" + # Таймфрейм подставляется из САМИХ строк, а не заявляется константой: раньше здесь стояла -# фиксированная «минутные свечи», пока панель пересобирала минутки в пятиминутки. -trade_window.source.candles_tf: - ru: "свечи %{min} мин" - en: "%{min}-minute candles" - es: "velas de %{min} min" +# фиксированная «минутные свечи», пока панель пересобирала минутки в пятиминутки. Свечи теперь +# всегда называют причину, по которой окно показывает их вместо тиков. +trade_window.source.candles_ticks_pending: + ru: "свечи %{min} мин, тики ещё грузятся" + en: "%{min}-minute candles, still fetching ticks" + es: "velas de %{min} min, cargando ticks todavía" + +trade_window.source.candles_no_route: + ru: "свечи %{min} мин — у %{brand} нет тиков" + en: "%{min}-minute candles - %{brand} has no public ticks" + es: "velas de %{min} min - %{brand} no ofrece ticks" + +trade_window.source.candles_retention: + ru: "свечи %{min} мин — биржа хранит тики только %{hours} ч" + en: "%{min}-minute candles - the exchange keeps ticks for only %{hours} h" + es: "velas de %{min} min - el exchange guarda ticks solo %{hours} h" + +trade_window.source.candles_no_trades: + ru: "свечи %{min} мин — сделок за это время не было" + en: "%{min}-minute candles - the exchange had no trades here" + es: "velas de %{min} min - el exchange no tuvo operaciones aquí" + +trade_window.source.candles_failed: + ru: "свечи %{min} мин — тики не загрузились" + en: "%{min}-minute candles - could not load ticks" + es: "velas de %{min} min - no se pudieron cargar los ticks" trade_window.empty.no_data: ru: "За это время по рынку нет данных." diff --git a/tools/gen_replica.py b/tools/gen_replica.py index 1f239c3e..c7c205d9 100644 --- a/tools/gen_replica.py +++ b/tools/gen_replica.py @@ -1,90 +1,275 @@ """Build a synthetic MoonTerminal report replica for Analytics timing measurements. -Usage: python gen_replica.py [span_days] +Usage: python gen_replica.py [span_days] [--analyze] -Writes /data/reports.sqlite and /data/strategies.sqlite (the Windows -data-directory layout crates/moon-core/src/config/paths.rs resolves). -Schema mirrors crates/moon-core/src/db/rep.rs (orders_rep + REP_INDEXES) and the columns -crates/moon-core/src/db/analytics reads. The supplied data directory must be disposable: the -generator recreates its report and strategy database files. +Writes the complete Windows data-directory layout `crates/moon-core/src/config/paths.rs` +resolves: `/data/reports.sqlite`, `strategies.sqlite`, `valuation.sqlite` and +`klines.sqlite`, plus a `/data/fixture.json` manifest. The supplied data directory +must be disposable: the generator recreates every one of those files. + +`reports.sqlite`'s schema mirrors every column `crates/moon-core/src/db/report_read.rs`'s +`DISPLAY_COLUMNS` projects (see `crates/moon-core/src/db/rep.rs` for the real replica's own +create-table, which starts from three columns and grows the rest dynamically from the core's +`ReportSchema` — this generator's hand-written column set plays that same role for the fixture). +`strategies.sqlite` mirrors the full writer schema in `crates/moon-core/src/strat_db/write.rs`. +`valuation.sqlite` mirrors `crates/moon-core/src/db/valuation/mod.rs::open_store`'s DDL exactly. +`klines.sqlite` is seeded by a separate Rust helper (`crates/moon-core/examples/gen_klines.rs`) +through `KlineCache`'s own production write API — never re-implemented here, see `_build_klines`. + +`--analyze` is opt-in and defaults to off: production never runs SQLite `ANALYZE` on this +database, so a fixture that always carried `sqlite_stat1` statistics the user's replica does not +have would let every measured query plan diverge from the one shipped. + +After generation the fixture is checked against hard bounds (row/core/coin/group cardinality, the +`reports.sqlite` size band, and the special-case ratios below) and the process exits non-zero on a +miss instead of silently shipping an undersized or ratio-drifted replica. """ +import json import os import random import sqlite3 +import subprocess import sys -TUNER_COLS = "lev,dmark,pricebug,hvol,hvolf,dvol,vd1m,bvsvratio,d24h,d3h,da1m,d5s,btc1hdelta,exchange1hdelta,btc24hdelta,exchange24hdelta,btc5mdelta,dbtc1m,d1h,d15m,d5m,d1m,pump1h,dump1h".split(",") +TUNER_COLS = ["lev", "dmark", "pricebug", "hvol", "hvolf", "dvol", "vd1m", "bvsvratio", "d24h", "d3h", "da1m", "d5s", "btc1hdelta", "exchange1hdelta", "btc24hdelta", "exchange24hdelta", "btc5mdelta", "dbtc1m", "d1h", "d15m", "d5m", "d1m", "pump1h", "dump1h"] +# Every DISPLAY_COLUMNS entry that is a real stored column rather than one of report_read.rs's +# four SYNTHETIC (computed) columns (profitpct, valuation_profit_usdt, valuation_rate, +# valuation_rate_source) — see report_read.rs:18-73 and :108-125. +DATE_COLS = ["closedate", "buydate", "sellsetdate"] REAL_COLS = [ "profitbtc", "spentbtc", "boughtq", "buyprice", "sellprice", + "quantity", "gainedbtc", "takeprofitlag", ] -DATE_COLS = ["closedate", "buydate"] -INT_COLS = ["isshort", "emulator", "deleted", "strategyid", "basecurrency"] -TEXT_COLS = ["coin", "sellreason", "channelname", "signaltype", "comment"] +INT_COLS = [ + "isshort", "emulator", "deleted", "strategyid", "basecurrency", + "id", "taskid", "source", "channel", "status", "last_update_at", +] +TEXT_COLS = ["coin", "sellreason", "channelname", "signaltype", "comment", "exorderid", "fname"] + +STRATEGIES_PER_CORE = 120 +COINS_TARGET = 400 +_REAL_TICKERS = ["BTC", "ETH", "SOL", "XRP", "ADA", "DOGE", "AVAX", "LINK", "DOT", "MATIC", "LTC", "BCH", "ATOM", "UNI", "ETC", "FIL", "APT", "ARB", "OP", "NEAR"] +LEV_CHOICES = [1, 5, 10, 25] -COINS = [f"{a}USDT" for a in - "BTC ETH SOL XRP ADA DOGE AVAX LINK DOT MATIC LTC BCH ATOM UNI ETC FIL APT ARB OP NEAR".split()] +# Special-case ratios (1b): expected fraction of rows falling into each mutually exclusive +# non-normal bucket, drawn from one categorical roll per row so the buckets never overlap. +LIQUIDATION_RATIO = 0.0005 +FUNDING_RATIO = 0.003 +NULLDATE_RATIO = 0.0005 +EMULATOR_RATIO = 0.02 +DELETED_RATIO = 0.005 +LIQUIDATION_OWNER = "ExternalOwner" -def build(data_dir, cores, rows, span_days=400): - """Create a disposable report and strategy replica with deterministic synthetic data. + +def _coin_universe(n=COINS_TARGET): + """Build a deterministic coin ticker universe wide enough for cardinality targets. Args: - data_dir: Root whose `data` child receives recreated SQLite databases. - cores: Number of synthetic core identities to generate. - rows: Number of synthetic report rows to generate. - span_days: Number of days covered by the generated close times. + n: Total distinct coin identities to produce. + + Returns: + The 20 original real tickers (unchanged, for anything keying off them) followed by + synthetic fillers up to `n`. + """ + coins = [f"{a}USDT" for a in _REAL_TICKERS] + i = 0 + while len(coins) < n: + coins.append(f"ALT{i:03d}USDT") + i += 1 + return coins[:n] + + +def _core_name(core): + """Format: + core -> f"core-{core:03d}", the same label the real replica writes per core. + """ + return f"core-{core:03d}" + + +def _btc_cores(cores): + """Cores whose reports are BTC-quoted (`basecurrency = 0`); every other core is USDT (`1`). + + Args: + cores: Total synthetic core count. + + Returns: + Up to the first 3 core ids, verified against `db/quote.rs`'s `QuoteCurrency::btc()` + ordinal (0) and `QuoteCurrency::usdt()` ordinal (1). + """ + return {c for c in range(1, cores + 1) if c <= 3} + + +def _margin_cores(cores): + """Cores that write posted MARGIN (`notional / lev`) into `spentbtc` instead of full notional. + + Args: + cores: Total synthetic core count. + + Returns: + Up to 2 core ids distinct from `_btc_cores`, matching `db/analytics/basis.rs`'s + per-core margin-vs-notional probe. + """ + return {c for c in range(1, cores + 1) if 4 <= c <= 5} + + +def _rate_usdt_for_minute(minute_utc): + """Deterministic synthetic BTC/USDT spot rate for one closed minute. + + A pure function of `minute_utc` rather than an RNG draw, so the same minute always resolves + to the same rate across cores and reruns without needing a shared random-state thread through + the row loop. + + Args: + minute_utc: UTC minute start, in Unix seconds. + + Returns: + A plausible BTC/USDT rate that varies slowly with time. + """ + return 20_000.0 + float((minute_utc // 60) % 50_000) + + +def _build_reports(conn, cores, rows, span_days, rng, btc_cores, margin_cores): + """Populate `orders_rep` and collect everything the valuation pass and the shape gate need. + + Args: + conn: Open connection to the recreated `reports.sqlite`. + cores: Synthetic core count. + rows: Synthetic row count. + span_days: Days covered by generated close times. + rng: Seeded RNG shared with every other builder in this run. + btc_cores: Core ids whose rows are BTC-quoted. + margin_cores: Core ids that write margin instead of notional into `spentbtc`. + + Returns: + Stats dict consumed by `_build_valuation` and the shape gate: `cores_seen`, `coins_seen`, + `groups_seen`, per-special-case row counts, and the BTC-quoted closed rows to value. """ - db_dir = os.path.join(data_dir, "data") - os.makedirs(db_dir, exist_ok=True) - reports = os.path.join(db_dir, "reports.sqlite") - for suffix in ("", "-wal", "-shm"): - try: - os.remove(reports + suffix) - except OSError: - pass - conn = sqlite3.connect(reports) - conn.execute("PRAGMA journal_mode=WAL") - conn.execute("CREATE TABLE IF NOT EXISTS app_meta (key TEXT PRIMARY KEY, value TEXT NOT NULL)") - conn.execute("INSERT OR REPLACE INTO app_meta(key,value) VALUES('legacy_dropped','1')") cols = ["core_uid INTEGER NOT NULL", "core_name TEXT NOT NULL", "newrecid INTEGER NOT NULL"] cols += [f"{c} INTEGER" for c in DATE_COLS] cols += [f"{c} REAL" for c in REAL_COLS] cols += [f"{c} INTEGER" for c in INT_COLS] cols += [f"{c} TEXT" for c in TEXT_COLS] cols += [f"{c} REAL" for c in TUNER_COLS] - conn.execute( - "CREATE TABLE orders_rep (%s, PRIMARY KEY (core_uid, newrecid))" % ", ".join(cols) - ) + conn.execute("CREATE TABLE IF NOT EXISTS app_meta (key TEXT PRIMARY KEY, value TEXT NOT NULL)") + conn.execute("INSERT OR REPLACE INTO app_meta(key,value) VALUES('legacy_dropped','1')") + conn.execute(f"CREATE TABLE orders_rep ({', '.join(cols)}, PRIMARY KEY (core_uid, newrecid))") names = ["core_uid", "core_name", "newrecid"] + DATE_COLS + REAL_COLS + INT_COLS + TEXT_COLS + TUNER_COLS placeholders = ",".join("?" * len(names)) - sql = "INSERT INTO orders_rep (%s) VALUES (%s)" % (",".join(names), placeholders) + sql = f"INSERT INTO orders_rep ({','.join(names)}) VALUES ({placeholders})" - rng = random.Random(20260820) + coins = _coin_universe() end = 1_780_000_000 start = end - span_days * 86_400 # A real replica accumulates in close-date order, so its table pages carry that locality. # Reproduce it: without this the index range scan pays a random page fetch per row and the # measurement overstates every period read. closes = sorted(rng.randrange(start, end) for _ in range(rows)) - strategies_per_core = 12 + + stats = { + "cores_seen": set(), + "coins_seen": set(), + "groups_seen": set(), + "lev_seen": set(), + "n_emulator": 0, + "n_deleted": 0, + "n_funding": 0, + "n_liquidation": 0, + "n_nulldate": 0, + } + to_value = [] # (core_uid, newrecid, closedate, profitbtc, spentbtc, last_update_at) + batch = [] for i in range(rows): core = i % cores + 1 - close = closes[i] - buy = close - rng.randrange(60, 86_400) + stats["cores_seen"].add(core) + raw_close = closes[i] + + kind_roll = rng.random() + if kind_roll < LIQUIDATION_RATIO: + kind = "liquidation" + elif kind_roll < LIQUIDATION_RATIO + FUNDING_RATIO: + kind = "funding" + elif kind_roll < LIQUIDATION_RATIO + FUNDING_RATIO + NULLDATE_RATIO: + kind = "nulldate" + else: + kind = "normal" + + if kind == "funding": + buy = raw_close + sellreason = "Funding" + stats["n_funding"] += 1 + else: + buy = raw_close - rng.randrange(60, 86_400) + sellreason = "TakeProfit" + closedate_val = None if kind == "nulldate" else raw_close + if kind == "nulldate": + stats["n_nulldate"] += 1 + sellsetdate_val = raw_close + rng.randrange(0, 300) + profit = rng.gauss(0.4, 25.0) - spent = rng.uniform(20.0, 900.0) - sid = (i // cores) % strategies_per_core + 1 - row = [ - core, "core-%03d" % core, i, - close, buy, profit, spent, - rng.uniform(0.01, 100.0), rng.uniform(0.5, 70000.0), rng.uniform(0.5, 70000.0), - rng.randrange(2), 0, 0, sid, 1, - COINS[i % len(COINS)], "TakeProfit", "", "", "", + notional = rng.uniform(20.0, 900.0) + lev = rng.choice(LEV_CHOICES) + stats["lev_seen"].add(lev) + spent = notional / lev if core in margin_cores else notional + boughtq = rng.uniform(0.01, 100.0) + buyprice = rng.uniform(0.5, 70000.0) + sellprice = rng.uniform(0.5, 70000.0) + quantity = boughtq + gainedbtc = spent + profit + takeprofitlag = rng.uniform(0.0, 300.0) + + isshort = rng.randrange(2) + emulator = 1 if rng.random() < EMULATOR_RATIO else 0 + stats["n_emulator"] += emulator + deleted = 1 if rng.random() < DELETED_RATIO else 0 + stats["n_deleted"] += deleted + sid = (i // cores) % STRATEGIES_PER_CORE + 1 + channelname = "Core" + signaltype = "AutoSignal" + if kind == "liquidation": + sid = 0 + channelname = "LIQUIDATION" + signaltype = LIQUIDATION_OWNER + stats["n_liquidation"] += 1 + stats["groups_seen"].add((core, sid)) + basecurrency = 0 if core in btc_cores else 1 + taskid = rng.randrange(1, 999_999) + source_val = rng.randrange(0, 3) + channel_val = rng.randrange(0, 3) + status_val = rng.randrange(0, 4) + last_update_at = raw_close * 1000 + rng.randrange(0, 1000) + coin = coins[i % len(coins)] + stats["coins_seen"].add(coin) + # `exorderid`/`fname`/`comment` are sized to match the byte width real exchange order ids + # and order filenames carry (see the size-band gate below): a real replica's rows are not + # this narrow, and the row width is what the index range-scan measurement is timing. + exorderid = f"{(i + 1) * 137 + core:018d}" + fname = f"core{core:03d}_order_{i:08d}_synthetic_fixture_row_reference" + comment = ( + f"synthetic-fixture-note row={i} core={core} strategy={sid} " + f"closeref={raw_close} padding-{'x' * 96}" + ) + + tuner_vals = [ + float(lev) if col == "lev" else rng.uniform(-50.0, 50.0) for col in TUNER_COLS ] - row += [rng.uniform(-50.0, 50.0) for _ in TUNER_COLS] + + row = ( + [core, _core_name(core), i] + + [closedate_val, buy, sellsetdate_val] + + [profit, spent, boughtq, buyprice, sellprice, quantity, gainedbtc, takeprofitlag] + + [ + isshort, emulator, deleted, sid, basecurrency, + i, taskid, source_val, channel_val, status_val, last_update_at, + ] + + [coin, sellreason, channelname, signaltype, comment, exorderid, fname] + + tuner_vals + ) batch.append(row) + if basecurrency == 0 and closedate_val is not None: + to_value.append((core, i, closedate_val, profit, spent, last_update_at)) if len(batch) >= 20000: conn.executemany(sql, batch) batch = [] @@ -97,58 +282,425 @@ def build(data_dir, cores, rows, span_days=400): ("idx_rep_strat", "core_uid, strategyid, buydate"), ("idx_rep_strategy_close", "core_uid, strategyid, closedate"), ]: - conn.execute("CREATE INDEX IF NOT EXISTS %s ON orders_rep(%s)" % (name, index_cols)) - conn.commit() - conn.execute("ANALYZE") + conn.execute(f"CREATE INDEX IF NOT EXISTS {name} ON orders_rep({index_cols})") conn.commit() - conn.close() - strat_path = os.path.join(db_dir, "strategies.sqlite") + stats["to_value"] = to_value + return stats + + +def _build_strategies(strat_path, cores, rng): + """Recreate `strategies.sqlite` with the full production schema (`strat_db/write.rs`). + + Args: + strat_path: Destination `strategies.sqlite` path. + cores: Synthetic core count. + rng: Shared seeded RNG. + + Returns: + None. Keeps the 7 deliberate per-core sid special cases the enrichment path depends on: + 1 live+enabled, 2 live+disabled, 3 deleted, 4 nameless, 5 no current version, 6 two + versions, 7 no head row at all. + """ try: os.remove(strat_path) except OSError: pass s = sqlite3.connect(strat_path) s.execute( - "CREATE TABLE strategies (core_uid INTEGER, strategy_id INTEGER, name TEXT," - " deleted INTEGER DEFAULT 0, checked INTEGER DEFAULT 1," - " PRIMARY KEY (core_uid, strategy_id))" + "CREATE TABLE strategies (\n" + " core_uid INTEGER NOT NULL,\n" + " strategy_id INTEGER NOT NULL,\n" + " core_name TEXT NOT NULL DEFAULT '',\n" + " name TEXT NOT NULL DEFAULT '',\n" + " kind TEXT NOT NULL DEFAULT '',\n" + " kind_ordinal INTEGER NOT NULL DEFAULT 0,\n" + " folder_path TEXT NOT NULL DEFAULT '',\n" + " is_short INTEGER NOT NULL DEFAULT 0,\n" + " checked INTEGER NOT NULL DEFAULT 0,\n" + " server_ver INTEGER NOT NULL DEFAULT 0,\n" + " server_ms INTEGER NOT NULL DEFAULT 0,\n" + " deleted INTEGER NOT NULL DEFAULT 0,\n" + " content_hash INTEGER NOT NULL DEFAULT 0,\n" + " head_hash INTEGER NOT NULL DEFAULT 0,\n" + " updated_ms INTEGER NOT NULL DEFAULT 0,\n" + " PRIMARY KEY (core_uid, strategy_id))" ) + s.execute("CREATE INDEX IF NOT EXISTS idx_strat_name ON strategies(core_uid, name)") + s.execute("CREATE INDEX IF NOT EXISTS idx_strat_sid ON strategies(strategy_id)") s.execute( - "CREATE TABLE strategy_versions (core_uid INTEGER, strategy_id INTEGER," - " raw_json TEXT, valid_to INTEGER)" + "CREATE TABLE strategy_versions (\n" + " id INTEGER PRIMARY KEY,\n" + " core_uid INTEGER NOT NULL,\n" + " strategy_id INTEGER NOT NULL,\n" + " valid_from INTEGER NOT NULL,\n" + " valid_to INTEGER,\n" + " change_kind TEXT NOT NULL,\n" + " origin TEXT,\n" + " n_changed INTEGER NOT NULL DEFAULT 0,\n" + " ver_gap INTEGER NOT NULL DEFAULT 0,\n" + " server_ver INTEGER NOT NULL DEFAULT 0,\n" + " server_ms INTEGER NOT NULL DEFAULT 0,\n" + " checked_at INTEGER NOT NULL DEFAULT 0,\n" + " raw_json TEXT NOT NULL,\n" + " changed_json TEXT,\n" + " UNIQUE (core_uid, strategy_id, valid_from))" ) - # Deliberate variety, so an equivalence A/B over the enrichment path actually exercises it: - # sid 1 live+enabled, 2 live+disabled, 3 deleted (status but no lists), 4 NULL name (falls - # back to the bare id), 5 no current version at all, 6 TWO current versions, 7 no head row at - # all (traded, but the strategy database does not know it), the rest ordinary. + s.execute("CREATE TABLE app_meta (key TEXT PRIMARY KEY, value TEXT NOT NULL)") + heads = [] versions = [] + base_valid_from = 1_700_000_000_000 for core in range(1, cores + 1): - for sid in range(1, strategies_per_core + 1): + core_name = _core_name(core) + for sid in range(1, STRATEGIES_PER_CORE + 1): if sid == 7: - continue - name = None if sid == 4 else "Strat_%d_%d" % (core, sid) + continue # deliberate: traded, but the strategy database never learned of it + # Deliberate variety exercised by the enrichment path (kept from the earlier fixture): + # sid 1 live+enabled, 2 live+disabled, 3 deleted, 4 nameless (NOT NULL DEFAULT '' rules + # out a literal SQL NULL here; see the generator report for the resulting fallback + # caveat), 5 no current version, 6 two versions, the rest ordinary. + name = "" if sid == 4 else f"Strat_{core}_{sid}" deleted = 1 if sid == 3 else 0 checked = 0 if sid == 2 else 1 - heads.append((core, sid, name, deleted, checked)) + kind = "MoonShot" if sid % 2 else "Standard" + heads.append( + (core, sid, core_name, name, kind, 0, "", 0, checked, + sid, 0, deleted, 0, 0, 0) + ) if sid == 5: continue - raw = ('{"SignalType":"MoonShot","LastEditDate":"2026-01-0%d",' - '"CoinsBlackList":"BTC,ETH,btc_rp","CoinsWhiteList":"SOL"}' % (sid % 9 + 1)) - versions.append((core, sid, raw, None)) + valid_from = base_valid_from + core * 1_000_000 + sid * 1_000 + raw = ( + f'{{"SignalType":"MoonShot","LastEditDate":"2026-01-0{sid % 9 + 1}",' + f'"CoinsBlackList":"BTC,ETH,btc_rp","CoinsWhiteList":"SOL"}}' + ) if sid == 6: - versions.append((core, sid, - '{"SignalType":"Second","LastEditDate":"2025-12-31",' - '"CoinsBlackList":"XRP","CoinsWhiteList":""}', None)) - s.executemany("INSERT INTO strategies VALUES (?,?,?,?,?)", heads) - s.executemany("INSERT INTO strategy_versions VALUES (?,?,?,?)", versions) + second_valid_from = valid_from + 500 + versions.append( + (core, sid, valid_from, second_valid_from, "created", None, 0, 0, + 1, 0, 0, raw, None) + ) + second_raw = ('{"SignalType":"Second","LastEditDate":"2025-12-31",' + '"CoinsBlackList":"XRP","CoinsWhiteList":""}') + changed = '{"CoinsBlackList":{"old":"BTC,ETH,btc_rp","new":"XRP"}}' + versions.append( + (core, sid, second_valid_from, None, "params", "local", 1, 0, + 2, 0, 0, second_raw, changed) + ) + else: + versions.append( + (core, sid, valid_from, None, "created", None, 0, 0, 1, 0, 0, raw, None) + ) + s.executemany( + "INSERT INTO strategies (core_uid, strategy_id, core_name, name, kind, kind_ordinal," + " folder_path, is_short, checked, server_ver, server_ms, deleted, content_hash," + " head_hash, updated_ms) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)", + heads, + ) + s.executemany( + "INSERT INTO strategy_versions (core_uid, strategy_id, valid_from, valid_to," + " change_kind, origin, n_changed, ver_gap, server_ver, server_ms, checked_at," + " raw_json, changed_json) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?)", + versions, + ) s.commit() s.close() - size = os.path.getsize(reports) / 1024 / 1024 - print("[OK] %s: %d rows, %d cores, %.1f MB" % (reports, rows, cores, size)) + + +def _build_valuation(valuation_path, to_value): + """Recreate `valuation.sqlite` with the exact reader-facing DDL from `valuation/mod.rs::open_store`. + + Args: + valuation_path: Destination `valuation.sqlite` path. + to_value: `(core_uid, newrecid, closedate, profitbtc, spentbtc, last_update_at)` rows for + every closed BTC-quoted report row, collected while `_build_reports` ran. + + Returns: + None. Writes one `trade_values` row per input row, keyed `(0, core_uid, newrecid)` + (`TradeSource::Typed.code() == 0`), plus the `rates` row each one resolves against. + """ + for suffix in ("", "-wal", "-shm"): + try: + os.remove(valuation_path + suffix) + except OSError: + pass + v = sqlite3.connect(valuation_path) + v.execute("PRAGMA journal_mode=WAL") + v.executescript( + "CREATE TABLE IF NOT EXISTS rates (\n" + " algorithm_version INTEGER NOT NULL,\n" + " quote_ordinal INTEGER NOT NULL,\n" + " minute_utc INTEGER NOT NULL,\n" + " resolved_minute_utc INTEGER NOT NULL,\n" + " rate_usdt REAL NOT NULL,\n" + " price_basis INTEGER NOT NULL,\n" + " provider TEXT NOT NULL,\n" + " symbol TEXT NOT NULL,\n" + " orientation INTEGER NOT NULL,\n" + " candle_open_ms INTEGER NOT NULL,\n" + " candle_close_ms INTEGER NOT NULL,\n" + " leg1_rate REAL NOT NULL,\n" + " leg2_provider TEXT,\n" + " leg2_symbol TEXT,\n" + " leg2_orientation INTEGER,\n" + " leg2_rate REAL,\n" + " fetched_at_ms INTEGER NOT NULL,\n" + " PRIMARY KEY (algorithm_version, quote_ordinal, minute_utc)\n" + " );\n" + " CREATE TABLE IF NOT EXISTS rate_searches (\n" + " algorithm_version INTEGER NOT NULL,\n" + " quote_ordinal INTEGER NOT NULL,\n" + " minute_utc INTEGER NOT NULL,\n" + " searched_through_minute INTEGER NOT NULL,\n" + " next_retry_at_ms INTEGER NOT NULL,\n" + " attempts INTEGER NOT NULL,\n" + " updated_at_ms INTEGER NOT NULL,\n" + " PRIMARY KEY (algorithm_version, quote_ordinal, minute_utc)\n" + " );\n" + " CREATE INDEX IF NOT EXISTS idx_rate_searches_retry\n" + " ON rate_searches (algorithm_version, next_retry_at_ms);\n" + " CREATE TABLE IF NOT EXISTS trade_values (\n" + " source_kind INTEGER NOT NULL,\n" + " core_uid INTEGER NOT NULL,\n" + " row_id INTEGER NOT NULL,\n" + " algorithm_version INTEGER NOT NULL,\n" + " closedate INTEGER NOT NULL,\n" + " quote_ordinal INTEGER NOT NULL,\n" + " profit_quote REAL NOT NULL,\n" + " spent_quote REAL,\n" + " rate_minute_utc INTEGER NOT NULL,\n" + " rate_usdt REAL NOT NULL,\n" + " profit_usdt REAL NOT NULL,\n" + " spent_usdt REAL,\n" + " valued_at_ms INTEGER NOT NULL,\n" + " PRIMARY KEY (source_kind, core_uid, row_id)\n" + " );\n" + " CREATE INDEX IF NOT EXISTS idx_trade_values_inputs\n" + " ON trade_values (algorithm_version, quote_ordinal, rate_minute_utc);" + ) + + ALGORITHM_VERSION = 2 + QUOTE_ORDINAL_BTC = 0 + SOURCE_KIND_TYPED = 0 + + rate_rows = [] + trade_rows = [] + for core_uid, newrecid, closedate, profit, spent, last_update_at in to_value: + minute_utc = (closedate // 60) * 60 + rate_usdt = _rate_usdt_for_minute(minute_utc) + candle_open_ms = minute_utc * 1000 + rate_rows.append(( + ALGORITHM_VERSION, QUOTE_ORDINAL_BTC, minute_utc, minute_utc, rate_usdt, + 0, "synthetic", "BTCUSDT", 0, candle_open_ms, candle_open_ms + 59_999, + rate_usdt, None, None, None, None, last_update_at, + )) + profit_usdt = profit * rate_usdt + spent_usdt = spent * rate_usdt + trade_rows.append(( + SOURCE_KIND_TYPED, core_uid, newrecid, ALGORITHM_VERSION, closedate, + QUOTE_ORDINAL_BTC, profit, spent, minute_utc, rate_usdt, profit_usdt, + spent_usdt, last_update_at, + )) + + v.executemany( + "INSERT OR IGNORE INTO rates (algorithm_version, quote_ordinal, minute_utc," + " resolved_minute_utc, rate_usdt, price_basis, provider, symbol, orientation," + " candle_open_ms, candle_close_ms, leg1_rate, leg2_provider, leg2_symbol," + " leg2_orientation, leg2_rate, fetched_at_ms) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)", + rate_rows, + ) + v.executemany( + "INSERT INTO trade_values (source_kind, core_uid, row_id, algorithm_version," + " closedate, quote_ordinal, profit_quote, spent_quote, rate_minute_utc, rate_usdt," + " profit_usdt, spent_usdt, valued_at_ms) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?)", + trade_rows, + ) + v.commit() + v.close() + + +def _build_klines(data_dir, span_days): + """Seed `data/klines.sqlite` through `KlineCache`'s own production write API. + + Never re-implemented in Python: `chunks_v2`'s packed row codec (`pack_rows_v2`) is private to + `market/kline_cache.rs`, and a second Python encoder of that binary format would be a second + authority no drift check could prove equivalent to the first (rejected in review). Instead + this shells out to a small Rust example that opens the cache and calls + `KlineCache::merge_batch_blocking` directly. + + Args: + data_dir: Root whose `data` child receives the recreated kline cache. + span_days: Days the synthetic candle series should cover. + + Returns: + None. Raises `subprocess.CalledProcessError` if the helper build or run fails. + """ + repo_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + cmd = [ + "cargo", "run", "--release", "-p", "moon-core", "--example", "gen_klines", + "--", os.path.abspath(data_dir), str(span_days), + ] + subprocess.run(cmd, cwd=repo_root, check=True) + + +def _check_shape(rows, cores, span_days, analyze, reports_bytes, stats): + """Hard shape gate (1g): fail loudly rather than shipping an undersized or ratio-drifted + fixture that would still produce a ranking and a cut line. + + Bounds are derived from the CLI arguments actually supplied, not hardcoded to one invocation, + so a small development run does not spuriously fail; every check below — cardinality, + size-band and ratio — scales to `rows` and applies at every size, so a tiny sparse fixture + cannot pass the gate by virtue of being tiny. + + Args: + rows, cores, span_days, analyze: The generator's own resolved arguments. + reports_bytes: Size of the produced `reports.sqlite` in bytes. + stats: The dict `_build_reports` returned. + + Returns: + None on success. + + Raises: + SystemExit(1) with a description of the first bound the fixture missed. + """ + failures = [] + + if len(stats["cores_seen"]) != cores: + failures.append( + f"core cardinality: expected {cores}, got {len(stats['cores_seen'])}" + ) + expected_coins = min(COINS_TARGET, rows) + if len(stats["coins_seen"]) != expected_coins: + failures.append( + f"coin cardinality: expected {expected_coins}, got {len(stats['coins_seen'])}" + ) + expected_groups = cores * STRATEGIES_PER_CORE + # A liquidation row's forced strategyid=0 can add one extra (core, 0) group per core beyond + # the ordinary 1..STRATEGIES_PER_CORE groups; both the plain and +cores counts are accepted. + if not (expected_groups <= len(stats["groups_seen"]) <= expected_groups + cores): + failures.append( + f"strategy-group cardinality: expected ~{expected_groups}, " + f"got {len(stats['groups_seen'])}" + ) + if not set(stats["lev_seen"]) <= set(LEV_CHOICES): + failures.append(f"lev values outside {LEV_CHOICES}: {stats['lev_seen']}") + + min_bytes_per_row, max_bytes_per_row = 650, 1000 + low, high = rows * min_bytes_per_row, rows * max_bytes_per_row + if not (low <= reports_bytes <= high): + failures.append( + f"reports.sqlite size band: expected {low / 1e6:.0f}-{high / 1e6:.0f} MB " + f"at {rows} rows, got {reports_bytes / 1e6:.1f} MB" + ) + + def ratio_ok(count, ratio, label): + """Append a failure when one observed special-case count escapes its scaled band.""" + expected = rows * ratio + # Wide multiplicative tolerance: this gate exists to catch a broken generator (an + # off-by-a-lot bug), not to police RNG sampling noise at the target row count. The + # scale-independent "+5" floor keeps a small fixture's near-zero expected count from + # rejecting on ordinary RNG noise. + if not (expected * 0.3 <= count <= expected * 3.0 + 5): + failures.append( + f"{label} ratio: expected ~{expected:.0f} rows ({ratio:.3%}), got {count}" + ) + + ratio_ok(stats["n_emulator"], EMULATOR_RATIO, "emulator") + ratio_ok(stats["n_deleted"], DELETED_RATIO, "deleted") + ratio_ok(stats["n_funding"], FUNDING_RATIO, "Funding") + ratio_ok(stats["n_liquidation"], LIQUIDATION_RATIO, "LIQUIDATION") + ratio_ok(stats["n_nulldate"], NULLDATE_RATIO, "NULL closedate") + + if failures: + sys.exit("[FAIL] fixture shape gate:\n " + "\n ".join(failures)) + + +def build(data_dir, cores, rows, span_days=400, analyze=False): + """Create a disposable report, strategy, valuation and kline replica with deterministic data. + + Args: + data_dir: Root whose `data` child receives recreated SQLite databases. + cores: Number of synthetic core identities to generate. + rows: Number of synthetic report rows to generate. + span_days: Number of days covered by the generated close times. + analyze: Whether to run SQLite `ANALYZE` on the finished `reports.sqlite` (opt-in; off by + default, since production never runs it — see the module docstring). + + Returns: + None. Writes the fixture databases and manifest below `data_dir`. + + Raises: + subprocess.CalledProcessError: If the production-API kline helper fails. + SystemExit: If the generated fixture violates a required shape bound. + """ + db_dir = os.path.join(data_dir, "data") + os.makedirs(db_dir, exist_ok=True) + reports = os.path.join(db_dir, "reports.sqlite") + for suffix in ("", "-wal", "-shm"): + try: + os.remove(reports + suffix) + except OSError: + pass + + seed = 20260820 + rng = random.Random(seed) + conn = sqlite3.connect(reports) + conn.execute("PRAGMA journal_mode=WAL") + btc_cores = _btc_cores(cores) + margin_cores = _margin_cores(cores) + stats = _build_reports(conn, cores, rows, span_days, rng, btc_cores, margin_cores) + if analyze: + conn.execute("ANALYZE") + conn.commit() + conn.close() + + strat_path = os.path.join(db_dir, "strategies.sqlite") + _build_strategies(strat_path, cores, rng) + + valuation_path = os.path.join(db_dir, "valuation.sqlite") + _build_valuation(valuation_path, stats["to_value"]) + + _build_klines(data_dir, span_days) + + reports_bytes = os.path.getsize(reports) + manifest = { + "rows": rows, + "cores": cores, + "strategy_groups": len(stats["groups_seen"]), + "coins": len(stats["coins_seen"]), + "span_days": span_days, + "reports_bytes": reports_bytes, + "analyzed": analyze, + "seed": seed, + "btc_cores": sorted(btc_cores), + "margin_cores": sorted(margin_cores), + "n_emulator": stats["n_emulator"], + "n_deleted": stats["n_deleted"], + "n_funding": stats["n_funding"], + "n_liquidation": stats["n_liquidation"], + "n_nulldate": stats["n_nulldate"], + } + with open(os.path.join(db_dir, "fixture.json"), "w", encoding="utf-8") as f: + json.dump(manifest, f, indent=2) + + _check_shape(rows, cores, span_days, analyze, reports_bytes, stats) + + print( + f"[OK] {reports}: {rows} rows, {cores} cores, " + f"{reports_bytes / 1024 / 1024:.1f} MB, " + f"{manifest['strategy_groups']} strategy groups, {manifest['coins']} coins" + ) if __name__ == "__main__": - build(sys.argv[1], int(sys.argv[2]), int(sys.argv[3]), - int(sys.argv[4]) if len(sys.argv) > 4 else 400) + argv = sys.argv[1:] + analyze_flag = "--analyze" in argv + argv = [a for a in argv if a != "--analyze"] + if len(argv) < 3: + sys.exit("usage: gen_replica.py [span_days] [--analyze]") + build( + argv[0], int(argv[1]), int(argv[2]), + int(argv[3]) if len(argv) > 3 else 400, + analyze=analyze_flag, + )