diff --git a/crates/freshell-codex/src/sidecar_test_support.rs b/crates/freshell-codex/src/sidecar_test_support.rs index c1ebd803f..82c016a2c 100644 --- a/crates/freshell-codex/src/sidecar_test_support.rs +++ b/crates/freshell-codex/src/sidecar_test_support.rs @@ -131,7 +131,7 @@ pub(crate) fn store_in(dir: &tempfile::TempDir) -> Arc { /// The committed fake app-server fixture (repo-owned test harness). pub(crate) fn fake_app_server_fixture() -> std::path::PathBuf { - std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) .join("../../test/fixtures/coding-cli/codex-app-server/fake-app-server.mjs") } diff --git a/crates/freshell-codex/tests/launch_lifecycle.rs b/crates/freshell-codex/tests/launch_lifecycle.rs index 4dc4f440a..ba7abaef9 100644 --- a/crates/freshell-codex/tests/launch_lifecycle.rs +++ b/crates/freshell-codex/tests/launch_lifecycle.rs @@ -753,10 +753,11 @@ async fn manager_exit_for_unknown_terminal_is_a_noop() { // ── D-C-R sidecar planning budget (S5.e precondition) ───────────────────────────── /// A [`FakeRuntime`]-shaped runtime whose `ensure_ready` blocks on a shared -/// [`tokio::sync::Notify`] so plans stay in flight until the test releases -/// them — the knob that keeps budget permits occupied. +/// zero-permit semaphore so plans stay in flight until the test releases +/// them. Release permits survive the handoff to a plan that has not entered +/// `ensure_ready` yet. struct BlockingRuntime { - release: Arc, + release: Arc, } impl CodexLaunchRuntime for BlockingRuntime { @@ -765,7 +766,11 @@ impl CodexLaunchRuntime for BlockingRuntime { cwd: Option, ) -> BoxFuture<'_, Result> { Box::pin(async move { - self.release.notified().await; + self.release + .acquire() + .await + .expect("test release gate remains open") + .forget(); // Released: stand up the file's real loopback echo upstream so // the plan completes against a real socket. let inner = FakeRuntime::start().await; @@ -788,9 +793,9 @@ impl CodexLaunchRuntime for BlockingRuntime { fn blocking_test_runtime_factory() -> ( freshell_codex::launch_lifecycle::CodexRuntimeFactory, - Arc, + Arc, ) { - let release = Arc::new(tokio::sync::Notify::new()); + let release = Arc::new(tokio::sync::Semaphore::new(0)); let factory_release = release.clone(); let factory: freshell_codex::launch_lifecycle::CodexRuntimeFactory = Box::new(move |_plan| { let rt = Arc::new(BlockingRuntime { @@ -840,9 +845,11 @@ async fn third_concurrent_plan_fails_fast_on_the_sidecar_budget() { err.to_string().contains("planning budget exhausted"), "{err}" ); - release.notify_waiters(); - let _ = a.await; - let _ = b.await; + release.add_permits(2); + let launch_a = a.await.expect("join").expect("first plan completes"); + let launch_b = b.await.expect("join").expect("second plan completes"); + manager.discard(launch_a).await; + manager.discard(launch_b).await; } // ── graceful restore/resume S1 (P2): restore-class plans queue, never die ───────── @@ -1060,20 +1067,20 @@ async fn restore_class_queue_overflow_fails_loud_as_queue_full() { ), "{err}" ); - // Drain: release the parked plans (BlockingRuntime parks on a Notify; - // the queued waiter parks again after the holder finishes, so notify twice). - release.notify_waiters(); + // Release both plans before either join. The queued waiter enters the + // runtime only after the holder gives up the planning permit. + release.add_permits(2); let launch = holder.await.expect("join").expect("holder plan completes"); manager.discard(launch).await; - release.notify_waiters(); let launch2 = queued.await.expect("join").expect("queued plan completes"); manager.discard(launch2).await; + assert_eq!(manager.plan_queue_depth(), 0, "plan queue drained"); } // ── the spawn integration leg: real child + real proxy + fake TUI ───────────────── fn fake_app_server_command() -> String { - let fixture = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + let fixture = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) .join("../../test/fixtures/coding-cli/codex-app-server/fake-app-server.mjs"); format!("node {}", fixture.display()) } @@ -1441,7 +1448,7 @@ async fn plan_retry_spawns_fresh_after_claimed_reattach_ensure_ready_fails() { // `sidecar_reconcile_tests::spawn_own_fake_app_server`; test binaries // cannot share code — the repo's copy-with-attribution convention). let survivor_ownership = "codex-sidecar-a7000003-cccc-4ccc-8ccc-cccccccccccc"; - let fixture = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + let fixture = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) .join("../../test/fixtures/coding-cli/codex-app-server/fake-app-server.mjs"); let bind_unused_ws_url = || { let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("bind ephemeral port"); diff --git a/crates/freshell-server/src/extensions.rs b/crates/freshell-server/src/extensions.rs index b430bc15e..c0c9f0e02 100644 --- a/crates/freshell-server/src/extensions.rs +++ b/crates/freshell-server/src/extensions.rs @@ -608,74 +608,11 @@ mod tests { // actually fire (absence from the registry alone was also true of the // old lenient port). // - // Capture strategy (matching freshell-freshagent's documented - // investigation): a set_global_default subscriber installed EXACTLY - // ONCE per test binary via OnceLock. Thread-local set_default proved - // nondeterministic under parallel `cargo test` (callsite interest - // caching); the global layer observes every event, and this test - // filters by its unique temp manifest_path. - use std::collections::BTreeMap; - use std::sync::{Arc, Mutex, OnceLock}; - use tracing::field::{Field, Visit}; - use tracing::{Event, Level, Subscriber}; - use tracing_subscriber::layer::{Context, Layer, SubscriberExt}; - - struct Captured { - message: String, - fields: BTreeMap, - } - #[derive(Default)] - struct V { - message: String, - fields: BTreeMap, - } - impl Visit for V { - fn record_debug(&mut self, f: &Field, v: &dyn std::fmt::Debug) { - let r = format!("{v:?}"); - if f.name() == "message" { - self.message = r; - } else { - self.fields.insert(f.name().into(), r); - } - } - fn record_str(&mut self, f: &Field, v: &str) { - if f.name() == "message" { - self.message = v.into(); - } else { - self.fields.insert(f.name().into(), v.into()); - } - } - } - struct CaptureLayer { - events: Arc>>, - } - impl Layer for CaptureLayer { - fn on_event(&self, event: &Event<'_>, _ctx: Context<'_, S>) { - if *event.metadata().level() != Level::WARN { - return; - } - let mut v = V::default(); - event.record(&mut v); - self.events.lock().expect("capture lock").push(Captured { - message: v.message, - fields: v.fields, - }); - } - } - - static GLOBAL_EVENTS: OnceLock>>> = OnceLock::new(); - let events = GLOBAL_EVENTS.get_or_init(|| { - let events = Arc::new(Mutex::new(Vec::new())); - let layer = CaptureLayer { - events: Arc::clone(&events), - }; - // Ignore the error case: some OTHER test installed a global - // subscriber first — then this assertion would fail noisily - // below, but no freshell-server test does that today. - let _ = - tracing::subscriber::set_global_default(tracing_subscriber::registry().with(layer)); - events - }); + // A shared process-global collector avoids competing test modules + // installing subscribers with disconnected event buffers. Thread-local + // set_default proved nondeterministic under parallel cargo test because + // of callsite interest caching; this test filters by its unique path. + let events = crate::test_trace_capture::captured_trace_events(); let root = tmp(); let bad_json = root.join("bad-json"); @@ -693,15 +630,14 @@ mod tests { let root_marker = root.display().to_string(); let mine: Vec = events - .lock() - .expect("capture lock") - .iter() + .snapshot() + .into_iter() .filter(|e| { e.fields .get("manifest_path") .is_some_and(|p| p.contains(&root_marker)) }) - .map(|e| e.message.clone()) + .map(|e| e.message) .collect(); assert_eq!( mine.len(), diff --git a/crates/freshell-server/src/main.rs b/crates/freshell-server/src/main.rs index 5dfcd1f49..694da8c56 100644 --- a/crates/freshell-server/src/main.rs +++ b/crates/freshell-server/src/main.rs @@ -80,6 +80,8 @@ pub(crate) mod test_clock_gate; mod test_clock_router; #[cfg(test)] pub(crate) mod test_env_lock; +#[cfg(test)] +mod test_trace_capture; mod updater; use std::net::IpAddr; @@ -2745,6 +2747,7 @@ async fn main() -> ExitCode { metadata: session_metadata_store.clone(), // STATUS-STRIP: sessions.cloned pages are client-ordered per instance. server_instance: Arc::clone(&server_instance_id), + collision_signatures: Arc::default(), // Unified agent names (Task 7 review M1): captured AFTER the boot // consolidation above ran — once the receipt committed, a scoped // coding-agent row's displayed title never consults the migrated diff --git a/crates/freshell-server/src/session_directory.rs b/crates/freshell-server/src/session_directory.rs index 20c0e6719..18490a793 100644 --- a/crates/freshell-server/src/session_directory.rs +++ b/crates/freshell-server/src/session_directory.rs @@ -48,6 +48,7 @@ use axum::{ Json, Router, }; use base64::Engine as _; +use freshell_sessions::codex_segments::CodexUnresolvedIdentity; use freshell_sessions::directory_index::{IndexedSession, SessionIndex}; // SESSION-07: the `userMessages`/`fullText` tier file-content search // (`apply_file_search`, below) -- ports `server/session-directory/file-search.ts`. @@ -88,6 +89,10 @@ pub struct SessionDirectoryState { /// STATUS-STRIP: stamped on every session-directory page (`serverInstance`); /// clients order pages by `snapshotSeq` only within one instance. pub server_instance: Arc, + /// Process-lifetime full collision signatures. Cloned route state shares + /// this gate so repeated polls log only once for each complete source set. + pub(crate) collision_signatures: + Arc>>, /// Task 20 (read-join): the SESSION-06 metadata store /// (`session-metadata.json`, same `.freshell` home dir as the POST route) /// whose `sessionType` tags [`apply_session_metadata`] overlays onto @@ -105,6 +110,12 @@ pub struct SessionDirectoryState { pub legacy_name_migration_completed: bool, } +/// Canonical sorted `(provider:sessionId, sorted source paths)` evidence for +/// one complete identity-collision snapshot. Kept in process memory so a +/// later distinct full source set is logged even when its bounded sample is +/// unchanged. +pub(crate) type CollisionSignature = Vec<(String, Vec)>; + /// One directory item, typed for the sort/filter/cursor derivation. Serialized to /// the `SessionDirectoryItem` shape by [`DirItem::to_value`]. #[derive(Debug, Clone)] @@ -156,12 +167,10 @@ struct DirItem { /// response has never carried `titleSource`; exposing it would be a /// separate parity decision). title_source: Option, - /// SESSION-07: the on-disk transcript to scan for the `userMessages`/ - /// `fullText` tiers (`IndexedSession::source_file`). Internal only -- - /// never serialized (`to_value` never reads it), mirroring - /// `sourceFiles.get(key)` (`session-directory/service.ts:164-173`), which - /// is looked up server-side and never sent to the client either. - source_file: Option, + /// Private on-disk transcript paths searched for `userMessages` and + /// `fullText`. A composed Codex row carries every chronological segment; + /// other file-backed rows carry their single transcript. Never serialized. + source_files: Vec, /// STATUS-STRIP: live token usage (`SessionDirectoryItem.tokenUsage`, /// `shared/read-models.ts`; Node's `CodingCliSession.tokenUsage`, /// `coding-cli/types.ts:190`). Powers the fresh-agent strip's context @@ -559,15 +568,38 @@ async fn session_directory( // the query (visibility filters, search, cursor paging) still compose // freshly PER REQUEST, same as before -- only the expensive filesystem // scan itself is now cached. - let items: Vec = match &state.session_index { - Some(index) => index - .snapshot() - .await - .iter() - .map(dir_item_from_indexed) - .collect(), - None => Vec::new(), - }; + let (items, unresolved_codex_identities): (Vec, Arc>) = + match &state.session_index { + Some(index) => { + let snapshot = index + .snapshot_with_failures_and_unresolved_codex_identities() + .await; + let items = snapshot + .sessions + .iter() + .map(|indexed| { + let source_files = if indexed.provider == "codex" { + snapshot + .codex_segment_paths + .get(&indexed.session_id) + .filter(|paths| { + !paths.is_empty() + && paths.last() == indexed.source_file.as_ref() + }) + .cloned() + .unwrap_or_else(|| { + indexed.source_file.clone().into_iter().collect() + }) + } else { + indexed.source_file.clone().into_iter().collect() + }; + dir_item_from_indexed_with_source_files(indexed, source_files) + }) + .collect(); + (items, snapshot.unresolved_codex_identities) + } + None => (Vec::new(), Arc::new(Vec::new())), + }; // STATUS-STRIP: assign the monotonic snapshot sequence AFTER the index // snapshot is captured — captured order is authoritative, and a seq // assigned pre-await would interleave with concurrent requests. @@ -602,9 +634,10 @@ async fn session_directory( ) .await }; + let session_overrides = state.settings.session_overrides(); let items = apply_session_overrides( items, - &state.settings.session_overrides(), + &session_overrides, state.legacy_name_migration_completed, &kilroy_only_lanes, ); @@ -628,20 +661,30 @@ async fn session_directory( .max() .unwrap_or(0) .max(0); - let collisions = persisted_identity_collisions(&items); + let collisions = merge_unresolved_codex_identity_collisions( + persisted_identity_collisions(&items), + &unresolved_codex_identities, + &session_overrides, + ); let identity_collision = if !collisions.is_empty() { let log_summary = persisted_identity_collision_log_summary(&collisions); let collision_samples_json = serde_json::to_string(&log_summary.samples).unwrap_or_else(|_| "[]".to_string()); - tracing::error!( - target: "freshell_server::session_directory", - collision_count = log_summary.collision_count, - duplicate_item_count = log_summary.duplicate_item_count, - collision_sample_count = log_summary.samples.len(), - collision_samples_truncated = log_summary.collision_samples_truncated, - collision_samples_json = %collision_samples_json, - "session_directory_identity_collision" - ); + let signature = persisted_identity_collision_signature(&collisions); + let signature_id = persisted_identity_collision_signature_id(&signature); + let should_log = state.collision_signatures.lock().unwrap().insert(signature); + if should_log { + tracing::error!( + target: "freshell_server::session_directory", + collision_signature_id = %signature_id, + collision_count = log_summary.collision_count, + duplicate_item_count = log_summary.duplicate_item_count, + collision_sample_count = log_summary.samples.len(), + collision_samples_truncated = log_summary.collision_samples_truncated, + collision_samples_json = %collision_samples_json, + "session_directory_identity_collision" + ); + } Some(( collisions .iter() @@ -732,6 +775,7 @@ async fn session_directory( struct PersistedIdentityCollision { key: String, source_files: Vec, + duplicate_item_count: usize, } const IDENTITY_COLLISION_KEY_SAMPLE_LIMIT: usize = 20; @@ -797,25 +841,114 @@ fn persisted_identity_collisions(items: &[DirItem]) -> Vec = indices .into_iter() - .map(|index| { - items[index] - .source_file - .as_deref() - .map(|path| path.to_string_lossy().into_owned()) - .unwrap_or_else(|| "".to_string()) + .flat_map(|index| { + if items[index].source_files.is_empty() { + vec!["".to_string()] + } else { + items[index] + .source_files + .iter() + .map(|path| path.to_string_lossy().into_owned()) + .collect() + } }) .collect(); source_files.sort(); + source_files.dedup(); Some(PersistedIdentityCollision { key: format!("{provider}:{session_id}"), source_files, + duplicate_item_count, }) }) .collect() } +/// Merge route-visible duplicate rows with same-generation Codex identity +/// evidence. A non-renderable member still contributes its complete path and +/// quarantines any renderable row with the same canonical Codex id. +fn merge_unresolved_codex_identity_collisions( + collisions: Vec, + unresolved: &[CodexUnresolvedIdentity], + overrides: &Map, +) -> Vec { + let mut by_key: std::collections::BTreeMap< + String, + (std::collections::BTreeSet, usize), + > = std::collections::BTreeMap::new(); + for collision in collisions { + let (paths, count) = by_key.entry(collision.key).or_default(); + paths.extend(collision.source_files); + *count = (*count).max(collision.duplicate_item_count); + } + for group in unresolved { + if group.paths.len() < 2 + || codex_identity_is_canonically_soft_deleted(&group.session_id, overrides) + { + continue; + } + let key = format!("codex:{}", group.session_id); + let (paths, count) = by_key.entry(key).or_default(); + // Preserve proven row multiplicity and add only evidence members + // whose paths are not already represented by those rows. + for path in &group.paths { + if paths.insert(path.to_string_lossy().into_owned()) { + *count += 1; + } + } + } + by_key + .into_iter() + .map( + |(key, (paths, duplicate_item_count))| PersistedIdentityCollision { + key, + source_files: paths.into_iter().collect(), + duplicate_item_count, + }, + ) + .collect() +} + +fn codex_identity_is_canonically_soft_deleted( + session_id: &str, + overrides: &Map, +) -> bool { + overrides + .get(&format!("codex:{session_id}")) + .and_then(Value::as_object) + .and_then(|override_row| override_row.get("deleted")) + .and_then(Value::as_bool) + .unwrap_or(false) +} + +fn persisted_identity_collision_signature( + collisions: &[PersistedIdentityCollision], +) -> CollisionSignature { + let mut signature: CollisionSignature = collisions + .iter() + .map(|collision| (collision.key.clone(), collision.source_files.clone())) + .collect(); + signature.sort(); + for (_, paths) in &mut signature { + paths.sort(); + paths.dedup(); + } + signature +} + +fn persisted_identity_collision_signature_id(signature: &CollisionSignature) -> String { + use sha2::{Digest, Sha256}; + + let canonical = serde_json::to_vec(signature) + .expect("a collision signature made only of strings always serializes"); + format!("{:x}", Sha256::digest(canonical)) +} + /// Build a deterministic, bounded diagnostic sample for the collision log. /// Counts cover the complete collision set; only local-path context is /// sampled so a corrupt corpus cannot create an unbounded single JSONL event. @@ -842,7 +975,7 @@ fn persisted_identity_collision_log_summary( collision_count: collisions.len(), duplicate_item_count: collisions .iter() - .map(|collision| collision.source_files.len()) + .map(|collision| collision.duplicate_item_count) .sum(), samples, collision_samples_truncated: collisions.len() > IDENTITY_COLLISION_KEY_SAMPLE_LIMIT, @@ -952,7 +1085,16 @@ pub(crate) fn codex_home(home: &Path) -> PathBuf { /// take their defaults here, exactly as `item_from_meta` did before the /// index existed -- `apply_session_overrides` / `apply_title_search` overlay /// them afterwards, unchanged. +#[cfg(test)] fn dir_item_from_indexed(idx: &IndexedSession) -> DirItem { + let source_files = idx.source_file.clone().into_iter().collect(); + dir_item_from_indexed_with_source_files(idx, source_files) +} + +fn dir_item_from_indexed_with_source_files( + idx: &IndexedSession, + source_files: Vec, +) -> DirItem { DirItem { session_id: idx.session_id.clone(), legacy_session_id: idx.legacy_session_id.clone(), @@ -974,7 +1116,7 @@ fn dir_item_from_indexed(idx: &IndexedSession) -> DirItem { live_terminal_only: false, session_type: None, title_source: idx.title_source.clone(), - source_file: idx.source_file.clone(), + source_files, token_usage: idx.token_usage.clone(), // Provenance is overlay-derived (`apply_session_overrides`), never // parsed from the transcript. @@ -1140,7 +1282,7 @@ fn item_from_meta( live_terminal_only: false, session_type: None, title_source: meta.title_source.clone(), - source_file, + source_files: source_file.into_iter().collect(), token_usage: None, title_overridden: false, provider_title: None, @@ -1516,7 +1658,7 @@ fn build_live_terminal_session_item( // parsed title source (Node's `buildLiveTerminalSessionItem` sets no // `titleSource` either, `service.ts:110-130`). title_source: None, - source_file: None, + source_files: Vec::new(), // PARITY NOTE: Rust's `TerminalIdentity` carries no token usage, so a // live-terminal-only row reports none here — unlike Node, whose // `TerminalMeta` carries `tokenUsage`. Fresh-agent pane sessions are @@ -1629,7 +1771,7 @@ mod join_tests { live_terminal_only: false, session_type: None, title_source: None, - source_file: None, + source_files: Vec::new(), token_usage: None, title_overridden: false, provider_title: None, @@ -2069,7 +2211,7 @@ struct FileSearchOutcome { /// "more exist" without this function ever scanning the entire remaining /// list (unlike the title tier, which does). /// -/// An item with no [`DirItem::source_file`] (a live-terminal-only item, or a +/// An item with no [`DirItem::source_files`] (a live-terminal-only item, or a /// provider with no per-file source -- opencode/amplifier) or an unsupported /// `provider` is skipped WITHOUT counting against the scan budget, mirroring /// `service.ts:191-195`'s `if (!sourceFile) continue` / `if (!provider) continue` @@ -2086,35 +2228,42 @@ fn apply_file_search( let mut partial = false; let mut partial_reason: Option<&'static str> = None; - for item in items { + 'items: for item in items { if results.len() > limit { break; } - if scanned >= max_scan { - partial = true; - partial_reason = Some("budget"); - break; - } - let Some(source_file) = item.source_file.clone() else { - continue; - }; if !matches!(item.provider.as_str(), "claude" | "codex") { continue; } - scanned += 1; - - match search_session_file(&source_file, &item.provider, query_text, tier) { - Ok(Some(m)) => { - let mut matched = item; - matched.matched_in = Some(m.matched_in.to_string()); - matched.snippet = Some(m.snippet); - results.push(matched); + if item.source_files.is_empty() { + continue; + } + + for source_file in &item.source_files { + if results.len() > limit { + break 'items; } - Ok(None) => {} - Err(_) => { + if scanned >= max_scan { partial = true; - if partial_reason.is_none() { - partial_reason = Some("io_error"); + partial_reason = Some("budget"); + break 'items; + } + scanned += 1; + + match search_session_file(source_file, &item.provider, query_text, tier) { + Ok(Some(m)) => { + let mut matched = item.clone(); + matched.matched_in = Some(m.matched_in.to_string()); + matched.snippet = Some(m.snippet); + results.push(matched); + break; + } + Ok(None) => {} + Err(_) => { + partial = true; + if partial_reason.is_none() { + partial_reason = Some("io_error"); + } } } } @@ -2153,6 +2302,7 @@ fn apply_title_search(mut item: DirItem, query_text: &str) -> Option { #[cfg(test)] mod tests { use super::*; + use crate::test_trace_capture::{CapturedTraceEvent, CapturedTraceEvents}; use std::time::Duration; fn fixtures_dir() -> PathBuf { @@ -2585,7 +2735,7 @@ mod tests { live_terminal_only: false, session_type: None, title_source: None, - source_file: None, + source_files: Vec::new(), token_usage: Some(freshell_sessions::meta::TokenSummary { input_tokens: 10, output_tokens: 5, @@ -2859,7 +3009,7 @@ mod tests { live_terminal_only: false, session_type: None, title_source: None, - source_file: None, + source_files: Vec::new(), token_usage: None, title_overridden: false, provider_title: None, @@ -3078,7 +3228,7 @@ mod tests { live_terminal_only: false, session_type: None, title_source: None, - source_file: None, + source_files: Vec::new(), token_usage: None, title_overridden: false, provider_title: None, @@ -3133,7 +3283,7 @@ mod tests { live_terminal_only: false, session_type: None, title_source: None, - source_file: None, + source_files: Vec::new(), token_usage: None, title_overridden: false, provider_title: None, @@ -3200,7 +3350,7 @@ mod tests { live_terminal_only: false, session_type: None, title_source: None, - source_file: None, + source_files: Vec::new(), token_usage: None, title_overridden: false, provider_title: None, @@ -3276,7 +3426,7 @@ mod tests { live_terminal_only: false, session_type: None, title_source: None, - source_file: None, + source_files: Vec::new(), token_usage: None, title_overridden: false, provider_title: None, @@ -3347,7 +3497,7 @@ mod tests { live_terminal_only: false, session_type: None, title_source: None, - source_file: None, + source_files: Vec::new(), token_usage: None, title_overridden: false, provider_title: None, @@ -3399,7 +3549,7 @@ mod tests { live_terminal_only: false, session_type: None, title_source: title_source.map(str::to_string), - source_file: None, + source_files: Vec::new(), token_usage: None, title_overridden: false, provider_title: None, @@ -3543,7 +3693,7 @@ mod tests { live_terminal_only: false, session_type: None, title_source: None, - source_file: None, + source_files: Vec::new(), token_usage: None, title_overridden: false, provider_title: None, @@ -3719,10 +3869,1119 @@ mod tests { identity, metadata: crate::session_metadata::SessionMetadataStore::new(home.join(".freshell")), server_instance: std::sync::Arc::new("srv-test".to_string()), + collision_signatures: Default::default(), legacy_name_migration_completed: false, }) } + fn codex_fixtures() -> (String, String) { + let fixture_dir = + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../test/fixtures/coding-cli/codex"); + let older = std::fs::read_to_string( + fixture_dir.join("multi-file-continuation-older.sanitized.jsonl"), + ) + .unwrap() + .replace( + r#""type":"event_msg","payload":{"type":"user_message","message":"Older first request"}"#, + r#""type":"response_item","payload":{"type":"message","role":"user","content":[{"type":"input_text","text":"Older first request"}]}"#, + ); + let newer = std::fs::read_to_string( + fixture_dir.join("multi-file-continuation-newer.sanitized.jsonl"), + ) + .unwrap() + .replace( + r#""type":"event_msg","payload":{"type":"user_message","message":"Continuation title"}"#, + r#""type":"response_item","payload":{"type":"message","role":"user","content":[{"type":"input_text","text":"Continuation title"}]}"#, + ); + (older, newer) + } + + fn shift_codex_fixture_timestamps(content: &str, offset_seconds: i64) -> String { + let records = content + .lines() + .map(|line| { + let mut record: Value = serde_json::from_str(line).unwrap(); + let timestamp = + chrono::DateTime::parse_from_rfc3339(record["timestamp"].as_str().unwrap()) + .unwrap() + + chrono::Duration::seconds(offset_seconds); + record["timestamp"] = + json!(timestamp.to_rfc3339_opts(chrono::SecondsFormat::Micros, true,)); + serde_json::to_string(&record).unwrap() + }) + .collect::>() + .join("\n"); + format!("{records}\n") + } + + fn write_codex_segments(home: &Path, older: &str, newer: &str) -> (PathBuf, PathBuf) { + let sessions = home.join(".codex").join("sessions"); + std::fs::create_dir_all(&sessions).unwrap(); + let older_path = sessions.join("z-older-rollout.jsonl"); + let newer_path = sessions.join("a-newer-rollout.jsonl"); + std::fs::write(&older_path, older).unwrap(); + std::fs::write(&newer_path, newer).unwrap(); + (older_path, newer_path) + } + + fn codex_session_directory_app( + home: &Path, + cache_path: Option, + identity: freshell_ws::identity::TerminalIdentityRegistry, + ) -> (Router, Arc) { + let source = Arc::new(CodexSource::new(home.join(".codex"))); + let index = Arc::new(SessionIndex::with_ttl_and_cache_path( + vec![source], + Duration::from_secs(3600), + cache_path, + )); + let app = router(SessionDirectoryState { + auth_token: Arc::new("tok".to_string()), + settings: crate::settings_store::SettingsStore::load(Some(home), vec!["codex".into()]), + session_index: Some(Arc::clone(&index)), + identity, + metadata: crate::session_metadata::SessionMetadataStore::new(home.join(".freshell")), + server_instance: Arc::new("srv-codex-test".to_string()), + collision_signatures: Default::default(), + legacy_name_migration_completed: false, + }); + (app, index) + } + + #[tokio::test] + async fn codex_multi_file_route_searches_each_segment_once() { + let home = unique_temp_dir(); + let (older, newer) = codex_fixtures(); + write_codex_segments(&home, &older, &newer); + let (app, _index) = codex_session_directory_app( + &home, + None, + freshell_ws::identity::TerminalIdentityRegistry::new(), + ); + + let base = "/api/session-directory?priority=visible&includeNonInteractive=1"; + let page = get_directory_page(&app, base).await; + let session_id = "b7936c10-4935-441c-837c-c1f33cafec2d"; + let rows = page["items"].as_array().unwrap(); + assert_eq!( + rows.len(), + 1, + "the accepted continuation is one logical row" + ); + assert_eq!(rows[0]["sessionId"], json!(session_id)); + assert_eq!( + rows[0]["createdAt"], + json!( + chrono::DateTime::parse_from_rfc3339("2026-10-03T00:00:00.000Z") + .unwrap() + .timestamp_millis() + ) + ); + assert_eq!( + rows[0]["lastActivityAt"], + json!( + chrono::DateTime::parse_from_rfc3339("2026-10-03T00:00:10.004Z") + .unwrap() + .timestamp_millis() + ) + ); + assert!(page.get("integrityError").is_none()); + + for (needle, tier, matched_in) in [ + ("Older first request", "userMessages", "userMessage"), + ("Continuation title", "userMessages", "userMessage"), + ("Older assistant summary", "fullText", "assistantMessage"), + ("Continuation summary", "fullText", "assistantMessage"), + ] { + let query = format!("{base}&query={}&tier={tier}", needle.replace(' ', "%20")); + let result = get_directory_page(&app, &query).await; + let matches = result["items"].as_array().unwrap(); + assert_eq!(matches.len(), 1, "segment needle {needle:?}: {result}"); + assert_eq!(matches[0]["sessionId"], json!(session_id)); + assert_eq!(matches[0]["matchedIn"], json!(matched_in)); + assert!(matches[0]["snippet"].as_str().unwrap().contains(needle)); + } + + let overlapping = + get_directory_page(&app, &format!("{base}&query=summary&tier=fullText")).await; + assert_eq!( + overlapping["items"].as_array().unwrap().len(), + 1, + "a term found in multiple continuation segments returns one logical row" + ); + + std::fs::remove_dir_all(&home).ok(); + } + + #[tokio::test(flavor = "current_thread")] + async fn codex_multi_file_route_keeps_match_and_reports_io_error() { + let home = unique_temp_dir(); + let (older, newer) = codex_fixtures(); + let (older_path, _newer_path) = write_codex_segments(&home, &older, &newer); + let (app, _index) = codex_session_directory_app( + &home, + None, + freshell_ws::identity::TerminalIdentityRegistry::new(), + ); + let base = "/api/session-directory?priority=visible&includeNonInteractive=1"; + let warm = get_directory_page(&app, base).await; + assert_eq!(warm["items"].as_array().unwrap().len(), 1); + + std::fs::remove_file(older_path).unwrap(); + let page = get_directory_page( + &app, + &format!("{base}&query=Continuation%20title&tier=userMessages&limit=1"), + ) + .await; + + let rows = page["items"].as_array().unwrap(); + assert_eq!(rows.len(), 1); + assert_eq!( + rows[0]["sessionId"], + json!("b7936c10-4935-441c-837c-c1f33cafec2d") + ); + assert_eq!(rows[0]["matchedIn"], json!("userMessage")); + assert_eq!(page["partial"], json!(true)); + assert_eq!(page["partialReason"], json!("io_error")); + + std::fs::remove_dir_all(&home).ok(); + } + + #[tokio::test(flavor = "current_thread")] + async fn codex_multi_file_route_reports_budget_when_match_falls_after_scan_limit() { + let home = unique_temp_dir(); + let (older, _) = codex_fixtures(); + let sessions = home.join(".codex").join("sessions"); + std::fs::create_dir_all(&sessions).unwrap(); + for index in 0..11 { + let user_message = if index == 10 { + "rrx7-budget-target".to_string() + } else { + format!("rrx7-unmatched-segment-{index}") + }; + let content = older.replace("Older first request", &user_message); + let content = shift_codex_fixture_timestamps(&content, index * 60); + std::fs::write(sessions.join(format!("rollout-{index:02}.jsonl")), content).unwrap(); + } + let (app, _index) = codex_session_directory_app( + &home, + None, + freshell_ws::identity::TerminalIdentityRegistry::new(), + ); + let base = "/api/session-directory?priority=visible&includeNonInteractive=1"; + let warm = get_directory_page(&app, base).await; + assert_eq!(warm["items"].as_array().unwrap().len(), 1); + + let target_match = get_directory_page( + &app, + &format!("{base}&query=rrx7-budget-target&tier=userMessages&limit=2"), + ) + .await; + assert_eq!(target_match["items"].as_array().unwrap().len(), 1); + assert_eq!(target_match["items"][0]["matchedIn"], json!("userMessage")); + assert!(target_match.get("partialReason").is_none()); + + let budget_limited = get_directory_page( + &app, + &format!("{base}&query=rrx7-budget-target&tier=userMessages&limit=1"), + ) + .await; + assert!(budget_limited["items"].as_array().unwrap().is_empty()); + assert_eq!(budget_limited["partial"], json!(true)); + assert_eq!(budget_limited["partialReason"], json!("budget")); + + std::fs::remove_dir_all(&home).ok(); + } + + #[tokio::test(flavor = "current_thread")] + async fn composed_codex_collision_keeps_a_same_id_filename_fallback_path() { + let home = unique_temp_dir(); + let (older, newer) = codex_fixtures(); + let (older_path, newer_path) = write_codex_segments(&home, &older, &newer); + let session_id = "b7936c10-4935-441c-837c-c1f33cafec2d"; + let sessions = home.join(".codex").join("sessions"); + let fallback_path = + sessions.join(format!("rollout-2026-10-03T00-00-11-{session_id}.jsonl")); + std::fs::write( + &fallback_path, + concat!( + "{\"timestamp\":\"2026-10-03T00:00:11.000Z\",\"type\":\"session_meta\",\"payload\":{\"cwd\":\"/sanitized/project\"}}\n", + "{\"timestamp\":\"2026-10-03T00:00:12.000Z\",\"type\":\"event_msg\",\"payload\":{\"type\":\"user_message\",\"message\":\"Filename fallback duplicate\"}}\n", + ), + ) + .unwrap(); + + let events = collision_trace_events(); + let (app, _index) = codex_session_directory_app( + &home, + None, + freshell_ws::identity::TerminalIdentityRegistry::new(), + ); + let page = get_directory_page( + &app, + "/api/session-directory?priority=visible&includeNonInteractive=1", + ) + .await; + + assert_eq!(page["integrityError"]["collisionCount"], json!(1)); + assert_eq!(page["integrityError"]["duplicateItemCount"], json!(2)); + assert!(page["items"].as_array().unwrap().is_empty()); + assert!(!serde_json::to_string(&page) + .unwrap() + .contains(&sessions.to_string_lossy().to_string())); + + let captured = collision_events_for_home(&events, &home); + assert_eq!(captured.len(), 1); + let samples: Vec = + serde_json::from_str(&decoded_trace_field(&captured[0], "collision_samples_json")) + .unwrap(); + let mut expected_paths = [older_path, newer_path, fallback_path]; + expected_paths.sort(); + assert_eq!(samples[0]["source_file_count"], json!(3)); + assert_eq!( + samples[0]["source_files"], + json!(expected_paths + .iter() + .map(|path| path.to_string_lossy().into_owned()) + .collect::>()) + ); + + std::fs::remove_dir_all(&home).ok(); + } + + async fn assert_equal_unknown_codex_classification_is_quarantined( + field: &str, + known_value: &str, + ) { + let home = unique_temp_dir(); + let (older, newer) = codex_fixtures(); + let known = format!(r#""{field}":"{known_value}""#); + let unknown = format!(r#""{field}":"unsupported-{field}""#); + let older = older.replace(&known, &unknown); + let newer = newer.replace(&known, &unknown); + write_codex_segments(&home, &older, &newer); + let session_id = "b7936c10-4935-441c-837c-c1f33cafec2d"; + let healthy_id = "11111111-2222-4333-8444-555555555555"; + std::fs::write( + home.join(".codex").join("sessions").join("healthy.jsonl"), + newer.replace(session_id, healthy_id), + ) + .unwrap(); + let (app, index) = codex_session_directory_app( + &home, + None, + freshell_ws::identity::TerminalIdentityRegistry::new(), + ); + let snapshot = index + .snapshot_with_failures_and_unresolved_codex_identities() + .await; + assert_eq!( + snapshot.sessions.len(), + 3, + "{field}: keep tolerant display rows" + ); + assert_eq!(snapshot.unresolved_codex_identities.len(), 1); + assert_eq!( + snapshot.unresolved_codex_identities[0].session_id, + session_id + ); + assert_eq!(snapshot.unresolved_codex_identities[0].paths.len(), 2); + + let base = "/api/session-directory?priority=visible&includeNonInteractive=1"; + let page = get_directory_page(&app, base).await; + assert_eq!( + page["integrityError"], + json!({ + "kind": "identity_collision", + "collisionCount": 1, + "duplicateItemCount": 2, + }), + "{field}: unknown classifications cannot certify a continuation" + ); + assert_eq!(page["partial"], json!(true)); + let rows = page["items"].as_array().unwrap(); + assert_eq!(rows.len(), 1); + assert_eq!(rows[0]["sessionId"], json!(healthy_id)); + assert_eq!(rows[0]["title"], json!("Continuation title")); + + let filtered = get_directory_page( + &app, + &format!("{base}&query=Older%20first%20request&tier=userMessages&limit=1"), + ) + .await; + assert!(filtered["items"].as_array().unwrap().is_empty()); + assert_eq!(filtered["integrityError"], page["integrityError"]); + std::fs::remove_dir_all(&home).ok(); + } + + #[tokio::test] + async fn codex_multi_file_route_quarantines_equal_unknown_history_modes() { + assert_equal_unknown_codex_classification_is_quarantined("history_mode", "paginated").await; + } + + #[tokio::test] + async fn codex_multi_file_route_quarantines_equal_unknown_thread_sources() { + assert_equal_unknown_codex_classification_is_quarantined("thread_source", "user").await; + } + + #[test] + fn codex_multi_file_apply_file_search_keeps_match_after_io_error() { + let home = unique_temp_dir(); + let (older, _) = codex_fixtures(); + let sessions = home.join("sessions"); + std::fs::create_dir_all(&sessions).unwrap(); + let missing_path = sessions.join("missing.jsonl"); + let readable_path = sessions.join("readable.jsonl"); + std::fs::write(&readable_path, older).unwrap(); + let mut item = dir_item_from_indexed(&static_indexed_session( + "codex", + "b7936c10-4935-441c-837c-c1f33cafec2d", + missing_path.to_str().unwrap(), + 100, + )); + item.source_files.push(readable_path); + + let result = apply_file_search( + vec![item], + "Older first request", + FileSearchTier::UserMessages, + 1, + ); + assert!(result.partial); + assert_eq!(result.partial_reason, Some("io_error")); + assert_eq!(result.items.len(), 1); + assert_eq!( + result.items[0].session_id, + "b7936c10-4935-441c-837c-c1f33cafec2d" + ); + assert_eq!(result.items[0].matched_in.as_deref(), Some("userMessage")); + assert!(result.items[0] + .snippet + .as_deref() + .unwrap() + .contains("Older first request")); + + std::fs::remove_dir_all(&home).ok(); + } + + #[test] + fn codex_multi_file_apply_file_search_counts_segments_against_shared_budget() { + let home = unique_temp_dir(); + let (older, _) = codex_fixtures(); + let sessions = home.join("sessions"); + std::fs::create_dir_all(&sessions).unwrap(); + let unmatched = older.replace("Older first request", "unmatched segment text"); + let first_path = sessions.join("segment-00.jsonl"); + std::fs::write(&first_path, &unmatched).unwrap(); + let mut item = dir_item_from_indexed(&static_indexed_session( + "codex", + "same-logical-session", + first_path.to_str().unwrap(), + 100, + )); + for index in 1..10 { + let path = sessions.join(format!("segment-{index:02}.jsonl")); + std::fs::write(&path, &unmatched).unwrap(); + item.source_files.push(path); + } + let eleventh_path = sessions.join("segment-10.jsonl"); + std::fs::write(&eleventh_path, older).unwrap(); + item.source_files.push(eleventh_path); + + let result = apply_file_search( + vec![item], + "Older first request", + FileSearchTier::UserMessages, + 1, + ); + assert!( + result.items.is_empty(), + "the eleventh segment is outside the scan budget" + ); + assert!(result.partial); + assert_eq!(result.partial_reason, Some("budget")); + + std::fs::remove_dir_all(&home).ok(); + } + + #[tokio::test] + async fn codex_multi_file_route_preserves_collision_quarantine_for_a_large_copy() { + let home = unique_temp_dir(); + let (older, newer) = codex_fixtures(); + let large = copied_large_codex_transcript(&older); + assert_eq!(large.lines().count(), 2001); + let sessions = home.join(".codex").join("sessions"); + std::fs::create_dir_all(&sessions).unwrap(); + for name in ["copy-a.jsonl", "copy-b.jsonl"] { + std::fs::write(sessions.join(name), &large).unwrap(); + } + let healthy_id = "11111111-2222-4333-8444-555555555555"; + std::fs::write( + sessions.join("healthy.jsonl"), + newer.replace("b7936c10-4935-441c-837c-c1f33cafec2d", healthy_id), + ) + .unwrap(); + + let identity = freshell_ws::identity::TerminalIdentityRegistry::new(); + identity.upsert( + "term-conflicted-codex", + Some("codex"), + Some("b7936c10-4935-441c-837c-c1f33cafec2d"), + Some("/live/codex"), + 2_000, + ); + let (app, _index) = codex_session_directory_app(&home, None, identity); + let page = get_directory_page( + &app, + "/api/session-directory?priority=visible&includeNonInteractive=1", + ) + .await; + assert_eq!( + page["integrityError"], + json!({ + "kind": "identity_collision", + "collisionCount": 1, + "duplicateItemCount": 2, + }) + ); + let rows = page["items"].as_array().unwrap(); + assert!(rows + .iter() + .any(|item| { item["provider"] == "codex" && item["sessionId"] == healthy_id })); + assert!(rows.iter().any(|item| { + item["provider"] == "codex" + && item["sessionId"] == "b7936c10-4935-441c-837c-c1f33cafec2d" + && item["runningTerminalId"] == "term-conflicted-codex" + })); + assert!(!serde_json::to_string(&page) + .unwrap() + .contains(&sessions.to_string_lossy().to_string())); + + let limited = get_directory_page( + &app, + "/api/session-directory?priority=visible&includeNonInteractive=1&limit=1", + ) + .await; + assert_eq!(limited["integrityError"]["collisionCount"], json!(1)); + + let filtered = get_directory_page( + &app, + "/api/session-directory?priority=visible&includeNonInteractive=1&query=absent-needle&limit=1", + ) + .await; + assert_eq!(filtered["items"].as_array().unwrap().len(), 0); + assert_eq!(filtered["integrityError"]["collisionCount"], json!(1)); + + std::fs::remove_dir_all(&home).ok(); + } + + #[tokio::test] + async fn amplifier_rows_without_source_files_keep_identity_collision_quarantined() { + let home = unique_temp_dir(); + let amplifier_home = home.join(".amplifier"); + let sessions = amplifier_home + .join("projects") + .join("project") + .join("sessions"); + for (directory, session_id) in [ + ("copy-a", "shared-amplifier-id"), + ("copy-b", "shared-amplifier-id"), + ("healthy", "healthy-amplifier-id"), + ] { + let session_dir = sessions.join(directory); + std::fs::create_dir_all(&session_dir).unwrap(); + std::fs::write( + session_dir.join("metadata.json"), + json!({ + "session_id": session_id, + "working_dir": "/project", + "created": "2026-10-03T00:00:00.000Z", + "name": directory, + }) + .to_string(), + ) + .unwrap(); + } + let source = freshell_sessions::amplifier::AmplifierSource::new(amplifier_home); + let discovered = source.scan(); + assert_eq!(discovered.len(), 3); + assert!(discovered.iter().all(|item| item.source_file.is_none())); + let app = router(SessionDirectoryState { + auth_token: Arc::new("tok".to_string()), + settings: crate::settings_store::SettingsStore::load( + Some(&home), + vec!["amplifier".into()], + ), + session_index: Some(Arc::new(test_session_index(vec![Arc::new(source)]))), + identity: freshell_ws::identity::TerminalIdentityRegistry::new(), + metadata: crate::session_metadata::SessionMetadataStore::new(home.join(".freshell")), + server_instance: Arc::new("srv-amplifier-test".to_string()), + collision_signatures: Default::default(), + legacy_name_migration_completed: false, + }); + + let base = "/api/session-directory?priority=visible&includeNonInteractive=1"; + let page = get_directory_page(&app, base).await; + assert_eq!( + page["integrityError"], + json!({ + "kind": "identity_collision", + "collisionCount": 1, + "duplicateItemCount": 2, + }) + ); + assert_eq!(page["partial"], json!(true)); + let rows = page["items"].as_array().unwrap(); + assert_eq!(rows.len(), 1, "only the healthy row remains visible"); + assert_eq!(rows[0]["sessionId"], json!("healthy-amplifier-id")); + + let filtered = get_directory_page(&app, &format!("{base}&query=missing&limit=1")).await; + assert!(filtered["items"].as_array().unwrap().is_empty()); + assert_eq!(filtered["integrityError"], page["integrityError"]); + + std::fs::remove_dir_all(&home).ok(); + } + + #[tokio::test(flavor = "current_thread")] + async fn persisted_identity_collision_counts_partially_overlapping_codex_members() { + let home = unique_temp_dir(); + let sessions = home.join(".codex").join("sessions"); + std::fs::create_dir_all(&sessions).unwrap(); + let session_id = "b7936c10-4935-441c-837c-c1f33cafec2d"; + let visible_path = sessions.join("a.jsonl"); + let hidden_path = sessions.join("b.jsonl"); + let fallback_path = + sessions.join(format!("rollout-2026-10-03T00-00-00-{session_id}.jsonl")); + for (path, payload) in [ + ( + &visible_path, + json!({ "id": session_id, "session_id": session_id, "cwd": "/p" }), + ), + ( + &hidden_path, + json!({ "id": session_id, "session_id": session_id }), + ), + (&fallback_path, json!({ "cwd": "/p" })), + ] { + std::fs::write( + path, + format!( + "{}\n", + json!({ + "timestamp": "2026-10-03T00:00:00.000Z", + "type": "session_meta", + "payload": payload, + }) + ), + ) + .unwrap(); + } + let events = collision_trace_events(); + let (app, index) = codex_session_directory_app( + &home, + None, + freshell_ws::identity::TerminalIdentityRegistry::new(), + ); + let snapshot = index + .snapshot_with_failures_and_unresolved_codex_identities() + .await; + assert_eq!(snapshot.sessions.len(), 2); + assert!(snapshot + .sessions + .iter() + .all(|item| item.session_id == session_id)); + assert_eq!( + snapshot + .sessions + .iter() + .filter_map(|item| item.source_file.clone()) + .collect::>(), + [visible_path.clone(), fallback_path.clone()] + .into_iter() + .collect() + ); + assert_eq!(snapshot.unresolved_codex_identities.len(), 1); + assert_eq!( + snapshot.unresolved_codex_identities[0].session_id, + session_id + ); + assert_eq!( + snapshot.unresolved_codex_identities[0].paths, + vec![visible_path.clone(), hidden_path.clone()] + ); + + let page = get_directory_page( + &app, + "/api/session-directory?priority=visible&includeNonInteractive=1", + ) + .await; + assert_eq!(page["integrityError"]["duplicateItemCount"], json!(3)); + assert_eq!(page["integrityError"]["collisionCount"], json!(1)); + assert_eq!(page["partial"], json!(true)); + assert!(page["items"].as_array().unwrap().is_empty()); + let captured = collision_events_for_home(&events, &home); + assert_eq!(captured.len(), 1); + assert_eq!(captured[0].level, tracing::Level::ERROR); + assert_eq!( + decoded_trace_field(&captured[0], "duplicate_item_count"), + "3" + ); + let samples: Vec = + serde_json::from_str(&decoded_trace_field(&captured[0], "collision_samples_json")) + .unwrap(); + assert_eq!(samples[0]["source_file_count"], json!(3)); + assert_eq!( + samples[0]["source_files"], + json!([visible_path, hidden_path, fallback_path]) + ); + + std::fs::remove_dir_all(&home).ok(); + } + + #[tokio::test(flavor = "current_thread")] + async fn persisted_identity_collision_quarantines_renderable_rows_for_hidden_codex_members() { + let home = unique_temp_dir(); + let (older, newer) = codex_fixtures(); + let cwdless = newer.replace("\"cwd\":\"/sanitized/project\",", ""); + let (older_path, hidden_path) = write_codex_segments(&home, &older, &cwdless); + let sessions = home.join(".codex").join("sessions"); + let healthy_id = "11111111-2222-4333-8444-555555555555"; + std::fs::write( + sessions.join("healthy.jsonl"), + newer.replace("b7936c10-4935-441c-837c-c1f33cafec2d", healthy_id), + ) + .unwrap(); + let cache_dir = unique_temp_dir(); + std::fs::create_dir_all(&cache_dir).unwrap(); + let cache_path = cache_dir.join("rust-session-cache.json"); + let events = collision_trace_events(); + let (app, index) = codex_session_directory_app( + &home, + Some(cache_path.clone()), + freshell_ws::identity::TerminalIdentityRegistry::new(), + ); + + let first = get_directory_page( + &app, + "/api/session-directory?priority=visible&includeNonInteractive=1&query=absent-needle&limit=1", + ) + .await; + assert_eq!(first["items"].as_array().unwrap().len(), 0); + assert_eq!( + first["integrityError"], + json!({ + "kind": "identity_collision", + "collisionCount": 1, + "duplicateItemCount": 2, + }) + ); + let mut cache_persisted = false; + for _ in 0..40 { + if std::fs::metadata(&cache_path).is_ok() { + cache_persisted = true; + break; + } + tokio::time::sleep(Duration::from_millis(50)).await; + } + assert!(cache_persisted, "the index cache persisted within 2s"); + let response_text = serde_json::to_string(&first).unwrap(); + assert!(!response_text.contains(&older_path.to_string_lossy().to_string())); + assert!(!response_text.contains(&hidden_path.to_string_lossy().to_string())); + assert!(!response_text.contains(healthy_id)); + let initial_events = collision_events_for_home(&events, &home); + assert_eq!( + initial_events.len(), + 1, + "the initial collision signature is captured once" + ); + assert_eq!(initial_events[0].level, tracing::Level::ERROR); + let initial_sample = &initial_events[0].fields["collision_samples_json"]; + assert!(initial_sample.contains(&older_path.to_string_lossy().to_string())); + assert!(initial_sample.contains(&hidden_path.to_string_lossy().to_string())); + + std::fs::remove_file(&hidden_path).unwrap(); + index.mark_provider_dirty("codex"); + let deadline = std::time::Instant::now() + Duration::from_secs(5); + loop { + let refreshed = get_directory_page( + &app, + "/api/session-directory?priority=visible&includeNonInteractive=1", + ) + .await; + if refreshed.get("integrityError").is_none() { + assert!(refreshed["items"].as_array().unwrap().iter().any(|item| { + item["provider"] == "codex" + && item["sessionId"] == "b7936c10-4935-441c-837c-c1f33cafec2d" + })); + break; + } + assert!( + std::time::Instant::now() < deadline, + "removed evidence must clear" + ); + tokio::time::sleep(Duration::from_millis(5)).await; + } + + let moved_hidden_path = sessions.join("zz-moved-hidden.jsonl"); + std::fs::write(&moved_hidden_path, &cwdless).unwrap(); + index.mark_provider_dirty("codex"); + let moved = loop { + let page = get_directory_page( + &app, + "/api/session-directory?priority=visible&includeNonInteractive=1", + ) + .await; + if page.get("integrityError").is_some() { + break page; + } + assert!( + std::time::Instant::now() < deadline, + "moved evidence must be seen" + ); + tokio::time::sleep(Duration::from_millis(5)).await; + }; + assert_eq!(moved["integrityError"]["duplicateItemCount"], json!(2)); + let moved_events = collision_events_for_home(&events, &home); + assert_eq!( + moved_events.len(), + 2, + "the moved path changes the full signature" + ); + assert!(moved_events[1].fields["collision_samples_json"] + .contains(&moved_hidden_path.to_string_lossy().to_string())); + + drop(app); + drop(index); + let (reloaded_app, _reloaded_index) = codex_session_directory_app( + &home, + Some(cache_path), + freshell_ws::identity::TerminalIdentityRegistry::new(), + ); + let after_reload = get_directory_page( + &reloaded_app, + "/api/session-directory?priority=visible&includeNonInteractive=1", + ) + .await; + assert_eq!(after_reload["integrityError"]["collisionCount"], json!(1)); + assert!(!serde_json::to_string(&after_reload) + .unwrap() + .contains(&moved_hidden_path.to_string_lossy().to_string())); + + std::fs::remove_dir_all(&home).ok(); + std::fs::remove_dir_all(&cache_dir).ok(); + } + + #[tokio::test(flavor = "current_thread")] + async fn soft_deleted_codex_identity_suppresses_unresolved_collision_until_restored() { + let home = unique_temp_dir(); + let (older, newer) = codex_fixtures(); + let cwdless = newer.replace("\"cwd\":\"/sanitized/project\",", ""); + let (_older_path, _hidden_path) = write_codex_segments(&home, &older, &cwdless); + let session_id = "b7936c10-4935-441c-837c-c1f33cafec2d"; + let events = collision_trace_events(); + let settings = + crate::settings_store::SettingsStore::load(Some(&home), vec!["codex".into()]); + settings + .patch_session_override( + &format!("codex:{session_id}"), + &[("deleted", Some(json!(true)))], + ) + .await; + let index = Arc::new(SessionIndex::with_ttl_and_cache_path( + vec![Arc::new(CodexSource::new(home.join(".codex")))], + Duration::from_secs(3600), + None, + )); + let app = router(SessionDirectoryState { + auth_token: Arc::new("tok".to_string()), + settings: settings.clone(), + session_index: Some(Arc::clone(&index)), + identity: freshell_ws::identity::TerminalIdentityRegistry::new(), + metadata: crate::session_metadata::SessionMetadataStore::new(home.join(".freshell")), + server_instance: Arc::new("srv-soft-delete-test".to_string()), + collision_signatures: Default::default(), + legacy_name_migration_completed: false, + }); + let uri = "/api/session-directory?priority=visible&includeNonInteractive=1"; + + let deleted = get_directory_page(&app, uri).await; + assert!(deleted["items"].as_array().unwrap().is_empty()); + assert!(deleted.get("integrityError").is_none()); + assert!(collision_events_for_home(&events, &home).is_empty()); + let snapshot = index + .snapshot_with_failures_and_unresolved_codex_identities() + .await; + assert_eq!(snapshot.unresolved_codex_identities.len(), 1); + assert_eq!( + snapshot.unresolved_codex_identities[0].session_id, + session_id + ); + + settings + .patch_session_override(&format!("codex:{session_id}"), &[("deleted", None)]) + .await; + let restored = get_directory_page(&app, uri).await; + assert!(restored["items"].as_array().unwrap().is_empty()); + assert_eq!( + restored["integrityError"], + json!({ + "kind": "identity_collision", + "collisionCount": 1, + "duplicateItemCount": 2, + }) + ); + let restored_events = collision_events_for_home(&events, &home); + assert_eq!(restored_events.len(), 1); + assert_eq!(restored_events[0].level, tracing::Level::ERROR); + + std::fs::remove_dir_all(&home).ok(); + } + + #[tokio::test(flavor = "current_thread")] + async fn query_free_directory_page_ignores_provider_scan_failure_overlay() { + let home = unique_temp_dir(); + let (older, newer) = codex_fixtures(); + write_codex_segments(&home, &older, &newer); + let opencode_home = home.join("opencode-data"); + std::fs::create_dir_all(&opencode_home).unwrap(); + std::fs::write(opencode_home.join("opencode.db"), b"not a sqlite database").unwrap(); + + let settings = crate::settings_store::SettingsStore::load( + Some(&home), + vec!["codex".into(), "opencode".into()], + ); + let index = Arc::new(test_session_index(vec![ + Arc::new(CodexSource::new(home.join(".codex"))) as Arc, + Arc::new(OpencodeSource::new(opencode_home)) as Arc, + ])); + let app = router(SessionDirectoryState { + auth_token: Arc::new("tok".to_string()), + settings, + session_index: Some(Arc::clone(&index)), + identity: freshell_ws::identity::TerminalIdentityRegistry::new(), + metadata: crate::session_metadata::SessionMetadataStore::new(home.join(".freshell")), + server_instance: Arc::new("srv-scan-failure-test".to_string()), + collision_signatures: Default::default(), + legacy_name_migration_completed: false, + }); + + let page = get_directory_page( + &app, + "/api/session-directory?priority=visible&includeNonInteractive=1", + ) + .await; + let snapshot = index + .snapshot_with_failures_and_unresolved_codex_identities() + .await; + assert!(snapshot + .scan_failures + .iter() + .any(|provider| provider == "opencode")); + assert!(page.get("partial").is_none()); + assert!(page.get("partialReason").is_none()); + let rows = page["items"].as_array().unwrap(); + assert_eq!(rows.len(), 1); + assert_eq!( + rows[0]["sessionId"], + json!("b7936c10-4935-441c-837c-c1f33cafec2d") + ); + + std::fs::remove_dir_all(&home).ok(); + } + + #[tokio::test(flavor = "current_thread")] + async fn persisted_identity_collision_logs_once_per_full_source_signature() { + let home = unique_temp_dir(); + let (older, _) = codex_fixtures(); + let large = copied_large_codex_transcript(&older); + let sessions = home.join(".codex").join("sessions"); + std::fs::create_dir_all(&sessions).unwrap(); + for index in 0..6 { + std::fs::write(sessions.join(format!("collision-{index:02}.jsonl")), &large).unwrap(); + } + let events = collision_trace_events(); + let (app, index) = codex_session_directory_app( + &home, + None, + freshell_ws::identity::TerminalIdentityRegistry::new(), + ); + let uri = "/api/session-directory?priority=visible&includeNonInteractive=1"; + + let (first, second, third) = tokio::join!( + get_directory_page(&app, uri), + get_directory_page(&app, uri), + get_directory_page(&app, uri), + ); + for page in [&first, &second, &third] { + assert_eq!(page["integrityError"]["duplicateItemCount"], json!(6)); + assert!(!serde_json::to_string(page) + .unwrap() + .contains(&sessions.to_string_lossy().to_string())); + } + let first_events = collision_events_for_home(&events, &home); + assert_eq!( + first_events.len(), + 1, + "concurrent identical requests should emit one collision event" + ); + assert_eq!(first_events[0].level, tracing::Level::ERROR); + let initial_id = decoded_trace_field(&first_events[0], "collision_signature_id"); + assert!(!initial_id.is_empty()); + let initial_samples = decoded_trace_field(&first_events[0], "collision_samples_json"); + let parsed_samples: Vec = serde_json::from_str(&initial_samples).unwrap(); + assert_eq!(parsed_samples.len(), 1); + assert_eq!( + parsed_samples[0]["source_files"].as_array().unwrap().len(), + 4 + ); + assert_eq!(parsed_samples[0]["source_files_truncated"], json!(true)); + + let mut changes = index.subscribe_changes(); + let generation_before_unchanged_refresh = *changes.borrow_and_update(); + index.mark_provider_dirty("codex"); + // A generation watch cannot signal completion when an unchanged + // refresh publishes nothing, so wait on the index's refresh barrier. + tokio::time::timeout( + Duration::from_secs(5), + index.wait_for_refresh_idle_for_test(), + ) + .await + .expect("marked unchanged refresh completes"); + assert!(!index.has_dirty(), "the marked Codex refresh was consumed"); + let generation_after_unchanged_refresh = *changes.borrow_and_update(); + assert_eq!( + generation_after_unchanged_refresh, generation_before_unchanged_refresh, + "an unchanged refresh must not advance the session generation" + ); + let unchanged = get_directory_page(&app, uri).await; + assert_eq!(unchanged["integrityError"]["duplicateItemCount"], json!(6)); + assert_eq!( + collision_events_for_home(&events, &home).len(), + 1, + "unchanged refresh is suppressed" + ); + + let old_beyond_sample = sessions.join("collision-05.jsonl"); + let moved_beyond_sample = sessions.join("zz-collision-05-moved.jsonl"); + std::fs::rename(&old_beyond_sample, &moved_beyond_sample).unwrap(); + index.mark_provider_dirty("codex"); + let deadline = std::time::Instant::now() + Duration::from_secs(5); + loop { + let snapshot = index + .snapshot_with_failures_and_unresolved_codex_identities() + .await; + let group = snapshot + .unresolved_codex_identities + .iter() + .find(|group| group.session_id == "b7936c10-4935-441c-837c-c1f33cafec2d"); + if group.is_some_and(|group| { + group.paths.contains(&moved_beyond_sample) + && !group.paths.contains(&old_beyond_sample) + }) { + break; + } + assert!( + std::time::Instant::now() < deadline, + "Codex provider refresh must publish the renamed member path" + ); + tokio::time::sleep(Duration::from_millis(5)).await; + } + let changed = get_directory_page(&app, uri).await; + assert_eq!(changed["integrityError"]["duplicateItemCount"], json!(6)); + let refreshed_group = index + .unresolved_codex_identities() + .into_iter() + .find(|group| group.session_id == "b7936c10-4935-441c-837c-c1f33cafec2d") + .expect("the changed unresolved group remains indexed"); + assert!(refreshed_group.paths.contains(&moved_beyond_sample)); + assert!(!refreshed_group.paths.contains(&old_beyond_sample)); + + let all_events = collision_events_for_home(&events, &home); + assert_eq!( + all_events.len(), + 2, + "one event is emitted for each new full signature" + ); + let changed_id = decoded_trace_field(&all_events[1], "collision_signature_id"); + assert_ne!(initial_id, changed_id); + assert_eq!( + decoded_trace_field(&all_events[1], "collision_samples_json"), + initial_samples, + "the changed path is beyond the bounded samples, so only the full signature id changes" + ); + assert!(decoded_trace_field(&all_events[1], "collision_samples_json").len() < 2048); + + std::fs::remove_dir_all(&home).ok(); + } + + async fn get_directory_page(app: &Router, uri: &str) -> Value { + use axum::http::Request; + use tower::ServiceExt; + + let response = app + .clone() + .oneshot( + Request::builder() + .method("GET") + .uri(uri) + .header("x-auth-token", "tok") + .body(axum::body::Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), axum::http::StatusCode::OK); + let body = axum::body::to_bytes(response.into_body(), usize::MAX) + .await + .unwrap(); + serde_json::from_slice(&body).unwrap() + } + + fn copied_large_codex_transcript(content: &str) -> String { + let lines: Vec<&str> = content.lines().collect(); + let mut copied = String::from(lines[0]); + copied.push('\n'); + for index in 0..2000usize { + let template = + serde_json::from_str::(lines[1 + index % (lines.len() - 1)]).unwrap(); + let timestamp_seconds = index + 1; + let timestamp = format!( + "2026-10-03T{:02}:{:02}:{:02}.000Z", + timestamp_seconds / 3600, + (timestamp_seconds / 60) % 60, + timestamp_seconds % 60, + ); + let mut record = template; + record["timestamp"] = Value::String(timestamp); + record["ordinal"] = Value::from(index as u64); + copied.push_str(&serde_json::to_string(&record).unwrap()); + copied.push('\n'); + } + copied + } + + fn collision_trace_events() -> CapturedTraceEvents { + crate::test_trace_capture::captured_trace_events() + } + + fn collision_events_for_home( + events: &CapturedTraceEvents, + home: &Path, + ) -> Vec { + let marker = home.to_string_lossy(); + events + .snapshot() + .into_iter() + .filter(|event| event.fields.contains_key("collision_samples_json")) + .filter(|event| decoded_trace_field(event, "collision_samples_json").contains(&*marker)) + .collect() + } + + fn decoded_trace_field(event: &CapturedTraceEvent, name: &str) -> String { + let encoded = event.fields.get(name).unwrap(); + serde_json::from_str::(encoded).unwrap_or_else(|_| encoded.clone()) + } + /// Comparable projection of either `DirItem` or `IndexedSession`, keyed /// the same way, for the B-T1 differential assertion (the two types are /// deliberately distinct -- one server-local, one in `freshell_sessions` @@ -3829,6 +5088,7 @@ mod tests { identity: freshell_ws::identity::TerminalIdentityRegistry::new(), metadata: crate::session_metadata::SessionMetadataStore::new(home.join(".freshell")), server_instance: std::sync::Arc::new("srv-test".to_string()), + collision_signatures: Default::default(), legacy_name_migration_completed: false, }; let app = router(state); @@ -3949,6 +5209,7 @@ mod tests { identity: freshell_ws::identity::TerminalIdentityRegistry::new(), metadata: crate::session_metadata::SessionMetadataStore::new(home.join(".freshell")), server_instance: std::sync::Arc::new("srv-test".to_string()), + collision_signatures: Default::default(), legacy_name_migration_completed: names.migration_completed(), }); let resp = app @@ -4050,6 +5311,7 @@ mod tests { identity, metadata, server_instance: Arc::new("srv-test".to_string()), + collision_signatures: Default::default(), legacy_name_migration_completed: true, }); let resp = app @@ -4146,6 +5408,7 @@ mod tests { identity, metadata, server_instance: Arc::new("srv-test".to_string()), + collision_signatures: Default::default(), legacy_name_migration_completed: true, }); let resp = app @@ -4560,6 +5823,7 @@ mod tests { // proves the join reads the persisted file, not shared memory. metadata: crate::session_metadata::SessionMetadataStore::new(home.join(".freshell")), server_instance: std::sync::Arc::new("srv-test".to_string()), + collision_signatures: Default::default(), legacy_name_migration_completed: false, }; let app = router(state); @@ -4638,6 +5902,7 @@ mod tests { identity: freshell_ws::identity::TerminalIdentityRegistry::new(), metadata: crate::session_metadata::SessionMetadataStore::new(home.join(".freshell")), server_instance: std::sync::Arc::new("srv-test".to_string()), + collision_signatures: Default::default(), legacy_name_migration_completed: false, }; let app = router(state); @@ -4698,6 +5963,7 @@ mod tests { identity: freshell_ws::identity::TerminalIdentityRegistry::new(), metadata: crate::session_metadata::SessionMetadataStore::new(home.join(".freshell")), server_instance: std::sync::Arc::new("srv-test".to_string()), + collision_signatures: Default::default(), legacy_name_migration_completed: false, }; let app = router(state); @@ -4743,6 +6009,7 @@ mod tests { // missing file (empty metadata), matching the no-home page. metadata: crate::session_metadata::SessionMetadataStore::new(unique_temp_dir()), server_instance: std::sync::Arc::new("srv-test".to_string()), + collision_signatures: Default::default(), legacy_name_migration_completed: false, }; let app = router(state); @@ -4827,6 +6094,7 @@ mod tests { identity: freshell_ws::identity::TerminalIdentityRegistry::new(), metadata: crate::session_metadata::SessionMetadataStore::new(home.join(".freshell")), server_instance: std::sync::Arc::new("srv-test".to_string()), + collision_signatures: Default::default(), legacy_name_migration_completed: false, }; let app = router(state); @@ -4933,6 +6201,7 @@ mod tests { identity: freshell_ws::identity::TerminalIdentityRegistry::new(), metadata: crate::session_metadata::SessionMetadataStore::new(home.join(".freshell")), server_instance: std::sync::Arc::new("srv-test".to_string()), + collision_signatures: Default::default(), legacy_name_migration_completed: false, }; let app = router(state); @@ -5032,6 +6301,7 @@ mod tests { identity: freshell_ws::identity::TerminalIdentityRegistry::new(), metadata: crate::session_metadata::SessionMetadataStore::new(home.join(".freshell")), server_instance: std::sync::Arc::new("srv-test".to_string()), + collision_signatures: Default::default(), legacy_name_migration_completed: false, }; let app = router(state); @@ -5128,6 +6398,7 @@ mod tests { identity: freshell_ws::identity::TerminalIdentityRegistry::new(), metadata: crate::session_metadata::SessionMetadataStore::new(home.join(".freshell")), server_instance: std::sync::Arc::new("srv-test".to_string()), + collision_signatures: Default::default(), legacy_name_migration_completed: false, }; let app = router(state); @@ -5266,6 +6537,7 @@ mod tests { identity: freshell_ws::identity::TerminalIdentityRegistry::new(), metadata: crate::session_metadata::SessionMetadataStore::new(home.join(".freshell")), server_instance: std::sync::Arc::new("srv-test".to_string()), + collision_signatures: Default::default(), legacy_name_migration_completed: false, }; let app = router(state); @@ -5405,7 +6677,7 @@ mod tests { let items = list_claude_sessions(&claude_home(&home)); assert_eq!(items.len(), 1); assert!( - items[0].source_file.is_some(), + !items[0].source_files.is_empty(), "a real session file must carry a source_file for tier search" ); @@ -5654,7 +6926,7 @@ mod tests { live_terminal_only: false, session_type: None, title_source: None, - source_file: None, + source_files: Vec::new(), token_usage: None, title_overridden: false, provider_title: None, diff --git a/crates/freshell-server/src/sessions_tests.rs b/crates/freshell-server/src/sessions_tests.rs index 151853454..f6a32a7ce 100644 --- a/crates/freshell-server/src/sessions_tests.rs +++ b/crates/freshell-server/src/sessions_tests.rs @@ -762,6 +762,7 @@ async fn patch_override_is_visible_through_session_directory_overlay() { identity: dir_identity, metadata: crate::session_metadata::SessionMetadataStore::new(home.join(".freshell")), server_instance: std::sync::Arc::new("srv-test".to_string()), + collision_signatures: Default::default(), legacy_name_migration_completed: false, }); let dir_resp = dir_app @@ -1036,6 +1037,7 @@ async fn deleted_session_disappears_from_session_directory_overlay() { identity: freshell_ws::identity::TerminalIdentityRegistry::new(), metadata: crate::session_metadata::SessionMetadataStore::new(home.join(".freshell")), server_instance: std::sync::Arc::new("srv-test".to_string()), + collision_signatures: Default::default(), legacy_name_migration_completed: false, }); diff --git a/crates/freshell-server/src/test_trace_capture.rs b/crates/freshell-server/src/test_trace_capture.rs new file mode 100644 index 000000000..5726fb43b --- /dev/null +++ b/crates/freshell-server/src/test_trace_capture.rs @@ -0,0 +1,104 @@ +use std::collections::BTreeMap; +use std::sync::{Arc, Mutex, OnceLock}; + +#[derive(Clone, Debug)] +pub(crate) struct CapturedTraceEvent { + pub(crate) level: tracing::Level, + pub(crate) message: String, + pub(crate) fields: BTreeMap, +} + +#[derive(Clone, Default)] +pub(crate) struct CapturedTraceEvents(Arc>>); + +impl CapturedTraceEvents { + pub(crate) fn snapshot(&self) -> Vec { + self.0.lock().unwrap().clone() + } +} + +/// One process-wide collector is shared by tests that need to inspect tracing +/// events. Keeping the subscriber and buffer here prevents modules from racing +/// to install separate global subscribers with disconnected event buffers. +pub(crate) fn captured_trace_events() -> CapturedTraceEvents { + use tracing_subscriber::prelude::*; + + static GLOBAL_EVENTS: OnceLock = OnceLock::new(); + GLOBAL_EVENTS + .get_or_init(|| { + let events = CapturedTraceEvents::default(); + let subscriber = tracing_subscriber::registry().with(CaptureTraceLayer { + events: events.clone(), + }); + tracing::subscriber::set_global_default(subscriber) + .expect("test trace capture must install the sole test subscriber"); + events + }) + .clone() +} + +struct CaptureTraceLayer { + events: CapturedTraceEvents, +} + +#[derive(Default)] +struct TraceFieldVisitor { + message: String, + fields: BTreeMap, +} + +impl tracing::field::Visit for TraceFieldVisitor { + fn record_debug(&mut self, field: &tracing::field::Field, value: &dyn std::fmt::Debug) { + let value = format!("{value:?}"); + if field.name() == "message" { + self.message = value.clone(); + } + self.fields.insert(field.name().to_string(), value); + } + + fn record_str(&mut self, field: &tracing::field::Field, value: &str) { + if field.name() == "message" { + self.message = value.to_string(); + } + self.fields + .insert(field.name().to_string(), value.to_string()); + } +} + +impl tracing_subscriber::Layer for CaptureTraceLayer +where + S: tracing::Subscriber, +{ + fn on_event( + &self, + event: &tracing::Event<'_>, + _context: tracing_subscriber::layer::Context<'_, S>, + ) { + let metadata = event.metadata(); + let level = *metadata.level(); + let target = metadata.target(); + let is_extension_warning = + target == "freshell_server::extensions" && level == tracing::Level::WARN; + let is_session_directory_error = + target == "freshell_server::session_directory" && level == tracing::Level::ERROR; + if !is_extension_warning && !is_session_directory_error { + return; + } + + let mut visitor = TraceFieldVisitor::default(); + event.record(&mut visitor); + if is_session_directory_error + && !visitor + .message + .contains("session_directory_identity_collision") + { + return; + } + + self.events.0.lock().unwrap().push(CapturedTraceEvent { + level, + message: visitor.message, + fields: visitor.fields, + }); + } +} diff --git a/crates/freshell-sessions/src/codex_segments.rs b/crates/freshell-sessions/src/codex_segments.rs new file mode 100644 index 000000000..63aaf47c6 --- /dev/null +++ b/crates/freshell-sessions/src/codex_segments.rs @@ -0,0 +1,663 @@ +//! Structural evidence and composition for Codex rollout continuations. +//! +//! The regular Codex parser is intentionally tolerant because it produces a +//! useful single-file display row from imperfect history. This module has a +//! stricter, separate scan: only a complete, owned file with known metadata +//! and a monotonic persisted-record interval can participate in a multi-file +//! session. The original per-file evidence stays in the directory-index +//! cache so a later refresh can reconsider the whole identity group. + +use std::collections::{BTreeMap, HashMap}; +use std::path::PathBuf; + +use chrono::DateTime; +use serde_json::Value; + +use crate::directory_index::IndexedSession; + +const REQUIRED_METADATA: [&str; 6] = [ + "cwd", + "source", + "thread_source", + "cli_version", + "originator", + "history_mode", +]; + +/// Codex 0.156's persisted `RolloutItemWire` variants. A continuation scan +/// must count every persisted record when establishing a complete interval, +/// even when the display parser does not interpret that record's payload. +const KNOWN_RECORD_TYPES: [&str; 12] = [ + "session_meta", + "response_item", + "inter_agent_communication", + "inter_agent_communication_metadata", + "compacted", + "turn_context", + "token_usage_record", + "world_state", + "retained_context", + "security_risk_score", + "event_msg", + "realtime_item", +]; + +/// Per-file evidence retained in `FileEntry` independently of its renderable +/// `IndexedSession`. The id is sourced only from transcript metadata; a +/// filename-derived fallback is never continuation evidence. +#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] +pub struct CodexFileEvidence { + /// The first known embedded id, even when ownership is incomplete. This + /// lets the index quarantine a cwd-less or malformed sibling with its + /// renderable same-id file. + pub session_id: Option, + pub first_line_owned: bool, + pub first_header_id: Option, + pub first_header_session_id: Option, + /// Exact structured values from the first session header. Missing values + /// remain missing and therefore cannot be treated as matching evidence. + pub required_metadata: BTreeMap, + /// Every recognized fork, parent, subagent, and referenced-prefix marker + /// from each top-level `session_meta` record, with its exact JSON value. + pub lineage_markers: Vec, + pub header_count: usize, + pub scan_complete: bool, + pub record_count: usize, + pub interval: Option, + /// A short, bounded set of structural reasons. This is internal cache + /// evidence, never serialized into the client session-directory response. + pub scan_errors: Vec, +} + +#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] +pub struct CodexLineageMarker { + pub name: String, + pub value: Value, + pub header_index: usize, +} + +/// Outer persisted-record bounds. Timestamps are stored as Unix nanoseconds +/// so millisecond ties and any supported sub-millisecond precision survive +/// cache serialization and strict cross-file comparison. +#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +pub struct CodexRecordInterval { + pub start_nanos: i64, + pub end_nanos: i64, +} + +#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] +#[serde(tag = "reason", content = "detail", rename_all = "snake_case")] +pub enum CodexSegmentScanError { + MissingFinalNewline, + EmptyRecord, + MalformedJson, + InvalidRecordShape, + InvalidUtf8, + UnknownRecordType(String), + MissingTimestamp, + InvalidTimestamp, + TimestampRegression, + InvalidOrdinal, + OrdinalRegression, +} + +/// Known-id members that could not be safely composed. All paths stay in the +/// Rust index; callers must not put them on the client wire. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct CodexUnresolvedIdentity { + pub session_id: String, + pub paths: Vec, +} + +/// Result of applying the same composition policy to direct scans and the +/// cached session-index snapshot. +#[derive(Debug, Default)] +pub struct CodexComposition { + pub items: Vec, + pub unresolved_identities: Vec, + /// Chronological paths for accepted multi-file rows, keyed by canonical + /// embedded id. This sidecar feeds bounded downstream search without + /// changing `source_file`'s existing meaning. + pub segment_paths: HashMap>, +} + +/// One per-file input to [`compose_codex_segments`]. +#[derive(Debug, Clone)] +pub struct CodexSegmentEntry { + pub path: PathBuf, + pub item: Option, + pub evidence: Option, +} + +/// Scan the complete JSONL stream for evidence suitable for continuation +/// composition. The display parser remains a separate, tolerant projection. +pub fn scan_codex_file_evidence(content: &str) -> CodexFileEvidence { + let mut errors = Vec::new(); + let mut headers = Vec::::new(); + let mut session_id = None; + let mut min_record_nanos = None::; + let mut max_record_nanos = None::; + let mut previous_record_nanos = None::; + let mut previous_ordinal = None::; + let mut post_header_records = 0usize; + + if !content.ends_with('\n') { + push_error(&mut errors, CodexSegmentScanError::MissingFinalNewline); + } + + let lines: Vec<&str> = content + .split_terminator('\n') + .map(|line| line.strip_suffix('\r').unwrap_or(line)) + .collect(); + + for (line_index, line) in lines.iter().enumerate() { + if line.is_empty() { + push_error(&mut errors, CodexSegmentScanError::EmptyRecord); + continue; + } + + let value: Value = match serde_json::from_str(line) { + Ok(value) => value, + Err(_) => { + push_error(&mut errors, CodexSegmentScanError::MalformedJson); + continue; + } + }; + let Some(record) = value.as_object() else { + push_error(&mut errors, CodexSegmentScanError::InvalidRecordShape); + continue; + }; + let Some(record_type) = record.get("type").and_then(Value::as_str) else { + push_error(&mut errors, CodexSegmentScanError::InvalidRecordShape); + continue; + }; + if !KNOWN_RECORD_TYPES.contains(&record_type) { + push_error( + &mut errors, + CodexSegmentScanError::UnknownRecordType(record_type.to_string()), + ); + } + + let timestamp = match record.get("timestamp") { + None => { + push_error(&mut errors, CodexSegmentScanError::MissingTimestamp); + None + } + Some(value) => match value.as_str().and_then(parse_rfc3339_nanos) { + Some(nanos) => Some(nanos), + None => { + push_error(&mut errors, CodexSegmentScanError::InvalidTimestamp); + None + } + }, + }; + + if line_index == 0 { + if let Some(timestamp) = timestamp { + // The first session header timestamps creation, not a + // persisted post-header write, so it is deliberately omitted + // from the segment's chronology interval. + let _header_timestamp_nanos = timestamp; + } + } else { + post_header_records = post_header_records.saturating_add(1); + if let Some(nanos) = timestamp { + if previous_record_nanos.is_some_and(|previous| nanos < previous) { + push_error(&mut errors, CodexSegmentScanError::TimestampRegression); + } + previous_record_nanos = Some(nanos); + min_record_nanos = Some(min_record_nanos.map_or(nanos, |min| min.min(nanos))); + max_record_nanos = Some(max_record_nanos.map_or(nanos, |max| max.max(nanos))); + } + } + + if let Some(ordinal_value) = record.get("ordinal") { + match ordinal_value.as_u64() { + Some(ordinal) => { + if previous_ordinal.is_some_and(|previous| ordinal <= previous) { + push_error(&mut errors, CodexSegmentScanError::OrdinalRegression); + } + previous_ordinal = Some(ordinal); + } + None => push_error(&mut errors, CodexSegmentScanError::InvalidOrdinal), + } + } + + if record_type == "session_meta" { + let payload = record.get("payload").and_then(Value::as_object); + let header = payload.map(|payload| { + let id = payload.get("id").cloned(); + let root_id = payload.get("session_id").cloned(); + let metadata = REQUIRED_METADATA + .iter() + .filter_map(|key| { + payload + .get(*key) + .cloned() + .map(|value| ((*key).to_string(), value)) + }) + .collect::>(); + let lineage_markers = collect_lineage_markers(payload, headers.len()); + let known_id = id + .as_ref() + .and_then(Value::as_str) + .filter(|id| !id.is_empty()) + .or_else(|| { + root_id + .as_ref() + .and_then(Value::as_str) + .filter(|id| !id.is_empty()) + }) + .map(str::to_owned); + if session_id.is_none() { + session_id = known_id; + } + HeaderRecord { + id, + session_id: root_id, + metadata, + lineage_markers, + } + }); + if let Some(header) = header { + headers.push(header); + } else { + push_error(&mut errors, CodexSegmentScanError::InvalidRecordShape); + } + } + } + + let first_header = headers.first(); + let first_line_owned = lines + .first() + .and_then(|line| serde_json::from_str::(line).ok()) + .and_then(|record| { + (record.get("type").and_then(Value::as_str) == Some("session_meta")) + .then(|| record.get("payload").and_then(Value::as_object).is_some()) + }) + .unwrap_or(false); + let first_header_id = first_header.and_then(|header| header.id.clone()); + let first_header_session_id = first_header.and_then(|header| header.session_id.clone()); + let identity_matches = first_header.is_some_and(|header| { + header + .id + .as_ref() + .and_then(Value::as_str) + .is_some_and(|id| !id.is_empty()) + && header + .session_id + .as_ref() + .and_then(Value::as_str) + .is_some_and(|id| !id.is_empty()) + && header.id == header.session_id + }); + if first_line_owned && !identity_matches { + push_error(&mut errors, CodexSegmentScanError::InvalidRecordShape); + } + + let required_metadata = first_header + .map(|header| header.metadata.clone()) + .unwrap_or_default(); + let mut lineage_markers = Vec::new(); + for header in &headers { + lineage_markers.extend(header.lineage_markers.iter().cloned()); + } + + let interval = match (post_header_records, min_record_nanos, max_record_nanos) { + (count, Some(start_nanos), Some(end_nanos)) if count > 0 => Some(CodexRecordInterval { + start_nanos, + end_nanos, + }), + _ => None, + }; + let scan_complete = errors.is_empty() && interval.is_some(); + + CodexFileEvidence { + session_id, + first_line_owned, + first_header_id, + first_header_session_id, + required_metadata, + lineage_markers, + header_count: headers.len(), + scan_complete, + record_count: lines.len(), + interval, + scan_errors: errors, + } +} + +/// Byte-oriented wrapper used for file reads. The display parser continues +/// to use lossy UTF-8 decoding, but replacement characters cannot certify a +/// complete persisted stream for composition. +pub fn scan_codex_file_bytes_evidence(bytes: &[u8]) -> CodexFileEvidence { + match std::str::from_utf8(bytes) { + Ok(content) => scan_codex_file_evidence(content), + Err(_) => { + let content = String::from_utf8_lossy(bytes); + let mut evidence = scan_codex_file_evidence(&content); + evidence.scan_complete = false; + push_error( + &mut evidence.scan_errors, + CodexSegmentScanError::InvalidUtf8, + ); + evidence + } + } +} + +/// Compose only groups for which every discovered same-id member has +/// complete ownership, compatible required metadata, no recognized lineage +/// marker, and a strictly disjoint persisted-record interval. A failed group +/// keeps every renderable row and publishes every known member path for +/// quarantine. +pub fn compose_codex_segments(entries: Vec) -> CodexComposition { + let mut groups = BTreeMap::>::new(); + let mut ungrouped = Vec::new(); + for entry in entries { + if let Some(session_id) = entry + .evidence + .as_ref() + .and_then(|evidence| evidence.session_id.clone()) + { + groups.entry(session_id).or_default().push(entry); + } else if let Some(item) = entry.item { + ungrouped.push(item); + } + } + + let mut composition = CodexComposition { + items: ungrouped, + ..CodexComposition::default() + }; + for (session_id, mut members) in groups { + if members.len() == 1 { + if let Some(item) = members.pop().and_then(|member| member.item) { + composition.items.push(item); + } + continue; + } + + let mut ordered = members; + ordered.sort_by(|left, right| { + let left_interval = left + .evidence + .as_ref() + .and_then(|evidence| evidence.interval); + let right_interval = right + .evidence + .as_ref() + .and_then(|evidence| evidence.interval); + left_interval + .map(|interval| interval.start_nanos) + .cmp(&right_interval.map(|interval| interval.start_nanos)) + .then_with(|| left.path.cmp(&right.path)) + }); + + if can_compose_group(&session_id, &ordered) { + let paths: Vec = ordered.iter().map(|member| member.path.clone()).collect(); + let items: Vec = ordered + .iter() + .filter_map(|member| member.item.clone()) + .collect(); + composition.segment_paths.insert(session_id, paths); + composition.items.push(merge_ordered_items(&ordered, items)); + } else { + let mut paths: Vec = + ordered.iter().map(|member| member.path.clone()).collect(); + paths.sort(); + composition + .unresolved_identities + .push(CodexUnresolvedIdentity { session_id, paths }); + composition + .items + .extend(ordered.into_iter().filter_map(|member| member.item)); + } + } + composition + .unresolved_identities + .sort_by(|left, right| left.session_id.cmp(&right.session_id)); + composition +} + +fn can_compose_group(session_id: &str, members: &[CodexSegmentEntry]) -> bool { + let mut previous_end = None; + let mut matching_metadata: Option<&BTreeMap> = None; + + for member in members { + let Some(evidence) = member.evidence.as_ref() else { + return false; + }; + let Some(item) = member.item.as_ref() else { + return false; + }; + if !evidence.first_line_owned + || evidence.header_count != 1 + || !evidence.scan_complete + || evidence.session_id.as_deref() != Some(session_id) + || evidence.first_header_id.as_ref().and_then(Value::as_str) != Some(session_id) + || evidence + .first_header_session_id + .as_ref() + .and_then(Value::as_str) + != Some(session_id) + || !evidence.lineage_markers.is_empty() + || item.provider != "codex" + || item.session_id != session_id + || item.is_subagent + { + return false; + } + if !REQUIRED_METADATA.iter().all(|key| { + evidence + .required_metadata + .get(*key) + .is_some_and(|value| match *key { + "cwd" | "cli_version" | "originator" => { + value.as_str().is_some_and(|text| !text.trim().is_empty()) + } + "source" => supported_root_session_source(value), + // Only the reported user thread with paginated history + // certifies continuation; other classifications remain displayable. + "thread_source" => value.as_str() == Some("user"), + "history_mode" => value.as_str() == Some("paginated"), + _ => false, + }) + }) { + return false; + } + let Some(interval) = evidence.interval else { + return false; + }; + if previous_end.is_some_and(|end| end >= interval.start_nanos) { + return false; + } + previous_end = Some(interval.end_nanos); + + if let Some(previous) = matching_metadata { + if previous != &evidence.required_metadata { + return false; + } + } else { + matching_metadata = Some(&evidence.required_metadata); + } + if item.cwd.as_deref() + != evidence + .required_metadata + .get("cwd") + .and_then(Value::as_str) + { + return false; + } + } + true +} + +fn merge_ordered_items( + members: &[CodexSegmentEntry], + items: Vec, +) -> IndexedSession { + let mut merged = items + .last() + .expect("a composable group has one renderable row per member") + .clone(); + merged.created_at = items.iter().filter_map(|item| item.created_at).min(); + merged.last_activity_at = items + .iter() + .map(|item| item.last_activity_at) + .max() + .unwrap_or(0); + merged.first_user_message = items + .iter() + .find_map(|item| nonempty(item.first_user_message.as_ref()).cloned()); + merged.title = items + .iter() + .find_map(|item| nonempty(item.title.as_ref()).cloned()); + merged.summary = items + .iter() + .find_map(|item| nonempty(item.summary.as_ref()).cloned()); + merged.token_usage = items.iter().rev().find_map(|item| item.token_usage.clone()); + merged.source_file = members.last().map(|member| member.path.clone()); + merged +} + +fn nonempty(value: Option<&String>) -> Option<&String> { + value.filter(|value| !value.trim().is_empty()) +} + +/// Validate the 0.156 `SessionSource` shapes we can safely compare for root +/// Codex sessions. Unknown and internal/subagent sources do not certify a +/// user-visible continuation; custom sources are supported only in their +/// serialized enum form and are compared exactly across members. +fn supported_root_session_source(value: &Value) -> bool { + match value { + Value::String(source) => matches!(source.as_str(), "cli" | "vscode" | "exec" | "mcp"), + Value::Object(fields) => { + fields.len() == 1 + && fields + .get("custom") + .and_then(Value::as_str) + .is_some_and(|source| !source.trim().is_empty()) + } + _ => false, + } +} + +fn collect_lineage_markers( + payload: &serde_json::Map, + header_index: usize, +) -> Vec { + const LINEAGE_FIELDS: [&str; 10] = [ + "forked_from_id", + "forked_from_ordinal_exclusive", + "parent_thread_id", + "parent_id", + "subagent_history_start_ordinal", + "agent_nickname", + "agent_role", + "agent_path", + "history_base", + "is_subagent", + ]; + + let mut markers = Vec::new(); + for name in LINEAGE_FIELDS { + if let Some(value) = payload.get(name) { + markers.push(CodexLineageMarker { + name: name.to_string(), + value: value.clone(), + header_index, + }); + } + } + if let Some(source) = payload.get("source") { + if let Some(subagent) = source.as_object().and_then(|source| source.get("subagent")) { + markers.push(CodexLineageMarker { + name: "source.subagent".to_string(), + value: subagent.clone(), + header_index, + }); + } + } + markers +} + +fn parse_rfc3339_nanos(value: &str) -> Option { + DateTime::parse_from_rfc3339(value) + .ok()? + .timestamp_nanos_opt() +} + +fn push_error(errors: &mut Vec, error: CodexSegmentScanError) { + const MAX_SCAN_ERRORS: usize = 8; + if errors.len() < MAX_SCAN_ERRORS { + errors.push(error); + } +} + +struct HeaderRecord { + id: Option, + session_id: Option, + metadata: BTreeMap, + lineage_markers: Vec, +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn interval_keeps_timestamp_precision_and_accepts_within_file_ties() { + let content = concat!( + "{\"timestamp\":\"2026-10-03T00:00:00.000Z\",\"type\":\"session_meta\",\"payload\":{\"id\":\"s\",\"session_id\":\"s\"}}\n", + "{\"timestamp\":\"2026-10-03T00:00:01.123456Z\",\"type\":\"event_msg\",\"payload\":{}}\n", + "{\"timestamp\":\"2026-10-03T00:00:01.123456Z\",\"type\":\"compacted\",\"payload\":{}}\n", + ); + let evidence = scan_codex_file_evidence(content); + assert!(evidence.scan_complete); + let interval = evidence.interval.unwrap(); + assert_eq!(interval.start_nanos, interval.end_nanos); + assert_eq!(interval.start_nanos % 1_000_000, 456_000); + } + + #[test] + fn record_scan_rejects_unknown_and_truncated_lines() { + let unknown = concat!( + "{\"timestamp\":\"2026-10-03T00:00:00.000Z\",\"type\":\"session_meta\",\"payload\":{\"id\":\"s\",\"session_id\":\"s\"}}\n", + "{\"timestamp\":\"2026-10-03T00:00:01Z\",\"type\":\"future_variant\",\"payload\":{}}\n", + ); + let evidence = scan_codex_file_evidence(unknown); + assert!(!evidence.scan_complete); + assert!(evidence + .scan_errors + .contains(&CodexSegmentScanError::UnknownRecordType( + "future_variant".to_string() + ))); + + let truncated = unknown.trim_end(); + let evidence = scan_codex_file_evidence(truncated); + assert!(!evidence.scan_complete); + assert!(evidence + .scan_errors + .contains(&CodexSegmentScanError::MissingFinalNewline)); + } + + #[test] + fn byte_scan_does_not_certify_lossy_utf8_replacements() { + let mut bytes = concat!( + "{\"timestamp\":\"2026-10-03T00:00:00.000Z\",\"type\":\"session_meta\",\"payload\":{\"id\":\"s\",\"session_id\":\"s\"}}\n", + "{\"timestamp\":\"2026-10-03T00:00:01.000Z\",\"type\":\"event_msg\",\"payload\":{\"type\":\"user_message\",\"message\":\"" + ) + .as_bytes() + .to_vec(); + bytes.push(0xff); + bytes.extend_from_slice(b"\"}}\n"); + + let evidence = scan_codex_file_bytes_evidence(&bytes); + assert_eq!(evidence.session_id.as_deref(), Some("s")); + assert!(!evidence.scan_complete); + assert!(evidence + .scan_errors + .contains(&CodexSegmentScanError::InvalidUtf8)); + } +} diff --git a/crates/freshell-sessions/src/directory_index.rs b/crates/freshell-sessions/src/directory_index.rs index 04cf7c966..32fe20b52 100644 --- a/crates/freshell-sessions/src/directory_index.rs +++ b/crates/freshell-sessions/src/directory_index.rs @@ -41,6 +41,10 @@ use std::time::{Duration, Instant}; use tokio::sync::Mutex as AsyncMutex; +use crate::codex_segments::{ + compose_codex_segments, scan_codex_file_bytes_evidence, CodexComposition, CodexFileEvidence, + CodexSegmentEntry, CodexUnresolvedIdentity, +}; use crate::meta::ParsedSessionMeta; use crate::provider_layout::ProviderLayout; use crate::{parse_codex_session_content, parse_session_content, ParseSessionOptions}; @@ -142,6 +146,24 @@ impl IndexedSession { } } +/// Related values from one published [`SessionIndex`] generation. The row +/// collection and potentially large Codex sidecars are shared through `Arc`s. +/// Consumers that need rows alongside scan status, unresolved Codex +/// identities, or accepted source paths should use the coherent accessor so +/// they do not mix values from separate refreshes. +#[derive(Debug, Clone)] +pub struct SessionIndexSnapshot { + /// Provider rows captured from this generation. + pub sessions: Arc>, + /// Providers whose listing attempt failed during this generation. + pub scan_failures: Vec, + /// Same-id Codex file groups that could not safely be composed. + pub unresolved_codex_identities: Arc>, + /// Chronological source paths for accepted multi-file Codex rows, keyed + /// by their canonical embedded session id. + pub codex_segment_paths: Arc>>, +} + /// One discovered file: its absolute path plus the stat facts (`mtime`/`size`) /// [`SessionIndex`]'s incremental cache uses to decide whether it needs /// re-parsing. Stat-only — no file content is read to produce this. @@ -202,6 +224,16 @@ pub trait SessionSource: Send + Sync { /// never panics. fn parse(&self, path: &Path) -> Option; + /// Parse Codex's strict per-file composition evidence alongside the + /// display row. Other providers keep the default path and never carry + /// Codex-specific cache state. + fn parse_with_codex_evidence( + &self, + path: &Path, + ) -> (Option, Option) { + (self.parse(path), None) + } + /// Batch C: direct-listed sources (opencode's single sqlite db, which /// enumerates MANY sessions in ONE query rather than one file per /// session) can't fit the per-file `discover`/`parse` cache — there's no @@ -557,10 +589,19 @@ impl CodexSource { /// call, ignoring any incremental cache. Test/perf use only — mirrors /// `ClaudeSource::scan()`. pub fn scan(&self) -> Vec { - self.discover() + let entries = self + .discover() .into_iter() - .filter_map(|stat| self.parse(&stat.path)) - .collect() + .map(|stat| { + let (item, evidence) = self.parse_with_codex_evidence(&stat.path); + CodexSegmentEntry { + path: stat.path, + item, + evidence, + } + }) + .collect(); + compose_codex_segments(entries).items } } @@ -586,6 +627,13 @@ impl SessionSource for CodexSource { fn parse(&self, path: &Path) -> Option { parse_codex_file(path) } + + fn parse_with_codex_evidence( + &self, + path: &Path, + ) -> (Option, Option) { + parse_codex_file_with_evidence(path) + } } /// Stat every `.jsonl` under `/sessions`, recursively. @@ -649,7 +697,24 @@ fn walk_jsonl_recursive(dir: &Path, out: &mut Vec) { /// gate (:756, :1124) applies to every provider, not just claude. fn parse_codex_file(path: &Path) -> Option { let content = String::from_utf8_lossy(&std::fs::read(path).ok()?).into_owned(); - let meta = parse_codex_session_content(&content); + parse_codex_content(&content, path) +} + +fn parse_codex_file_with_evidence( + path: &Path, +) -> (Option, Option) { + let Ok(bytes) = std::fs::read(path) else { + return (None, None); + }; + let content = String::from_utf8_lossy(&bytes).into_owned(); + ( + parse_codex_content(&content, path), + Some(scan_codex_file_bytes_evidence(&bytes)), + ) +} + +fn parse_codex_content(content: &str, path: &Path) -> Option { + let meta = parse_codex_session_content(content); meta.cwd.as_ref()?; let fallback = extract_codex_session_id_from_filename(path); let session_id = meta.session_id.clone().unwrap_or(fallback); @@ -981,6 +1046,11 @@ struct FileEntry { mtime_ms: i64, size: u64, item: Option, + /// Strict Codex identity/interval evidence, stored independently from + /// `item` so a known same-id file excluded from rendering still blocks a + /// partial composition. + #[serde(default)] + codex_evidence: Option, } /// The cached, TTL-refreshed session index composed from one or more @@ -1109,6 +1179,23 @@ struct CachedSnapshot { /// published THIS generation (`getScanFailures` parity — see /// [`SessionIndex::scan_failures`]). scan_failures: HashSet, + /// Known same-id Codex file groups that were not composed. Published + /// atomically with rows so quarantine sees one generation of evidence. + unresolved_codex_identities: Arc>, + /// Chronological source paths for accepted multi-file rows, keyed by + /// canonical embedded id. This remains internal to Rust consumers. + codex_segment_paths: Arc>>, +} + +/// Fields copied together from one published generation. The identity list +/// and Codex path map stay behind `Arc`s here so consumers that only need +/// rows and failures do not clone either sidecar. +#[derive(Default)] +struct SnapshotRead { + items: Arc>, + scan_failures: Vec, + unresolved_codex_identities: Arc>, + codex_segment_paths: Arc>>, } /// Bookkeeping for the persistent parse-cache's opportunistic-save gating @@ -1222,6 +1309,29 @@ impl SessionIndex { .unwrap_or_default() } + /// Read unresolved known-id Codex groups from the currently published + /// generation. This is a short point-in-time read with the same + /// stale-while-revalidate behavior as `snapshot()`; paths are Rust-only + /// evidence and are never part of the client session-directory payload. + pub fn unresolved_codex_identities(&self) -> Vec { + self.snapshot + .lock() + .unwrap() + .as_ref() + .map(|snapshot| snapshot.unresolved_codex_identities.as_ref().clone()) + .unwrap_or_default() + } + + /// Chronological file paths for an accepted multi-file Codex row. A + /// single-file row continues to use `IndexedSession::source_file`. + pub fn codex_segment_paths(&self, session_id: &str) -> Option> { + self.snapshot + .lock() + .unwrap() + .as_ref() + .and_then(|snapshot| snapshot.codex_segment_paths.get(session_id).cloned()) + } + /// Fire-and-forget refresh (`requestRefresh` parity): gives a degraded /// response's Retry a chance to converge once a failed provider recovers. /// No-op if a sweep is already running. @@ -1351,6 +1461,15 @@ impl SessionIndex { self.change_rx.clone() } + /// Test seam for proving that an explicitly requested refresh completed + /// even when it publishes identical contents and leaves the generation + /// unchanged. This waits for the current sweep to release its lock; it + /// does not start a refresh. + #[doc(hidden)] + pub async fn wait_for_refresh_idle_for_test(&self) { + drop(self.refresh_lock.lock().await); + } + /// Whether any dirty paths or providers are pending. pub fn has_dirty(&self) -> bool { has_dirty_parts(&self.dirty_paths, &self.dirty_providers) @@ -1379,8 +1498,36 @@ impl SessionIndex { /// stale failures. Same stale-while-revalidate semantics as /// [`Self::snapshot`]. pub async fn snapshot_with_failures(&self) -> (Arc>, Vec) { - if let Some(pair) = self.cached_pair(true) { - return pair; + let snapshot = self.snapshot_read().await; + (snapshot.items, snapshot.scan_failures) + } + + /// [`Self::snapshot_with_failures`] plus unresolved Codex identity + /// evidence and accepted Codex source paths from the SAME published + /// generation, read under ONE lock acquisition. Session-directory + /// consumers that combine rows with quarantine evidence or source paths + /// must use this accessor rather than pairing separate snapshot lookups. + /// Stale-while-revalidate timing is unchanged: a stale snapshot is + /// returned immediately while refresh runs in the background, and only + /// a cold cache waits for its first refresh. + pub async fn snapshot_with_failures_and_unresolved_codex_identities( + &self, + ) -> SessionIndexSnapshot { + let snapshot = self.snapshot_read().await; + SessionIndexSnapshot { + sessions: snapshot.items, + scan_failures: snapshot.scan_failures, + unresolved_codex_identities: snapshot.unresolved_codex_identities, + codex_segment_paths: snapshot.codex_segment_paths, + } + } + + /// Return one coherent published generation, refreshing according to the + /// same cold-cache and stale-while-revalidate policy for all public + /// snapshot accessors. + async fn snapshot_read(&self) -> SnapshotRead { + if let Some(snapshot) = self.cached_snapshot_read(true) { + return snapshot; } // Stale or absent. Try to become this round's sweeper WITHOUT // blocking -- `try_lock_owned` never waits, so a caller that @@ -1388,18 +1535,19 @@ impl SessionIndex { // in-flight sweep. match Arc::clone(&self.refresh_lock).try_lock_owned() { Ok(guard) => { - if let Some(stale) = self.cached_pair(false) { + if let Some(stale) = self.cached_snapshot_read(false) { // Someone must read fresh data eventually, but not THIS // caller, and not by blocking anyone else either. self.spawn_background_refresh(guard); return stale; } // Truly cold: nothing to serve, so wait for the (only) sweep. - self.run_refresh_inline(guard).await + let _ = self.run_refresh_inline(guard).await; + self.cached_snapshot_read(false).unwrap_or_default() } Err(_) => { // Another caller is already sweeping this round. - if let Some(stale) = self.cached_pair(false) { + if let Some(stale) = self.cached_snapshot_read(false) { return stale; } // Truly cold AND racing another cold-start caller: wait for @@ -1407,9 +1555,9 @@ impl SessionIndex { // B-T5's "N concurrent misses -> 1 sweep" guarantee for the // cold-cache case). let guard = self.refresh_lock.lock().await; - let pair = self - .cached_pair(true) - .or_else(|| self.cached_pair(false)) + let snapshot = self + .cached_snapshot_read(true) + .or_else(|| self.cached_snapshot_read(false)) .unwrap_or_default(); drop(guard); // D5-1: this caller held `refresh_lock` (however briefly), @@ -1441,7 +1589,7 @@ impl SessionIndex { .await; } } - pair + snapshot } } } @@ -1480,6 +1628,15 @@ impl SessionIndex { /// await point). `require_fresh` applies the TTL window; `false` is the /// stale-while-revalidate read. fn cached_pair(&self, require_fresh: bool) -> Option<(Arc>, Vec)> { + self.cached_snapshot_read(require_fresh) + .map(|snapshot| (snapshot.items, snapshot.scan_failures)) + } + + /// The cached rows, scan failures, unresolved Codex identities, and + /// accepted Codex source paths from the SAME generation, read under ONE + /// lock acquisition. Sidecar `Arc`s are cloned while the snapshot lock + /// is held; the published values are immutable thereafter. + fn cached_snapshot_read(&self, require_fresh: bool) -> Option { let guard = self.snapshot.lock().unwrap(); match guard.as_ref() { Some(c) if !require_fresh || c.fetched_at.elapsed() < self.ttl => { @@ -1496,7 +1653,12 @@ impl SessionIndex { return None; } } - Some((Arc::clone(&c.items), sorted_names(&c.scan_failures))) + Some(SnapshotRead { + items: Arc::clone(&c.items), + scan_failures: sorted_names(&c.scan_failures), + unresolved_codex_identities: Arc::clone(&c.unresolved_codex_identities), + codex_segment_paths: Arc::clone(&c.codex_segment_paths), + }) } _ => None, } @@ -1789,7 +1951,7 @@ impl SessionIndex { .as_ref() .map(|c| c.scan_failures.clone()) .unwrap_or_default(); - let (items, changed, amplifier_root_dirs) = refresh_snapshot( + let refreshed = refresh_snapshot( &sources, &mut cache, &mut direct, @@ -1797,11 +1959,25 @@ impl SessionIndex { scoped_paths, scoped_providers, ); - (items, changed, failures, amplifier_root_dirs) + ( + refreshed.items, + refreshed.changed, + failures, + refreshed.amplifier_root_dirs, + refreshed.unresolved_codex_identities, + refreshed.codex_segment_paths, + ) } }) .await; - let (items, changed, failures, amplifier_root_dirs) = match sweep_result { + let ( + items, + changed, + failures, + amplifier_root_dirs, + unresolved_codex_identities, + codex_segment_paths, + ) = match sweep_result { Ok(result) => result, Err(join_err) => { // `discover`/`parse` are documented never-panic (every @@ -1828,6 +2004,8 @@ impl SessionIndex { } }; let items = Arc::new(items); + let unresolved_codex_identities = Arc::new(unresolved_codex_identities); + let codex_segment_paths = Arc::new(codex_segment_paths); let failure_names = sorted_names(&failures); { // ONE lock write publishes the snapshot AND its scan failures as @@ -1839,6 +2017,8 @@ impl SessionIndex { items: Arc::clone(&items), fetched_at: Instant::now(), scan_failures: failures, + unresolved_codex_identities, + codex_segment_paths, }); } // guard dropped here — never held across an .await. // Self-correction report (amplifier watch-reduction design @@ -2010,21 +2190,31 @@ fn parse_scoped_path( path: &Path, sources: &[Arc], provider_hint: Option<&str>, -) -> (Option, Option) { +) -> ( + Option, + Option, + Option, +) { if let Some(name) = provider_hint { if let Some(source) = sources.iter().find(|s| s.provider_name() == Some(name)) { - return (source.parse(path), Some(name.to_owned())); + let (item, evidence) = source.parse_with_codex_evidence(path); + return (item, Some(name.to_owned()), evidence); } } for source in sources { if source.direct_change_token().is_some() { continue; } - if let Some(item) = source.parse(path) { - return (Some(item), source.provider_name().map(str::to_owned)); + let (item, evidence) = source.parse_with_codex_evidence(path); + if item.is_some() + || evidence + .as_ref() + .is_some_and(|evidence| evidence.session_id.is_some()) + { + return (item, source.provider_name().map(str::to_owned), evidence); } } - (None, None) + (None, None, None) } /// One incremental refresh sweep across all sources: @@ -2054,6 +2244,14 @@ fn parse_scoped_path( /// version of this module had (see the module doc comment). Analogously, a /// sweep over an unchanged direct-listed source costs 2 stats (db + db-wal), /// not a query. +struct RefreshedSnapshot { + items: Vec, + changed: usize, + amplifier_root_dirs: Option>, + unresolved_codex_identities: Vec, + codex_segment_paths: HashMap>, +} + fn refresh_snapshot( sources: &[Arc], cache: &mut HashMap, @@ -2061,7 +2259,7 @@ fn refresh_snapshot( scan_failures: &mut HashSet, scoped_paths: HashMap, scoped_providers: HashSet, -) -> (Vec, usize, Option>) { +) -> RefreshedSnapshot { let is_full = scoped_paths.is_empty() && scoped_providers.is_empty(); let mut discovered: std::collections::HashSet = std::collections::HashSet::new(); let mut fully_discovered_providers = HashSet::::new(); @@ -2216,7 +2414,7 @@ fn refresh_snapshot( } } } else { - let item = source.parse(&stat.path); + let (item, codex_evidence) = source.parse_with_codex_evidence(&stat.path); // A content-IDENTICAL rewrite (editor autosave, a repeated // provider write: same bytes, only mtime/size moved) // re-parses to exactly the cached item. Count ONLY a re-parse @@ -2226,9 +2424,11 @@ fn refresh_snapshot( // broadcast fans a spurious `sessions.changed` out to every // client. The stat bookkeeping (mtime_ms/size) is refreshed // either way so the NEXT sweep treats the file as unchanged. - let content_moved = cache - .get(&stat.path) - .is_none_or(|entry| entry.item != item || entry.source_name != source_name); + let content_moved = cache.get(&stat.path).is_none_or(|entry| { + entry.item != item + || entry.codex_evidence != codex_evidence + || entry.source_name != source_name + }); cache.insert( stat.path.clone(), FileEntry { @@ -2236,6 +2436,7 @@ fn refresh_snapshot( mtime_ms: stat.mtime_ms, size: stat.size, item, + codex_evidence, }, ); if content_moved { @@ -2271,7 +2472,7 @@ fn refresh_snapshot( .get(path) .is_some_and(|e| e.mtime_ms == stat.mtime_ms && e.size == stat.size); if !unchanged { - let (item, resolved_source) = + let (item, resolved_source, codex_evidence) = parse_scoped_path(path, sources, Some(watcher_provider)); // Same content-identical-rewrite rule as the discover // arm above: a watcher-scoped re-parse whose parsed @@ -2279,7 +2480,9 @@ fn refresh_snapshot( // change — else it bumps the generation and wakes a // spurious `sessions.changed` broadcast. let content_moved = cache.get(path).is_none_or(|entry| { - entry.item != item || entry.source_name != resolved_source + entry.item != item + || entry.codex_evidence != codex_evidence + || entry.source_name != resolved_source }); cache.insert( path.clone(), @@ -2288,6 +2491,7 @@ fn refresh_snapshot( mtime_ms: stat.mtime_ms, size: stat.size, item, + codex_evidence, }, ); if content_moved { @@ -2357,10 +2561,22 @@ fn refresh_snapshot( } changed = changed.saturating_add(cache_len_before_prune - cache.len()); + let codex_entries = cache + .iter() + .filter(|(_, entry)| entry.source_name.as_deref() == Some("codex")) + .map(|(path, entry)| CodexSegmentEntry { + path: path.clone(), + item: entry.item.clone(), + evidence: entry.codex_evidence.clone(), + }) + .collect(); + let codex_composition: CodexComposition = compose_codex_segments(codex_entries); let mut items: Vec = cache .values() + .filter(|entry| entry.source_name.as_deref() != Some("codex")) .filter_map(|entry| entry.item.clone()) .collect(); + items.extend(codex_composition.items.iter().cloned()); for entry in direct_cache.values() { items.extend(entry.items.iter().cloned()); } @@ -2369,7 +2585,13 @@ fn refresh_snapshot( .cmp(&a.last_activity_at) .then_with(|| b.key().cmp(&a.key())) }); - (items, changed, amplifier_root_dirs) + RefreshedSnapshot { + items, + changed, + amplifier_root_dirs, + unresolved_codex_identities: codex_composition.unresolved_identities, + codex_segment_paths: codex_composition.segment_paths, + } } // -- Persistent parse cache (self-hosting-readiness bake-in, "kill the cold @@ -2406,10 +2628,9 @@ fn refresh_snapshot( /// Schema version for the persisted parse-cache file. Bump on any format /// change so an old (or a future, if this ever needs to roll back) file is /// cleanly discarded -- never partially or incorrectly deserialized into a -/// mismatched shape. (v2: `IndexedSession.token_usage` added — a v1 cache -/// would load every session with `token_usage: None` forever, hiding usage -/// data that already exists on disk from the fresh-agent strip meter.) -const CACHE_SCHEMA_VERSION: u32 = 2; +/// mismatched shape. (v3: per-file Codex continuation evidence added; v2 +/// entries do not have enough information to certify multi-file groups.) +const CACHE_SCHEMA_VERSION: u32 = 3; /// See "File location"/"Filename" above. const CACHE_FILENAME: &str = "rust-session-cache.json"; @@ -3588,6 +3809,8 @@ pub(crate) mod tests { items: Arc::clone(&good_snapshot), fetched_at: Instant::now(), scan_failures: HashSet::new(), + unresolved_codex_identities: Arc::new(Vec::new()), + codex_segment_paths: Arc::new(HashMap::new()), }))); let file_cache = Arc::new(StdMutex::new(HashMap::new())); let direct_cache = Arc::new(StdMutex::new(HashMap::new())); @@ -3901,6 +4124,154 @@ pub(crate) mod tests { std::fs::read_to_string(path).unwrap() } + fn codex_continuation_fixtures() -> (String, String) { + let fixture_dir = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("../../test/fixtures/coding-cli/codex"); + ( + std::fs::read_to_string( + fixture_dir.join("multi-file-continuation-older.sanitized.jsonl"), + ) + .unwrap(), + std::fs::read_to_string( + fixture_dir.join("multi-file-continuation-newer.sanitized.jsonl"), + ) + .unwrap(), + ) + } + + fn copied_large_codex_transcript(content: &str) -> String { + let lines: Vec<&str> = content.lines().collect(); + let mut copied = String::from(lines[0]); + copied.push('\n'); + for index in 0..2000usize { + let template = + serde_json::from_str::(lines[1 + index % (lines.len() - 1)]) + .unwrap(); + let timestamp_seconds = index + 1; + let timestamp = format!( + "2026-10-03T{:02}:{:02}:{:02}.000Z", + timestamp_seconds / 3600, + (timestamp_seconds / 60) % 60, + timestamp_seconds % 60, + ); + let mut record = template; + record["timestamp"] = serde_json::Value::String(timestamp); + record["ordinal"] = serde_json::Value::from(index as u64); + copied.push_str(&serde_json::to_string(&record).unwrap()); + copied.push('\n'); + } + copied + } + + fn codex_continuation_fixture_at( + second: u64, + user_message: &str, + assistant_message: &str, + total_tokens: u64, + ) -> String { + let timestamp = |millis: u64| { + format!( + "2026-10-03T00:{:02}:{:02}.{:03}Z", + (second + millis / 1000) / 60, + (second + millis / 1000) % 60, + millis % 1000, + ) + }; + let id = "b7936c10-4935-441c-837c-c1f33cafec2d"; + let records = [ + serde_json::json!({ + "timestamp": timestamp(0), + "type": "session_meta", + "payload": { + "id": id, + "session_id": id, + "cwd": "/sanitized/project", + "source": "vscode", + "thread_source": "user", + "cli_version": "0.156.0", + "originator": "codex-vscode", + "history_mode": "paginated" + } + }), + serde_json::json!({ + "timestamp": timestamp(1), + "ordinal": 0, + "type": "event_msg", + "payload": {"type": "user_message", "message": user_message} + }), + serde_json::json!({ + "timestamp": timestamp(2), + "ordinal": 1, + "type": "response_item", + "payload": { + "type": "message", + "role": "assistant", + "content": [{"type": "output_text", "text": assistant_message}] + } + }), + serde_json::json!({ + "timestamp": timestamp(3), + "ordinal": 2, + "type": "event_msg", + "payload": { + "type": "token_count", + "info": { + "total_token_usage": { + "input_tokens": total_tokens, + "cached_input_tokens": 20, + "output_tokens": 10, + "total_tokens": total_tokens + 10 + }, + "last_token_usage": { + "input_tokens": total_tokens - 20, + "cached_input_tokens": 10, + "output_tokens": 8, + "total_tokens": total_tokens - 2 + }, + "model_context_window": 258400 + } + } + }), + serde_json::json!({ + "timestamp": timestamp(4), + "ordinal": 3, + "type": "world_state", + "payload": {"state": "sanitized"} + }), + ]; + records + .iter() + .map(|record| serde_json::to_string(record).unwrap()) + .collect::>() + .join("\n") + + "\n" + } + + fn write_codex_pair(home: &Path, older: &str, newer: &str) -> (PathBuf, PathBuf) { + let sessions = home.join(".codex").join("sessions"); + std::fs::create_dir_all(&sessions).unwrap(); + let older_path = sessions.join("z-older-rollout.jsonl"); + let newer_path = sessions.join("a-newer-rollout.jsonl"); + std::fs::write(&older_path, older).unwrap(); + std::fs::write(&newer_path, newer).unwrap(); + (older_path, newer_path) + } + + fn assert_uncomposed_same_id_rows(source: &CodexSource, label: &str) { + let rows = source.scan(); + assert_eq!( + rows.len(), + 2, + "{label}: keep both renderable rows for quarantine" + ); + assert!( + rows.iter().all(|row| { + row.session_id == "b7936c10-4935-441c-837c-c1f33cafec2d" && row.provider == "codex" + }), + "{label}: both rows must keep the shared embedded identity" + ); + } + /// A `/.codex/sessions/…` layout. `nested` controls whether the /// fixture is placed directly in `sessions/` or several levels deep /// (codex's real `sessions/YYYY/MM/DD/*.jsonl` layout) — proving @@ -3938,6 +4309,683 @@ pub(crate) mod tests { std::fs::remove_dir_all(codex_home.parent().unwrap()).ok(); } + #[tokio::test] + async fn codex_source_composes_the_reported_same_id_rollout_shape() { + let home = unique_temp_dir("codex-continuation-compose"); + let (older, newer) = codex_continuation_fixtures(); + let (older_path, newer_path) = write_codex_pair(&home, &older, &newer); + + let source = Arc::new(CodexSource::new(home.join(".codex"))); + let direct = source.scan(); + assert_eq!(direct.len(), 1, "a verified continuation is one session"); + assert_eq!(direct[0].session_id, "b7936c10-4935-441c-837c-c1f33cafec2d"); + assert_eq!(direct[0].source_file.as_deref(), Some(newer_path.as_path())); + + let index = + SessionIndex::with_ttl_and_cache_path(vec![source], Duration::from_secs(60), None); + let rows = index.snapshot().await; + assert_eq!( + rows.len(), + 1, + "the cached index uses the same composition policy" + ); + let row = &rows[0]; + assert_eq!(row.session_id, "b7936c10-4935-441c-837c-c1f33cafec2d"); + assert_eq!(row.source_file.as_deref(), Some(newer_path.as_path())); + assert_eq!( + index.codex_segment_paths(&row.session_id).as_deref(), + Some([older_path.clone(), newer_path.clone()].as_slice()), + "the index retains every accepted segment in persisted-record order" + ); + assert_eq!( + row.first_user_message.as_deref(), + Some("Older first request") + ); + assert_eq!(row.title.as_deref(), Some("Older first request")); + assert_eq!(row.summary.as_deref(), Some("Older assistant summary")); + assert_eq!( + row.token_usage.as_ref().map(|usage| usage.total_tokens), + Some(780) + ); + assert_eq!( + row.created_at, + crate::time::parse_timestamp_ms(&serde_json::json!("2026-10-03T00:00:00.000Z")) + ); + assert_eq!( + row.last_activity_at, + crate::time::parse_timestamp_ms(&serde_json::json!("2026-10-03T00:00:10.004Z")) + .unwrap() + ); + + std::fs::remove_dir_all(&home).ok(); + } + + async fn assert_equal_unknown_codex_classification_stays_unresolved( + field: &str, + known_value: &str, + ) { + let home = unique_temp_dir(&format!("codex-unknown-{field}")); + let (older, newer) = codex_continuation_fixtures(); + let known = format!(r#""{field}":"{known_value}""#); + let unknown = format!(r#""{field}":"unsupported-{field}""#); + let (older_path, newer_path) = write_codex_pair( + &home, + &older.replace(&known, &unknown), + &newer.replace(&known, &unknown), + ); + let source = Arc::new(CodexSource::new(home.join(".codex"))); + + let single_file = source.parse(&newer_path).unwrap(); + assert_eq!(single_file.title.as_deref(), Some("Continuation title")); + assert_eq!(single_file.summary.as_deref(), Some("Continuation summary")); + assert_uncomposed_same_id_rows(&source, field); + + let index = + SessionIndex::with_ttl_and_cache_path(vec![source], Duration::from_secs(60), None); + let snapshot = index + .snapshot_with_failures_and_unresolved_codex_identities() + .await; + assert_eq!( + snapshot.sessions.len(), + 2, + "{field}: do not compose unknown evidence" + ); + assert!(snapshot.scan_failures.is_empty()); + let session_id = "b7936c10-4935-441c-837c-c1f33cafec2d"; + assert_eq!( + snapshot.unresolved_codex_identities.as_ref(), + &vec![CodexUnresolvedIdentity { + session_id: session_id.to_string(), + paths: vec![newer_path, older_path], + }], + "{field}: preserve the whole unresolved identity for quarantine" + ); + assert!(snapshot.codex_segment_paths.get(session_id).is_none()); + std::fs::remove_dir_all(&home).ok(); + } + + #[tokio::test] + async fn codex_source_keeps_equal_unknown_history_modes_unresolved() { + assert_equal_unknown_codex_classification_stays_unresolved("history_mode", "paginated") + .await; + } + + #[tokio::test] + async fn codex_source_keeps_equal_unknown_thread_sources_unresolved() { + assert_equal_unknown_codex_classification_stays_unresolved("thread_source", "user").await; + } + + #[test] + fn codex_source_keeps_ambiguous_same_id_files_separate() { + let (older, valid_newer) = codex_continuation_fixtures(); + let large_copy = copied_large_codex_transcript(&older); + assert_eq!(large_copy.lines().count(), 2001); + + let mut cases = + vec![ + ( + "byte-identical copied files", + large_copy.clone(), + large_copy.clone(), + ), + ( + "overlapping or interleaved ranges", + older + .clone() + .replace("00:00:05.000000Z", "00:00:10.005000Z"), + valid_newer.clone(), + ), + ( + "equal cross-file boundary", + older.clone(), + valid_newer.replace("00:00:10.001000Z", "00:00:05.000000Z"), + ), + ( + "malformed JSONL", + older.clone(), + format!("{valid_newer}{{broken json}}\n"), + ), + ( + "unterminated JSONL", + older.clone(), + valid_newer.trim_end().to_string(), + ), + ( + "unknown top-level variant", + older.clone(), + valid_newer.replace( + "\"type\":\"retained_context\"", + "\"type\":\"future_variant\"", + ), + ), + ( + "missing outer timestamp", + older.clone(), + valid_newer.replace( + "{\"timestamp\":\"2026-10-03T00:00:10.009000Z\",\"ordinal\":8,", + "{\"ordinal\":8,", + ), + ), + ( + "invalid outer timestamp", + older.clone(), + valid_newer.replace("00:00:10.009000Z", "not-a-time"), + ), + ( + "timestamp regression", + older.clone(), + valid_newer.replace("00:00:10.009000Z", "00:00:10.003000Z"), + ), + ( + "invalid ordinal", + older.clone(), + valid_newer.replace( + "\"timestamp\":\"2026-10-03T00:00:10.009000Z\",\"ordinal\":8", + "\"timestamp\":\"2026-10-03T00:00:10.009000Z\",\"ordinal\":\"invalid\"", + ), + ), + ( + "regressing ordinal", + older.clone(), + valid_newer.replace( + "\"timestamp\":\"2026-10-03T00:00:10.009000Z\",\"ordinal\":8", + "\"timestamp\":\"2026-10-03T00:00:10.009000Z\",\"ordinal\":6", + ), + ), + ( + "conflicting embedded ownership", + older.clone(), + valid_newer.replace( + "\"session_id\":\"b7936c10-4935-441c-837c-c1f33cafec2d\"", + "\"session_id\":\"00000000-0000-4000-8000-000000000000\"", + ), + ), + ( + "conflicting cwd", + older.clone(), + valid_newer.replace("/sanitized/project", "/different/project"), + ), + ( + "conflicting source", + older.clone(), + valid_newer.replace("\"source\":\"vscode\"", "\"source\":\"cli\""), + ), + ( + "unsupported source shape", + older.replace("\"source\":\"vscode\"", "\"source\":{}"), + valid_newer.replace("\"source\":\"vscode\"", "\"source\":{}"), + ), + ( + "unknown source value", + older.replace( + "\"source\":\"vscode\"", + "\"source\":\"future_source\"", + ), + valid_newer.replace("\"source\":\"vscode\"", "\"source\":\"future_source\""), + ), + ( + "conflicting thread source", + older.clone(), + valid_newer.replace( + "\"thread_source\":\"user\"", + "\"thread_source\":\"subagent\"", + ), + ), + ( + "subagent thread classification", + older.replace("\"thread_source\":\"user\"", "\"thread_source\":\"subagent\""), + valid_newer + .replace("\"thread_source\":\"user\"", "\"thread_source\":\"subagent\""), + ), + ( + "conflicting CLI version", + older.clone(), + valid_newer.replace("\"cli_version\":\"0.156.0\"", "\"cli_version\":\"0.155.0\""), + ), + ( + "conflicting originator", + older.clone(), + valid_newer.replace( + "\"originator\":\"codex-vscode\"", + "\"originator\":\"codex-cli\"", + ), + ), + ( + "conflicting history mode", + older.clone(), + valid_newer.replace( + "\"history_mode\":\"paginated\"", + "\"history_mode\":\"legacy\"", + ), + ), + ( + "missing required metadata", + older.clone(), + valid_newer.replace("\"originator\":\"codex-vscode\",", ""), + ), + ( + "fork id evidence", + older.clone(), + valid_newer.replace( + "\"history_mode\":\"paginated\"", + "\"history_mode\":\"paginated\",\"forked_from_id\":\"parent\"", + ), + ), + ( + "fork ordinal evidence", + older.clone(), + valid_newer.replace( + "\"history_mode\":\"paginated\"", + "\"history_mode\":\"paginated\",\"forked_from_ordinal_exclusive\":5", + ), + ), + ( + "parent evidence", + older.clone(), + valid_newer.replace( + "\"history_mode\":\"paginated\"", + "\"history_mode\":\"paginated\",\"parent_thread_id\":\"parent\"", + ), + ), + ( + "subagent evidence", + older.clone(), + valid_newer.replace( + "\"history_mode\":\"paginated\"", + "\"history_mode\":\"paginated\",\"subagent_history_start_ordinal\":0", + ), + ), + ( + "referenced history prefix evidence", + older.clone(), + valid_newer.replace( + "\"history_mode\":\"paginated\"", + "\"history_mode\":\"paginated\",\"history_base\":{\"rollout_id\":\"prefix\"}", + ), + ), + ]; + let mut non_first_line = String::from( + "{\"timestamp\":\"2026-10-03T00:00:09.999Z\",\"type\":\"event_msg\",\"payload\":{\"type\":\"task_started\"}}\n", + ); + non_first_line.push_str(&valid_newer); + cases.push(("header after first line", older.clone(), non_first_line)); + + let header = valid_newer.lines().next().unwrap(); + let later_header = header + .replace("00:00:10.000Z", "00:00:10.010Z") + .replace( + "\"type\":\"session_meta\"", + "\"ordinal\":9,\"type\":\"session_meta\"", + ) + .replace("/sanitized/project", "/contradictory/project"); + let mut contradictory_header = valid_newer.clone(); + contradictory_header.push_str(&later_header); + contradictory_header.push('\n'); + cases.push(( + "later contradictory metadata", + older.clone(), + contradictory_header, + )); + + let home = unique_temp_dir("codex-continuation-ambiguous"); + let source = CodexSource::new(home.join(".codex")); + for (label, older_content, newer_content) in cases { + write_codex_pair(&home, &older_content, &newer_content); + assert_uncomposed_same_id_rows(&source, label); + } + std::fs::remove_dir_all(&home).ok(); + } + + #[tokio::test] + async fn codex_source_keeps_cwdless_same_id_members_in_unresolved_identity_sidecar() { + let home = unique_temp_dir("codex-continuation-hidden-member"); + let (older, newer) = codex_continuation_fixtures(); + let cwdless = newer.replace("\"cwd\":\"/sanitized/project\",", ""); + let (older_path, newer_path) = write_codex_pair(&home, &older, &cwdless); + let source = Arc::new(CodexSource::new(home.join(".codex"))); + let index = + SessionIndex::with_ttl_and_cache_path(vec![source], Duration::from_secs(60), None); + + let rows = index.snapshot().await; + assert_eq!(rows.len(), 1, "the cwd-less segment remains non-renderable"); + assert_eq!(rows[0].source_file.as_deref(), Some(older_path.as_path())); + assert_eq!( + index.unresolved_codex_identities(), + vec![CodexUnresolvedIdentity { + session_id: "b7936c10-4935-441c-837c-c1f33cafec2d".to_string(), + paths: vec![newer_path, older_path], + }], + "the hidden member prevents partial composition and remains available to quarantine" + ); + std::fs::remove_dir_all(&home).ok(); + } + + #[tokio::test] + async fn snapshot_with_codex_identity_evidence_returns_one_coherent_generation() { + let home = unique_temp_dir("codex-continuation-snapshot-generation"); + let (older, newer) = codex_continuation_fixtures(); + let (older_path, newer_path) = write_codex_pair( + &home, + &older, + &newer.replace("\"cwd\":\"/sanitized/project\",", ""), + ); + let source = Arc::new(CodexSource::new(home.join(".codex"))); + let index = + SessionIndex::with_ttl_and_cache_path(vec![source], Duration::from_secs(60), None); + + let snapshot = index + .snapshot_with_failures_and_unresolved_codex_identities() + .await; + let rows = &snapshot.sessions; + assert_eq!(rows.len(), 1, "the cwd-less segment is evidence-only"); + assert!(snapshot.scan_failures.is_empty()); + assert_eq!(rows[0].source_file.as_deref(), Some(older_path.as_path())); + assert_eq!( + snapshot.unresolved_codex_identities.as_ref().clone(), + vec![CodexUnresolvedIdentity { + session_id: "b7936c10-4935-441c-837c-c1f33cafec2d".to_string(), + paths: vec![newer_path.clone(), older_path.clone()], + }], + "rows and identity evidence must come from the same published generation" + ); + + std::fs::remove_file(&newer_path).unwrap(); + index.mark_provider_dirty("codex"); + let deadline = std::time::Instant::now() + Duration::from_secs(5); + loop { + let snapshot = index + .snapshot_with_failures_and_unresolved_codex_identities() + .await; + if snapshot.unresolved_codex_identities.is_empty() { + assert!(snapshot.scan_failures.is_empty()); + assert_eq!(snapshot.sessions.len(), 1); + assert_eq!( + snapshot.sessions[0].source_file.as_deref(), + Some(older_path.as_path()) + ); + break; + } + assert_eq!( + snapshot.sessions.len(), + 1, + "stale reads must stay on the old generation" + ); + assert!(snapshot.scan_failures.is_empty()); + assert_eq!( + snapshot.unresolved_codex_identities[0].paths, + vec![newer_path.clone(), older_path.clone()] + ); + assert!( + std::time::Instant::now() < deadline, + "the recovery generation must be published after the dirty refresh" + ); + tokio::time::sleep(Duration::from_millis(2)).await; + } + + std::fs::remove_dir_all(&home).ok(); + } + + #[tokio::test] + async fn snapshot_with_codex_paths_keeps_rows_and_paths_in_one_generation() { + let home = unique_temp_dir("codex-continuation-path-generation"); + let (older, newer) = codex_continuation_fixtures(); + let (older_path, newer_path) = write_codex_pair(&home, &older, &newer); + let moved_path = newer_path.with_file_name("moved-newer-rollout.jsonl"); + let source = Arc::new(CodexSource::new(home.join(".codex"))); + let index = + SessionIndex::with_ttl_and_cache_path(vec![source], Duration::from_secs(60), None); + let session_id = "b7936c10-4935-441c-837c-c1f33cafec2d"; + + let snapshot = index + .snapshot_with_failures_and_unresolved_codex_identities() + .await; + assert!(snapshot.scan_failures.is_empty()); + assert!(snapshot.unresolved_codex_identities.is_empty()); + assert_eq!(snapshot.sessions.len(), 1); + assert_eq!( + snapshot.sessions[0].source_file.as_deref(), + Some(newer_path.as_path()) + ); + assert_eq!( + snapshot.codex_segment_paths.get(session_id), + Some(&vec![older_path.clone(), newer_path.clone()]), + "the accepted source paths must belong to the same published generation as the composed row" + ); + + std::fs::rename(&newer_path, &moved_path).unwrap(); + index.mark_provider_dirty("codex"); + let deadline = std::time::Instant::now() + Duration::from_secs(5); + loop { + let snapshot = index + .snapshot_with_failures_and_unresolved_codex_identities() + .await; + assert!(snapshot.scan_failures.is_empty()); + assert!(snapshot.unresolved_codex_identities.is_empty()); + assert_eq!(snapshot.sessions.len(), 1); + let paths = snapshot + .codex_segment_paths + .get(session_id) + .expect("the accepted continuation keeps its source path list"); + assert_eq!(paths[0], older_path); + if paths[1] == newer_path { + assert_eq!( + snapshot.sessions[0].source_file.as_deref(), + Some(newer_path.as_path()) + ); + } else if paths[1] == moved_path { + assert_eq!( + snapshot.sessions[0].source_file.as_deref(), + Some(moved_path.as_path()) + ); + break; + } else { + panic!("unexpected source paths in published generation: {paths:?}"); + } + + assert!( + std::time::Instant::now() < deadline, + "the renamed path must eventually be published with its matching composed row" + ); + tokio::time::sleep(Duration::from_millis(2)).await; + } + + std::fs::remove_dir_all(&home).ok(); + } + + #[tokio::test] + async fn codex_continuation_composition_survives_refresh_and_cache_reload() { + let home = unique_temp_dir("codex-continuation-refresh"); + let (older, newer) = codex_continuation_fixtures(); + let (older_path, newer_path) = write_codex_pair(&home, &older, &newer); + let newest_path = home + .join(".codex") + .join("sessions") + .join("m-latest-rollout.jsonl"); + let cache_dir = unique_temp_dir("codex-continuation-refresh-cache"); + let cache_path = cache_path_in(&cache_dir); + let source = Arc::new(CodexSource::new(home.join(".codex"))); + let index = SessionIndex::with_ttl_and_cache_path( + vec![source.clone()], + Duration::from_secs(3600), + Some(cache_path.clone()), + ); + let first = index.snapshot().await; + assert_eq!(first.len(), 1); + assert_eq!( + index.codex_segment_paths(&first[0].session_id), + Some(vec![older_path.clone(), newer_path.clone()]) + ); + + let first_generation = *index.subscribe_changes().borrow(); + let newest_before_append = source.parse(&newer_path).unwrap(); + use std::io::Write; + let mut newest_file = std::fs::OpenOptions::new() + .append(true) + .open(&newer_path) + .unwrap(); + writeln!( + newest_file, + "{{\"timestamp\":\"2026-10-03T00:00:10.010000Z\",\"ordinal\":9,\"type\":\"world_state\",\"payload\":{{\"state\":\"appended\"}}}}" + ) + .unwrap(); + drop(newest_file); + assert_eq!( + source.parse(&newer_path).unwrap(), + newest_before_append, + "the append changes evidence but not the display row" + ); + index.mark_dirty(&[(newer_path.clone(), "codex".to_string())]); + assert!( + wait_until(Duration::from_secs(3), || { + index + .file_cache + .lock() + .unwrap() + .get(&newer_path) + .is_some_and(|entry| { + entry + .codex_evidence + .as_ref() + .and_then(|evidence| evidence.interval) + .is_some_and(|interval| { + interval.end_nanos + == chrono::DateTime::parse_from_rfc3339( + "2026-10-03T00:00:10.010000Z", + ) + .unwrap() + .timestamp_nanos_opt() + .unwrap() + }) + }) + }) + .await, + "the append refreshes persisted-record evidence" + ); + assert!( + *index.subscribe_changes().borrow() > first_generation, + "an evidence-only change advances the published generation" + ); + assert_eq!(index.persist_state.lock().unwrap().changed_since_save, 1); + let after_append = index.snapshot().await; + assert_eq!(after_append.len(), 1); + assert_eq!( + index.codex_segment_paths(&after_append[0].session_id), + Some(vec![older_path.clone(), newer_path.clone()]) + ); + assert_eq!( + after_append[0].source_file.as_deref(), + Some(newer_path.as_path()) + ); + + std::fs::write( + &newest_path, + codex_continuation_fixture_at( + 20, + "Latest continuation request", + "Latest continuation summary", + 1800, + ), + ) + .unwrap(); + index.mark_provider_dirty("codex"); + assert!( + wait_until(Duration::from_secs(3), || { + index + .codex_segment_paths("b7936c10-4935-441c-837c-c1f33cafec2d") + .is_some_and(|paths| { + paths == vec![older_path.clone(), newer_path.clone(), newest_path.clone()] + }) + }) + .await, + "the new segment is composed with cached siblings in transcript order" + ); + let after_third = index.snapshot().await; + assert_eq!(after_third.len(), 1); + assert_eq!( + after_third[0].source_file.as_deref(), + Some(newest_path.as_path()) + ); + assert_eq!(after_third[0].title.as_deref(), Some("Older first request")); + assert_eq!( + after_third[0].summary.as_deref(), + Some("Older assistant summary") + ); + assert_eq!( + after_third[0] + .token_usage + .as_ref() + .map(|usage| usage.total_tokens), + Some(1798) + ); + + save_cache_file(&cache_path, &index.file_cache.lock().unwrap()).unwrap(); + let reloaded = SessionIndex::with_ttl_and_cache_path( + vec![Arc::new(CodexSource::new(home.join(".codex")))], + Duration::from_secs(3600), + Some(cache_path.clone()), + ); + assert_eq!( + reloaded.file_cache.lock().unwrap().len(), + 3, + "all path-keyed entries reload from the serialized cache" + ); + let reloaded_rows = reloaded.snapshot().await; + assert_eq!(reloaded_rows.len(), 1); + assert_eq!( + reloaded.codex_segment_paths(&reloaded_rows[0].session_id), + Some(vec![ + older_path.clone(), + newer_path.clone(), + newest_path.clone() + ]) + ); + + let old_schema_path = cache_path_in(&cache_dir.join("old-schema")); + save_cache_file(&old_schema_path, &index.file_cache.lock().unwrap()).unwrap(); + let mut old_schema: serde_json::Value = + serde_json::from_slice(&std::fs::read(&old_schema_path).unwrap()).unwrap(); + old_schema["schema_version"] = serde_json::Value::from(2); + std::fs::write(&old_schema_path, serde_json::to_vec(&old_schema).unwrap()).unwrap(); + let old_schema_index = SessionIndex::with_ttl_and_cache_path( + vec![Arc::new(CodexSource::new(home.join(".codex")))], + Duration::from_secs(3600), + Some(old_schema_path), + ); + assert!(old_schema_index.file_cache.lock().unwrap().is_empty()); + assert_eq!(old_schema_index.snapshot().await.len(), 1); + assert!(old_schema_index + .file_cache + .lock() + .unwrap() + .values() + .all(|entry| entry.codex_evidence.is_some())); + + std::fs::remove_file(&older_path).unwrap(); + reloaded.mark_provider_dirty("codex"); + assert!( + wait_until(Duration::from_secs(3), || { + reloaded + .codex_segment_paths("b7936c10-4935-441c-837c-c1f33cafec2d") + .is_some_and(|paths| paths == vec![newer_path.clone(), newest_path.clone()]) + }) + .await, + "the delete refresh removes the stale oldest path" + ); + let after_delete = reloaded.snapshot().await; + assert_eq!(after_delete.len(), 1); + assert_eq!( + after_delete[0].source_file.as_deref(), + Some(newest_path.as_path()) + ); + assert!(!reloaded + .codex_segment_paths(&after_delete[0].session_id) + .unwrap() + .contains(&older_path)); + + std::fs::remove_dir_all(&home).ok(); + std::fs::remove_dir_all(&cache_dir).ok(); + } + #[test] fn codex_source_discovers_nested_yyyy_mm_dd_sessions() { let codex_home = codex_home_with_fixture("codexsrc-nested", true); @@ -5138,6 +6186,7 @@ pub(crate) mod tests { mtime_ms: matching_stat.mtime_ms, size: matching_stat.size, item: source.parse(&matching_path), + codex_evidence: None, }, ), ( @@ -5147,6 +6196,7 @@ pub(crate) mod tests { mtime_ms: child_stat.mtime_ms, size: child_stat.size, item: Some(legacy_child), + codex_evidence: None, }, ), ]); @@ -5363,6 +6413,7 @@ pub(crate) mod tests { mtime_ms: canonical_stat.mtime_ms, size: canonical_stat.size, item: source.parse(&canonical_path), + codex_evidence: None, }, ), ( @@ -5372,6 +6423,7 @@ pub(crate) mod tests { mtime_ms: nested_stat.mtime_ms, size: nested_stat.size, item: source.parse(&nested_path), + codex_evidence: None, }, ), ]); @@ -5462,6 +6514,7 @@ pub(crate) mod tests { mtime_ms: 1, size: 2, item: None, + codex_evidence: None, }, ); let result = save_cache_file(&path, &cache); @@ -6856,7 +7909,10 @@ pub(crate) mod tests { ) .unwrap(); index.mark_dirty(&[(metadata.clone(), "amplifier".to_string())]); - assert!(wait_until(Duration::from_secs(2), || !index.has_dirty()).await); + // A cleared dirty map means the sweep took the mark, but the parsed + // row may not have been published yet. + index.wait_for_refresh_idle_for_test().await; + assert!(!index.has_dirty()); let snap2 = index.snapshot().await; let row2 = snap2.iter().find(|s| s.provider == "amplifier").unwrap(); @@ -6879,7 +7935,8 @@ pub(crate) mod tests { // A steady second scoped mark (no file movement) re-parses nothing, // proving the folded-vs-folded cache keys match (no raw-fold thrash). index.mark_dirty(&[(metadata.clone(), "amplifier".to_string())]); - assert!(wait_until(Duration::from_secs(2), || !index.has_dirty()).await); + index.wait_for_refresh_idle_for_test().await; + assert!(!index.has_dirty()); assert_eq!( parse_calls.load(Ordering::SeqCst), 2, @@ -6897,7 +7954,8 @@ pub(crate) mod tests { ) .unwrap(); index.mark_dirty(&[(metadata.clone(), "amplifier".to_string())]); - assert!(wait_until(Duration::from_secs(2), || !index.has_dirty()).await); + index.wait_for_refresh_idle_for_test().await; + assert!(!index.has_dirty()); let snap3 = index.snapshot().await; let row3 = snap3.iter().find(|s| s.provider == "amplifier").unwrap(); diff --git a/crates/freshell-sessions/src/lib.rs b/crates/freshell-sessions/src/lib.rs index 353d66810..70832c032 100644 --- a/crates/freshell-sessions/src/lib.rs +++ b/crates/freshell-sessions/src/lib.rs @@ -21,6 +21,7 @@ pub mod amplifier; pub mod amplifier_stub; pub mod bundle_config; pub mod codex_locator; +pub mod codex_segments; pub mod directory_index; pub mod meta; pub mod opencode_locator; diff --git a/crates/freshell-sessions/src/parse/codex.rs b/crates/freshell-sessions/src/parse/codex.rs index 36b8f7822..942675604 100644 --- a/crates/freshell-sessions/src/parse/codex.rs +++ b/crates/freshell-sessions/src/parse/codex.rs @@ -563,4 +563,25 @@ mod tests { let meta = parse_codex_session_content(content); assert_eq!(meta.is_subagent, Some(true)); } + + #[test] + fn display_projection_remains_tolerant_of_rollout_only_records() { + let path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")).join( + "../../test/fixtures/coding-cli/codex/multi-file-continuation-newer.sanitized.jsonl", + ); + let content = std::fs::read_to_string(path).unwrap(); + let meta = parse_codex_session_content(&content); + + assert_eq!( + meta.session_id.as_deref(), + Some("b7936c10-4935-441c-837c-c1f33cafec2d") + ); + assert_eq!(meta.cwd.as_deref(), Some("/sanitized/project")); + assert_eq!(meta.title.as_deref(), Some("Continuation title")); + assert_eq!(meta.summary.as_deref(), Some("Continuation summary")); + assert_eq!( + meta.token_usage.as_ref().map(|usage| usage.total_tokens), + Some(780) + ); + } } diff --git a/crates/freshell-ws/tests/term09_output_queue.rs b/crates/freshell-ws/tests/term09_output_queue.rs index 3a7fd8ac7..6e690b056 100644 --- a/crates/freshell-ws/tests/term09_output_queue.rs +++ b/crates/freshell-ws/tests/term09_output_queue.rs @@ -867,23 +867,14 @@ async fn eviction_and_supersede_without_sends_still_close() { // moment the monitor fires, and an event emitted before the subscriber // exists is lost for good (tracing dispatch is not replayed). let events = global_capture(); - // Same deliberately-inverted injection as the progressing-client test: - // the queue may hold bytes over the threshold, making the monitor's - // window observable. The 30 s harness ping interval keeps the per-send - // write timeout at 60 s — far beyond this test's bounds — so a closure - // observed here is the monitor's, not the send timeout's. + let captured_from = events.lock().expect("capture lock").len(); + // The queue may hold bytes over the threshold, making the monitor's + // window observable. A small cap makes the first eviction an observable + // synchronization point even when the PTY producer is heavily contended. + // The 30 s harness ping interval keeps the per-send write timeout at + // 60 s — far beyond the monitor window. let term09 = Term09Config { - queue_max_bytes: 8 * 1024 * 1024, - // 32 KiB, not the earlier 1 MiB: premise-neutral (the eviction floor - // stays at the 8 MiB queue cap, far above this, so "bytes shrink via - // eviction while over threshold" is preserved) but it shrinks the - // load-sensitive step — the post-supersede-discard refill — 32x. - // Under concurrent full-suite gates the flood shell's production - // rate collapses, and a 1 MiB refill (≈10k flood lines) can take - // many minutes-to-forever: the monitor honestly has not seen - // over-threshold yet. 32 KiB (≈350 lines) keeps the window - // observable on a contended box; the observation loop below is - // production-progress-driven, so the two together are load-immune. + queue_max_bytes: 256 * 1024, catastrophic_buffered_bytes: 32 * 1024, catastrophic_stall_ms: 2_000, }; @@ -901,8 +892,7 @@ async fn eviction_and_supersede_without_sends_still_close() { tokio::time::sleep(Duration::from_millis(200)).await; let marker = "FLOOD-DONE-MARKER"; - // ~60 MB keeps production alive for several seconds past the mid-stall - // supersede below (the queue must refill after the discard). + // ~60 MB leaves ample output to refill the queue after supersede. let flood = flood_command(600_000, marker); creator @@ -917,22 +907,57 @@ async fn eviction_and_supersede_without_sends_still_close() { .await .expect("send flood input"); - // Do NOT read from the stuck socket at all while the stall window runs — - // reading would drain the kernel buffers and count as send progress. - // First the queue fills and the in-flight frame wedges (eviction keeps - // shrinking queue bytes with zero completed sends). - tokio::time::sleep(Duration::from_millis(1_000)).await; + // Wait for an actual eviction before superseding. A fixed sleep can run + // before the contended PTY has produced any output, so it cannot prove + // that this scenario exercised queue overflow. + let spill_deadline = tokio::time::Instant::now() + Duration::from_secs(300); + loop { + if events + .lock() + .expect("capture lock") + .iter() + .skip(captured_from) + .any(|e| e.message == "ws.terminal_stream.queue_overflow_spill") + { + break; + } + assert!( + tokio::time::Instant::now() < spill_deadline, + "the stuck client's queue did not evict within 300 s; the PTY \ + flood may not be producing or the writer may not be receiving it" + ); + tokio::time::sleep(Duration::from_millis(20)).await; + } - // Mid-stall, re-attach the stuck client: the superseding attach DISCARDS + // Re-attach the stuck client: the superseding attach DISCARDS // its entire queued output (a byte reduction that is NOT a send). The // monitor may honestly reset on the below-threshold fall, but the still- // producing flood refills the queue and the window must close the // connection — eviction/supersede alone must never keep it alive. attach(&mut stuck, &terminal_id, "attach-stuck-supersede").await; - // Keep not reading through the refill + a full stall window (+margin): - // zero successful sends the whole time. - tokio::time::sleep(Duration::from_millis(4_000)).await; + // Keep the socket unread until the monitor decides. Reading after a + // fixed four-second sleep, before a contended producer refills the queue, + // lets socket sends complete and resets the very no-send window this + // test intends to prove. + let decision_deadline = tokio::time::Instant::now() + Duration::from_secs(300); + loop { + if events + .lock() + .expect("capture lock") + .iter() + .skip(captured_from) + .any(|e| e.message == "ws.terminal_stream.catastrophic_close") + { + break; + } + assert!( + tokio::time::Instant::now() < decision_deadline, + "the monitor made no decision within 300 s after eviction and \ + supersede while the stuck socket remained unread" + ); + tokio::time::sleep(Duration::from_millis(50)).await; + } // NOW resume reading: the connection should already be terminated (the // catastrophic monitor fired while we were silent). Attribution does @@ -943,61 +968,19 @@ async fn eviction_and_supersede_without_sends_still_close() { // (exactly one ws.terminal_stream.catastrophic_close event); a write- // timeout or keepalive close would produce none. // - // The observation is PROGRESS-BASED (standing test discipline: a - // wall-clock budget must never fail working code). This test's waits - // starved out three times under concurrent full-suite gates before the - // design converged: a fixed 60 s window (twice), then a 300 s - // decision cap whose real victim was the FLOOD — under extreme - // starvation the shell's production rate collapses and the - // post-discard refill (which must re-cross the injected threshold) - // had not happened yet, so the monitor was honestly still waiting - // (not dead). The threshold injection is therefore sized for the - // contended-box reality (32 KiB re-crosses at even ~1% of focused - // production rate), and the wait below tracks the monitor's own - // decision record in the capture. The only fixed bound before the - // decision is a true-stall cap: 300 s with no decision event at all — - // a dead monitor or a flood starved below ~100 B/s of production, at - // which point no gate finishes anyway. A slow-but-producing box can - // never trip it. After the decision, a 120 s delivery cap catches a - // wedged teardown. - let decision_deadline = tokio::time::Instant::now() + Duration::from_secs(300); - let mut decided_at: Option = None; + // After the decision, a 120 s delivery cap catches a wedged teardown. + let close_deadline = tokio::time::Instant::now() + Duration::from_secs(120); loop { - let now = tokio::time::Instant::now(); - if decided_at.is_none() - && events - .lock() - .expect("capture lock") - .iter() - .any(|e| e.message == "ws.terminal_stream.catastrophic_close") - { - decided_at = Some(now); - } - if let Some(at) = decided_at { - assert!( - at.elapsed() < Duration::from_secs(120), - "the monitor decided but its close was never delivered to the \ - now-reading client within 120 s — a wedged teardown, not a \ - slow box" - ); - } else { - assert!( - now < decision_deadline, - "no monitor decision in the capture within 300 s — a dead \ - monitor or a flood starved below ~100 B/s of production (a \ - slow-but-producing box can never trip this)" - ); - } - let tick = if decided_at.is_some() { - Duration::from_millis(50) - } else { - Duration::from_millis(250) - }; + assert!( + tokio::time::Instant::now() < close_deadline, + "the monitor decided but the now-reading client did not see \ + stream termination within 120 s" + ); // The loop's only non-panic exit is the close observation itself: // every other path is a stall-cap panic carrying the failure's // exact context, so the connection-must-close requirement is // enforced structurally. - match tokio::time::timeout(tick, stuck.next()).await { + match tokio::time::timeout(Duration::from_millis(50), stuck.next()).await { Ok(Some(Ok(WsMessage::Close(_)))) | Ok(None) | Ok(Some(Err(_))) => break, Ok(Some(Ok(_))) => {} Err(_) => continue, // tick elapsed with no frame: re-check progress @@ -1016,6 +999,7 @@ async fn eviction_and_supersede_without_sends_still_close() { .lock() .expect("capture lock") .iter() + .skip(captured_from) .filter(|e| e.message == "ws.terminal_stream.catastrophic_close") .cloned() .collect(); diff --git a/docs/plans/2026-10-03-kata-rrx7-rollout-continuation.md b/docs/plans/2026-10-03-kata-rrx7-rollout-continuation.md new file mode 100644 index 000000000..a8984c175 --- /dev/null +++ b/docs/plans/2026-10-03-kata-rrx7-rollout-continuation.md @@ -0,0 +1,484 @@ +# Codex Multi-File Session Continuations Implementation Plan + +> **For agentic workers:** Execute this plan task by task with a fresh +> implementer and a specification-plus-quality review after every task. Track +> progress with the checkbox steps below. + +## User Request + +### Requested result +Fix KataTracker `rrx7` so verified Codex continuations across rollout files appear as one complete session while ambiguous duplicates remain quarantined. Correct the Claude Code CLI review launch, assess every readable finding preserved in rejected RRX7 delta reports and every finding from the restarted delta reviews, repair all substantiated in-scope findings, and continue the bounded delta Fresh Eyes loop. + +### Explicit constraints +- Track and fix KataTracker item `rrx7`. +- Use a `gpt-6-sol` subagent with xhigh reasoning for root-cause investigation. +- Follow the “the-usual” workflow. +- Launch Claude reviews through the Claude Code CLI from the command line; diagnose inherited authentication routing and do not call the Anthropic API directly. +- Deliberately update the installed The Usual copy along with its canonical repository. +- Do not add a recurring behavioral test for the The Usual auth fix; validate it with a test call. +- Create and use a dedicated `.worktrees/` worktree from `origin/main`; the user authorized bypassing the earlier base-test gate to create it. +- Assess every readable finding in the prior rejected delta-review reports; do not discard substantive feedback solely because of report formatting. +- Assess every finding from each restarted delta review and resolve all substantiated in-scope findings before the next review round. +- Do not broaden this work into redesigning `bundle-the-usual`. +- Use red-green-refactor testing and do not weaken or skip tests. +- Do not create a PR without explicit approval or deploy/restart production. + +### Accepted tradeoffs and residuals +- None stated. + +## Stage 2 Scope Decisions and Residuals +- Restore the existing Freshell resume action by publishing one row with the canonical Codex session ID. Do not change the separate resumed-activity file watcher in this repair; the Kata identifies row quarantine as the resume blocker, and the watcher can still attach to an older file or miss late materialization, with no proven causal link to that symptom. +- Use a conservative structural acceptance rule based on the exact reported pair. The pair's physical creation provenance is not present in file contents, so composition cannot prove that provenance from files alone. Do not claim universal classification of every possible same-ID file history. +- Codex's provider-indexed history for this exact session was not inspected. Keep resume by canonical ID and do not claim this repair verifies provider-side history reconstruction. +- Lifetime collision-signature suppression may suppress the same signature if it clears and recurs before process restart. Distinct source-file sets remain separately diagnosable. + +**Goal:** The reported Codex session, continued across sequential rollout files with one session ID, appears once in sidebar and history, exposes all source files to bounded transcript search, carries the earliest creation and latest activity times, and produces no integrity alert. A copied or otherwise ambiguous same-ID group remains quarantined, including when a member cannot render because required metadata is missing, while unchanged collisions do not produce per-request ERROR logs. + +**Architecture:** Preserve the path-keyed per-file cache, retain identity evidence independently of renderable rows, and compose rows at snapshot publication only when ownership, metadata, lineage markers, and complete persisted-record write intervals agree. Publish unresolved identity groups alongside rows in the same snapshot generation so the directory route can quarantine and log every known member, including files without a renderable row. Carry every accepted source path into bounded directory search, and suppress repeated collision logs by the full signature while including a stable full-signature identifier in each event. The visible row continues to resume through the canonical session ID. + +**Tech Stack:** Rust workspace (`freshell-sessions`, `freshell-server`), React/TypeScript client, Vitest, Rust tests, and Playwright browser tests. + +## Global Constraints + +- Work only in `/home/dan/code/freshell/.worktrees/kata-rrx7-rollout-continuation` on `the-usual/kata-rrx7-rollout-continuation`, based on fetched `origin/main` `a531d63b32ef6442b3d64340115d8d8c2561717d`. +- Keep the work focused on Codex rollout continuations. Keep provider files read-only; do not edit or move files under a real Codex home. +- Preserve the existing session-directory wire shape and its additive `integrityError` object. Do not expose on-disk paths to clients. +- Preserve quarantine for copied, overlapping, interleaved, forked, incomplete, or metadata-conflicting same-ID groups. Never select a path based only on filename order or filename UUID suffix. +- Use JSONL structured logs with severity. Repeated polls for an unchanged collision signature must not produce repeated ERROR events; a different full source-file set must have a different stable log identifier. Same-signature recurrence during one process lifetime may remain suppressed. +- Use the repository's pinned pnpm 10.34.5, frozen dependency state, and coordinated test entrypoints. For broad agent-launched gates, set `GCLOUD_ROBOT_REQUIRE=1` and respect the shared test coordinator. +- Before browser verification, use the configured `FRESHELL_E2E_BACKEND`. If it is unset, ask the user to choose local or cloud as required by `AGENTS.md`; do not silently switch backends. Confirm the selected browser spec actually ran and was not cloud-skipped. +- Do not add prose/config hash tests. Tests must exercise session indexing, search, collision behavior, UI behavior, and the canonical-ID resume action boundary. +- Do not create a PR, merge, push, deploy, restart the live server, or clean up this worktree. + +--- + +**TDD order:** Before any production implementation, complete Task 3 Steps 1–2 +as the browser-level red preflight. Then complete Task 1 and Task 2 in order, +writing and running each task's failing tests before its production changes. +Return to Task 3 Step 3 onward after Tasks 1–2 to refine and verify the green +browser regression. Do not treat a browser test run after the directory fix +as proof of the original red state. + +### Task 1: Compose Only Structurally Supported Codex Continuation Segments + +**Files:** +- Create: `crates/freshell-sessions/src/codex_segments.rs` +- Modify: `crates/freshell-sessions/src/lib.rs` +- Modify: `crates/freshell-sessions/src/parse/codex.rs` +- Modify: `crates/freshell-sessions/src/directory_index.rs` +- Create: `test/fixtures/coding-cli/codex/multi-file-continuation-older.sanitized.jsonl` +- Create: `test/fixtures/coding-cli/codex/multi-file-continuation-newer.sanitized.jsonl` +- Test: `crates/freshell-sessions/src/directory_index.rs` CodexSource and SessionIndex tests + +**Interfaces:** +- Consumes: `parse_codex_session_content(&str) -> ParsedSessionMeta`; path-keyed `FileEntry` cache; `IndexedSession` snapshot rows. +- Produces: serde-serialized per-file Codex identity/segment evidence in `FileEntry`, independent of `item: Option`, plus a same-generation unresolved-identity sidecar from `SessionIndex`. A snapshot composition helper keeps one `IndexedSession` for a structurally accepted continuation group. Retain the existing `source_file` as the deterministic latest-segment representative for compatibility, and retain all per-segment evidence and paths in chronological order for downstream consumers. + +- [x] **Step 1: Write the failing behavioral tests and sanitized fixtures** + +Add `codex_source_composes_the_reported_same_id_rollout_shape` using sanitized +fixture files that model the issue-listed pair without copying private +transcript text. Both have first-line `session_meta`, equal `payload.id` and +`payload.session_id`, matching known `cwd`, `source`, `thread_source`, +`cli_version`, `originator`, and paginated history mode; neither has fork, +parent, subagent, or referenced-prefix metadata. Give the filenames lexical +order opposite to event order. Include timestamped records from the Codex +0.156 persisted variants seen in the pair, including `compacted` and +inter-agent metadata, plus user/assistant messages and distinct token +snapshots. Include the observed within-file timestamp ties and a strict gap +between the complete persisted-record write intervals. Build a `SessionIndex` +over those files and assert one row, the stable canonical ID, earliest +`created_at`, latest `last_activity_at`, earliest nonempty +`first_user_message`, title and summary from their earliest segments with +substantive data, latest token snapshot, and both source paths ordered by +persisted-record time. Assert `CodexSource::scan()` +uses the same composition policy. + +Add `codex_source_keeps_ambiguous_same_id_files_separate` for byte-identical +copies, overlapping/interleaved ranges, equal cross-file boundaries, +malformed or unterminated JSONL, unknown top-level variants, missing/invalid +outer timestamps, timestamp regressions, invalid/regressing ordinals, +non-first-line or inconsistent ownership, later contradictory metadata, +conflicting required metadata, and fork/parent/subagent/history-base evidence. +Assert every renderable ambiguous member remains a separate same-ID row so +the server can quarantine the whole identity. Also assert that a file with a +known matching ID but missing `cwd` is retained as non-renderable identity +evidence, prevents partial composition, and appears in the snapshot's +unresolved-identity sidecar with its renderable sibling. Generate a +2,001-line copied transcript from the small sanitized fixture in the test +helper instead of checking in a needlessly large file. + +Add `codex_continuation_composition_survives_refresh_and_cache_reload`: +after the first two files compose, append to the newest file, create a third +valid continuation, reload from the persisted cache, then delete one segment. +After every refresh assert one correctly ordered row and no stale source path. +Assert that a cache written with the old schema is discarded and reparsed. + +- [x] **Step 2: Run the tests and verify the intended failure** + +Run: + +```bash +pnpm run test:integration -p freshell-sessions directory_index::tests::codex +``` + +Expected: the new continuation test fails because two same-ID file rows are +published instead of one; ambiguous cases and existing single-file tests +continue to demonstrate their current behavior. + +- [x] **Step 3: Add the minimal evidence and composition implementation** + +In the new `codex_segments` module define serde-compatible per-file identity +and interval evidence. Store that evidence on the path-keyed `FileEntry` +independently of `item`, so a known identity survives when the normal parser +returns no renderable row (for example, missing `cwd`). Keep the existing +single-file display parser's `Option` behavior. Require the +first line to be a valid `session_meta` and validate +`payload.id == payload.session_id` for multi-file composition. Retain exact +structured values for required matching metadata (`cwd`, `source`, +`thread_source`, `cli_version`, `originator`, and `history_mode`) and every +recognized fork, parent, subagent, fork-ordinal, and `history_base` marker. +Compare these fields explicitly; absent, unknown, conflicting, or +unsupported evidence prevents composition. Filename-derived IDs remain a +single-file fallback only and never prove a continuation. + +Implement a dedicated full-record evidence scan separate from the tolerant +display parser. Validate every complete JSONL line against the known 0.156 +top-level rollout variants, including `compacted` and inter-agent metadata; +require a valid outer timestamp on every post-header record; preserve parsed +timestamp precision; compute extrema; track physical order; and reject +malformed/truncated lines, unknown variants, missing/invalid timestamps, +timestamp regressions, contradictory later identity/lineage records, and +invalid or regressing ordinals when present. Within-file timestamp ties are +valid and physical order remains available; ordinal values restart per file +and do not order files. Define the range as the persisted-record write +interval, excluding only the first header timestamp. Do not flatten +compaction or checkpoint payloads into invented original-history timestamps. +Keep the existing tolerant display parsing behavior for single-file +projections. + +For a multi-file group, compose only when every discovered member has +first-line ownership with `payload.id == payload.session_id == session_id`, +matching known required metadata, compatible version/originator/history-mode +metadata, no known fork, parent, subagent, fork-ordinal, or referenced-prefix +evidence, and a complete persisted-record write interval. This policy admits +the issue-listed structural case; it does not prove physical writer +provenance or every possible same-ID history. Sort by interval extrema and +require each previous end to be strictly earlier than the next start. If any +member conflicts, is non-renderable, or lacks complete evidence, publish no +partial composition: preserve all renderable same-ID rows for quarantine and +retain known-ID evidence for hidden members. Build an unresolved-identity +sidecar from the full file cache after each full or scoped reconciliation; +it must include all known member paths for every uncomposed same-ID group +with at least two members. Publish it atomically with the rows and scan +failures in `CachedSnapshot`, and expose it through a narrow `SessionIndex` +read method that preserves the existing stale-while-revalidate behavior. +Evidence-only additions, changes, and removals must advance refresh change +detection and persistence accounting. Persist the evidence with `FileEntry` +and bump `CACHE_SCHEMA_VERSION`. Filename suffixes, mtime, and traversal +order never establish chronology. + +For an accepted group set `created_at` to the earliest segment, activity to +the latest segment, `first_user_message` to the earliest nonempty segment, +title and summary from their respective earliest segments with substantive +data, and token usage from the newest snapshot without summing cumulative +counters. +Preserve existing single-file results. Bump `CACHE_SCHEMA_VERSION` so cached +rows lacking evidence cannot remain unmerged indefinitely. Compose after each +full or scoped cache reconciliation and sort the final rows once as before. + +- [x] **Step 4: Run the focused tests** + +Run: + +```bash +pnpm run test:integration -p freshell-sessions directory_index::tests::codex +``` + +Expected: all new grouping, refusal, chronology, and metadata assertions pass. + +- [x] **Step 5: Refactor while green** + +Keep the grouping decision in one Codex-specific helper used by both +`CodexSource::scan()` and `SessionIndex` publication. Keep evidence parsing +separate from provider-agnostic `ParsedSessionMeta` behavior, and verify that +single-file Codex and non-Codex `IndexedSession` projections remain unchanged. + +- [x] **Step 6: Run impacted-test verification** + +Run: + +```bash +pnpm run test:integration -p freshell-sessions directory_index::tests +pnpm run test:integration -p freshell-sessions --test malformed_data_quarantine codex_ +pnpm run test:integration -p freshell-sessions --test codex_fixture_parity +``` + +Expected: all Codex indexing, cache, malformed-input quarantine, and parser +parity tests pass. + +- [x] **Step 7: Commit the task** + +```bash +git add crates/freshell-sessions/src/codex_segments.rs crates/freshell-sessions/src/lib.rs crates/freshell-sessions/src/parse/codex.rs crates/freshell-sessions/src/directory_index.rs test/fixtures/coding-cli/codex/multi-file-continuation-older.sanitized.jsonl test/fixtures/coding-cli/codex/multi-file-continuation-newer.sanitized.jsonl +git commit -m "fix(sessions): merge verified Codex continuations" +``` + +### Task 2: Preserve Full Search and Collision Behavior for Composed Rows + +**Files:** +- Modify: `crates/freshell-server/src/session_directory.rs` +- Modify: `src/components/Sidebar.tsx` +- Modify: `src/components/HistoryView.tsx` +- Test: `crates/freshell-server/src/session_directory.rs` +- Test: `test/unit/client/components/Sidebar.test.tsx` +- Test: `test/unit/client/components/HistoryView.a11y.test.tsx` + +**Interfaces:** +- Consumes: one coherent `SessionIndex` snapshot of rendered `IndexedSession` rows and unresolved same-ID groups; existing `search_session_file` user/full-text tiers; existing `SessionDirectoryState` and `integrityError` response shape. +- Produces: private `DirItem` source-path collection; a bounded multi-file search that returns at most one logical row; route quarantine/logging that unions rendered row paths with unresolved index evidence; a process-shared collision-signature gate that emits one structured event per newly observed full collision signature. + +- [x] **Step 1: Write failing route, logging, and UI behavior tests** + +Add a real route test that seeds the sequential fixture through +`CodexSource`/`SessionIndex`, requests the session-directory endpoint, and +asserts one row, earliest `createdAt`, latest `lastActivityAt`, and no +`integrityError`. Search +using a distinct user-message needle from each segment and an assistant +needle from each segment; each response must contain that same logical row +once with the correct tier and snippet. Search with a needle present in both +segments and assert one row. Include one unreadable segment plus a match in +the other segment and assert that the match remains while the result reports +`partialReason: "io_error"`. Exhaust the shared segment-scan budget midway +through a logical row and assert `partialReason: "budget"`. + +Add a focused `apply_file_search` test with one missing segment path and one +readable matching segment to prove that search continues after the I/O +failure while preserving the partial result. Keep this unit seam separate +from the route test because the real index prunes deleted files when it +refreshes. + +Add a production-index route test that copies the same 2,001-line Codex file +to a second path and asserts both persisted rows are absent, the existing +additive `integrityError` remains, healthy rows and matching live placeholders +remain, and pagination/filtering cannot hide the conflict. + +Add a second production-index route test with exactly one renderable Codex +row and one same-ID file whose first-line identity is valid but whose `cwd` +is missing. Assert the rendered row is quarantined, `integrityError` is +present, a healthy unrelated row remains, and neither the response nor its +items expose source paths. Capture the collision event and assert its full +signature includes both the renderable and hidden member paths. Remove the +hidden file and assert a refreshed request restores the row without stale +integrity evidence. Then move the hidden member to a different path and +reopen the persistent index cache; assert the same identity is still +quarantined and the refreshed full signature reflects the new path. This +covers evidence-only path changes through refresh, persistence, and cache +reload rather than only testing the pure collision helper. + +Capture actual tracing events while issuing concurrent and repeated identical +requests, performing an unchanged refresh, and changing one colliding source +path to a different path beyond the diagnostic sample. Assert one ERROR event +per distinct full signature, none for repeated polls/unchanged refresh, and +different stable signature identifiers for the changed full path sets. Do +not require the same signature to log again after a clear interval in the +same process. Assert event samples stay bounded, the full signature controls +deduplication, and source paths never enter the wire response. + +Update both existing alert behavior tests to assert the alert remains +accessible and dismissible. Do not test exact prose. Replace the advice to +remove or rename provider files with a neutral instruction to check server +logs for the conflict details. + +Add a focused Sidebar open-action test showing that a composed row still +passes its canonical `sessionId` and provider into the existing session-open +flow. Use the existing command-construction unit test to cover +`codex resume `; do not launch Codex from the browser test. + +- [x] **Step 2: Run the tests and verify the intended failure** + +Run: + +```bash +pnpm run test:server session_directory::tests::codex_multi_file +pnpm run test:server session_directory::tests::persisted_identity_collision +pnpm run test:vitest run test/unit/client/components/Sidebar.test.tsx test/unit/client/components/HistoryView.a11y.test.tsx --config config/vitest/vitest.config.ts +``` + +Expected: route coverage already sees one composed row from Task 1, while +multi-file search cannot yet find the older segment, a renderable row plus +same-ID non-renderable evidence is not yet quarantined by the route, and +repeated requests still emit repeated collision events without a +full-signature identifier. The copied-file route should continue to +quarantine both rows. + +- [x] **Step 3: Add multi-source search and collision-state logging** + +Transfer all Codex segment paths to `DirItem`; keep the representative path +private and never serialize any path. Search sources in chronological order +using the existing user/full-text parser, stop after the first matching +segment, and return a logical row once. Count each file read against the +existing `limit * 10` scan ceiling so many segments cannot bypass the bound. +If a segment read fails, continue searching later segments; retain any match +and report `partialReason: "io_error"`. Preserve paging order, snippets, tier +semantics, and existing `partial` wire behavior. + +Merge the same-generation unresolved identity groups from `SessionIndex` +with row-derived collisions before filtering. Treat their complete member +paths as persisted sources even when only one or none of those files has a +renderable row; quarantine every rendered row for that identity and include +the evidence-only members in the collision signature and diagnostic counts. +Do not synthesize a visible row solely to represent a non-renderable file. + +Store process-lifetime full collision signatures in shared +`SessionDirectoryState` memory as an +`Arc>>`. Atomically insert the complete +sorted signature and log only when insertion is new; do not reset the set +when a collision clears. Derive a stable collision identifier (for example, +a SHA-256 digest) from the full canonical signature, include it in the +structured ERROR event, and keep the existing bounded path sample. Different +full signatures must not become indistinguishable when they differ beyond +the sample. Update Sidebar and History View alert copy to direct users to +server logs without suggesting edits to Codex-owned files. Keep the wire +shape unchanged. + +- [x] **Step 4: Run the focused tests** + +Run: + +```bash +pnpm run test:server session_directory::tests::codex_multi_file +pnpm run test:server session_directory::tests::persisted_identity_collision +pnpm run test:server session_directory::tests::tier_ +pnpm run test:vitest run test/unit/client/components/Sidebar.test.tsx test/unit/client/components/HistoryView.a11y.test.tsx --config config/vitest/vitest.config.ts +``` + +Expected: multi-source search, copied-file quarantine, distinct full-signature +logging, alert behavior, canonical-ID row opening, and existing tier/collision +tests pass. + +- [x] **Step 5: Refactor while green** + +Keep collision-signature construction separate from response sampling and +ensure the mutex protects the full-signature insert as one atomic operation. +Remove any obsolete singular-path search branch once all persisted rows use +the source-path collection. + +- [x] **Step 6: Run impacted-test verification** + +Run: + +```bash +pnpm run test:server session_directory::tests +pnpm run test:vitest run test/unit/client/components/Sidebar.test.tsx test/unit/client/components/HistoryView.a11y.test.tsx test/unit/shared/session-directory-schema.test.ts --config config/vitest/vitest.config.ts +``` + +Expected: the complete session-directory Rust module and both alert +components plus the unchanged wire-schema contract pass. + +- [x] **Step 7: Commit the task** + +```bash +git add crates/freshell-server/src/session_directory.rs src/components/Sidebar.tsx src/components/HistoryView.tsx test/unit/client/components/Sidebar.test.tsx test/unit/client/components/HistoryView.a11y.test.tsx +git commit -m "fix(session-directory): search Codex continuation segments" +``` + +### Task 3: Prove the Repaired Session Through the Browser + +**Files:** +- Modify: `test/e2e-browser/helpers/session-corpus/codex.ts` +- Modify: `test/e2e-browser/specs/session-directory-matrix.spec.ts` + +**Interfaces:** +- Consumes: the production index, directory route/search behavior, integrity alert, and canonical-ID row-opening action from Tasks 1–2. +- Produces: cloud-legal browser coverage using isolated test homes and the real session-directory route; no route response mocking or provider launch. + +- [x] **Step 1: Write the failing browser regression** + +Add a Codex corpus helper that writes both continuation files into an +isolated test home. Write separate positive and collision browser cases. The +positive home contains only the continuation pair; assert through the actual +API that it appears once with earliest `createdAt`, latest `lastActivityAt`, +and no `integrityError`, then assert one visible row in Sidebar and History. +Use the Sidebar `userMessages` tier with distinct text from each segment to +prove older and newer transcript search. Do not require transcript search +from History View's local metadata filter. + +The separate collision home contains only a copied same-ID pair plus one +healthy unrelated session. Assert the copied pair remains hidden, the +existing integrity alert is visible and dismissible in Sidebar and History, +and the healthy row remains visible. Do not use an expected-failure marker +or a `CLOUD_SKIP_SPECS` exemption. The resume action boundary is covered by +the focused Sidebar test in Task 2; do not launch Codex in the browser suite. +The existing `createE2eServerHandle` fixture still owns the original matrix +cases. For the new RRX7 cases, use `bootCodexBrowserPage` to start an owned +`RustServer`; its pre-start `setupHome` callback must seed that server's +isolated home before provider discovery begins. Do not mock the +session-directory route. + +- [x] **Step 2: Run the test and verify the intended failure** + +Run: + +```bash +GCLOUD_ROBOT_REQUIRE=1 pnpm run test:e2e test/e2e-browser/specs/session-directory-matrix.spec.ts +``` + +Run this red preflight before any production edits. Expected: the current +index produces a response-wide collision alert and hides the positive +continuation row. The collision-only case should continue to show the +quarantine. The selected backend must execute the named spec with a nonzero +test count. + +- [x] **Step 3: Run the focused browser test** + +Run: + +```bash +GCLOUD_ROBOT_REQUIRE=1 pnpm run test:e2e test/e2e-browser/specs/session-directory-matrix.spec.ts +``` + +Expected: the named browser spec runs on the configured backend and passes +with one positive continuation row, no positive-case alert, successful +Sidebar transcript search across both source files, and a separate copied +file case with a visible alert and intact healthy row. + +- [x] **Step 4: Refactor while green** + +Keep the existing single-file Codex corpus helper behavior intact; share only +small fixture-writing helpers that make session IDs and event ranges explicit. + +- [x] **Step 5: Run impacted-test verification** + +Run: + +```bash +GCLOUD_ROBOT_REQUIRE=1 pnpm run test:e2e test/e2e-browser/specs/session-directory-matrix.spec.ts +``` + +Expected: the entire named matrix spec passes with this regression included; +the test runner reports executed tests rather than a skipped cloud spec or +empty filter. + +- [x] **Step 6: Commit the task** + +```bash +git add test/e2e-browser/helpers/session-corpus/codex.ts test/e2e-browser/specs/session-directory-matrix.spec.ts +git commit -m "test(e2e): cover Codex multi-file continuations" +``` + +## Final Verification + +After all three task commits and task reviews pass, run the repository's +coordinated full check once at final `HEAD`: + +```bash +GCLOUD_ROBOT_REQUIRE=1 pnpm run check +``` + +Also rerun the named browser spec on the configured `FRESHELL_E2E_BACKEND` +after the last source change. If any browser run was part of the configured +full check, adopt that result only when it executed this exact spec at final +`HEAD`; otherwise run the named spec separately. Record exact commands, +commit SHA, test totals, configured backend, and any skipped scopes in the +progress ledger. Do not deploy or create a PR as part of this plan. diff --git a/src/components/HistoryView.tsx b/src/components/HistoryView.tsx index bda955fc2..52d96689e 100644 --- a/src/components/HistoryView.tsx +++ b/src/components/HistoryView.tsx @@ -4,7 +4,7 @@ import type { CodingCliProviderName, CodingCliSession, ProjectGroup } from '@/st import type { RootState } from '@/store/store' import { removeSessionFromProjects, toggleProjectExpanded } from '@/store/sessionsSlice' import { api } from '@/lib/api' -import { activateSessionSurface, fetchSessionWindow } from '@/store/sessionsThunks' +import { activateSessionSurface, fetchSessionWindow, type FetchSessionWindowResult } from '@/store/sessionsThunks' import { openSessionTab } from '@/store/tabsSlice' import { applySessionRenameCascade } from '@/store/titleSync' import { receiveSessionNameProjections, receiveSessionNames } from '@/store/sessionNamesSlice' @@ -69,8 +69,10 @@ export default function HistoryView({ onOpenSession }: { onOpenSession?: () => v const expandedProjects = useAppSelector((s) => s.sessions.expandedProjects) const sessionNames = useAppSelector((s) => s.sessionNames) const historyWindow = useAppSelector((s) => s.sessions.windows?.history) - const projects = useAppSelector((s) => s.sessions.windows?.history?.projects ?? s.sessions.projects) - const topLevelSessionCount = useAppSelector((s) => s.sessions.projects?.length ?? 0) + const projects = useAppSelector((s) => { + const window = s.sessions.windows?.history + return typeof window?.lastLoadedAt === 'number' ? window.projects : s.sessions.projects + }) const [filter, setFilter] = useState('') const [loading, setLoading] = useState(false) const [mobileSessionSheet, setMobileSessionSheet] = useState(null) @@ -81,13 +83,24 @@ export default function HistoryView({ onOpenSession }: { onOpenSession?: () => v const [dismissedIntegrityCount, setDismissedIntegrityCount] = useState(null) useEffect(() => { - if (historyWindow || topLevelSessionCount > 0) return - dispatch(activateSessionSurface('history')) - void dispatch(fetchSessionWindow({ + if (typeof store.getState().sessions.windows?.history?.lastLoadedAt === 'number') return + + let mounted = true + const request = dispatch(fetchSessionWindow({ surface: 'history', priority: 'visible', - }) as any) - }, [dispatch, historyWindow, topLevelSessionCount]) + }) as any) as Promise + void request.then((result) => { + const hasCommittedHistory = typeof store.getState().sessions.windows?.history?.lastLoadedAt === 'number' + if (mounted && result.ok && hasCommittedHistory && store.getState().sessions.activeSurface !== 'history') { + dispatch(activateSessionSurface('history')) + } + }) + + return () => { + mounted = false + } + }, [dispatch, store]) const filtered = useMemo(() => { const q = filter.trim().toLowerCase() @@ -287,7 +300,7 @@ export default function HistoryView({ onOpenSession }: { onOpenSession?: () => v