From 8342ad97bf4b703fc0ff27f4070cab32ccfda869 Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Wed, 9 Sep 2026 17:14:58 +0000 Subject: [PATCH 1/6] refactor(store-runtime): move writer gates and lifecycle handles --- Cargo.lock | 1 + crates/tracedecay-cli/src/main.rs | 4 +- crates/tracedecay-store-runtime/Cargo.toml | 1 + crates/tracedecay-store-runtime/src/lib.rs | 12 ++++++ .../src/semantic_artifact_gc.rs} | 33 ++++++-------- .../src/standalone_session.rs | 43 +++++++++++++++++++ .../src}/store_shutdown.rs | 43 +++++++++++-------- .../src/writer_gate.rs} | 40 ++++++++--------- crates/tracedecay/src/daemon.rs | 5 --- crates/tracedecay/src/daemon/bootstrap.rs | 9 ++-- crates/tracedecay/src/daemon/branch_admin.rs | 6 +-- .../daemon/branch_admin/project_retirement.rs | 8 ++-- .../branch_admin/remote_recovery_lifecycle.rs | 2 +- .../tracedecay/src/daemon/engine/shutdown.rs | 2 +- .../src/daemon/project_server_lifecycle.rs | 2 +- .../src/daemon/shutdown_orchestration.rs | 2 +- .../vector_retention_tests.rs | 2 +- .../tracedecay/src/daemon/tests/lifecycle.rs | 2 +- .../tracedecay/src/project_store_runtime.rs | 32 +++----------- 19 files changed, 138 insertions(+), 111 deletions(-) rename crates/{tracedecay/src/daemon/maintenance_tasks.rs => tracedecay-store-runtime/src/semantic_artifact_gc.rs} (74%) create mode 100644 crates/tracedecay-store-runtime/src/standalone_session.rs rename crates/{tracedecay/src/daemon => tracedecay-store-runtime/src}/store_shutdown.rs (92%) rename crates/{tracedecay/src/daemon/store_writer_gate.rs => tracedecay-store-runtime/src/writer_gate.rs} (89%) diff --git a/Cargo.lock b/Cargo.lock index 5af6f5193d..4bfa7ee4b1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7189,6 +7189,7 @@ dependencies = [ "tracedecay-code-index-runtime", "tracedecay-contracts", "tracedecay-daemon-identity", + "tracedecay-daemon-service", "tracedecay-domain", "tracedecay-global-db", "tracedecay-graph-db", diff --git a/crates/tracedecay-cli/src/main.rs b/crates/tracedecay-cli/src/main.rs index 3a483ccf4f..8eaeec3cd2 100644 --- a/crates/tracedecay-cli/src/main.rs +++ b/crates/tracedecay-cli/src/main.rs @@ -1371,7 +1371,7 @@ async fn dispatch_runtime_command(command: Commands) -> tracedecay_domain::error // The MCP server is long-lived, so it may run the detached // structured-row backfill sweep; one-shot CLI/hook processes never // do (they would drop the sweep mid-parse on exit). - tracedecay::daemon::mark_process_long_lived_for_session_maintenance(); + tracedecay_store_runtime::mark_process_long_lived_for_session_maintenance(); hotpath::future!(serve_cmd::run_serve(path, timings), label = "cli.serve.run").await?; } Commands::Daemon { action } => { @@ -1392,7 +1392,7 @@ async fn dispatch_daemon_command(action: DaemonAction) -> tracedecay_domain::err remote_tls_key, } => { // Long-lived host: allowed to run the structured-row sweep. - tracedecay::daemon::mark_process_long_lived_for_session_maintenance(); + tracedecay_store_runtime::mark_process_long_lived_for_session_maintenance(); let socket_path = tracedecay_daemon_control::socket_path_or_default(socket)?; let remote_tls = tracedecay_daemon_control::RemoteBrainTlsConfig::from_optional_parts( remote_listen, diff --git a/crates/tracedecay-store-runtime/Cargo.toml b/crates/tracedecay-store-runtime/Cargo.toml index c43ec48b8a..877d58c0c6 100644 --- a/crates/tracedecay-store-runtime/Cargo.toml +++ b/crates/tracedecay-store-runtime/Cargo.toml @@ -38,6 +38,7 @@ tokio = { version = "1", features = ["full"] } tracing = "0.1" tracedecay-contracts = { path = "../tracedecay-contracts", version = "0.1.0" } tracedecay-automation-runtime = { path = "../tracedecay-automation-runtime", version = "0.1.0" } +tracedecay-daemon-service = { path = "../tracedecay-daemon-service", version = "0.1.0" } tracedecay-code-index = { path = "../tracedecay-code-index", version = "0.1.0", default-features = false } tracedecay-code-index-retention = { path = "../tracedecay-code-index-retention", version = "0.1.0" } tracedecay-code-index-runtime = { path = "../tracedecay-code-index-runtime", version = "0.1.0", default-features = false } diff --git a/crates/tracedecay-store-runtime/src/lib.rs b/crates/tracedecay-store-runtime/src/lib.rs index f50dbeafb4..a6bbe9cc7e 100644 --- a/crates/tracedecay-store-runtime/src/lib.rs +++ b/crates/tracedecay-store-runtime/src/lib.rs @@ -28,8 +28,12 @@ pub mod remote_credentials; pub mod remote_query; pub mod remote_replay_transaction; +pub mod semantic_artifact_gc; pub mod session_registry; +pub mod standalone_session; pub mod store_locator_resolver; +pub mod store_shutdown; +pub mod writer_gate; mod schema; @@ -41,6 +45,9 @@ pub use remote_credentials::{ pub use remote_query::DaemonRemoteExactObservationQueryPortV1; pub use remote_replay_transaction::DaemonRemoteReplayTransactionAuthorityV1; pub use schema::register_registered_schema_installer; +pub use semantic_artifact_gc::{ + SemanticArtifactGcMaintenanceTask, spawn_semantic_artifact_gc_maintenance, +}; #[cfg(any(test, feature = "test-helpers"))] pub use session_registry::maintenance::RegisteredSchemaConvergenceTestGate; pub use session_registry::maintenance::{ @@ -52,3 +59,8 @@ pub use session_registry::{ mark_process_long_lived_for_session_maintenance, open_user_memory_db, process_runtime_generation, registry_open_error, release_process_allocator_memory, }; +pub use standalone_session::join_standalone_session_registry; +pub use store_shutdown::{ + ShutdownTaskOutcome, ShutdownTaskReceipt, ShutdownTaskStatus, join_shutdown_tasks_until, +}; +pub use writer_gate::{StoreWriterClass, StoreWriterGates, WriterAdmissionGuard, WriterScope}; diff --git a/crates/tracedecay/src/daemon/maintenance_tasks.rs b/crates/tracedecay-store-runtime/src/semantic_artifact_gc.rs similarity index 74% rename from crates/tracedecay/src/daemon/maintenance_tasks.rs rename to crates/tracedecay-store-runtime/src/semantic_artifact_gc.rs index 01dc0a8353..ef4851f824 100644 --- a/crates/tracedecay/src/daemon/maintenance_tasks.rs +++ b/crates/tracedecay-store-runtime/src/semantic_artifact_gc.rs @@ -1,29 +1,22 @@ -//! Background maintenance owned by the daemon root. -//! -//! Long-lived-process opt-in for session-store maintenance, and the periodic -//! semantic artifact GC whose task is joined during daemon shutdown. +//! Periodic semantic-artifact GC whose task handle is joined during shutdown. use std::sync::Arc; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; -use super::*; +use tokio::task::JoinHandle; -/// Enables background maintenance only for long-lived daemon/MCP processes. -/// -/// Session-store mounts retain the registered database authority for the -/// lifetime of each maintenance task. One-shot commands never enable it. -pub fn mark_process_long_lived_for_session_maintenance() { - tracedecay_store_runtime::mark_process_long_lived_for_session_maintenance(); -} +use crate::DaemonSessionRuntimeRegistryV1; const SEMANTIC_ARTIFACT_GC_PERIOD: Duration = Duration::from_hours(24); +/// Admitted handle for the process-wide semantic artifact GC task. #[derive(Clone)] -pub(super) struct SemanticArtifactGcMaintenanceTask { +pub struct SemanticArtifactGcMaintenanceTask { task: Arc>>>, } impl SemanticArtifactGcMaintenanceTask { - pub(super) fn cancel(&self) { + pub fn cancel(&self) { if let Ok(task) = self.task.try_lock() && let Some(task) = task.as_ref() { @@ -32,7 +25,7 @@ impl SemanticArtifactGcMaintenanceTask { } #[hotpath::skip] - pub(super) async fn shutdown(self) -> std::result::Result<(), String> { + pub async fn shutdown(self) -> std::result::Result<(), String> { let mut retained = self.task.lock().await; let Some(task) = retained.as_mut() else { return Ok(()); @@ -54,8 +47,9 @@ impl Drop for SemanticArtifactGcMaintenanceTask { } } -pub(super) fn spawn_semantic_artifact_gc_maintenance( - registry: Arc, +/// Spawn the admitted semantic-artifact GC task for a live session registry. +pub fn spawn_semantic_artifact_gc_maintenance( + registry: Arc, ) -> SemanticArtifactGcMaintenanceTask { let task = tokio::spawn(hotpath::future!( async move { @@ -73,9 +67,6 @@ pub(super) fn spawn_semantic_artifact_gc_maintenance( .duration_since(UNIX_EPOCH) .unwrap_or_default() .as_secs(); - // The task-lifetime future above measures the whole loop; this - // wall span is one GC sweep, the unit a hang or cost regression - // is diagnosed against. let receipts = hotpath::measure_block!( "daemon.maintenance.semantic_artifact_gc_sweep", owner.run_daemon_artifact_gc(now_unix) @@ -88,7 +79,7 @@ pub(super) fn spawn_semantic_artifact_gc_maintenance( Err(_) => { hotpath::gauge!("daemon.maintenance.semantic_artifact_gc.failed_total") .inc(1_u64); - log_daemon_event( + crate::session_registry::log_store_runtime_event( "semantic_artifact_gc", &[("outcome", "retry_next_interval".to_owned())], ); diff --git a/crates/tracedecay-store-runtime/src/standalone_session.rs b/crates/tracedecay-store-runtime/src/standalone_session.rs new file mode 100644 index 0000000000..64747df190 --- /dev/null +++ b/crates/tracedecay-store-runtime/src/standalone_session.rs @@ -0,0 +1,43 @@ +//! Process-global owner for standalone session-runtime registries. +//! +//! Direct init/open still has a single writer for the profile session-relation +//! graph (an exclusive Grafeo file lock). A second independent registry on the +//! same profile cannot open that store. Concurrent opens in one process join +//! the live registry; entries are weak so close-then-reopen constructs a +//! fresh mount after the last holder drops. +//! +//! The composition root stores the returned lease and registers runtime ports +//! before joining. + +use std::path::PathBuf; +use std::sync::{Arc, LazyLock}; + +use tokio::sync::Mutex as AsyncMutex; +use tracedecay_daemon_identity::profile_identity::LocalProfileIdentityAuthorityV1; +use tracedecay_domain::errors::Result; +use tracedecay_runtime_core::weak_registry::WeakRegistry; + +use crate::DaemonSessionRuntimeRegistryV1; + +static STANDALONE_SESSION_REGISTRIES: LazyLock< + AsyncMutex>, +> = LazyLock::new(|| AsyncMutex::new(WeakRegistry::new())); + +/// Join the process-wide standalone session registry for `identity`. +/// +/// Returns a live lease the caller stores. Port registration stays in the +/// composition root so this crate never names root wiring. +#[hotpath::measure(label = "lifecycle.join_session_registry", future = true)] +pub async fn join_standalone_session_registry( + identity: LocalProfileIdentityAuthorityV1, +) -> Result> { + let profile_key = + tracedecay_runtime_core::lifecycle_lease::canonical_or_original(identity.profile_root()); + let registries = STANDALONE_SESSION_REGISTRIES.lock().await; + if let Some(registry) = registries.get_live(&profile_key) { + return Ok(registry); + } + let registry = Arc::new(DaemonSessionRuntimeRegistryV1::open(identity).await?); + registries.insert(profile_key, ®istry); + Ok(registry) +} diff --git a/crates/tracedecay/src/daemon/store_shutdown.rs b/crates/tracedecay-store-runtime/src/store_shutdown.rs similarity index 92% rename from crates/tracedecay/src/daemon/store_shutdown.rs rename to crates/tracedecay-store-runtime/src/store_shutdown.rs index 0ee7bdff02..db15a041cf 100644 --- a/crates/tracedecay/src/daemon/store_shutdown.rs +++ b/crates/tracedecay-store-runtime/src/store_shutdown.rs @@ -1,4 +1,4 @@ -//! Typed receipts and bounded joins for named daemon shutdown tasks. +//! Typed receipts and bounded joins for named store-runtime shutdown tasks. //! //! `join_shutdown_tasks_until` reserves an abort budget inside the caller's //! deadline: tasks get the cooperative window first, then stragglers are @@ -8,24 +8,25 @@ use std::collections::HashMap; use std::future::Future; -use super::DAEMON_TASK_ABORT_DEADLINE; -use super::shutdown_coordination::ShutdownStatus; +use tracedecay_daemon_service::ShutdownStatus; +use tracedecay_runtime_core::DAEMON_TASK_ABORT_DEADLINE; -pub(super) type ShutdownTaskStatus = ShutdownStatus; +pub type ShutdownTaskStatus = ShutdownStatus; #[derive(Clone, Debug, Eq, PartialEq)] -pub(super) struct ShutdownTaskOutcome { - pub(super) owner: String, - pub(super) status: ShutdownTaskStatus, +pub struct ShutdownTaskOutcome { + pub owner: String, + pub status: ShutdownTaskStatus, } #[derive(Clone, Debug, Default, Eq, PartialEq)] -pub(super) struct ShutdownTaskReceipt { - pub(super) outcomes: Vec, +pub struct ShutdownTaskReceipt { + pub outcomes: Vec, } impl ShutdownTaskReceipt { - pub(super) fn failed(owner: impl Into, error: impl Into) -> Self { + #[must_use] + pub fn failed(owner: impl Into, error: impl Into) -> Self { Self { outcomes: vec![ShutdownTaskOutcome { owner: owner.into(), @@ -34,7 +35,8 @@ impl ShutdownTaskReceipt { } } - pub(super) fn timed_out(owner: impl Into) -> Self { + #[must_use] + pub fn timed_out(owner: impl Into) -> Self { Self { outcomes: vec![ShutdownTaskOutcome { owner: owner.into(), @@ -43,14 +45,15 @@ impl ShutdownTaskReceipt { } } - pub(super) fn is_clean(&self) -> bool { + #[must_use] + pub fn is_clean(&self) -> bool { self.outcomes .iter() .all(|outcome| outcome.status == ShutdownTaskStatus::Clean) } - #[cfg(test)] - pub(super) fn status(&self) -> ShutdownTaskStatus { + #[must_use] + pub fn status(&self) -> ShutdownTaskStatus { let failures = self .outcomes .iter() @@ -72,11 +75,11 @@ impl ShutdownTaskReceipt { } } - pub(super) fn extend(&mut self, mut other: Self) { + pub fn extend(&mut self, mut other: Self) { self.outcomes.append(&mut other.outcomes); } - pub(super) fn retain_failures_from(&mut self, failures: &[ShutdownTaskOutcome]) { + pub fn retain_failures_from(&mut self, failures: &[ShutdownTaskOutcome]) { for failure in failures { let ShutdownTaskStatus::Failed(prior_error) = &failure.status else { continue; @@ -103,14 +106,16 @@ impl ShutdownTaskReceipt { } } - pub(super) fn failed_count(&self) -> usize { + #[must_use] + pub fn failed_count(&self) -> usize { self.outcomes .iter() .filter(|outcome| matches!(outcome.status, ShutdownTaskStatus::Failed(_))) .count() } - pub(super) fn timed_out_count(&self) -> usize { + #[must_use] + pub fn timed_out_count(&self) -> usize { self.outcomes .iter() .filter(|outcome| outcome.status == ShutdownTaskStatus::TimedOut) @@ -118,7 +123,7 @@ impl ShutdownTaskReceipt { } } -pub(super) async fn join_shutdown_tasks_until( +pub async fn join_shutdown_tasks_until( deadline: tokio::time::Instant, tasks: Tasks, ) -> ShutdownTaskReceipt diff --git a/crates/tracedecay/src/daemon/store_writer_gate.rs b/crates/tracedecay-store-runtime/src/writer_gate.rs similarity index 89% rename from crates/tracedecay/src/daemon/store_writer_gate.rs rename to crates/tracedecay-store-runtime/src/writer_gate.rs index e0dba6541f..3f104f7b04 100644 --- a/crates/tracedecay/src/daemon/store_writer_gate.rs +++ b/crates/tracedecay-store-runtime/src/writer_gate.rs @@ -1,6 +1,6 @@ -//! Per-store daemon writer gates. +//! Per-store writer gates. //! -//! # Why this is not one daemon-wide mutex +//! # Why this is not one process-wide mutex //! //! Writer administration used to be a single process-wide `Mutex`. Its own //! comment conceded that "a background refresh or a generation rebuild can hold @@ -30,11 +30,11 @@ //! * **Daemon scope still excludes everything.** A `Daemon` acquisition takes //! `daemon.write()`, which no store-scoped acquisition can hold concurrently. //! * **Owner and Content are deliberately concurrent.** They are disjoint -//! concerns: `Owner` mutates the daemon's in-memory owner/scheduler -//! bookkeeping for a store, `Content` writes index rows into a store that is -//! already open. They were only serialized before because there was one gate. -//! Content writes were never exclusive against the rest of the daemon anyway -//! — hook writes and memory writes go straight to the store without taking +//! concerns: `Owner` mutates in-memory owner/scheduler bookkeeping for a +//! store, `Content` writes index rows into a store that is already open. +//! They were only serialized before because there was one gate. Content +//! writes were never exclusive against the rest of the process anyway — +//! hook writes and memory writes go straight to the store without taking //! this gate at all — so admitting an owner-bookkeeping mutation beside a //! sync adds no writer that did not already exist. @@ -46,9 +46,9 @@ use tokio::sync::{Mutex, OwnedMutexGuard, OwnedRwLockReadGuard, OwnedRwLockWrite /// What one writer acquisition is allowed to do to a store. #[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub(super) enum StoreWriterClass { - /// Mutates the daemon's owner/scheduler bookkeeping for the store (project - /// open, owner rekey, scheduler start/stop). Serialized against itself. +pub enum StoreWriterClass { + /// Mutates owner/scheduler bookkeeping for the store (project open, owner + /// rekey, scheduler start/stop). Serialized against itself. Owner, /// Writes index content into an already-open store (git-watch sync, /// background refresh). Serialized against itself. @@ -57,8 +57,8 @@ pub(super) enum StoreWriterClass { /// Which lane a writer acquisition takes. #[derive(Clone, Debug)] -pub(super) enum WriterScope { - /// Daemon-wide exclusion. Reserved for operations that sweep every mounted +pub enum WriterScope { + /// Process-wide exclusion. Reserved for operations that sweep every mounted /// store, or whose store cannot be resolved. Daemon, /// Exclusion scoped to one store family, keyed by its canonical `data_root`. @@ -72,7 +72,8 @@ impl WriterScope { /// Store-scoped acquisition for `data_root`. The caller is responsible for /// passing a canonical path; [`StoreWriterGates`] keys on it verbatim so /// that a mismatched key can never silently split a store's gate. - pub(super) fn store(data_root: impl Into, class: StoreWriterClass) -> Self { + #[must_use] + pub fn store(data_root: impl Into, class: StoreWriterClass) -> Self { Self::Store { data_root: data_root.into(), class, @@ -102,7 +103,7 @@ impl StoreGate { /// /// Dropping this releases the whole hierarchy. The fields are never read; they /// exist to pin the guards. -pub(super) struct WriterAdmissionGuard { +pub struct WriterAdmissionGuard { _class: Option>, _daemon: DaemonGuard, /// Keeps the store's gate alive for as long as it is held, so a registry @@ -119,9 +120,8 @@ enum DaemonGuard { Exclusive(OwnedRwLockWriteGuard<()>), } -/// The daemon's writer-gate registry: one daemon-wide lane plus one lane set -/// per store family. -pub(super) struct StoreWriterGates { +/// Writer-gate registry: one process-wide lane plus one lane set per store family. +pub struct StoreWriterGates { daemon: Arc>, stores: std::sync::Mutex>>, } @@ -159,7 +159,7 @@ impl StoreWriterGates { /// Number of live store gates. Test-only observability for the isolation /// proofs. #[cfg(test)] - pub(super) fn live_store_gates(&self) -> usize { + pub fn live_store_gates(&self) -> usize { let mut stores = self .stores .lock() @@ -170,7 +170,7 @@ impl StoreWriterGates { /// Acquires admission for `scope`, waiting as long as necessary. #[hotpath::skip] - pub(super) async fn acquire(&self, scope: &WriterScope) -> WriterAdmissionGuard { + pub async fn acquire(&self, scope: &WriterScope) -> WriterAdmissionGuard { match scope { WriterScope::Daemon => WriterAdmissionGuard { _class: None, @@ -206,7 +206,7 @@ impl StoreWriterGates { /// Acquires admission only if every level is free right now. #[hotpath::measure(label = "daemon.writer_gate.try_acquire")] - pub(super) fn try_acquire(&self, scope: &WriterScope) -> Option { + pub fn try_acquire(&self, scope: &WriterScope) -> Option { match scope { WriterScope::Daemon => Some(WriterAdmissionGuard { _class: None, diff --git a/crates/tracedecay/src/daemon.rs b/crates/tracedecay/src/daemon.rs index 9335f785dc..dd2a6831b0 100644 --- a/crates/tracedecay/src/daemon.rs +++ b/crates/tracedecay/src/daemon.rs @@ -264,7 +264,6 @@ mod shutdown_orchestration; mod shutdown_watchdog; #[cfg(feature = "hotpath")] pub use shutdown_watchdog::install_hotpath_shutdown_finalizer; -mod store_shutdown; pub(crate) use core_admission::*; pub use core_client::*; pub(crate) use core_doctor::*; @@ -312,9 +311,6 @@ use lsp_sessions::{ settle_pending_lsp_workspace_mutation, update_connection_lsp_sessions, }; mod maintenance; -mod maintenance_tasks; -pub use maintenance_tasks::mark_process_long_lived_for_session_maintenance; -use maintenance_tasks::spawn_semantic_artifact_gc_maintenance; pub mod pr_autotrack; mod production_harness; mod store_maintenance; @@ -401,7 +397,6 @@ pub(crate) mod session_runtime_tests; #[cfg(test)] pub(crate) mod store_runtime_tests; -mod store_writer_gate; mod wire_io; #[cfg(test)] mod work_evidence_retrieval_tests; diff --git a/crates/tracedecay/src/daemon/bootstrap.rs b/crates/tracedecay/src/daemon/bootstrap.rs index bb00440b86..80a8290279 100644 --- a/crates/tracedecay/src/daemon/bootstrap.rs +++ b/crates/tracedecay/src/daemon/bootstrap.rs @@ -10,6 +10,7 @@ use tokio::task::JoinSet; #[cfg(unix)] use tracedecay_code_index_runtime::{GitWatchMaintenanceWakeV1, git_watch}; use tracedecay_daemon_control::RemoteBrainTlsConfig; +use tracedecay_store_runtime::spawn_semantic_artifact_gc_maintenance; use tracedecay_daemon_identity::authority; use tracedecay_domain::errors::{Result, TraceDecayError}; use tracedecay_runtime_core::DAEMON_SHUTDOWN_DEADLINE; @@ -454,7 +455,7 @@ fn log_background_shutdown_receipt(receipt: &shutdown_coordination::ShutdownRece } } -fn log_project_server_shutdown_receipt(receipt: &store_shutdown::ShutdownTaskReceipt) { +fn log_project_server_shutdown_receipt(receipt: &tracedecay_store_runtime::ShutdownTaskReceipt) { if receipt.is_clean() { return; } @@ -468,9 +469,9 @@ fn log_project_server_shutdown_receipt(receipt: &store_shutdown::ShutdownTaskRec ); for outcome in &receipt.outcomes { let status = match outcome.status { - store_shutdown::ShutdownTaskStatus::Clean => continue, - store_shutdown::ShutdownTaskStatus::Failed(_) => "failed", - store_shutdown::ShutdownTaskStatus::TimedOut => "timed_out", + tracedecay_store_runtime::ShutdownTaskStatus::Clean => continue, + tracedecay_store_runtime::ShutdownTaskStatus::Failed(_) => "failed", + tracedecay_store_runtime::ShutdownTaskStatus::TimedOut => "timed_out", }; log_daemon_event( "daemon_shutdown", diff --git a/crates/tracedecay/src/daemon/branch_admin.rs b/crates/tracedecay/src/daemon/branch_admin.rs index a849b55407..03ad405855 100644 --- a/crates/tracedecay/src/daemon/branch_admin.rs +++ b/crates/tracedecay/src/daemon/branch_admin.rs @@ -24,8 +24,8 @@ use super::profile_host_admission_replay::{ }; #[cfg(unix)] use super::scheduler::{AutomationSchedulerHandle, MaintenanceTaskTermination}; -use super::store_writer_gate::StoreWriterGates; -pub(super) use super::store_writer_gate::{StoreWriterClass, WriterScope}; +use tracedecay_store_runtime::StoreWriterGates; +pub(super) use tracedecay_store_runtime::{StoreWriterClass, WriterScope}; use super::{DaemonHandshake, DatabaseOwnerRegistry, write_json_rpc_response}; use tracedecay_code_index_runtime::git_transactions::DaemonGitIndexTransactionServiceRegistry; use tracedecay_daemon_identity::{authority, profile_identity}; @@ -432,7 +432,7 @@ impl ProfileHostAdmissionBootstrapContext { /// administration cannot prove ownership against stale daemon state. /// /// Writer admission itself is *per store* — see -/// [`store_writer_gate`](super::store_writer_gate) for the hierarchy and the +/// [`tracedecay_store_runtime::writer_gate`] for the hierarchy and the /// exclusivity argument. The proof branch administration performs is computed /// from one store family's database paths, so a writer on another store can /// never invalidate it; a single daemon-wide gate only meant a sync of project diff --git a/crates/tracedecay/src/daemon/branch_admin/project_retirement.rs b/crates/tracedecay/src/daemon/branch_admin/project_retirement.rs index 0eb6d70dd6..82945f732c 100644 --- a/crates/tracedecay/src/daemon/branch_admin/project_retirement.rs +++ b/crates/tracedecay/src/daemon/branch_admin/project_retirement.rs @@ -1,6 +1,6 @@ use std::sync::Arc; -use super::super::store_shutdown::{ShutdownTaskOutcome, ShutdownTaskReceipt, ShutdownTaskStatus}; +use tracedecay_store_runtime::{ShutdownTaskOutcome, ShutdownTaskReceipt, ShutdownTaskStatus}; use super::{StoreAdministration, StoreOwnerKey}; pub(super) struct ProjectServerRetirement { @@ -126,14 +126,14 @@ pub(in crate::daemon) struct ProjectRetirementFenceV1 { // removed by this temporary recovery guard. _invocation: tracedecay_daemon_service::ProjectRuntimeRootQuiescenceV1, _project_open: crate::daemon::project_open_admission::ProjectOpenIdentityQuiescenceV1, - _writer: crate::daemon::store_writer_gate::WriterAdmissionGuard, + _writer: tracedecay_store_runtime::WriterAdmissionGuard, } impl ProjectRetirementFenceV1 { pub(super) fn new( invocation: tracedecay_daemon_service::ProjectRuntimeRootQuiescenceV1, project_open: crate::daemon::project_open_admission::ProjectOpenIdentityQuiescenceV1, - writer: crate::daemon::store_writer_gate::WriterAdmissionGuard, + writer: tracedecay_store_runtime::WriterAdmissionGuard, ) -> Self { Self { _invocation: invocation, @@ -470,7 +470,7 @@ mod tests { use super::*; use crate::daemon::project_server_lifecycle; - use crate::daemon::store_writer_gate::{StoreWriterClass, WriterScope}; + use tracedecay_store_runtime::{StoreWriterClass, WriterScope}; fn owner(project_id: &str) -> StoreOwnerKey { isolated_owner(std::path::Path::new("/profile"), project_id) diff --git a/crates/tracedecay/src/daemon/branch_admin/remote_recovery_lifecycle.rs b/crates/tracedecay/src/daemon/branch_admin/remote_recovery_lifecycle.rs index e682883441..57c87190c6 100644 --- a/crates/tracedecay/src/daemon/branch_admin/remote_recovery_lifecycle.rs +++ b/crates/tracedecay/src/daemon/branch_admin/remote_recovery_lifecycle.rs @@ -15,7 +15,7 @@ use super::{ DatabaseOwnerRegistry, StoreAdministration, StoreWriterClass, StoreWriterGates, WriterScope, }; use crate::daemon::maintenance::StoreTelemetrySamplingRegistry; -use crate::daemon::store_writer_gate::WriterAdmissionGuard; +use tracedecay_store_runtime::WriterAdmissionGuard; use tracedecay_daemon_identity::authority; use tracedecay_daemon_service::DaemonNativeIntegrationRuntimeRegistrar; use tracedecay_domain::errors::{Result, TraceDecayError}; diff --git a/crates/tracedecay/src/daemon/engine/shutdown.rs b/crates/tracedecay/src/daemon/engine/shutdown.rs index 03650a0010..131ee5ce2b 100644 --- a/crates/tracedecay/src/daemon/engine/shutdown.rs +++ b/crates/tracedecay/src/daemon/engine/shutdown.rs @@ -23,7 +23,7 @@ use crate::daemon::shutdown_coordination::{ShutdownOwner, ShutdownStatus}; use crate::daemon::shutdown_orchestration::{ DaemonShutdownPlan, DaemonShutdownReceipt, coordinate_daemon_shutdown, }; -use crate::daemon::store_shutdown::ShutdownTaskReceipt; +use tracedecay_store_runtime::ShutdownTaskReceipt; use crate::daemon::{log_daemon_event, project_open_tasks, shutdown_project_servers}; #[cfg(test)] use tracedecay_runtime_core::DAEMON_SHUTDOWN_DEADLINE; diff --git a/crates/tracedecay/src/daemon/project_server_lifecycle.rs b/crates/tracedecay/src/daemon/project_server_lifecycle.rs index 8a58dab6ce..3d6c293532 100644 --- a/crates/tracedecay/src/daemon/project_server_lifecycle.rs +++ b/crates/tracedecay/src/daemon/project_server_lifecycle.rs @@ -6,7 +6,7 @@ use super::profile_host_admission_replay::ProfileHostAdmissionBootstrapStatus; use super::shutdown_coordination::ShutdownStatus; -use super::store_shutdown::{ShutdownTaskOutcome, ShutdownTaskReceipt, join_shutdown_tasks_until}; +use tracedecay_store_runtime::{ShutdownTaskOutcome, ShutdownTaskReceipt, join_shutdown_tasks_until}; use super::*; use std::collections::HashSet; use tracedecay_daemon_identity::authority; diff --git a/crates/tracedecay/src/daemon/shutdown_orchestration.rs b/crates/tracedecay/src/daemon/shutdown_orchestration.rs index dfdad8cfd4..b46be43d9e 100644 --- a/crates/tracedecay/src/daemon/shutdown_orchestration.rs +++ b/crates/tracedecay/src/daemon/shutdown_orchestration.rs @@ -10,7 +10,7 @@ use super::shutdown_coordination::{ DrainingGauge, ShutdownOwner, ShutdownOwnerReceipt, ShutdownReceipt, ShutdownStatus, prepare_shutdown_owner_phases, }; -use super::store_shutdown::{ShutdownTaskOutcome, ShutdownTaskReceipt}; +use tracedecay_store_runtime::{ShutdownTaskOutcome, ShutdownTaskReceipt}; use super::{ DAEMON_BACKGROUND_DRAIN_DEADLINE, DAEMON_CLIENT_DRAIN_DEADLINE, DAEMON_PROJECT_SERVER_DRAIN_DEADLINE, DAEMON_STORE_CLOSE_RESERVE, DAEMON_TASK_ABORT_DEADLINE, diff --git a/crates/tracedecay/src/daemon/store_maintenance/vector_retention_tests.rs b/crates/tracedecay/src/daemon/store_maintenance/vector_retention_tests.rs index 437e6f0a69..d43462fea0 100644 --- a/crates/tracedecay/src/daemon/store_maintenance/vector_retention_tests.rs +++ b/crates/tracedecay/src/daemon/store_maintenance/vector_retention_tests.rs @@ -13,7 +13,7 @@ use crate::daemon::maintenance::{ SemanticVectorRetentionCensusOutcome, SemanticVectorRetentionReadV1, StoreTelemetrySamplingRegistry, }; -use crate::daemon::store_writer_gate::{StoreWriterGates, WriterScope}; +use tracedecay_store_runtime::{StoreWriterGates, WriterScope}; use crate::tracedecay::TraceDecay; use tracedecay_application::semantic_runtime::ProjectSemanticActivationExt; use tracedecay_code_index_retention::code_index_generations::{ diff --git a/crates/tracedecay/src/daemon/tests/lifecycle.rs b/crates/tracedecay/src/daemon/tests/lifecycle.rs index e08acf0b73..bae9d99345 100644 --- a/crates/tracedecay/src/daemon/tests/lifecycle.rs +++ b/crates/tracedecay/src/daemon/tests/lifecycle.rs @@ -1224,7 +1224,7 @@ async fn persistent_idle_client_closes_on_draining_without_timeout() { super::super::shutdown_orchestration::DaemonShutdownPlan::new( clients, Vec::new(), - |_| async { super::super::store_shutdown::ShutdownTaskReceipt::default() }, + |_| async { tracedecay_store_runtime::ShutdownTaskReceipt::default() }, ) }, ) diff --git a/crates/tracedecay/src/project_store_runtime.rs b/crates/tracedecay/src/project_store_runtime.rs index 74cff8543f..835c22380f 100644 --- a/crates/tracedecay/src/project_store_runtime.rs +++ b/crates/tracedecay/src/project_store_runtime.rs @@ -1,43 +1,21 @@ //! Root composition for standalone project-store runtime ownership. //! -//! Standalone init/open joins one daemon session registry per profile, then -//! hands the aggregate the concrete registry it and daemon/MCP callers use. +//! Standalone init/open registers runtime ports, then joins the process-global +//! session-registry owner and stores the returned lease on [`TraceDecay`]. -use std::path::PathBuf; -use std::sync::{Arc, LazyLock}; +use std::sync::Arc; -use tokio::sync::Mutex as AsyncMutex; use tracedecay_daemon_identity::profile_identity::LocalProfileIdentityAuthorityV1; use tracedecay_domain::errors::Result; -use tracedecay_runtime_core::weak_registry::WeakRegistry; - use tracedecay_store_runtime::DaemonSessionRuntimeRegistryV1; -/// One standalone session runtime registry per profile, process-wide. -/// -/// Direct init/open still has a single writer for the profile session-relation -/// graph (an exclusive Grafeo file lock). A second independent registry on the -/// same profile cannot open that store. Concurrent opens in one process join -/// the live registry; entries are weak so close-then-reopen constructs a -/// fresh mount after the last holder drops. -static STANDALONE_SESSION_REGISTRIES: LazyLock< - AsyncMutex>, -> = LazyLock::new(|| AsyncMutex::new(WeakRegistry::new())); - +/// Join the process-wide standalone session registry after root port registration. #[hotpath::measure(label = "lifecycle.join_session_registry", future = true)] pub(crate) async fn join_standalone_session_registry( identity: LocalProfileIdentityAuthorityV1, ) -> Result> { crate::register_runtime_ports()?; - let profile_key = - tracedecay_runtime_core::lifecycle_lease::canonical_or_original(identity.profile_root()); - let registries = STANDALONE_SESSION_REGISTRIES.lock().await; - if let Some(registry) = registries.get_live(&profile_key) { - return Ok(registry); - } - let registry = Arc::new(DaemonSessionRuntimeRegistryV1::open(identity).await?); - registries.insert(profile_key, ®istry); - Ok(registry) + tracedecay_store_runtime::join_standalone_session_registry(identity).await } impl crate::tracedecay::TraceDecay { From b498dad0083d89935279aac3da460a644a744e45 Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Wed, 9 Sep 2026 17:59:43 +0000 Subject: [PATCH 2/6] refactor(maintenance): extract store kernels and tick policy --- Cargo.lock | 9 + crates/tracedecay-maintenance/Cargo.toml | 9 + crates/tracedecay-maintenance/src/clock.rs | 12 + .../src/compaction_receipt.rs | 41 + .../src}/generation.rs | 51 +- crates/tracedecay-maintenance/src/lease.rs | 107 + crates/tracedecay-maintenance/src/lib.rs | 13 + crates/tracedecay-maintenance/src/loop_run.rs | 140 ++ .../src}/store_maintenance/graph_replay.rs | 47 +- .../src/store_maintenance/mod.rs | 1789 ++++++++++++++++ .../tracedecay-maintenance/src/telemetry.rs | 984 +++++++++ crates/tracedecay-maintenance/src/tick.rs | 184 ++ crates/tracedecay/src/daemon.rs | 6 +- crates/tracedecay/src/daemon/branch_admin.rs | 11 +- .../branch_admin/remote_recovery_lifecycle.rs | 4 +- crates/tracedecay/src/daemon/doctor_kernel.rs | 12 +- crates/tracedecay/src/daemon/maintenance.rs | 1388 +------------ .../generation_retention_test.rs | 109 +- .../src/daemon/project_composition.rs | 2 +- .../src/daemon/store_maintenance/mod.rs | 1790 +---------------- .../vector_retention_tests.rs | 71 +- 21 files changed, 3490 insertions(+), 3289 deletions(-) create mode 100644 crates/tracedecay-maintenance/src/clock.rs create mode 100644 crates/tracedecay-maintenance/src/compaction_receipt.rs rename crates/{tracedecay/src/daemon/maintenance => tracedecay-maintenance/src}/generation.rs (75%) create mode 100644 crates/tracedecay-maintenance/src/lease.rs create mode 100644 crates/tracedecay-maintenance/src/loop_run.rs rename crates/{tracedecay/src/daemon => tracedecay-maintenance/src}/store_maintenance/graph_replay.rs (90%) create mode 100644 crates/tracedecay-maintenance/src/store_maintenance/mod.rs create mode 100644 crates/tracedecay-maintenance/src/telemetry.rs create mode 100644 crates/tracedecay-maintenance/src/tick.rs diff --git a/Cargo.lock b/Cargo.lock index 4bfa7ee4b1..65ce0207ad 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6711,8 +6711,11 @@ dependencies = [ "sha2 0.11.0", "tempfile", "tokio", + "tracedecay-application", "tracedecay-automation", "tracedecay-code-index-retention", + "tracedecay-code-index-runtime", + "tracedecay-configuration", "tracedecay-contracts", "tracedecay-domain", "tracedecay-global-db", @@ -6721,6 +6724,12 @@ dependencies = [ "tracedecay-private-fs", "tracedecay-runtime-core", "tracedecay-rusqlite-runtime", + "tracedecay-semantic-contracts", + "tracedecay-session-memory", + "tracedecay-store", + "tracedecay-store-runtime", + "tracedecay-tool-catalog", + "tracing", ] [[package]] diff --git a/crates/tracedecay-maintenance/Cargo.toml b/crates/tracedecay-maintenance/Cargo.toml index be63ff4314..71b0852d97 100644 --- a/crates/tracedecay-maintenance/Cargo.toml +++ b/crates/tracedecay-maintenance/Cargo.toml @@ -25,8 +25,12 @@ serde_json = "1" sha2 = "0.11" tempfile = "3" tokio = { version = "1", features = ["rt", "macros", "time", "sync"] } +tracing = "0.1" +tracedecay-application = { path = "../tracedecay-application", version = "0.1.0" } tracedecay-contracts = { path = "../tracedecay-contracts", version = "0.1.0" } tracedecay-automation = { path = "../tracedecay-automation", version = "0.1.0" } +tracedecay-code-index-runtime = { path = "../tracedecay-code-index-runtime", version = "0.1.0", default-features = false } +tracedecay-configuration = { path = "../tracedecay-configuration", version = "0.1.0" } tracedecay-domain = { path = "../tracedecay-domain", version = "0.1.0" } tracedecay-global-db = { path = "../tracedecay-global-db", version = "0.1.0" } tracedecay-graph-db = { path = "../tracedecay-graph-db", version = "0.1.0" } @@ -35,6 +39,11 @@ tracedecay-private-fs = { path = "../tracedecay-private-fs", version = "0.1.0" } tracedecay-runtime-core = { path = "../tracedecay-runtime-core", version = "0.1.0" } tracedecay-rusqlite-runtime = { path = "../tracedecay-rusqlite-runtime", version = "0.1.0" } tracedecay-code-index-retention = { path = "../tracedecay-code-index-retention", version = "0.1.0" } +tracedecay-semantic-contracts.workspace = true +tracedecay-session-memory = { path = "../tracedecay-session-memory", version = "0.1.0" } +tracedecay-store = { path = "../tracedecay-store", version = "0.1.0" } +tracedecay-store-runtime = { path = "../tracedecay-store-runtime", version = "0.1.0" } +tracedecay-tool-catalog = { path = "../tracedecay-tool-catalog", version = "0.1.0" } [target.'cfg(any(all(target_os = "linux", target_env = "gnu"), target_os = "macos"))'.dependencies] libc = "0.2" diff --git a/crates/tracedecay-maintenance/src/clock.rs b/crates/tracedecay-maintenance/src/clock.rs new file mode 100644 index 0000000000..095d4c0ef6 --- /dev/null +++ b/crates/tracedecay-maintenance/src/clock.rs @@ -0,0 +1,12 @@ +//! Wall-clock helpers for retention cutoffs. + +use std::time::{SystemTime, UNIX_EPOCH}; + +/// Current Unix time in seconds, or a typed clock failure. +pub fn now_secs_i64() -> Result { + let seconds = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_err(|_| "system_clock_before_unix_epoch")? + .as_secs(); + i64::try_from(seconds).map_err(|_| "system_clock_out_of_range") +} diff --git a/crates/tracedecay-maintenance/src/compaction_receipt.rs b/crates/tracedecay-maintenance/src/compaction_receipt.rs new file mode 100644 index 0000000000..7b3ea6a084 --- /dev/null +++ b/crates/tracedecay-maintenance/src/compaction_receipt.rs @@ -0,0 +1,41 @@ +//! Live-compaction outcome → operator receipt. + +use crate::log_maintenance_event; +use crate::retention::live_compaction::LiveStoreCompactionOutcomeV1; + +/// Record one live-compaction outcome and return whether the store is healthy. +#[must_use] +pub fn record_live_compaction_outcome( + store_name: &'static str, + outcome: LiveStoreCompactionOutcomeV1, +) -> bool { + match outcome { + LiveStoreCompactionOutcomeV1::NotScheduled => true, + LiveStoreCompactionOutcomeV1::Compacted { + freelist_before, + freelist_after, + } => { + log_maintenance_event( + "retention_compaction", + &[ + ("store", store_name.to_owned()), + ( + "freed_pages", + freelist_before.saturating_sub(freelist_after).to_string(), + ), + ], + ); + true + } + LiveStoreCompactionOutcomeV1::Failed(failure) => { + log_maintenance_event( + "retention_degraded", + &[ + ("pass", "compaction".to_owned()), + ("failure", failure.as_str().to_owned()), + ], + ); + false + } + } +} diff --git a/crates/tracedecay/src/daemon/maintenance/generation.rs b/crates/tracedecay-maintenance/src/generation.rs similarity index 75% rename from crates/tracedecay/src/daemon/maintenance/generation.rs rename to crates/tracedecay-maintenance/src/generation.rs index 70798ab5f4..6fb4c10856 100644 --- a/crates/tracedecay/src/daemon/maintenance/generation.rs +++ b/crates/tracedecay-maintenance/src/generation.rs @@ -1,12 +1,16 @@ //! Ordered generation retention for one mounted project. -use super::{ - MaintenanceContinuation, MaintenanceTickOutcome, StoreTelemetrySamplingRegistry, - record_live_compaction_outcome, +use crate::compaction_receipt::record_live_compaction_outcome; +use crate::lease::ProjectStoreMaintenanceLeaseV1; +use crate::retention::branch_compaction::CompactionThresholdConfig; +use crate::store_maintenance::{ + CodeGenerationRetentionOutcomeV1, run_branch_compaction, run_code_generation_retention, + run_code_index_scope_reconciliation, run_semantic_vector_generation_retention, }; -use crate::daemon::store_maintenance::CodeGenerationRetentionOutcomeV1; +use crate::telemetry::StoreTelemetrySamplingRegistry; +use crate::tick::{MaintenanceContinuation, MaintenanceTickOutcome}; -/// Run the production generation-maintenance journey for one mounted project. +/// Run the production generation-maintenance journey for one admitted store lease. /// /// Vector generations converge before their source code generations can be /// collected. Scope deletion is admitted only from a complete @@ -24,21 +28,18 @@ use crate::daemon::store_maintenance::CodeGenerationRetentionOutcomeV1; /// draining a superseded backlog on the short cadence without re-running /// scope reconciliation or compaction. #[hotpath::measure(label = "daemon.maintenance.generation", future = true)] -pub(in crate::daemon) async fn run_project_generation_maintenance( - graph: &crate::tracedecay::TraceDecay, +pub async fn run_project_generation_maintenance( + lease: &ProjectStoreMaintenanceLeaseV1, code_index_schedulers: &tracedecay_code_index_runtime::code_index_scheduler::CodeIndexSchedulerRegistryV1, maintenance_observations: &StoreTelemetrySamplingRegistry, cancellation: &tracedecay_session_memory::context::CancellationToken, - retention: &crate::config::RetentionConfig, + compaction: Option<&CompactionThresholdConfig>, continuation: Option, ) -> MaintenanceTickOutcome { - // Each ordered phase gets its own wall span: the outer generation span is - // inclusive, so a slow tick is attributed to vector retention, code - // generation retention, scope reconciliation, or compaction — not guessed. let mut outcome = hotpath::measure_block!( "daemon.maintenance.vector_retention", - crate::daemon::store_maintenance::run_semantic_vector_generation_retention( - graph, + run_semantic_vector_generation_retention( + lease, code_index_schedulers, maintenance_observations, cancellation, @@ -54,8 +55,8 @@ pub(in crate::daemon) async fn run_project_generation_maintenance( } else { hotpath::measure_block!( "daemon.maintenance.code_generation_retention", - crate::daemon::store_maintenance::run_code_generation_retention( - graph, + run_code_generation_retention( + lease, code_index_schedulers, maintenance_observations, cancellation, @@ -82,12 +83,12 @@ pub(in crate::daemon) async fn run_project_generation_maintenance( if semantic_collection_complete && code_generation == CodeGenerationRetentionOutcomeV1::Complete && !cancellation.is_cancelled() - && maintenance_observations.semantic_vector_scope_collection_ready(graph.project_root()) + && maintenance_observations.semantic_vector_scope_collection_ready(lease.project_root()) { let scope_reconciled = hotpath::measure_block!( "daemon.maintenance.scope_reconciliation", - crate::daemon::store_maintenance::run_code_index_scope_reconciliation( - graph, + run_code_index_scope_reconciliation( + lease, code_index_schedulers, maintenance_observations, ) @@ -98,13 +99,13 @@ pub(in crate::daemon) async fn run_project_generation_maintenance( } } if !cancellation.is_cancelled() - && let Some(compaction) = &retention.compaction + && let Some(compaction) = compaction { hotpath::measure_block!("daemon.maintenance.compaction", { let project_compacted = record_live_compaction_outcome( - crate::config::DB_FILENAME, - tracedecay_maintenance::retention::live_compaction::compact_project_store( - graph.db(), + tracedecay_runtime_core::config::DB_FILENAME, + crate::retention::live_compaction::compact_project_store( + lease.graph_db(), compaction, ) .await, @@ -113,9 +114,7 @@ pub(in crate::daemon) async fn run_project_generation_maintenance( outcome = MaintenanceTickOutcome::Retry; } if !cancellation.is_cancelled() { - let branch_compacted = - crate::daemon::store_maintenance::run_branch_compaction(graph, compaction) - .await; + let branch_compacted = run_branch_compaction(lease, compaction).await; if !branch_compacted { outcome = MaintenanceTickOutcome::Retry; } @@ -125,8 +124,6 @@ pub(in crate::daemon) async fn run_project_generation_maintenance( finalize_generation_outcome(outcome, cancellation) } -/// Cancelled and degraded ticks are recorded too: a maintenance lane that -/// silently retries forever is exactly the waste being diagnosed. fn finalize_generation_outcome( outcome: MaintenanceTickOutcome, cancellation: &tracedecay_session_memory::context::CancellationToken, diff --git a/crates/tracedecay-maintenance/src/lease.rs b/crates/tracedecay-maintenance/src/lease.rs new file mode 100644 index 0000000000..68fb8a19cf --- /dev/null +++ b/crates/tracedecay-maintenance/src/lease.rs @@ -0,0 +1,107 @@ +//! Admitted project-store lease for maintenance kernels. +//! +//! Callers extract these fields from a mounted project store. Kernels never +//! name the composition-root aggregate. + +use std::path::{Path, PathBuf}; +use std::sync::Arc; + +use tracedecay_configuration::ProjectConfigurationRuntime; +use tracedecay_domain::ProjectId; +use tracedecay_domain::errors::Result; +use tracedecay_global_db::RegisteredGlobalDbLeaseV1; +use tracedecay_runtime_core::db::Database; +use tracedecay_runtime_core::storage::{self, StoreLayout}; +use tracedecay_store_runtime::DaemonSessionRuntimeRegistryV1; + +/// Registered store lease for one mounted project's maintenance journey. +#[derive(Clone)] +pub struct ProjectStoreMaintenanceLeaseV1 { + project_root: PathBuf, + store_layout: StoreLayout, + graph_db: Database, + store_runtime: Arc, + configuration_runtime: Arc, + profile_database: RegisteredGlobalDbLeaseV1, +} + +impl ProjectStoreMaintenanceLeaseV1 { + #[must_use] + pub fn new( + project_root: PathBuf, + store_layout: StoreLayout, + graph_db: Database, + store_runtime: Arc, + configuration_runtime: Arc, + profile_database: RegisteredGlobalDbLeaseV1, + ) -> Self { + Self { + project_root, + store_layout, + graph_db, + store_runtime, + configuration_runtime, + profile_database, + } + } + + #[must_use] + pub fn project_root(&self) -> &Path { + &self.project_root + } + + #[must_use] + pub fn store_layout(&self) -> &StoreLayout { + &self.store_layout + } + + #[must_use] + pub fn graph_db(&self) -> &Database { + &self.graph_db + } + + #[must_use] + pub fn store_runtime(&self) -> &Arc { + &self.store_runtime + } + + #[must_use] + pub fn configuration_runtime(&self) -> &Arc { + &self.configuration_runtime + } + + #[must_use] + pub fn profile_database(&self) -> &RegisteredGlobalDbLeaseV1 { + &self.profile_database + } +} + +/// Filter candidate roots to those whose on-disk identity names `project_id`. +pub fn enrolled_project_roots( + candidates: impl IntoIterator, + project_id: &ProjectId, +) -> Result> { + let mut candidates = candidates.into_iter().collect::>(); + candidates.sort(); + candidates.dedup(); + + let mut roots = Vec::new(); + for candidate in candidates { + let candidate = tracedecay_runtime_core::worktree::repository_identity_root(&candidate) + .unwrap_or(candidate); + let Ok(canonical) = candidate.canonicalize() else { + continue; + }; + if roots.contains(&canonical) { + continue; + } + let named_id = match storage::read_repository_identity_marker(&canonical)? { + Some(marker) => marker.project_id, + None => storage::default_profile_project_id(&canonical), + }; + if named_id == project_id.as_str() { + roots.push(canonical); + } + } + Ok(roots) +} diff --git a/crates/tracedecay-maintenance/src/lib.rs b/crates/tracedecay-maintenance/src/lib.rs index d024d708e6..5e7acf9b53 100644 --- a/crates/tracedecay-maintenance/src/lib.rs +++ b/crates/tracedecay-maintenance/src/lib.rs @@ -45,5 +45,18 @@ #![allow(clippy::large_futures)] #![allow(unreachable_pub)] +pub mod clock; +pub mod compaction_receipt; +pub mod generation; +pub mod lease; +pub mod loop_run; pub mod profile_backup; pub mod retention; +pub mod store_maintenance; +pub mod telemetry; +pub mod tick; + +/// Operator-log line for a maintenance kernel. Callers supply structured fields. +pub fn log_maintenance_event(event: &str, fields: &[(&str, String)]) { + tracing::info!(target: "tracedecay_maintenance", event, ?fields, "maintenance event"); +} diff --git a/crates/tracedecay-maintenance/src/loop_run.rs b/crates/tracedecay-maintenance/src/loop_run.rs new file mode 100644 index 0000000000..3ff5b81b82 --- /dev/null +++ b/crates/tracedecay-maintenance/src/loop_run.rs @@ -0,0 +1,140 @@ +//! Cadence loop that drives admitted maintenance ticks. + +use std::future::Future; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::time::Duration; + +use tokio::sync::Notify; + +use crate::tick::{ + CadenceInstant, MaintenanceCadence, MaintenanceContinuation, MaintenanceTickOutcome, +}; + +static MAINTENANCE_FUTURES_ACTIVE: AtomicUsize = AtomicUsize::new(0); + +/// Process-wide count of live maintenance loops. Tests isolate overlapping loops. +#[must_use] +pub fn maintenance_futures_active() -> usize { + MAINTENANCE_FUTURES_ACTIVE.load(Ordering::SeqCst) +} + +struct MaintenanceLifecycleInstrumentation; + +impl MaintenanceLifecycleInstrumentation { + fn new() -> Self { + let active = MAINTENANCE_FUTURES_ACTIVE.fetch_add(1, Ordering::SeqCst) + 1; + hotpath::gauge!("daemon_maintenance_futures_active").set(active); + Self + } + + fn record_outcome(&self, outcome: MaintenanceTickOutcome) { + match outcome { + MaintenanceTickOutcome::Complete => { + hotpath::gauge!("daemon_maintenance_outcome_complete").inc(1.0); + } + MaintenanceTickOutcome::Continue(MaintenanceContinuation::SemanticVectorRetention) => { + hotpath::gauge!("daemon_maintenance_outcome_semantic_vector_progress").inc(1.0); + } + MaintenanceTickOutcome::Continue(MaintenanceContinuation::CodeGenerationRetention) => { + hotpath::gauge!("daemon_maintenance_outcome_code_generation_progress").inc(1.0); + } + MaintenanceTickOutcome::Retry => { + hotpath::gauge!("daemon_maintenance_outcome_retry").inc(1.0); + } + } + } + + fn record_cancellation(&self) { + hotpath::gauge!("daemon_maintenance_outcome_cancelled").inc(1.0); + } +} + +impl Drop for MaintenanceLifecycleInstrumentation { + fn drop(&mut self) { + let active = MAINTENANCE_FUTURES_ACTIVE + .fetch_sub(1, Ordering::SeqCst) + .saturating_sub(1); + hotpath::gauge!("daemon_maintenance_futures_active").set(active); + } +} + +struct MaintenancePhaseInstrumentation { + continuation: Option, +} + +impl MaintenancePhaseInstrumentation { + fn new(continuation: Option) -> Self { + match continuation { + Some(MaintenanceContinuation::SemanticVectorRetention) => { + hotpath::gauge!("daemon_maintenance_phase_semantic_vector_active").inc(1.0); + } + Some(MaintenanceContinuation::CodeGenerationRetention) => { + hotpath::gauge!("daemon_maintenance_phase_code_generation_active").inc(1.0); + } + None => { + hotpath::gauge!("daemon_maintenance_phase_full_tick_active").inc(1.0); + } + } + Self { continuation } + } +} + +impl Drop for MaintenancePhaseInstrumentation { + fn drop(&mut self) { + match self.continuation { + Some(MaintenanceContinuation::SemanticVectorRetention) => { + hotpath::gauge!("daemon_maintenance_phase_semantic_vector_active").inc(-1.0); + } + Some(MaintenanceContinuation::CodeGenerationRetention) => { + hotpath::gauge!("daemon_maintenance_phase_code_generation_active").inc(-1.0); + } + None => { + hotpath::gauge!("daemon_maintenance_phase_full_tick_active").inc(-1.0); + } + } + } +} + +/// Park on cancel / wake / cadence, then run the next admitted tick. +pub async fn run_maintenance_loop( + cancellation: &tracedecay_session_memory::context::CancellationToken, + wake: &Notify, + interval: Duration, + mut run_tick: F, +) where + F: FnMut(Option) -> Fut, + Fut: Future, +{ + let lifecycle = MaintenanceLifecycleInstrumentation::new(); + let mut cadence = MaintenanceCadence::new(interval); + let mut deadline = CadenceInstant::now() + cadence.retry_delay(); + let mut continuation = None; + loop { + tokio::select! { + biased; + () = cancellation.cancelled() => { + lifecycle.record_cancellation(); + break; + } + () = wake.notified() => {} + () = tokio::time::sleep_until(deadline) => {} + } + if cancellation.is_cancelled() { + lifecycle.record_cancellation(); + break; + } + let now = CadenceInstant::now(); + if now < deadline || !cadence.reserve(now) { + continue; + } + let _phase = MaintenancePhaseInstrumentation::new(continuation); + let outcome = run_tick(continuation).await; + if cancellation.is_cancelled() { + lifecycle.record_cancellation(); + break; + } + lifecycle.record_outcome(outcome); + continuation = outcome.continuation(); + deadline = cadence.finish(CadenceInstant::now(), outcome); + } +} diff --git a/crates/tracedecay/src/daemon/store_maintenance/graph_replay.rs b/crates/tracedecay-maintenance/src/store_maintenance/graph_replay.rs similarity index 90% rename from crates/tracedecay/src/daemon/store_maintenance/graph_replay.rs rename to crates/tracedecay-maintenance/src/store_maintenance/graph_replay.rs index e8d211f8bc..27d7474bfd 100644 --- a/crates/tracedecay/src/daemon/store_maintenance/graph_replay.rs +++ b/crates/tracedecay-maintenance/src/store_maintenance/graph_replay.rs @@ -1,12 +1,13 @@ use std::path::Path; -use super::{TraceDecay, log_daemon_event}; +use crate::lease::ProjectStoreMaintenanceLeaseV1; +use crate::log_maintenance_event; use tracedecay_code_index_retention::code_index_generations::{ code_generation_graph_replay_release_page, complete_code_generation_graph_replay_release, try_acquire_code_generation_store_lock, }; -pub(super) enum ReconcileOutcome { +pub enum ReconcileOutcome { /// Every queued release event has been consumed. Complete, /// The bounded page was served and more queued release events remain; the @@ -36,8 +37,8 @@ fn retire_generation_read_bundle(store_root: &Path, generation_file: &str) -> Re .map_err(|error| error.to_string()) } -pub(super) fn log_code_generation_retention_degraded( - observations: &crate::daemon::maintenance::StoreTelemetrySamplingRegistry, +pub fn log_code_generation_retention_degraded( + observations: &crate::telemetry::StoreTelemetrySamplingRegistry, project_root: &Path, failure: &str, ) { @@ -47,8 +48,8 @@ pub(super) fn log_code_generation_retention_degraded( /// Shared deferral for a held graph-replay pool: the outer probe and the /// collection executor's typed busy result arm the same backoff and must /// not keep the daemon writer gate. -pub(super) fn defer_graph_replay_pool_busy( - observations: &crate::daemon::maintenance::StoreTelemetrySamplingRegistry, +pub fn defer_graph_replay_pool_busy( + observations: &crate::telemetry::StoreTelemetrySamplingRegistry, project_root: &Path, ) -> super::CodeGenerationRetentionOutcomeV1 { observations.record_graph_replay_release_unhealthy(project_root); @@ -62,12 +63,12 @@ pub(super) fn defer_graph_replay_pool_busy( /// on every retention tick with no way to tell an unregistered graph shard /// from a pool-lock deadline from a conflict. fn log_code_generation_retention_degraded_with_error( - observations: &crate::daemon::maintenance::StoreTelemetrySamplingRegistry, + observations: &crate::telemetry::StoreTelemetrySamplingRegistry, failure: &str, error: &dyn std::fmt::Debug, ) { observations.mark_loud_retention_log(); - log_daemon_event( + log_maintenance_event( "retention_degraded", &[ ("pass", "code_generations".to_string()), @@ -103,7 +104,7 @@ fn release_failure_is_runtime_unhealthy(error: &tracedecay_graph_db::GraphDbErro /// probe-to-execute window defers with `GraphReplayPoolBusy` instead of /// pinning the daemon writer gate. #[hotpath::measure(label = "daemon.git.maintenance.replay_pool_probe")] -pub(super) fn replay_pool_is_held(replay_pool_root: &Path) -> bool { +pub fn replay_pool_is_held(replay_pool_root: &Path) -> bool { if !replay_pool_root.is_dir() { return false; } @@ -120,16 +121,16 @@ pub(super) fn replay_pool_is_held(replay_pool_root: &Path) -> bool { } #[hotpath::measure(label = "daemon.git.maintenance.graph_replay_release", future = true)] -pub(super) async fn reconcile_graph_replay_releases( - graph: &TraceDecay, +pub async fn reconcile_graph_replay_releases( + lease: &ProjectStoreMaintenanceLeaseV1, store_root: &Path, - observations: &crate::daemon::maintenance::StoreTelemetrySamplingRegistry, + observations: &crate::telemetry::StoreTelemetrySamplingRegistry, cancellation: &tracedecay_session_memory::context::CancellationToken, ) -> ReconcileOutcome { - let Some(project_id) = graph.hook_store_layout().identity.project_id.as_ref() else { + let Some(project_id) = lease.store_layout().identity.project_id.as_ref() else { log_code_generation_retention_degraded( observations, - graph.project_root(), + lease.project_root(), "graph_replay_project_identity_unavailable", ); return ReconcileOutcome::Failed; @@ -139,13 +140,13 @@ pub(super) async fn reconcile_graph_replay_releases( Err(_) => { log_code_generation_retention_degraded( observations, - graph.project_root(), + lease.project_root(), "graph_replay_project_identity_invalid", ); return ReconcileOutcome::Failed; } }; - let project_root = graph.project_root(); + let project_root = lease.project_root(); // A runtime that answered its last attempts with deadline or // unavailability failures is skipped for the bounded backoff window // instead of being polled — and timed out against — on every tick. The @@ -156,11 +157,11 @@ pub(super) async fn reconcile_graph_replay_releases( return ReconcileOutcome::Deferred; } let staging_cursor = observations.graph_staging_release_cursor(project_root); - let staging_release = graph - .store_runtime_registry() + let staging_release = lease + .store_runtime() .release_one_sealed_generation_staging_rows( project_id.clone(), - graph.db(), + lease.graph_db(), cancellation, staging_cursor, ) @@ -203,11 +204,11 @@ pub(super) async fn reconcile_graph_replay_releases( if cancellation.is_cancelled() { return ReconcileOutcome::Failed; } - match graph - .store_runtime_registry() + match lease + .store_runtime() .reconcile_deleted_code_generation_graph_replays( project_id.clone(), - graph.db(), + lease.graph_db(), &release.generation.generation_id, &release.generation.generation_file, cancellation, @@ -223,7 +224,7 @@ pub(super) async fn reconcile_graph_replay_releases( retire_generation_read_bundle(store_root, &release.generation.generation_file) { observations.mark_loud_retention_log(); - log_daemon_event( + log_maintenance_event( "retention_degraded", &[ ("pass", "code_generations".to_string()), diff --git a/crates/tracedecay-maintenance/src/store_maintenance/mod.rs b/crates/tracedecay-maintenance/src/store_maintenance/mod.rs new file mode 100644 index 0000000000..1874c66140 --- /dev/null +++ b/crates/tracedecay-maintenance/src/store_maintenance/mod.rs @@ -0,0 +1,1789 @@ +//! Retention, compaction, and garbage-collection operations run by the daemon +//! maintenance owner. +//! +//! Every operation that opens or garbage-collects a store lives here so its +//! [`StoreAdministration`] lifetime is kept separate from the watcher state +//! machine. The git watcher itself never opens or mutates a store: it routes +//! exact-frontier freshness requests to the code-index scheduler and wakes the +//! maintenance owner. + +use std::path::{Path, PathBuf}; + +use crate::clock::now_secs_i64; +use crate::lease::ProjectStoreMaintenanceLeaseV1; +use crate::log_maintenance_event; +use crate::retention::branch_compaction::CompactionThresholdConfig; +use crate::telemetry::StoreTelemetrySamplingRegistry; +use crate::tick::MaintenanceTickOutcome; +use tracedecay_application::semantic_runtime::ProjectSemanticActivationExt; +use tracedecay_code_index_runtime::code_index_scheduler::CodeIndexSchedulerRegistryV1; +use tracedecay_semantic_contracts::SemanticConfig; + +mod graph_replay; +use graph_replay::{defer_graph_replay_pool_busy, log_code_generation_retention_degraded}; + +struct ScopeRootProofInputsV1 { + live_roots: std::collections::BTreeSet, + registered_roots: + tracedecay_code_index_retention::code_index_generations::ScopeRootAuthorityReceiptV1, + git_worktrees: + tracedecay_code_index_retention::code_index_generations::ScopeRootAuthorityReceiptV1, + mounted_leases: + tracedecay_code_index_retention::code_index_generations::ScopeRootAuthorityReceiptV1, + configuration_roots: + tracedecay_code_index_retention::code_index_generations::ScopeRootAuthorityReceiptV1, + vector_census: + tracedecay_code_index_retention::code_index_generations::ScopeRootAuthorityReceiptV1, + vector_dependencies: + tracedecay_code_index_retention::code_index_generations::ScopeRootAuthorityReceiptV1, + vector_sources: std::collections::BTreeSet, +} + +impl ScopeRootProofInputsV1 { + fn bind_candidate( + &self, + scope_hash: String, + source_scope: tracedecay_store::StoreShardIdV1, + vector_revision: tracedecay_store::SemanticVectorStageCensusRevision, + ) -> Result< + tracedecay_code_index_retention::code_index_generations::ScopeRootLivenessProofV1, + &'static str, + > { + let live_scope_hashes = self + .live_roots + .iter() + .map(|root| { + tracedecay_code_index_retention::code_index_generations::code_index_scope_hash(root) + }) + .collect(); + tracedecay_code_index_retention::code_index_generations::ScopeRootLivenessProofV1::new( + live_scope_hashes, + self.registered_roots.clone(), + self.git_worktrees.clone(), + self.mounted_leases.clone(), + self.configuration_roots.clone(), + self.vector_census.clone(), + self.vector_dependencies.clone(), + tracedecay_code_index_retention::code_index_generations::ScopeRootCandidateBindingV1 { + scope_hash, + source_scope, + vector_census_revision: vector_revision.get().to_string(), + live: false, + }, + ) + .map_err(|_| "scope_liveness_proof_invalid") + } +} + +/// Advance one bounded project-wide semantic-vector retention page. +/// +/// The maintenance observation registry carries the stage cursor across ticks. +/// A mutating action resets the cursor because the returned census described +/// pre-action state; a no-action page advances it, and end-of-census publishes +/// only fixed-size aggregate counts for Doctor. +#[hotpath::measure( + label = "daemon.git.maintenance.semantic_vector_retention", + future = true +)] +pub async fn run_semantic_vector_generation_retention( + lease: &ProjectStoreMaintenanceLeaseV1, + schedulers: &CodeIndexSchedulerRegistryV1, + observations: &StoreTelemetrySamplingRegistry, + cancellation: &tracedecay_session_memory::context::CancellationToken, +) -> MaintenanceTickOutcome { + let root = lease.project_root(); + if cancellation.is_cancelled() { + observations.record_semantic_vector_retention_failure(root); + log_semantic_vector_retention_degraded(observations, root, "retention_cancelled"); + return MaintenanceTickOutcome::Retry; + } + let Some(configuration) = lease + .configuration_runtime() + .semantic_configuration_inventory_authority() + else { + // The activation coordinator is not seated. Whether that is the + // ordinary default-off state or a project-open overlap is decided by + // the durable semantic configuration, never by mount timing: a + // committed retrieval profile means a coordinator is expected + // imminently, so the pass stays retryable on the short cadence + // instead of pinning quiet and making the first census wait a full + // maintenance interval. + return match lease.configuration_runtime().client().current().await { + Ok(runtime_configuration) + if semantic_retrieval_profiles_disabled( + &runtime_configuration.config().semantic, + ) => + { + // Default off: no committed active or rollback retrieval + // profile, so no census will ever complete. Pin the typed + // unseated read for the code-generation pass and succeed + // quietly instead of resetting to Unknown and re-logging a + // degraded retry loop every tick. + observations.record_semantic_vector_retention_unseated(root); + MaintenanceTickOutcome::Complete + } + Ok(_) => { + observations.record_semantic_vector_retention_failure(root); + log_semantic_vector_retention_degraded( + observations, + root, + "configuration_inventory_unavailable", + ); + MaintenanceTickOutcome::Retry + } + Err(_) => { + observations.record_semantic_vector_retention_failure(root); + log_semantic_vector_retention_degraded( + observations, + root, + "runtime_configuration_unavailable", + ); + MaintenanceTickOutcome::Retry + } + }; + }; + let after = observations.semantic_vector_retention_cursor(root); + match tracedecay_code_index_runtime::code_index_scheduler::semantic_vector_graph::retire_one_project_vector_generation( + schedulers, + root, + &configuration, + after, + ) + .await + { + tracedecay_code_index_runtime::code_index_scheduler::semantic_vector_graph::ProjectSemanticVectorRetentionStep::Ready( + census, + ) => { + let convergence_pending = census.continuation.is_some() + || matches!( + census.action, + tracedecay_graph_db::SemanticVectorRetentionAction::Retired(_) + | tracedecay_graph_db::SemanticVectorRetentionAction::Finalized(_) + | tracedecay_graph_db::SemanticVectorRetentionAction::CancelledRemoved(_) + ); + if let Some(failure) = observations + .record_semantic_vector_retention_census(root, &census) + .as_failure_label() + { + log_semantic_vector_retention_degraded(observations, root, failure); + return MaintenanceTickOutcome::Retry; + } + if !matches!( + census.action, + tracedecay_graph_db::SemanticVectorRetentionAction::None + ) { + log_maintenance_event( + "retention_semantic_vector_generations", + &[ + ("project", root.display().to_string()), + ("action", format!("{:?}", census.action)), + ], + ); + } + if convergence_pending { + MaintenanceTickOutcome::Continue( + crate::tick::MaintenanceContinuation::SemanticVectorRetention, + ) + } else { + MaintenanceTickOutcome::Complete + } + } + tracedecay_code_index_runtime::code_index_scheduler::semantic_vector_graph::ProjectSemanticVectorRetentionStep::ResetRequired( + reason, + ) => { + observations.record_semantic_vector_retention_failure(root); + log_semantic_vector_retention_degraded( + observations, + root, + &format!("reset_required:{reason}"), + ); + MaintenanceTickOutcome::Retry + } + tracedecay_code_index_runtime::code_index_scheduler::semantic_vector_graph::ProjectSemanticVectorRetentionStep::Corrupt( + reason, + ) => { + observations.record_semantic_vector_retention_failure(root); + log_semantic_vector_retention_degraded(observations, root, &format!("corrupt:{reason}")); + MaintenanceTickOutcome::Retry + } + tracedecay_code_index_runtime::code_index_scheduler::semantic_vector_graph::ProjectSemanticVectorRetentionStep::Unavailable( + reason, + ) => { + observations.record_semantic_vector_retention_failure(root); + log_semantic_vector_retention_degraded( + observations, + root, + &format!("unavailable:{reason}"), + ); + MaintenanceTickOutcome::Retry + } + tracedecay_code_index_runtime::code_index_scheduler::semantic_vector_graph::ProjectSemanticVectorRetentionStep::Denied( + reason, + ) => { + observations.record_semantic_vector_retention_failure(root); + log_semantic_vector_retention_degraded(observations, root, &format!("denied:{reason}")); + MaintenanceTickOutcome::Retry + } + } +} + +/// Semantic retrieval is genuinely disabled only when the durable +/// configuration commits neither an active nor a rollback retrieval profile. +/// A committed profile with an unseated activation coordinator is a transient +/// (or genuinely degraded) state that must stay retryable, not a quiet pin. +pub fn semantic_retrieval_profiles_disabled(semantic: &SemanticConfig) -> bool { + semantic.active_profile.is_none() && semantic.rollback_profile.is_none() +} + +fn log_semantic_vector_retention_degraded( + observations: &StoreTelemetrySamplingRegistry, + project_root: &Path, + failure: &str, +) { + observations.emit_retention_degraded(project_root, "semantic_vector_generations", failure); +} + +/// Vector protection inventory for one code-generation retention pass. +/// +/// `Online` carries the exact vector pin set read from the mounted code +/// graph plus the authorities needed to re-verify it under the writer freeze. +/// `SemanticUnseated` is the ordinary default-off state: no semantic runtime +/// is seated, no census will ever exist, and the pass sweeps under the +/// offline protection set without reporting a degradation. `CensusScanning` +/// is in-progress: the bounded census is still paging toward its exact pin +/// set, so the pass defers instead of planning against a mid-scan inventory. +/// `Offline` is a typed degradation for an unreadable vector inventory: the +/// live pin set is unknown, so the pass reports and retains every source +/// rather than planning against an offline protection set that cannot name +/// the sources a mounted activation lease binds. `Refused` is fail-closed +/// for the same reason: the vector authority reported reset/corrupt/denied +/// and no sweep may run. +pub enum VectorRetentionInventoryV1 { + Online { + sources: std::collections::BTreeSet, + configuration: + tracedecay_application::semantic_runtime::ProductionSemanticRetrievalConfigurationStoreV1, + expected_vector_revision: tracedecay_store::SemanticVectorStageCensusRevision, + }, + SemanticUnseated, + CensusScanning, + Offline { + reason: String, + }, + Refused { + reason: String, + }, +} + +impl VectorRetentionInventoryV1 { + /// The `retention_degraded` failure the code-generation pass reports for + /// this inventory, or `None` for states that are ordinary journeys and + /// must stay quiet on every pass: an online inventory, a daemon whose + /// semantic runtime is not seated (the default-off state), and a census + /// still paging toward its exact pin set. + pub fn degraded_reason(&self) -> Option { + match self { + Self::Online { .. } | Self::SemanticUnseated | Self::CensusScanning => None, + Self::Offline { reason } => Some(format!("vector_inventory_offline:{reason}")), + Self::Refused { reason } => Some(reason.clone()), + } + } +} + +pub async fn resolve_vector_retention_inventory( + lease: &ProjectStoreMaintenanceLeaseV1, + schedulers: &CodeIndexSchedulerRegistryV1, + observations: &StoreTelemetrySamplingRegistry, +) -> VectorRetentionInventoryV1 { + // A mounted provider can still own vector activation leases when a census + // or configuration read fails. Distinguish that refusal from an absent + // provider; neither unknown state proves its source generations are dead. + let vector_provider = schedulers + .semantic_vector_graph_provider(lease.project_root()) + .await; + let vector_provider_mounted = vector_provider.is_some(); + let unavailable = |reason: String| { + if vector_provider_mounted { + VectorRetentionInventoryV1::Refused { reason } + } else { + VectorRetentionInventoryV1::Offline { reason } + } + }; + let expected_vector_revision = match observations + .semantic_vector_retention_read(lease.project_root()) + { + crate::telemetry::SemanticVectorRetentionReadV1::Observed { receipt } => receipt.revision, + crate::telemetry::SemanticVectorRetentionReadV1::SemanticUnseated => { + let Some(provider) = vector_provider.as_ref() else { + return VectorRetentionInventoryV1::SemanticUnseated; + }; + // Providers are mounted even with semantic search disabled. + // An exact empty first page proves there are no retained stages; + // a nonempty page must never be mistaken for disabled liveness. + let empty = async { + let retained = provider + .graph_for_current() + .await + .map_err(|error| error.to_string())?; + let store = tracedecay_application::store::vector_generations::GraphVectorGenerationStoreV1::read_only(&retained) + .await + .map_err(|error| error.to_string())?; + let census = store + .project_stage_census(std::sync::Arc::clone(retained.cancellation())) + .await + .map_err(|error| error.to_string())?; + Ok::<_, String>(census.records.is_empty() + && census.continuation.is_none() + && census.complete_receipt.is_some()) + } + .await; + return match empty { + Ok(true) => VectorRetentionInventoryV1::SemanticUnseated, + Ok(false) => VectorRetentionInventoryV1::Refused { + reason: "unseated_semantic_vector_stages_remain".to_owned(), + }, + Err(reason) => VectorRetentionInventoryV1::Refused { reason }, + }; + } + crate::telemetry::SemanticVectorRetentionReadV1::Scanning => { + return VectorRetentionInventoryV1::CensusScanning; + } + crate::telemetry::SemanticVectorRetentionReadV1::Unknown => { + return unavailable("vector_census_incomplete".to_owned()); + } + }; + let Some(configuration) = lease + .configuration_runtime() + .semantic_configuration_inventory_authority() + else { + return unavailable("configuration_inventory_unavailable".to_owned()); + }; + let project_root = lease.store_layout().project_root.clone(); + let sources = tracedecay_code_index_runtime::code_index_scheduler::semantic_vector_graph::project_vector_readable_sources( + schedulers, + &project_root, + &configuration, + expected_vector_revision, + ) + .await; + match classify_vector_readable_sources(sources, configuration, expected_vector_revision) { + VectorRetentionInventoryV1::Offline { reason } => unavailable(reason), + inventory => inventory, + } +} + +/// Map the mounted graph's readable-source read onto the retention inventory: +/// unavailable is the typed offline degradation, while reset, corrupt, and +/// denied are refusals. Both retain every source: an inventory that cannot be +/// read cannot prove which sources a mounted activation lease binds. +pub fn classify_vector_readable_sources( + sources: tracedecay_code_index_runtime::code_index_scheduler::semantic_vector_graph::ProjectVectorReadableSources, + configuration: tracedecay_application::semantic_runtime::ProductionSemanticRetrievalConfigurationStoreV1, + expected_vector_revision: tracedecay_store::SemanticVectorStageCensusRevision, +) -> VectorRetentionInventoryV1 { + match sources { + tracedecay_code_index_runtime::code_index_scheduler::semantic_vector_graph::ProjectVectorReadableSources::Ready { + sources, + .. + } => VectorRetentionInventoryV1::Online { + sources, + configuration, + expected_vector_revision, + }, + tracedecay_code_index_runtime::code_index_scheduler::semantic_vector_graph::ProjectVectorReadableSources::Unavailable( + reason, + ) => VectorRetentionInventoryV1::Offline { + reason: format!("vector_graph_unavailable:{reason}"), + }, + tracedecay_code_index_runtime::code_index_scheduler::semantic_vector_graph::ProjectVectorReadableSources::ResetRequired( + reason, + ) => VectorRetentionInventoryV1::Refused { + reason: format!("vector_graph_reset_required:{reason}"), + }, + tracedecay_code_index_runtime::code_index_scheduler::semantic_vector_graph::ProjectVectorReadableSources::Corrupt( + reason, + ) => VectorRetentionInventoryV1::Refused { + reason: format!("vector_graph_corrupt:{reason}"), + }, + tracedecay_code_index_runtime::code_index_scheduler::semantic_vector_graph::ProjectVectorReadableSources::Denied( + reason, + ) => VectorRetentionInventoryV1::Refused { + reason: format!("vector_graph_denied:{reason}"), + }, + } +} + +/// Outcome of one bounded code-generation retention pass. +/// +/// `MoreWork` reports bounded progress with a remaining backlog — another +/// collectable superseded generation, or unconsumed graph-replay release +/// evidence — so the maintenance owner keeps the short cadence until the +/// store converges instead of parking multi-GiB debris behind the full +/// maintenance interval. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum CodeGenerationRetentionOutcomeV1 { + Complete, + MoreWork, + Failed, +} + +/// Collect superseded code-index generations for one mounted project. +/// +/// Sealed generations are ordinary files, so no database retention or +/// compaction pass reclaims them. This runs on the ordinary maintenance cadence +/// and is independent of the semantic projection lane: the only previous caller +/// sat inside legacy vector migration, so a profile with semantic search +/// disabled never collected anything and grew without bound. +/// +/// Vector-readable source generations are pinned through the mounted code +/// graph when it is resolvable. A daemon without a seated semantic runtime +/// (the default-off state) sweeps under the offline protection set as its +/// ordinary quiet journey, and an in-progress census defers the sweep until +/// its exact pin set is complete. When the vector inventory is unreadable — +/// saturated capacity, failed activation, nothing serving, or a census reset +/// by a failure or mutation — the pass reports its degradation and collects +/// nothing: the offline protection set (active pointer head, durable pointer +/// index, rollback floor, and the serving generation) cannot name the exact +/// source generations a mounted vector activation lease binds, so sweeping +/// under it deleted a live vector source. Reset, corrupt, and denied vector +/// authorities stay fail-closed for the same reason. +#[hotpath::measure( + label = "daemon.git.maintenance.code_generation_retention", + future = true +)] +pub async fn run_code_generation_retention( + lease: &ProjectStoreMaintenanceLeaseV1, + schedulers: &CodeIndexSchedulerRegistryV1, + observations: &StoreTelemetrySamplingRegistry, + cancellation: &tracedecay_session_memory::context::CancellationToken, +) -> CodeGenerationRetentionOutcomeV1 { + if cancellation.is_cancelled() { + log_code_generation_retention_degraded( + observations, + lease.project_root(), + "retention_cancelled", + ); + return CodeGenerationRetentionOutcomeV1::Failed; + } + let layout = lease.store_layout(); + let store_root = tracedecay_code_index_retention::code_index_generations::code_index_store_root( + &layout.data_root, + &layout.project_root, + ); + // A store directory that never materialized has nothing to sweep. A store + // *without* an active pointer is different: it is crash debris from a + // publish that never reached its pointer write (an OOM-killed rebuild is + // the ordinary cause), and the planner collects it as a typed unpublished + // store — before this, such orphaned partial generations were unreachable + // by every retention pass while their worktree root stayed live. + if !store_root.is_dir() { + return CodeGenerationRetentionOutcomeV1::Complete; + } + let vector_inventory = + resolve_vector_retention_inventory(lease, schedulers, observations).await; + apply_code_generation_retention( + lease, + schedulers, + observations, + vector_inventory, + cancellation, + ) + .await +} + +/// The offline protection pin: the generation the mounted scheduler is +/// currently serving, when one is mounted at all. +#[hotpath::measure( + label = "daemon.git.maintenance.serving_generation_pins", + future = true +)] +async fn serving_generation_pins( + schedulers: &CodeIndexSchedulerRegistryV1, + project_root: &Path, +) -> std::collections::BTreeSet { + let mut pins = std::collections::BTreeSet::new(); + if let Some(scope) = schedulers.serving_code_scope(project_root).await + && let Some(serving) = scope.serving_generation + { + pins.insert(serving.manifest().generation_id.clone()); + } + // A clean restart whose retained revision-7 head recovered serves through + // the text projection and never seats a second copy of its sealed + // generation, so the sealed slot alone under-reports what is live. Pin + // the level that actually serves or retention collects it out from under + // the route. + if let Some(text) = schedulers.latest_text_serving_for_root(project_root).await { + pins.insert(text.metadata().manifest().generation_id.clone()); + } + pins +} + +/// Execute one code-generation retention pass against a resolved vector +/// inventory. Emitting `retention_degraded` is decided exclusively by +/// [`VectorRetentionInventoryV1::degraded_reason`], so quiet states cannot be +/// reintroduced into the degraded log by a divergent match arm. +#[hotpath::measure( + label = "daemon.git.maintenance.code_generation_retention_apply", + future = true +)] +pub async fn apply_code_generation_retention( + lease: &ProjectStoreMaintenanceLeaseV1, + schedulers: &CodeIndexSchedulerRegistryV1, + observations: &StoreTelemetrySamplingRegistry, + vector_inventory: VectorRetentionInventoryV1, + cancellation: &tracedecay_session_memory::context::CancellationToken, +) -> CodeGenerationRetentionOutcomeV1 { + use tracedecay_code_index_retention::code_index_generations::{ + CodeGenerationRetentionErrorV1, CodeGenerationRetentionModeV1, + DEFAULT_SUPERSEDED_GENERATION_FLOOR, execute_code_generation_retention_cancellable, + prepare_next_code_generation_retention_cancellable, + }; + let layout = lease.store_layout(); + let store_root = tracedecay_code_index_retention::code_index_generations::code_index_store_root( + &layout.data_root, + &layout.project_root, + ); + // Retired generations stay reachable for graph replay through the replay + // pool; retention hard-links each one there before its release event + // becomes durable, and the replay reconciler deletes pool entries once + // the graph confirms it no longer needs them. + let graph_replay_pool_root = lease + .graph_db() + .database_path() + .with_extension("graph-replay"); + if let Some(failure) = vector_inventory.degraded_reason() { + log_code_generation_retention_degraded(observations, lease.project_root(), &failure); + } + // Published vectors live in the mounted code graph. When the graph is + // resolvable, its inventory is the exact vector pin set. Without a seated + // semantic runtime the durable configuration is canonical proof that no + // vector stage can pin a source, so that journey sweeps under the offline + // protection set (active pointer head, durable pointer index, rollback + // floor, plus the serving generation). A paging census defers: its exact + // pin set arrives when the scan completes, and the vector retention pass + // already keeps the retry cadence short while paging. + // + // An unreadable vector inventory is fail-closed. The offline protection + // set names the serving generation, never the exact source generations a + // mounted vector activation lease still binds, so planning against it + // while the inventory is unknown collected a live vector source + // (production journey cc-5583). "Unknown" is retained, not swept: the + // pass reports its degradation and collects nothing until an exact — or + // canonically empty — inventory is readable again. Reset, corrupt, and + // denied vector authorities stay fail-closed for the same reason. + let (vector_readable_sources, inventory_mode) = match &vector_inventory { + VectorRetentionInventoryV1::Online { sources, .. } => (sources.clone(), "online"), + VectorRetentionInventoryV1::SemanticUnseated => ( + serving_generation_pins(schedulers, &layout.project_root).await, + "semantic_unseated", + ), + VectorRetentionInventoryV1::CensusScanning => { + return CodeGenerationRetentionOutcomeV1::Complete; + } + VectorRetentionInventoryV1::Offline { .. } | VectorRetentionInventoryV1::Refused { .. } => { + return CodeGenerationRetentionOutcomeV1::Failed; + } + }; + // A held replay pool makes every later phase of this pass fail closed: + // the release reconcile's pool acquisition would burn its whole + // graph-operation deadline discovering the holder (the live wedge logged + // that as `graph_replay_release_failed error=DeadlineExceeded` on every + // tick), and the collection executor would then contend for the same + // lock while holding the daemon writer gate. One non-blocking probe + // defers the pass for this tick instead — before the multi-GiB + // full-digest planning below is paid — and the executor's own checked + // acquire returns `GraphReplayPoolBusy` if a publisher wins the + // probe-to-execute window, so the writer gate is never pinned on a + // blocking flock. Both paths arm the same bounded release backoff. + if graph_replay::replay_pool_is_held(&graph_replay_pool_root) { + return defer_graph_replay_pool_busy(observations, lease.project_root()); + } + // Full digest verification routinely reads several GiB. Run it before + // entering the graph transaction and preserve the daemon shutdown token + // through the blocking boundary; the planner checks it after every bounded + // read chunk and creates no journal before verification completes. + let plan_root = store_root.clone(); + let plan_sources = vector_readable_sources.clone(); + let plan_cancellation = cancellation.clone(); + let plan_pool_root = graph_replay_pool_root.clone(); + let plan = tokio::task::spawn_blocking(move || { + prepare_next_code_generation_retention_cancellable( + &plan_root, + &plan_sources, + DEFAULT_SUPERSEDED_GENERATION_FLOOR, + &|| plan_cancellation.is_cancelled(), + Some(&plan_pool_root), + ) + }) + .await; + let plan = match plan { + Ok(Ok(plan)) => plan, + Ok(Err( + tracedecay_code_index_retention::code_index_generations::CodeGenerationRetentionErrorV1::Cancelled, + )) => { + log_code_generation_retention_degraded(observations, lease.project_root(), "retention_cancelled"); + return CodeGenerationRetentionOutcomeV1::Failed; + } + Ok(Err( + tracedecay_code_index_retention::code_index_generations::CodeGenerationRetentionErrorV1::GraphReplayPoolBusy, + )) => { + return defer_graph_replay_pool_busy(observations, lease.project_root()); + } + Ok(Err(error)) => { + // The bare label proved undiagnosable on a live profile: without + // the typed error, a pointer CAS loss under rebuild churn is + // indistinguishable from unrecognized-file or storage failures. + observations.mark_loud_retention_log(); + log_maintenance_event( + "retention_degraded", + &[ + ("pass", "code_generations".to_string()), + ("failure", "retention_plan_failed".to_string()), + ("error", error.to_string()), + ], + ); + return CodeGenerationRetentionOutcomeV1::Failed; + } + Err(_) => { + log_code_generation_retention_degraded(observations, lease.project_root(), "retention_task_panicked"); + return CodeGenerationRetentionOutcomeV1::Failed; + } + }; + // A failed, deferred, or retained replay reconcile keeps its durable + // release evidence for a later graph-available pass. Deleting newly + // planned files stays safe in every inventory mode — retention hard-links + // each retired generation into the replay pool before its release event + // becomes durable, so the graph can always finish its retirement later. + // The pass therefore keeps collecting instead of letting sealed + // generations and their multi-GiB text artifacts accumulate without bound + // whenever the graph is dark, wedged, or busy (a recurring + // `graph_replay_release_failed` used to abort every pass here and grew + // one store by tens of GiB in a single crash-rebuild night). A failure + // still reports degraded and fails the pass so the retry cadence stays + // short; a deferral fails the pass quietly under the bounded backoff. + let mut replay_reconcile_failed = false; + let mut release_backlog_remains = false; + let replay_reconcile_attemptable = match graph_replay::reconcile_graph_replay_releases( + lease, + &store_root, + observations, + cancellation, + ) + .await + { + graph_replay::ReconcileOutcome::Complete | graph_replay::ReconcileOutcome::Retained => true, + graph_replay::ReconcileOutcome::MoreWork => { + release_backlog_remains = true; + true + } + // A deferred or failed attempt must not be repeated by the + // post-collection reconcile below: the graph runtime already proved + // it cannot serve this tick. + graph_replay::ReconcileOutcome::Deferred | graph_replay::ReconcileOutcome::Failed => { + replay_reconcile_failed = true; + false + } + }; + if !plan.has_collectable_work() { + return if replay_reconcile_failed { + CodeGenerationRetentionOutcomeV1::Failed + } else if release_backlog_remains { + CodeGenerationRetentionOutcomeV1::MoreWork + } else { + CodeGenerationRetentionOutcomeV1::Complete + }; + } + if cancellation.is_cancelled() { + log_code_generation_retention_degraded( + observations, + lease.project_root(), + "retention_cancelled", + ); + return CodeGenerationRetentionOutcomeV1::Failed; + } + + // Freeze the vector writer, then re-read the committed active+rollback + // identities and their exact source generations. Graph head order is not + // retention authority: a newer unactivated candidate must not displace + // the configured generation from this fence. The unseated default-off + // sweep has no vector inventory to fence, so no freeze is taken there; + // every other non-online inventory already returned without collecting. + let vector_writer_freeze = if let VectorRetentionInventoryV1::Online { + configuration, + expected_vector_revision, + .. + } = &vector_inventory + { + let Some(vector_runtime) = + tracedecay_application::semantic_runtime::project_semantic_production_runtime( + &layout.project_root, + ) + else { + log_code_generation_retention_degraded( + observations, + lease.project_root(), + "vector_writer_unavailable", + ); + return CodeGenerationRetentionOutcomeV1::Failed; + }; + let vector_writer_freeze = vector_runtime.freeze_vector_mutations().await; + let pinned_vector_sources = + match tracedecay_code_index_runtime::code_index_scheduler::semantic_vector_graph::project_vector_readable_sources( + schedulers, + &layout.project_root, + configuration, + *expected_vector_revision, + ) + .await + { + tracedecay_code_index_runtime::code_index_scheduler::semantic_vector_graph::ProjectVectorReadableSources::Ready { + sources, + .. + } => sources, + tracedecay_code_index_runtime::code_index_scheduler::semantic_vector_graph::ProjectVectorReadableSources::ResetRequired( + reason, + ) => { + log_code_generation_retention_degraded(observations, lease.project_root(), &format!( + "vector_inventory_reset_required:{reason}" + )); + return CodeGenerationRetentionOutcomeV1::Failed; + } + tracedecay_code_index_runtime::code_index_scheduler::semantic_vector_graph::ProjectVectorReadableSources::Corrupt( + reason, + ) => { + log_code_generation_retention_degraded(observations, lease.project_root(), &format!( + "vector_inventory_corrupt:{reason}" + )); + return CodeGenerationRetentionOutcomeV1::Failed; + } + tracedecay_code_index_runtime::code_index_scheduler::semantic_vector_graph::ProjectVectorReadableSources::Unavailable( + reason, + ) => { + log_code_generation_retention_degraded(observations, lease.project_root(), &format!( + "vector_inventory_unavailable:{reason}" + )); + return CodeGenerationRetentionOutcomeV1::Failed; + } + tracedecay_code_index_runtime::code_index_scheduler::semantic_vector_graph::ProjectVectorReadableSources::Denied( + reason, + ) => { + log_code_generation_retention_degraded(observations, lease.project_root(), &format!( + "vector_inventory_denied:{reason}" + )); + return CodeGenerationRetentionOutcomeV1::Failed; + } + }; + if pinned_vector_sources != vector_readable_sources { + log_code_generation_retention_degraded( + observations, + lease.project_root(), + "vector_inventory_changed", + ); + return CodeGenerationRetentionOutcomeV1::Failed; + } + tracedecay_application::semantic_runtime::retain_project_semantic_code_sources( + &layout.project_root, + &pinned_vector_sources, + ); + for generation in &plan.collectable_generations { + match tracedecay_code_index_runtime::code_index_scheduler::semantic_vector_graph::project_vector_source_generation_is_live( + schedulers, + &layout.project_root, + &generation.generation_id, + *expected_vector_revision, + ) + .await + { + tracedecay_code_index_runtime::code_index_scheduler::semantic_vector_graph::ProjectSemanticVectorSourceLiveness::Ready( + true, + ) => { + // A pending, ready, published, or base-linked vector stage still + // reads this exact source. It was absent from the root-only + // planning inventory, so retain it and let vector convergence + // make the next maintenance tick eligible. + return CodeGenerationRetentionOutcomeV1::Complete; + } + tracedecay_code_index_runtime::code_index_scheduler::semantic_vector_graph::ProjectSemanticVectorSourceLiveness::Ready( + false, + ) => {} + tracedecay_code_index_runtime::code_index_scheduler::semantic_vector_graph::ProjectSemanticVectorSourceLiveness::Unavailable( + reason, + ) => { + log_code_generation_retention_degraded(observations, lease.project_root(), &format!( + "vector_source_liveness_unavailable:{reason}" + )); + return CodeGenerationRetentionOutcomeV1::Failed; + } + tracedecay_code_index_runtime::code_index_scheduler::semantic_vector_graph::ProjectSemanticVectorSourceLiveness::Denied( + reason, + ) => { + log_code_generation_retention_degraded(observations, lease.project_root(), &format!( + "vector_source_liveness_denied:{reason}" + )); + return CodeGenerationRetentionOutcomeV1::Failed; + } + tracedecay_code_index_runtime::code_index_scheduler::semantic_vector_graph::ProjectSemanticVectorSourceLiveness::ResetRequired( + reason, + ) => { + log_code_generation_retention_degraded(observations, lease.project_root(), &format!( + "vector_source_liveness_reset_required:{reason}" + )); + return CodeGenerationRetentionOutcomeV1::Failed; + } + tracedecay_code_index_runtime::code_index_scheduler::semantic_vector_graph::ProjectSemanticVectorSourceLiveness::Corrupt( + reason, + ) => { + log_code_generation_retention_degraded(observations, lease.project_root(), &format!( + "vector_source_liveness_corrupt:{reason}" + )); + return CodeGenerationRetentionOutcomeV1::Failed; + } + } + } + Some(vector_writer_freeze) + } else { + None + }; + if cancellation.is_cancelled() { + log_code_generation_retention_degraded( + observations, + lease.project_root(), + "retention_cancelled", + ); + return CodeGenerationRetentionOutcomeV1::Failed; + } + // `current_timestamp()` counts seconds; wrapping it in `UtcMicros` stamped + // every deletion receipt with a seconds value in a micros-typed field + // (live receipts read as 1970). The receipt is durable journal evidence, + // so it takes the canonical micros clock. + let completed_at = tracedecay_contracts::clock::now_micros(); + let execution_root = store_root.clone(); + let execution_pool_root = graph_replay_pool_root.clone(); + let execution_cancellation = cancellation.clone(); + let report = tokio::task::spawn_blocking(move || { + execute_code_generation_retention_cancellable( + &execution_root, + plan, + CodeGenerationRetentionModeV1::Apply, + completed_at, + Some(&execution_pool_root), + &|| execution_cancellation.is_cancelled(), + ) + }) + .await; + drop(vector_writer_freeze); + + match report { + Ok(Ok(report)) => { + let generation_reclaimed = report.receipt.as_ref().map_or_else( + || { + report + .deleted_generations + .iter() + .map(|generation| generation.size_bytes) + .sum() + }, + |receipt| receipt.reclaimed_bytes, + ); + let text_artifact_reclaimed = report.text_artifact_receipt.as_ref().map_or_else( + || { + report + .deleted_text_artifacts + .iter() + .map(|artifact| artifact.size_bytes) + .sum() + }, + |receipt| receipt.reclaimed_bytes, + ); + let reclaimed = generation_reclaimed.saturating_add(text_artifact_reclaimed); + if reclaimed > 0 { + log_maintenance_event( + "retention_code_generations", + &[ + ("store", "code-index-v1".to_string()), + ("mode", inventory_mode.to_string()), + ("bytes_reclaimed", reclaimed.to_string()), + ( + "generations_collected", + report.deleted_generations.len().to_string(), + ), + ( + "text_artifacts_collected", + report.deleted_text_artifacts.len().to_string(), + ), + ], + ); + } + // The just-collected generation queued fresh release evidence; + // offer it to the graph immediately — but only when this tick's + // earlier reconcile was actually served. A deferred or failed + // runtime must not be probed twice in one tick. + let mut release_reconcile_failed = replay_reconcile_failed; + if replay_reconcile_attemptable { + match graph_replay::reconcile_graph_replay_releases( + lease, + &store_root, + observations, + cancellation, + ) + .await + { + graph_replay::ReconcileOutcome::Complete + | graph_replay::ReconcileOutcome::Retained => {} + graph_replay::ReconcileOutcome::MoreWork => { + release_backlog_remains = true; + } + graph_replay::ReconcileOutcome::Deferred + | graph_replay::ReconcileOutcome::Failed => { + release_reconcile_failed = true; + } + } + } + if release_reconcile_failed { + CodeGenerationRetentionOutcomeV1::Failed + } else if release_backlog_remains + || !report.deleted_generations.is_empty() + || !report.deleted_text_artifacts.is_empty() + { + // Something was collected, so the next bounded census may find + // another collectable unit; stay on the short cadence until a + // pass proves the store converged. A census that finds nothing + // returns Complete one tick later at metadata cost only. + CodeGenerationRetentionOutcomeV1::MoreWork + } else { + CodeGenerationRetentionOutcomeV1::Complete + } + } + Ok(Err(CodeGenerationRetentionErrorV1::Cancelled)) => { + log_code_generation_retention_degraded( + observations, + lease.project_root(), + "retention_cancelled", + ); + CodeGenerationRetentionOutcomeV1::Failed + } + Ok(Err(CodeGenerationRetentionErrorV1::GraphReplayPoolBusy)) => { + defer_graph_replay_pool_busy(observations, lease.project_root()) + } + Ok(Err(error)) => { + // Same diagnosability contract as the plan failure above: the + // apply step's typed error names the exact refusal (CAS loss, + // unsafe state, storage) instead of a bare retry label. + observations.mark_loud_retention_log(); + log_maintenance_event( + "retention_degraded", + &[ + ("pass", "code_generations".to_string()), + ("failure", "retention_pass_failed".to_string()), + ("error", error.to_string()), + ], + ); + CodeGenerationRetentionOutcomeV1::Failed + } + Err(_) => { + log_code_generation_retention_degraded( + observations, + lease.project_root(), + "retention_task_panicked", + ); + CodeGenerationRetentionOutcomeV1::Failed + } + } +} + +async fn collect_scope_root_proof_inputs( + lease: &ProjectStoreMaintenanceLeaseV1, + schedulers: &CodeIndexSchedulerRegistryV1, + vector_receipt: &tracedecay_store::SemanticVectorProjectCensusReceipt, +) -> Result { + let layout = lease.store_layout(); + let project_id = layout + .identity + .project_id + .as_deref() + .ok_or("registered_project_identity_missing")?; + let registered = lease + .profile_database() + .registered_project_root_inventory(project_id) + .await + .map_err(|_| "registered_root_inventory_unavailable")? + .ok_or("registered_root_inventory_missing")?; + let registered_candidates = registered + .roots + .iter() + .map(PathBuf::from) + .collect::>(); + let project_id = tracedecay_domain::ProjectId::new(project_id.to_owned()) + .map_err(|_| "registered_project_identity_invalid")?; + let enrolled_roots = crate::lease::enrolled_project_roots(registered_candidates, &project_id) + .map_err(|_| "registered_enrollment_inventory_unavailable")?; + if enrolled_roots.is_empty() { + return Err("registered_enrollment_inventory_empty"); + } + let enrolled_material = enrolled_roots + .iter() + .map(|root| root.to_string_lossy().into_owned()) + .collect::>(); + let enrolled_digest = tracedecay_domain::canonical_sha256(&( + "tracedecay.registered-enrollment-root-inventory.v1", + registered.inventory_digest.as_str(), + &enrolled_material, + )) + .map_err(|_| "registered_enrollment_inventory_digest_failed")?; + let registered_receipt = + tracedecay_code_index_retention::code_index_generations::ScopeRootAuthorityReceiptV1 { + revision: registered.inventory_digest.as_str().to_owned(), + terminal_count: u64::try_from(enrolled_roots.len()) + .map_err(|_| "registered_enrollment_count_overflow")?, + digest: enrolled_digest.as_str().to_owned(), + }; + let mut live_roots = std::collections::BTreeSet::new(); + for root in enrolled_roots { + tracedecay_code_index_retention::code_index_generations::insert_live_root_variants( + &mut live_roots, + &root, + ); + } + + let project_root = lease.project_root().to_path_buf(); + let (git_roots, git_receipt) = tokio::task::spawn_blocking(move || { + tracedecay_code_index_retention::code_index_generations::git_worktree_scope_root_inventory( + &project_root, + ) + }) + .await + .map_err(|_| "git_worktree_inventory_task_panicked")??; + live_roots.extend(git_roots); + + let mounted = schedulers.scope_retention_mounted_roots().await?; + let mounted_count = u64::try_from(mounted.len()).map_err(|_| "mounted_root_count_overflow")?; + let mounted_material = mounted + .iter() + .map(|root| root.to_string_lossy().into_owned()) + .collect::>(); + let mounted_digest = tracedecay_domain::canonical_sha256(&( + "tracedecay.mounted-code-index-root-inventory.v1", + &mounted_material, + )) + .map_err(|_| "mounted_root_inventory_digest_failed")?; + let mounted_receipt = + tracedecay_code_index_retention::code_index_generations::ScopeRootAuthorityReceiptV1 { + revision: mounted_digest.as_str().to_owned(), + terminal_count: mounted_count, + digest: mounted_digest.as_str().to_owned(), + }; + for root in mounted { + tracedecay_code_index_retention::code_index_generations::insert_live_root_variants( + &mut live_roots, + &root, + ); + } + if live_roots.is_empty() { + return Err("scope_live_root_inventory_empty"); + } + + let configuration = lease + .configuration_runtime() + .semantic_configuration_inventory_authority() + .ok_or("configuration_inventory_unavailable")?; + let ( + vector_sources, + configuration_receipt, + configured_root_receipt, + ) = match tracedecay_code_index_runtime::code_index_scheduler::semantic_vector_graph::project_vector_readable_sources( + schedulers, + lease.project_root(), + &configuration, + vector_receipt.revision, + ) + .await + { + tracedecay_code_index_runtime::code_index_scheduler::semantic_vector_graph::ProjectVectorReadableSources::Ready { + sources, + configuration_receipt, + configured_root_receipt, + } => (sources, configuration_receipt, configured_root_receipt), + tracedecay_code_index_runtime::code_index_scheduler::semantic_vector_graph::ProjectVectorReadableSources::ResetRequired( + _, + ) => return Err("scope_vector_inventory_reset_required"), + tracedecay_code_index_runtime::code_index_scheduler::semantic_vector_graph::ProjectVectorReadableSources::Corrupt( + _, + ) => return Err("scope_vector_inventory_corrupt"), + tracedecay_code_index_runtime::code_index_scheduler::semantic_vector_graph::ProjectVectorReadableSources::Unavailable( + _, + ) => return Err("scope_vector_inventory_unavailable"), + tracedecay_code_index_runtime::code_index_scheduler::semantic_vector_graph::ProjectVectorReadableSources::Denied( + _, + ) => return Err("scope_vector_inventory_denied"), + }; + let configuration_roots = + tracedecay_code_index_retention::code_index_generations::ScopeRootAuthorityReceiptV1 { + revision: configuration_receipt + .revision() + .map_or_else(|| "absent".to_owned(), |revision| revision.to_string()), + terminal_count: configuration_receipt.root_binding_count(), + digest: configuration_receipt.inventory_digest().as_str().to_owned(), + }; + let vector_dependency_digest = tracedecay_domain::canonical_sha256(&( + "tracedecay.configured-vector-dependency-inventory.v1", + configured_root_receipt.root_digest().as_str(), + &vector_sources, + )) + .map_err(|_| "vector_dependency_inventory_digest_failed")?; + let vector_dependencies = + tracedecay_code_index_retention::code_index_generations::ScopeRootAuthorityReceiptV1 { + revision: configured_root_receipt + .revision() + .map_or_else(|| "absent".to_owned(), |revision| revision.to_string()), + terminal_count: configured_root_receipt.root_count(), + digest: vector_dependency_digest.as_str().to_owned(), + }; + let vector_count = vector_receipt + .counts + .pending + .checked_add(vector_receipt.counts.ready) + .and_then(|count| count.checked_add(vector_receipt.counts.published)) + .and_then(|count| count.checked_add(vector_receipt.counts.cancelled)) + .ok_or("vector_census_count_overflow")?; + let vector_census = + tracedecay_code_index_retention::code_index_generations::ScopeRootAuthorityReceiptV1 { + revision: vector_receipt.revision.get().to_string(), + terminal_count: vector_count, + digest: vector_receipt.record_digest.as_str().to_owned(), + }; + + Ok(ScopeRootProofInputsV1 { + live_roots, + registered_roots: registered_receipt, + git_worktrees: git_receipt, + mounted_leases: mounted_receipt, + configuration_roots, + vector_census, + vector_dependencies, + vector_sources, + }) +} + +/// Reconcile whole code-index *scope roots* for one mounted repository. +/// +/// Generation retention above is scoped to a single +/// `code-index-v1//` directory, and every caller +/// derives exactly one such scope from the root it was handed. Nothing has ever +/// enumerated the siblings, so a scope whose project root is gone — a deleted +/// agent worktree is the ordinary cause — is unreachable by any retention pass +/// and uncounted by any report. One large repository carried three scope +/// directories, two of them orphaned, holding 7.2 GiB nothing could see. +/// +/// The pass is fail-closed by construction. Git proves the complete registered +/// worktree set, while the revision-pinned semantic staging authority binds +/// every candidate physical scope hash to its exact logical source shard. A +/// candidate is collected only when both authorities say it is unreferenced; +/// missing, conflicting, or stale vector evidence collects nothing. +#[hotpath::measure(label = "daemon.git.maintenance.scope_reconciliation", future = true)] +pub async fn run_code_index_scope_reconciliation( + lease: &ProjectStoreMaintenanceLeaseV1, + schedulers: &CodeIndexSchedulerRegistryV1, + observations: &StoreTelemetrySamplingRegistry, +) -> bool { + use tracedecay_code_index_retention::code_index_generations::{ + CodeGenerationRetentionModeV1, DEFAULT_STRANDED_SCOPE_MINIMUM_AGE_SECS, + complete_scope_root_binding_cleanup, execute_scope_root_retention, + plan_scope_root_retention, plan_scope_root_retention_with_liveness_proof, + prepare_scope_root_binding_cleanup, recover_scope_root_binding_cleanup, + recover_scope_root_retention, + }; + + let layout = lease.store_layout(); + let store_root = + tracedecay_code_index_retention::code_index_generations::code_index_scope_store_root( + &layout.data_root, + ); + if !store_root.is_dir() { + return true; + } + + let recovery_root = store_root.clone(); + let pending_binding_cleanup = tokio::task::spawn_blocking(move || { + recover_scope_root_retention(&recovery_root) + .map_err(|_| "scope_reconciliation_recovery_failed")?; + recover_scope_root_binding_cleanup(&recovery_root) + .map_err(|_| "scope_binding_cleanup_recovery_failed") + }) + .await; + let pending_binding_cleanup = match pending_binding_cleanup { + Ok(Ok(pending)) => pending, + Ok(Err(failure)) => { + log_code_index_scope_reconciliation_degraded(failure); + return false; + } + Err(_) => { + log_code_index_scope_reconciliation_degraded("scope_reconciliation_task_panicked"); + return false; + } + }; + let vector_receipt = match observations.semantic_vector_retention_read(lease.project_root()) { + crate::telemetry::SemanticVectorRetentionReadV1::Observed { receipt } => receipt, + crate::telemetry::SemanticVectorRetentionReadV1::Unknown + | crate::telemetry::SemanticVectorRetentionReadV1::Scanning => { + log_code_index_scope_reconciliation_degraded("vector_census_incomplete"); + return false; + } + // Scope collection is gated on a complete post-convergence census, so + // an unseated semantic runtime can never reach this pass through the + // maintenance journey; refuse fail-closed with its own reason if a + // future caller ever does. + crate::telemetry::SemanticVectorRetentionReadV1::SemanticUnseated => { + log_code_index_scope_reconciliation_degraded("semantic_configuration_unseated"); + return false; + } + }; + if let Some(replay) = pending_binding_cleanup { + let Some(vector_runtime) = + tracedecay_application::semantic_runtime::project_semantic_production_runtime( + lease.project_root(), + ) + else { + log_code_index_scope_reconciliation_degraded("vector_writer_unavailable"); + return false; + }; + let _vector_writer = vector_runtime.freeze_vector_mutations().await; + let current_inputs = + match collect_scope_root_proof_inputs(lease, schedulers, &vector_receipt).await { + Ok(inputs) => inputs, + Err(failure) => { + log_code_index_scope_reconciliation_degraded(failure); + return false; + } + }; + match tracedecay_code_index_runtime::code_index_scheduler::semantic_vector_graph::project_vector_code_scope_is_live( + schedulers, + lease.project_root(), + &replay.scope_hash, + vector_receipt.revision, + ) + .await + { + tracedecay_code_index_runtime::code_index_scheduler::semantic_vector_graph::ProjectSemanticVectorCodeScopeLiveness::Ready { + source_scope, + live: false, + } => { + if source_scope != replay.source_scope { + log_code_index_scope_reconciliation_degraded( + "vector_scope_binding_replay_mismatch", + ); + return false; + } + let current_proof = match current_inputs.bind_candidate( + replay.scope_hash.clone(), + source_scope.clone(), + vector_receipt.revision, + ) { + Ok(proof) => proof, + Err(failure) => { + log_code_index_scope_reconciliation_degraded(failure); + return false; + } + }; + if current_proof != replay.liveness_proof { + log_code_index_scope_reconciliation_degraded( + "scope_binding_cleanup_authority_changed", + ); + return false; + } + match tracedecay_code_index_runtime::code_index_scheduler::semantic_vector_graph::remove_project_vector_code_scope_binding( + schedulers, + lease.project_root(), + &replay.scope_hash, + &source_scope, + vector_receipt.revision, + ) + .await + { + tracedecay_code_index_runtime::code_index_scheduler::semantic_vector_graph::ProjectSemanticVectorSourceLiveness::Ready(true) => {} + tracedecay_code_index_runtime::code_index_scheduler::semantic_vector_graph::ProjectSemanticVectorSourceLiveness::Ready(false) => { + log_code_index_scope_reconciliation_degraded( + "vector_scope_binding_not_removed", + ); + return false; + } + tracedecay_code_index_runtime::code_index_scheduler::semantic_vector_graph::ProjectSemanticVectorSourceLiveness::Unavailable(reason) => { + log_code_index_scope_reconciliation_degraded(&format!( + "vector_scope_binding_unavailable:{reason}" + )); + return false; + } + tracedecay_code_index_runtime::code_index_scheduler::semantic_vector_graph::ProjectSemanticVectorSourceLiveness::Denied(reason) => { + log_code_index_scope_reconciliation_degraded(&format!( + "vector_scope_binding_denied:{reason}" + )); + return false; + } + tracedecay_code_index_runtime::code_index_scheduler::semantic_vector_graph::ProjectSemanticVectorSourceLiveness::ResetRequired(reason) => { + log_code_index_scope_reconciliation_degraded(&format!( + "vector_scope_binding_reset_required:{reason}" + )); + return false; + } + tracedecay_code_index_runtime::code_index_scheduler::semantic_vector_graph::ProjectSemanticVectorSourceLiveness::Corrupt(reason) => { + log_code_index_scope_reconciliation_degraded(&format!( + "vector_scope_binding_corrupt:{reason}" + )); + return false; + } + } + } + tracedecay_code_index_runtime::code_index_scheduler::semantic_vector_graph::ProjectSemanticVectorCodeScopeLiveness::Missing => {} + tracedecay_code_index_runtime::code_index_scheduler::semantic_vector_graph::ProjectSemanticVectorCodeScopeLiveness::Ready { + live: true, + .. + } => { + log_code_index_scope_reconciliation_degraded("vector_scope_binding_still_live"); + return false; + } + tracedecay_code_index_runtime::code_index_scheduler::semantic_vector_graph::ProjectSemanticVectorCodeScopeLiveness::Unavailable( + reason, + ) => { + log_code_index_scope_reconciliation_degraded(&format!( + "vector_scope_unavailable:{reason}" + )); + return false; + } + tracedecay_code_index_runtime::code_index_scheduler::semantic_vector_graph::ProjectSemanticVectorCodeScopeLiveness::Denied( + reason, + ) => { + log_code_index_scope_reconciliation_degraded(&format!( + "vector_scope_denied:{reason}" + )); + return false; + } + tracedecay_code_index_runtime::code_index_scheduler::semantic_vector_graph::ProjectSemanticVectorCodeScopeLiveness::ResetRequired( + reason, + ) => { + log_code_index_scope_reconciliation_degraded(&format!( + "vector_scope_reset_required:{reason}" + )); + return false; + } + tracedecay_code_index_runtime::code_index_scheduler::semantic_vector_graph::ProjectSemanticVectorCodeScopeLiveness::Corrupt( + reason, + ) => { + log_code_index_scope_reconciliation_degraded(&format!( + "vector_scope_corrupt:{reason}" + )); + return false; + } + } + let completion_root = store_root.clone(); + let completion_replay = replay.clone(); + match tokio::task::spawn_blocking(move || { + complete_scope_root_binding_cleanup(&completion_root, &completion_replay) + .map_err(|_| "scope_binding_cleanup_completion_failed") + }) + .await + { + Ok(Ok(())) => {} + Ok(Err(failure)) => { + log_code_index_scope_reconciliation_degraded(failure); + return false; + } + Err(_) => { + log_code_index_scope_reconciliation_degraded("scope_reconciliation_task_panicked"); + return false; + } + } + // Removing a binding advances the vector census revision. Defer the + // next filesystem plan until maintenance has observed that revision. + return false; + } + + let now_secs = match now_secs_i64() { + Ok(now) => now, + Err(failure) => { + log_code_index_scope_reconciliation_degraded(failure); + return false; + } + }; + // Same micros-typed receipt contract as the code-generation pass above: + // `current_timestamp()` is a seconds clock and must not be stored as micros. + let completed_at = tracedecay_contracts::clock::now_micros(); + let Some(vector_runtime) = + tracedecay_application::semantic_runtime::project_semantic_production_runtime( + lease.project_root(), + ) + else { + log_code_index_scope_reconciliation_degraded("vector_writer_unavailable"); + return false; + }; + // Configuration activation, vector publication, and source-scope + // collection share this mutation fence. Root inventories are read twice + // under it and compared exactly before quarantine. + let _vector_writer = vector_runtime.freeze_vector_mutations().await; + let initial_inputs = + match collect_scope_root_proof_inputs(lease, schedulers, &vector_receipt).await { + Ok(inputs) => inputs, + Err(failure) => { + log_code_index_scope_reconciliation_degraded(failure); + return false; + } + }; + let plan_root = store_root.clone(); + let plan_live_roots = initial_inputs.live_roots.clone(); + let plan = tokio::task::spawn_blocking(move || { + recover_scope_root_retention(&plan_root) + .map_err(|_| "scope_reconciliation_recovery_failed")?; + plan_scope_root_retention( + &plan_root, + &plan_live_roots, + DEFAULT_STRANDED_SCOPE_MINIMUM_AGE_SECS, + now_secs, + ) + .map_err(|_| "scope_reconciliation_pass_failed") + }) + .await; + let plan = match plan { + Ok(Ok(plan)) => plan, + Ok(Err(failure)) => { + log_code_index_scope_reconciliation_degraded(failure); + return false; + } + Err(_) => { + log_code_index_scope_reconciliation_degraded("scope_reconciliation_task_panicked"); + return false; + } + }; + if plan.collectable_scopes.is_empty() { + return true; + } + + let candidates = plan.collectable_scopes.clone(); + let start = usize::try_from(now_secs) + .ok() + .map_or(0, |now| now % candidates.len()); + let mut selected = None; + const MAX_SCOPE_LIVENESS_CHECKS_PER_PASS: usize = 32; + for offset in 0..candidates.len().min(MAX_SCOPE_LIVENESS_CHECKS_PER_PASS) { + let candidate = &candidates[(start + offset) % candidates.len()]; + match tracedecay_code_index_runtime::code_index_scheduler::semantic_vector_graph::project_vector_code_scope_is_live( + schedulers, + lease.project_root(), + &candidate.scope_hash, + vector_receipt.revision, + ) + .await + { + tracedecay_code_index_runtime::code_index_scheduler::semantic_vector_graph::ProjectSemanticVectorCodeScopeLiveness::Ready { + source_scope, + live: false, + } => { + selected = Some((candidate.clone(), source_scope)); + break; + } + tracedecay_code_index_runtime::code_index_scheduler::semantic_vector_graph::ProjectSemanticVectorCodeScopeLiveness::Ready { + live: true, + .. + } => {} + tracedecay_code_index_runtime::code_index_scheduler::semantic_vector_graph::ProjectSemanticVectorCodeScopeLiveness::Missing => { + log_code_index_scope_reconciliation_degraded("vector_scope_binding_missing"); + return false; + } + tracedecay_code_index_runtime::code_index_scheduler::semantic_vector_graph::ProjectSemanticVectorCodeScopeLiveness::Unavailable( + reason, + ) => { + log_code_index_scope_reconciliation_degraded(&format!( + "vector_scope_unavailable:{reason}" + )); + return false; + } + tracedecay_code_index_runtime::code_index_scheduler::semantic_vector_graph::ProjectSemanticVectorCodeScopeLiveness::Denied( + reason, + ) => { + log_code_index_scope_reconciliation_degraded(&format!( + "vector_scope_denied:{reason}" + )); + return false; + } + tracedecay_code_index_runtime::code_index_scheduler::semantic_vector_graph::ProjectSemanticVectorCodeScopeLiveness::ResetRequired( + reason, + ) => { + log_code_index_scope_reconciliation_degraded(&format!( + "vector_scope_reset_required:{reason}" + )); + return false; + } + tracedecay_code_index_runtime::code_index_scheduler::semantic_vector_graph::ProjectSemanticVectorCodeScopeLiveness::Corrupt( + reason, + ) => { + log_code_index_scope_reconciliation_degraded(&format!( + "vector_scope_corrupt:{reason}" + )); + return false; + } + } + } + let Some((candidate, source_scope)) = selected else { + return true; + }; + let planned_proof = match initial_inputs.bind_candidate( + candidate.scope_hash.clone(), + source_scope.clone(), + vector_receipt.revision, + ) { + Ok(proof) => proof, + Err(failure) => { + log_code_index_scope_reconciliation_degraded(failure); + return false; + } + }; + let proof_root = store_root.clone(); + let proof_for_plan = planned_proof.clone(); + let plan = match tokio::task::spawn_blocking(move || { + plan_scope_root_retention_with_liveness_proof( + &proof_root, + proof_for_plan, + DEFAULT_STRANDED_SCOPE_MINIMUM_AGE_SECS, + now_secs, + ) + .map_err(|_| "scope_proof_bound_plan_failed") + }) + .await + { + Ok(Ok(plan)) => plan, + Ok(Err(failure)) => { + log_code_index_scope_reconciliation_degraded(failure); + return false; + } + Err(_) => { + log_code_index_scope_reconciliation_degraded("scope_reconciliation_task_panicked"); + return false; + } + }; + + // Re-read every terminal authority and the exact source binding after + // planning. This is the compare-and-swap immediately preceding quarantine. + let revalidated_inputs = + match collect_scope_root_proof_inputs(lease, schedulers, &vector_receipt).await { + Ok(inputs) => inputs, + Err(failure) => { + log_code_index_scope_reconciliation_degraded(failure); + return false; + } + }; + let revalidated_source_scope = + match tracedecay_code_index_runtime::code_index_scheduler::semantic_vector_graph::project_vector_code_scope_is_live( + schedulers, + lease.project_root(), + &candidate.scope_hash, + vector_receipt.revision, + ) + .await + { + tracedecay_code_index_runtime::code_index_scheduler::semantic_vector_graph::ProjectSemanticVectorCodeScopeLiveness::Ready { + source_scope, + live: false, + } => source_scope, + _ => { + log_code_index_scope_reconciliation_degraded( + "scope_candidate_changed_before_quarantine", + ); + return false; + } + }; + if revalidated_source_scope != source_scope { + log_code_index_scope_reconciliation_degraded( + "scope_candidate_binding_changed_before_quarantine", + ); + return false; + } + let revalidated_proof = match revalidated_inputs.bind_candidate( + candidate.scope_hash.clone(), + revalidated_source_scope, + vector_receipt.revision, + ) { + Ok(proof) => proof, + Err(failure) => { + log_code_index_scope_reconciliation_degraded(failure); + return false; + } + }; + if revalidated_proof != planned_proof { + log_code_index_scope_reconciliation_degraded( + "scope_liveness_authority_changed_before_quarantine", + ); + return false; + } + tracedecay_application::semantic_runtime::retain_project_semantic_code_sources( + lease.project_root(), + &revalidated_inputs.vector_sources, + ); + + let execute_root = + tracedecay_code_index_retention::code_index_generations::code_index_scope_store_root( + &layout.data_root, + ); + let intent_scope = candidate.scope_hash.clone(); + let intent_source_scope = source_scope.clone(); + let proof_for_execute = revalidated_proof.clone(); + let report = tokio::task::spawn_blocking(move || { + prepare_scope_root_binding_cleanup( + &execute_root, + &plan, + &intent_scope, + &intent_source_scope, + &proof_for_execute, + completed_at, + ) + .map_err(|_| "scope_binding_cleanup_prepare_failed")?; + execute_scope_root_retention( + &execute_root, + plan, + &proof_for_execute, + CodeGenerationRetentionModeV1::Apply, + now_secs, + completed_at, + ) + .map_err(|_| "scope_reconciliation_pass_failed") + }) + .await; + + match report { + Ok(Ok(report)) => { + let reclaimed = report + .receipt + .as_ref() + .map_or(0, |receipt| receipt.reclaimed_bytes); + if reclaimed > 0 || report.plan.stranded_scope_count() > 0 { + log_maintenance_event( + "retention_code_index_scopes", + &[ + ("store", "code-index-v1".to_string()), + ("live_scopes", report.plan.live_scope_count.to_string()), + ( + "stranded_scopes", + report.plan.stranded_scope_count().to_string(), + ), + ( + "stranded_bytes", + report.plan.stranded_scope_bytes().to_string(), + ), + ( + "retained_immature_scopes", + report.plan.retained_immature_scopes.len().to_string(), + ), + ( + "refused_scopes", + report.plan.refused_scopes.len().to_string(), + ), + ( + "collected_scopes", + report.collected_scopes.len().to_string(), + ), + ("bytes_reclaimed", reclaimed.to_string()), + ], + ); + } + // The durable intent is deliberately completed on the next cadence. + // A daemon restart at this boundary exercises exactly the same + // replay path as an ordinary subsequent tick. + report.collected_scopes.is_empty() + } + Ok(Err(failure)) => { + log_code_index_scope_reconciliation_degraded(failure); + false + } + Err(_) => { + log_code_index_scope_reconciliation_degraded("scope_reconciliation_task_panicked"); + false + } + } +} + +/// Durable failure visibility for scope reconciliation. Every refusal names why +/// so a fail-closed pass is never mistaken for "nothing was stranded". +fn log_code_index_scope_reconciliation_degraded(failure: &str) { + log_maintenance_event( + "retention_degraded", + &[ + ("pass", "code_index_scopes".to_string()), + ("failure", failure.to_string()), + ], + ); +} + +/// Runs bounded incremental-vacuum compaction over every tracked branch +/// database other than the one `cg` currently has mounted (the maintenance +/// owner compacts that store through its live-runtime authority). Best-effort +/// and independent per file: a busy or failing branch database never blocks +/// the rest, but keeps the maintenance cadence retry-eligible — see +/// `src/retention/branch_compaction.rs` for the compaction policy itself. +#[hotpath::measure(label = "daemon.git.maintenance.branch_compaction", future = true)] +pub async fn run_branch_compaction( + lease: &ProjectStoreMaintenanceLeaseV1, + config: &CompactionThresholdConfig, +) -> bool { + let layout = lease.store_layout(); + let Some(meta) = tracedecay_runtime_core::branch_meta::load_branch_meta(&layout.data_root) + else { + return true; + }; + let active_db_path = layout.graph_db_path.clone(); + let candidates = crate::retention::branch_compaction::select_branch_db_candidates( + &layout.data_root, + &meta, + &active_db_path, + ); + if candidates.is_empty() { + return true; + } + let report = crate::retention::branch_compaction::compact_branch_databases(&candidates, config); + if report.policy_invalid { + // Never silent: an out-of-range threshold disables the pass entirely + // and would otherwise be indistinguishable from "nothing to compact". + log_maintenance_event( + "retention_degraded", + &[ + ("pass", "branch_compaction".to_string()), + ("failure", "invalid_compaction_policy".to_string()), + ( + "free_page_ratio_threshold", + config.free_page_ratio_threshold.to_string(), + ), + ], + ); + return false; + } + if report.compacted.is_empty() && report.skipped.is_empty() { + return true; + } + let freed_pages: u64 = report + .compacted + .iter() + .map(|outcome| outcome.freed_pages) + .sum(); + let unreclaimable = report + .skipped + .iter() + .filter(|skip| { + skip.reason + == crate::retention::branch_compaction::BranchCompactionSkipReason::IncrementalVacuumUnavailable + }) + .count(); + log_maintenance_event( + "retention_branch_compaction", + &[ + ("project", lease.project_root().display().to_string()), + ("compacted", report.compacted.len().to_string()), + ("freed_pages", freed_pages.to_string()), + ("skipped", report.skipped.len().to_string()), + // Branch databases predating `auto_vacuum = INCREMENTAL`: their + // free pages need a full VACUUM this pass deliberately avoids. + ("unreclaimable", unreclaimable.to_string()), + ], + ); + branch_compaction_succeeded(&report) +} + +pub fn branch_compaction_succeeded( + report: &crate::retention::branch_compaction::BranchCompactionReport, +) -> bool { + !report.policy_invalid && report.skipped.is_empty() +} diff --git a/crates/tracedecay-maintenance/src/telemetry.rs b/crates/tracedecay-maintenance/src/telemetry.rs new file mode 100644 index 0000000000..1100072879 --- /dev/null +++ b/crates/tracedecay-maintenance/src/telemetry.rs @@ -0,0 +1,984 @@ +//! Store telemetry sampling and semantic-vector retention progress. +//! +//! Shared by the daemon maintenance loop and diagnostic projections. The +//! registry is a concrete owner, not a port. + +use std::collections::{BTreeMap, BTreeSet, HashMap}; +use std::path::{Path, PathBuf}; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::time::Duration; + +use tracedecay_contracts::request_identity::{GlobalRequestSurface, mint_global_request_id}; +use tracedecay_contracts::storage::{ + StorageByteSizeV1, StorageTelemetryFuture, StorageTelemetryReadV1, StoreKeyV1, + StoreSizeSampleV1, StoreSizeTelemetryPort, TableGrowthBaselinePendingV1, TableGrowthSampleV1, + TableGrowthTelemetryReadV1, TableNameV1, +}; +use tracedecay_contracts::{ + ApplicationContractError, CancellationContext, CapabilityGrantId, CapabilityGrantSnapshot, + Deadline, DisclosureClass, RequestAdmission, RequestContext, ResolvedScope, now_micros, +}; +use tracedecay_domain::{ManifestDigest, UtcMicros}; +use tracedecay_runtime_core::db::DatabaseStorageTelemetryHandle; +use tracedecay_tool_catalog::{CapabilityId, UseCaseId}; + +use crate::log_maintenance_event; +use crate::tick::MaintenanceTickOutcome; + +const STORAGE_TELEMETRY_CONTEXT_HORIZON_MICROS: i64 = 30_000_000; +const STORAGE_TELEMETRY_CAPABILITY: &str = "capability.application.storage.telemetry"; +const STORAGE_TELEMETRY_USE_CASE: &str = "use-case.application.storage.telemetry.read"; + +#[derive(Clone, Copy)] +pub struct TableWatermark { + bytes: StorageByteSizeV1, + observed_at: UtcMicros, +} + +#[derive(Clone, Copy, Eq, PartialEq)] +pub enum TableGrowthObservation { + Preview, + Advance, +} + +/// Store telemetry bound to the database's guarded read capability. +/// +/// The runtime-core handle retains the exact database client that issued it; +/// this daemon adapter must not unwrap that guard into a raw SQL handle just to +/// retain the maintenance-owned table-growth baseline. +#[derive(Clone)] +pub struct GuardedStoreTelemetryPort { + handle: DatabaseStorageTelemetryHandle, + store: StoreKeyV1, + scope: ResolvedScope, + reader_wait: Duration, + table_watermarks: Arc>>>, +} + +impl GuardedStoreTelemetryPort { + fn new( + handle: DatabaseStorageTelemetryHandle, + store: StoreKeyV1, + scope: ResolvedScope, + reader_wait: Duration, + ) -> Self { + Self { + handle, + store, + scope, + reader_wait, + table_watermarks: Arc::new(std::sync::Mutex::new(None)), + } + } + + fn admits(&self, context: &RequestContext, store: &StoreKeyV1) -> bool { + context.validate().is_ok() + && context.scope() == &self.scope + && store == &self.store + && context.admission_at(now_micros()) == RequestAdmission::Admitted + } + + fn for_scope(&self, scope: ResolvedScope) -> Self { + Self { + handle: self.handle.clone(), + store: self.store.clone(), + scope, + reader_wait: self.reader_wait, + table_watermarks: Arc::clone(&self.table_watermarks), + } + } + + fn rebind(&self, handle: DatabaseStorageTelemetryHandle, scope: ResolvedScope) -> Self { + Self { + handle, + store: self.store.clone(), + scope, + reader_wait: self.reader_wait, + table_watermarks: Arc::clone(&self.table_watermarks), + } + } + + pub fn preview_table_growth<'a>( + &'a self, + context: &'a RequestContext, + store: &'a StoreKeyV1, + ) -> StorageTelemetryFuture<'a, TableGrowthTelemetryReadV1> { + self.read_table_growth(context, store, TableGrowthObservation::Preview) + } + + fn read_table_growth<'a>( + &'a self, + context: &'a RequestContext, + store: &'a StoreKeyV1, + observation: TableGrowthObservation, + ) -> StorageTelemetryFuture<'a, TableGrowthTelemetryReadV1> { + Box::pin(hotpath::future!( + async move { + if !self.admits(context, store) { + return TableGrowthTelemetryReadV1::Denied { + store: store.clone(), + }; + } + let Ok(current) = self + .handle + .table_size_telemetry(self.reader_wait, || telemetry_interruption(context)) + else { + return TableGrowthTelemetryReadV1::Unknown { + store: store.clone(), + }; + }; + let observed_at = now_micros(); + let mut current_tables = BTreeMap::new(); + for sample in current { + let Ok(table) = TableNameV1::new(sample.table_name) else { + return TableGrowthTelemetryReadV1::Unknown { + store: store.clone(), + }; + }; + current_tables.insert(table, StorageByteSizeV1(sample.bytes)); + } + let mut watermarks = match self.table_watermarks.lock() { + Ok(watermarks) => watermarks, + Err(poisoned) => poisoned.into_inner(), + }; + compare_table_growth( + store, + current_tables, + observed_at, + &mut watermarks, + observation, + ) + }, + label = "daemon.maintenance.read_table_growth" + )) + } +} + +impl StoreSizeTelemetryPort for GuardedStoreTelemetryPort { + fn store_size<'a>( + &'a self, + context: &'a RequestContext, + store: &'a StoreKeyV1, + ) -> StorageTelemetryFuture<'a, StorageTelemetryReadV1> { + Box::pin(hotpath::future!( + async move { + if !self.admits(context, store) { + return StorageTelemetryReadV1::Denied { + store: store.clone(), + }; + } + let Ok(sample) = self + .handle + .store_size_telemetry(self.reader_wait, || telemetry_interruption(context)) + else { + return StorageTelemetryReadV1::Unknown { + store: store.clone(), + }; + }; + let sample = StoreSizeSampleV1 { + store: store.clone(), + page_size_bytes: sample.page_size_bytes, + page_count: sample.page_count, + freelist_pages: sample.freelist_pages, + observed_at: now_micros(), + }; + if sample.validate().is_err() { + return StorageTelemetryReadV1::Unknown { + store: store.clone(), + }; + } + StorageTelemetryReadV1::Observed { sample } + }, + label = "daemon.maintenance.read_store_size" + )) + } + + fn table_growth<'a>( + &'a self, + context: &'a RequestContext, + store: &'a StoreKeyV1, + ) -> StorageTelemetryFuture<'a, TableGrowthTelemetryReadV1> { + self.read_table_growth(context, store, TableGrowthObservation::Advance) + } +} + +#[hotpath::measure(label = "daemon.maintenance.compare_table_growth")] +pub fn compare_table_growth( + store: &StoreKeyV1, + current_tables: BTreeMap, + observed_at: UtcMicros, + watermarks: &mut Option>, + observation: TableGrowthObservation, +) -> TableGrowthTelemetryReadV1 { + let Some(previous_watermarks) = watermarks.as_ref() else { + if observation == TableGrowthObservation::Preview { + return TableGrowthTelemetryReadV1::Unknown { + store: store.clone(), + }; + } + let tables_observed = u64::try_from(current_tables.len()).unwrap_or(u64::MAX); + *watermarks = Some( + current_tables + .into_iter() + .map(|(table, bytes)| (table, TableWatermark { bytes, observed_at })) + .collect(), + ); + return TableGrowthTelemetryReadV1::BaselineEstablished { + store: store.clone(), + observed_at, + tables_observed, + }; + }; + + let mut growth = Vec::new(); + let mut baseline_pending = Vec::new(); + for (table, current_bytes) in ¤t_tables { + if let Some(previous) = previous_watermarks.get(table) { + let sample = TableGrowthSampleV1 { + store: store.clone(), + table: table.clone(), + previous_bytes: previous.bytes, + current_bytes: *current_bytes, + previous_observed_at: previous.observed_at, + current_observed_at: observed_at, + }; + if sample.validate().is_err() { + return TableGrowthTelemetryReadV1::Unknown { + store: store.clone(), + }; + } + growth.push(sample); + } else { + baseline_pending.push(TableGrowthBaselinePendingV1 { + store: store.clone(), + table: table.clone(), + current_bytes: *current_bytes, + observed_at, + }); + } + } + if observation == TableGrowthObservation::Advance { + *watermarks = Some( + current_tables + .into_iter() + .map(|(table, bytes)| (table, TableWatermark { bytes, observed_at })) + .collect(), + ); + } + TableGrowthTelemetryReadV1::Observed { + store: store.clone(), + samples: growth, + baseline_pending, + } +} + +fn telemetry_interruption( + context: &RequestContext, +) -> Option { + match context.admission_at(now_micros()) { + RequestAdmission::Admitted => None, + RequestAdmission::Cancelled => Some(tracedecay_store::UnavailableReasonV1::Cancelled), + RequestAdmission::TimedOut => Some(tracedecay_store::UnavailableReasonV1::DeadlineExceeded), + } +} + +#[derive(Clone)] +struct CachedStoreTelemetryPort { + scope: ResolvedScope, + store: StoreKeyV1, + port: GuardedStoreTelemetryPort, +} + +/// Daemon-owned table-growth baseline authority shared by maintenance and +/// read-only diagnostic projections. +#[derive(Clone, Default)] +pub struct StoreTelemetrySamplingRegistry { + ports: Arc>>, + semantic_vector_retention: + Arc>>, + graph_replay_release: Arc>>, + graph_staging_release: + Arc>>, + /// Last by-design retention operator line per lane and project. A + /// persistent unavailable-by-design condition logs once, then counts on + /// [`daemon.git.maintenance.retention_quiet_total`]; a state change or a + /// genuine anomaly emits again. + retention_operator_log: Arc>>, + loud_retention_this_tick: Arc, +} + +/// Operator-log lane for the once-then-quiet retention pin. +#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] +pub enum RetentionOperatorLogLaneV1 { + Semantic, + CodeGeneration, + Tick, +} + +#[derive(Clone, Debug, Eq, Hash, PartialEq)] +struct RetentionOperatorLogKeyV1 { + lane: RetentionOperatorLogLaneV1, + scope: PathBuf, +} + +/// Longest run of short-cadence ticks a project's graph-replay release +/// reconcile may be skipped after consecutive unhealthy attempts. At the +/// one-minute retry cadence this bounds post-recovery release latency to +/// roughly eight minutes while a wedged runtime is probed a handful of times +/// per hour instead of once per tick. +const GRAPH_REPLAY_RELEASE_BACKOFF_CAP_TICKS: u32 = 8; + +/// Per-project reconcile state for the graph-replay release queue. +/// +/// Release evidence is durable on disk, so none of this state guards +/// correctness: the cursor makes the queue walk incremental across ticks +/// (retained entries stop blocking later pages), and the backoff window +/// converts "retry a known-wedged graph runtime every tick" into a bounded +/// re-probe. Losing the state (restart, project retirement) only means the +/// next attempt starts from the front of the queue immediately. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +struct GraphReplayReleaseProgressV1 { + consecutive_unhealthy: u32, + skip_remaining: u32, + cursor: Option, +} + +#[derive(Clone, Copy, Default)] +pub struct StoreTelemetrySamplingOutcome { + pub observed: u64, + pub unavailable: u64, +} + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct SemanticVectorRetentionBacklogV1 { + pub pending: u64, + pub ready: u64, + pub published: u64, + pub cancelled: u64, +} + +impl SemanticVectorRetentionBacklogV1 { + pub fn from_receipt(receipt: &tracedecay_store::SemanticVectorProjectCensusReceipt) -> Self { + Self { + pending: receipt.counts.pending, + ready: receipt.counts.ready, + published: receipt.counts.published, + cancelled: receipt.counts.cancelled, + } + } +} + +/// Result of recording one semantic-vector retention census page. +/// +/// Rejected variants stay fail-closed: progress resets and no receipt is +/// accepted. `CensusCountOverflow` is the only true u64-sum overflow +/// (`receipt.validate()`); other rejects name the actual page defect. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum SemanticVectorRetentionCensusOutcome { + Accepted, + InconsistentPage, + IncompleteTerminalPage, + CensusCountOverflow, + ReceiptIdentityMismatch, +} + +impl SemanticVectorRetentionCensusOutcome { + #[hotpath::skip] + pub const fn as_failure_label(self) -> Option<&'static str> { + match self { + Self::Accepted => None, + Self::InconsistentPage => Some("inconsistent_page"), + Self::IncompleteTerminalPage => Some("incomplete_terminal_page"), + Self::CensusCountOverflow => Some("census_count_overflow"), + Self::ReceiptIdentityMismatch => Some("receipt_identity_mismatch"), + } + } +} + +// `Observed` is matched by field-destructuring across several call sites +// (doctor_kernel, git_watch/store_maintenance); boxing the receipt would +// ripple through all of them for a cold, infrequently-read maintenance +// status. +#[allow(clippy::large_enum_variant)] +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum SemanticVectorRetentionReadV1 { + Unknown, + /// The semantic runtime is not seated for this daemon, so no vector + /// census will ever start, let alone complete. This is the ordinary + /// default-off state, distinct from [`Self::Unknown`] (a census that has + /// not run yet or was reset by a failure or mutation). + SemanticUnseated, + Scanning, + Observed { + receipt: tracedecay_store::SemanticVectorProjectCensusReceipt, + }, +} + +#[derive(Clone, Debug, Default)] +struct SemanticVectorRetentionProgressV1 { + cursor: Option, + observed: Option, + scanning: bool, + semantic_unseated: bool, +} + +impl StoreTelemetrySamplingRegistry { + pub fn register_port( + &self, + path: &Path, + scope: &ResolvedScope, + open: impl FnOnce() -> Result, + ) -> bool { + let Some(store_name) = path.file_name().and_then(std::ffi::OsStr::to_str) else { + return false; + }; + let Ok(store) = StoreKeyV1::new(store_name.to_owned()) else { + return false; + }; + let Ok(handle) = open() else { + return false; + }; + let mut ports = self + .ports + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if let Some(cached) = ports.get_mut(path) { + cached.scope = scope.clone(); + cached.store = store; + cached.port = cached.port.rebind(handle, scope.clone()); + return true; + } + let port = GuardedStoreTelemetryPort::new( + handle, + store.clone(), + scope.clone(), + Duration::from_secs(5), + ); + ports.insert( + path.to_path_buf(), + CachedStoreTelemetryPort { + scope: scope.clone(), + store, + port, + }, + ); + true + } + + pub fn registered_port( + &self, + path: &Path, + scope: &ResolvedScope, + ) -> Option<(StoreKeyV1, GuardedStoreTelemetryPort)> { + let ports = self + .ports + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let cached = ports.get(path)?; + Some((cached.store.clone(), cached.port.for_scope(scope.clone()))) + } + + /// Release the telemetry client's exact database lease before the owning + /// project store is retired. Other project and profile sampling ports stay + /// mounted. + pub fn release_retained_handle(&self, path: &Path) { + self.ports + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .remove(path); + self.semantic_vector_retention + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .remove(path); + self.graph_replay_release + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .remove(path); + self.retention_operator_log + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .retain(|key, _| key.scope != path); + } + + pub fn release_retained_handles_for_shutdown(&self) { + self.ports + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .clear(); + self.semantic_vector_retention + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .clear(); + self.graph_replay_release + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .clear(); + self.retention_operator_log + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .clear(); + self.loud_retention_this_tick + .store(false, Ordering::Release); + } + + pub fn semantic_vector_retention_cursor( + &self, + project_root: &Path, + ) -> Option { + self.semantic_vector_retention + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .get(project_root) + .and_then(|progress| progress.cursor.clone()) + } + + pub fn retain_project_maintenance_state(&self, active_projects: &BTreeSet) { + self.semantic_vector_retention + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .retain(|project, _| active_projects.contains(project)); + self.graph_replay_release + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .retain(|project, _| active_projects.contains(project)); + self.graph_staging_release + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .retain(|project, _| active_projects.contains(project)); + self.retention_operator_log + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .retain(|key, _| { + key.scope.as_os_str().is_empty() || active_projects.contains(&key.scope) + }); + } + + /// Whether this tick may attempt the graph-replay release reconcile. + /// + /// Consecutive unhealthy attempts open a bounded skip window; each denied + /// tick burns one unit of it, so a wedged runtime is re-probed after at + /// most [`GRAPH_REPLAY_RELEASE_BACKOFF_CAP_TICKS`] short-cadence ticks + /// rather than being polled (and timing out) on every one. + pub fn graph_replay_release_attempt_admitted(&self, project_root: &Path) -> bool { + let mut progress = self + .graph_replay_release + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let Some(state) = progress.get_mut(project_root) else { + return true; + }; + if state.skip_remaining == 0 { + return true; + } + state.skip_remaining -= 1; + false + } + + /// Record a release attempt the graph runtime could not serve (deadline, + /// unavailability, or a held replay pool) and widen the skip window: + /// 1, 2, 4, then capped at [`GRAPH_REPLAY_RELEASE_BACKOFF_CAP_TICKS`]. + pub fn record_graph_replay_release_unhealthy(&self, project_root: &Path) { + let mut progress = self + .graph_replay_release + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let state = progress.entry(project_root.to_path_buf()).or_default(); + state.consecutive_unhealthy = state.consecutive_unhealthy.saturating_add(1); + state.skip_remaining = GRAPH_REPLAY_RELEASE_BACKOFF_CAP_TICKS + .min(1_u32 << state.consecutive_unhealthy.saturating_sub(1).min(3)); + } + + /// Record a served release attempt: close the skip window and advance the + /// durable-queue cursor to `continuation` (`None` restarts from the front + /// of the queue on the next attempt). + pub fn record_graph_replay_release_served( + &self, + project_root: &Path, + continuation: Option, + ) { + let mut progress = self + .graph_replay_release + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + match progress.entry(project_root.to_path_buf()) { + std::collections::hash_map::Entry::Occupied(mut entry) => { + if continuation.is_none() { + entry.remove(); + } else { + *entry.get_mut() = GraphReplayReleaseProgressV1 { + consecutive_unhealthy: 0, + skip_remaining: 0, + cursor: continuation, + }; + } + } + std::collections::hash_map::Entry::Vacant(entry) => { + if continuation.is_some() { + entry.insert(GraphReplayReleaseProgressV1 { + consecutive_unhealthy: 0, + skip_remaining: 0, + cursor: continuation, + }); + } + } + } + } + + /// The release-queue cursor recorded by the last served attempt. + pub fn graph_replay_release_cursor(&self, project_root: &Path) -> Option { + self.graph_replay_release + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .get(project_root) + .and_then(|state| state.cursor.clone()) + } + + pub fn graph_staging_release_cursor( + &self, + project_root: &Path, + ) -> Option { + self.graph_staging_release + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .get(project_root) + .cloned() + } + + pub fn record_graph_staging_release_cursor( + &self, + project_root: &Path, + cursor: Option, + ) { + let mut cursors = self + .graph_staging_release + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if let Some(cursor) = cursor { + cursors.insert(project_root.to_path_buf(), cursor); + } else { + cursors.remove(project_root); + } + } + + /// Open a fresh per-tick loud-vs-quiet window before any retention pass + /// emits. A genuine anomaly during the tick keeps the tick line loud. + pub fn begin_retention_tick_log_window(&self) { + self.loud_retention_this_tick + .store(false, Ordering::Release); + } + + /// Mark that this tick emitted a genuine retention anomaly. By-design + /// unavailable pins stay quiet; this forces the tick summary to stay loud. + pub fn mark_loud_retention_log(&self) { + self.loud_retention_this_tick.store(true, Ordering::Release); + } + + /// Whether this (lane, scope, detail) pair should emit an operator line. + /// + /// Identical repeats of a persistent by-design condition increment + /// `daemon.git.maintenance.retention_quiet_total` and stay silent. A + /// changed detail logs again. + pub fn admit_by_design_retention_log( + &self, + lane: RetentionOperatorLogLaneV1, + scope: &Path, + detail: &str, + ) -> bool { + let key = RetentionOperatorLogKeyV1 { + lane, + scope: scope.to_path_buf(), + }; + let mut states = self + .retention_operator_log + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if states.get(&key).map(String::as_str) == Some(detail) { + hotpath::gauge!("daemon.git.maintenance.retention_quiet_total").inc(1_u64); + return false; + } + states.insert(key, detail.to_owned()); + true + } + + pub fn clear_by_design_retention_log(&self, lane: RetentionOperatorLogLaneV1, scope: &Path) { + let key = RetentionOperatorLogKeyV1 { + lane, + scope: scope.to_path_buf(), + }; + self.retention_operator_log + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .remove(&key); + } + + /// Emit `retention_degraded` once per by-design state, or every time for + /// a genuine anomaly. Repeat by-design ticks count on the quiet gauge. + pub fn emit_retention_degraded(&self, project_root: &Path, pass: &'static str, failure: &str) { + let lane = match pass { + "semantic_vector_generations" => RetentionOperatorLogLaneV1::Semantic, + "code_generations" => RetentionOperatorLogLaneV1::CodeGeneration, + _ => { + self.mark_loud_retention_log(); + log_maintenance_event( + "retention_degraded", + &[("pass", pass.to_owned()), ("failure", failure.to_owned())], + ); + return; + } + }; + if retention_failure_is_by_design(lane, failure) { + if !self.admit_by_design_retention_log(lane, project_root, failure) { + return; + } + } else { + self.mark_loud_retention_log(); + self.clear_by_design_retention_log(lane, project_root); + } + log_maintenance_event( + "retention_degraded", + &[("pass", pass.to_owned()), ("failure", failure.to_owned())], + ); + } + + /// Whether the tick summary line should be written. Repeated by-design + /// `retry` ticks stay quiet; a loud anomaly or an outcome change logs. + pub fn admit_retention_tick_log(&self, outcome: MaintenanceTickOutcome) -> bool { + let detail = format!("{}:{}", outcome.succeeded(), outcome.label()); + let loud = self.loud_retention_this_tick.load(Ordering::Acquire); + if matches!(outcome, MaintenanceTickOutcome::Retry) && !loud { + return self.admit_by_design_retention_log( + RetentionOperatorLogLaneV1::Tick, + Path::new(""), + &detail, + ); + } + self.clear_by_design_retention_log(RetentionOperatorLogLaneV1::Tick, Path::new("")); + true + } + + pub fn record_semantic_vector_retention_failure(&self, project_root: &Path) { + self.semantic_vector_retention + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .insert( + project_root.to_path_buf(), + SemanticVectorRetentionProgressV1::default(), + ); + } + + /// Pin the project's census read to [`SemanticVectorRetentionReadV1::SemanticUnseated`]. + /// + /// The vector retention pass records this when the daemon has no seated + /// semantic runtime, so downstream passes can distinguish "no census will + /// ever exist" from a census that merely has not completed yet. + pub fn record_semantic_vector_retention_unseated(&self, project_root: &Path) { + self.semantic_vector_retention + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .insert( + project_root.to_path_buf(), + SemanticVectorRetentionProgressV1 { + semantic_unseated: true, + ..SemanticVectorRetentionProgressV1::default() + }, + ); + } + + pub fn record_semantic_vector_retention_census( + &self, + project_root: &Path, + census: &tracedecay_graph_db::SemanticVectorRetentionCensus, + ) -> SemanticVectorRetentionCensusOutcome { + use tracedecay_graph_db::SemanticVectorRetentionAction; + + let mut retention = self + .semantic_vector_retention + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let progress = retention.entry(project_root.to_path_buf()).or_default(); + // A census page can only come from a seated semantic runtime. + progress.semantic_unseated = false; + if matches!( + census.action, + SemanticVectorRetentionAction::Retired(_) + | SemanticVectorRetentionAction::Finalized(_) + | SemanticVectorRetentionAction::CancelledRemoved(_) + ) { + // The returned page describes the pre-action state. Restart from + // the beginning on the next tick instead of publishing stale sums. + *progress = SemanticVectorRetentionProgressV1::default(); + return SemanticVectorRetentionCensusOutcome::Accepted; + } + progress.cursor.clone_from(&census.continuation); + if census.continuation.is_some() { + if census.complete_receipt.is_some() { + *progress = SemanticVectorRetentionProgressV1::default(); + return SemanticVectorRetentionCensusOutcome::InconsistentPage; + } + progress.scanning = true; + progress.observed = None; + } else { + let Some(receipt) = census.complete_receipt.clone() else { + *progress = SemanticVectorRetentionProgressV1::default(); + return SemanticVectorRetentionCensusOutcome::IncompleteTerminalPage; + }; + if receipt.validate().is_err() { + *progress = SemanticVectorRetentionProgressV1::default(); + return SemanticVectorRetentionCensusOutcome::CensusCountOverflow; + } + if receipt.shard_id != census.shard_id || receipt.revision != census.revision { + *progress = SemanticVectorRetentionProgressV1::default(); + return SemanticVectorRetentionCensusOutcome::ReceiptIdentityMismatch; + } + progress.observed = Some(receipt); + progress.cursor = None; + progress.scanning = false; + } + SemanticVectorRetentionCensusOutcome::Accepted + } + + pub fn semantic_vector_retention_read( + &self, + project_root: &Path, + ) -> SemanticVectorRetentionReadV1 { + let retention = self + .semantic_vector_retention + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let Some(progress) = retention.get(project_root) else { + return SemanticVectorRetentionReadV1::Unknown; + }; + if progress.semantic_unseated { + return SemanticVectorRetentionReadV1::SemanticUnseated; + } + if progress.scanning { + return SemanticVectorRetentionReadV1::Scanning; + } + progress + .observed + .clone() + .map_or(SemanticVectorRetentionReadV1::Unknown, |receipt| { + SemanticVectorRetentionReadV1::Observed { receipt } + }) + } + + pub fn semantic_vector_scope_collection_ready(&self, project_root: &Path) -> bool { + matches!( + self.semantic_vector_retention_read(project_root), + SemanticVectorRetentionReadV1::Observed { + receipt: tracedecay_store::SemanticVectorProjectCensusReceipt { + counts: tracedecay_store::SemanticVectorStageCensusCounts { + pending: 0, + ready: 0, + published: _, + cancelled: 0, + }, + .. + }, + } + ) + } + + #[hotpath::measure(label = "daemon.maintenance.sample_store_telemetry", future = true)] + pub async fn advance_registered( + &self, + active_paths: &BTreeSet, + sampled_paths: &BTreeSet, + ) -> StoreTelemetrySamplingOutcome { + let ports = { + let mut ports = self + .ports + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + ports.retain(|path, _| active_paths.contains(path)); + ports + .iter() + .filter(|(path, _)| sampled_paths.contains(*path)) + .map(|(_, cached)| cached.clone()) + .collect::>() + }; + let mut outcome = StoreTelemetrySamplingOutcome::default(); + for cached in ports { + let Ok(context) = storage_telemetry_request_context(cached.scope.clone()) else { + outcome.unavailable = outcome.unavailable.saturating_add(1); + continue; + }; + match cached.port.table_growth(&context, &cached.store).await { + TableGrowthTelemetryReadV1::BaselineEstablished { .. } + | TableGrowthTelemetryReadV1::Observed { .. } => { + outcome.observed = outcome.observed.saturating_add(1); + } + TableGrowthTelemetryReadV1::Unsupported { .. } + | TableGrowthTelemetryReadV1::Denied { .. } + | TableGrowthTelemetryReadV1::Unknown { .. } => { + outcome.unavailable = outcome.unavailable.saturating_add(1); + } + } + } + outcome + } +} + +/// Persistent by-design retention conditions log once, then count on gauges. +/// Corrupt, reset, denied, and cancelled failures stay loud every attempt. +pub fn retention_failure_is_by_design(lane: RetentionOperatorLogLaneV1, failure: &str) -> bool { + match lane { + RetentionOperatorLogLaneV1::Semantic => { + failure == "configuration_inventory_unavailable" || failure.starts_with("unavailable:") + } + RetentionOperatorLogLaneV1::CodeGeneration => { + failure.starts_with("vector_inventory_offline:") + } + RetentionOperatorLogLaneV1::Tick => false, + } +} + +#[hotpath::measure(label = "daemon.maintenance.mint_telemetry_context")] +fn storage_telemetry_request_context( + scope: ResolvedScope, +) -> Result { + let observed_at = now_micros(); + let expires_at = tracedecay_domain::UtcMicros( + observed_at + .0 + .saturating_add(STORAGE_TELEMETRY_CONTEXT_HORIZON_MICROS), + ); + let request_id = + mint_global_request_id(GlobalRequestSurface::DaemonStorageTelemetry).map_err(|_| { + ApplicationContractError::Inconsistent { + field: "storage telemetry request identity", + } + })?; + let suffix = request_id.as_str().to_owned(); + let actor = tracedecay_domain::ActorId::new("actor.tracedecay-daemon-storage-telemetry")?; + let capability = CapabilityId::new(STORAGE_TELEMETRY_CAPABILITY.to_owned())?; + let use_case = UseCaseId::new(STORAGE_TELEMETRY_USE_CASE.to_owned())?; + let manifest: ManifestDigest = tracedecay_domain::canonical_sha256(&( + "tracedecay.daemon.storage-telemetry-grant.v1", + &scope, + &capability, + &use_case, + expires_at, + ))?; + let grant = CapabilityGrantSnapshot::new( + CapabilityGrantId::new(format!("grant.daemon.storage-telemetry.{suffix}"))?, + 1, + manifest, + actor.clone(), + observed_at, + expires_at, + scope.clone(), + BTreeSet::from([capability]), + BTreeSet::from([use_case]), + DisclosureClass::Metadata, + )?; + RequestContext::new( + actor, + scope, + grant, + request_id, + Deadline::new(expires_at)?, + CancellationContext::active(format!("cancel.daemon.storage-telemetry.{suffix}"))?, + ) +} diff --git a/crates/tracedecay-maintenance/src/tick.rs b/crates/tracedecay-maintenance/src/tick.rs new file mode 100644 index 0000000000..9863a304b1 --- /dev/null +++ b/crates/tracedecay-maintenance/src/tick.rs @@ -0,0 +1,184 @@ +//! Maintenance tick policy: continuation, cadence, and store-window selection. + +use std::time::Duration; + +/// Resume a bounded maintenance phase over the normal graph window. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum MaintenanceContinuation { + /// Resume the bounded semantic-vector phase over the normal graph window. + SemanticVectorRetention, + /// Resume bounded code-generation retention over the normal graph window. + CodeGenerationRetention, +} + +impl MaintenanceContinuation { + /// Two phases asking to continue collapse to the one whose continuation + /// tick still advances both. + #[must_use] + pub fn combine(self, other: Self) -> Self { + match (self, other) { + (Self::CodeGenerationRetention, _) | (_, Self::CodeGenerationRetention) => { + Self::CodeGenerationRetention + } + (Self::SemanticVectorRetention, Self::SemanticVectorRetention) => { + Self::SemanticVectorRetention + } + } + } +} + +/// Outcome of one maintenance tick or per-store unit. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum MaintenanceTickOutcome { + Complete, + Continue(MaintenanceContinuation), + Retry, +} + +impl MaintenanceTickOutcome { + #[must_use] + pub fn is_complete(self) -> bool { + self == Self::Complete + } + + #[must_use] + pub fn continuation(self) -> Option { + match self { + Self::Continue(continuation) => Some(continuation), + Self::Complete | Self::Retry => None, + } + } + + #[must_use] + pub fn succeeded(self) -> bool { + !matches!(self, Self::Retry) + } + + #[must_use] + pub fn label(self) -> &'static str { + match self { + Self::Complete => "complete", + Self::Continue(MaintenanceContinuation::SemanticVectorRetention) => { + "semantic_vector_progress" + } + Self::Continue(MaintenanceContinuation::CodeGenerationRetention) => { + "code_generation_progress" + } + Self::Retry => "retry", + } + } + + /// A failure wins over ordinary bounded progress so the next short tick + /// retries the complete maintenance journey. + #[must_use] + pub fn combine(self, other: Self) -> Self { + match (self, other) { + (Self::Retry, _) | (_, Self::Retry) => Self::Retry, + (Self::Continue(left), Self::Continue(right)) => Self::Continue(left.combine(right)), + (Self::Continue(continuation), Self::Complete) + | (Self::Complete, Self::Continue(continuation)) => Self::Continue(continuation), + (Self::Complete, Self::Complete) => Self::Complete, + } + } +} + +/// The maintenance loop parks on `tokio::time::sleep_until`, so every deadline +/// it derives must be measured on the same clock the timer wheel uses. +pub type CadenceInstant = tokio::time::Instant; + +/// Interval and retry-delay policy for the maintenance loop. +#[derive(Debug)] +pub struct MaintenanceCadence { + interval: Duration, + retry_delay: Duration, + not_before: Option, + in_flight: bool, +} + +impl MaintenanceCadence { + #[must_use] + pub fn new(interval: Duration) -> Self { + Self { + interval, + retry_delay: interval.min(Duration::from_mins(1)), + not_before: None, + in_flight: false, + } + } + + pub fn reserve(&mut self, now: CadenceInstant) -> bool { + if self.in_flight || self.not_before.is_some_and(|not_before| now < not_before) { + return false; + } + self.in_flight = true; + true + } + + pub fn finish( + &mut self, + now: CadenceInstant, + outcome: MaintenanceTickOutcome, + ) -> CadenceInstant { + self.in_flight = false; + let delay = match outcome { + MaintenanceTickOutcome::Complete => self.interval, + MaintenanceTickOutcome::Continue(_) | MaintenanceTickOutcome::Retry => self.retry_delay, + }; + let deadline = now + delay; + self.not_before = Some(deadline); + deadline + } + + #[must_use] + pub fn retry_delay(&self) -> Duration { + self.retry_delay + } +} + +/// Pure round-robin window selection over stably-sorted store keys. +#[must_use] +pub fn select_store_window( + keys: &[String], + after: Option<&str>, + budget: usize, +) -> (Vec, Option) { + let count = keys.len(); + if count == 0 || budget == 0 { + return (Vec::new(), after.map(str::to_owned)); + } + let start = match after { + Some(cursor) => keys.partition_point(|key| key.as_str() <= cursor) % count, + None => 0, + }; + let take = budget.min(count); + let indices = (0..take) + .map(|offset| (start + offset) % count) + .collect::>(); + let next = indices.last().map(|&index| keys[index].clone()); + (indices, next) +} + +#[must_use] +pub fn cursor_after_attempted_units( + keys: &[String], + window: &[usize], + attempted: usize, + prior: Option<&str>, +) -> Option { + attempted + .checked_sub(1) + .and_then(|last| window.get(last)) + .and_then(|&index| keys.get(index)) + .cloned() + .or_else(|| prior.map(str::to_owned)) +} + +/// Whether any retention or compaction window is configured. +#[must_use] +pub fn retention_maintenance_enabled( + orphan_store_gc_days: Option, + incident_debris_retention_days: Option, + compaction: bool, +) -> bool { + orphan_store_gc_days.is_some() || incident_debris_retention_days.is_some() || compaction +} diff --git a/crates/tracedecay/src/daemon.rs b/crates/tracedecay/src/daemon.rs index dd2a6831b0..5742c113c0 100644 --- a/crates/tracedecay/src/daemon.rs +++ b/crates/tracedecay/src/daemon.rs @@ -6,7 +6,7 @@ use std::sync::Arc; #[cfg(test)] use std::sync::atomic::AtomicUsize; use std::sync::atomic::{AtomicBool, Ordering}; -use std::time::{Instant, SystemTime, UNIX_EPOCH}; +use std::time::Instant; use serde_json::json; #[cfg(unix)] @@ -262,8 +262,6 @@ pub(crate) mod retained_test_support; mod shutdown_coordination; mod shutdown_orchestration; mod shutdown_watchdog; -#[cfg(feature = "hotpath")] -pub use shutdown_watchdog::install_hotpath_shutdown_finalizer; pub(crate) use core_admission::*; pub use core_client::*; pub(crate) use core_doctor::*; @@ -273,6 +271,8 @@ pub(crate) use core_lifecycle::*; pub use core_logging::*; pub use core_proxy::*; pub(crate) use shutdown_coordination::ShutdownStatus; +#[cfg(feature = "hotpath")] +pub use shutdown_watchdog::install_hotpath_shutdown_finalizer; mod github_credential_lifecycle; mod graph_resolution; use graph_resolution::retained_project_server_resolver; diff --git a/crates/tracedecay/src/daemon/branch_admin.rs b/crates/tracedecay/src/daemon/branch_admin.rs index 03ad405855..e2e73a3212 100644 --- a/crates/tracedecay/src/daemon/branch_admin.rs +++ b/crates/tracedecay/src/daemon/branch_admin.rs @@ -24,13 +24,13 @@ use super::profile_host_admission_replay::{ }; #[cfg(unix)] use super::scheduler::{AutomationSchedulerHandle, MaintenanceTaskTermination}; -use tracedecay_store_runtime::StoreWriterGates; -pub(super) use tracedecay_store_runtime::{StoreWriterClass, WriterScope}; use super::{DaemonHandshake, DatabaseOwnerRegistry, write_json_rpc_response}; use tracedecay_code_index_runtime::git_transactions::DaemonGitIndexTransactionServiceRegistry; use tracedecay_daemon_identity::{authority, profile_identity}; use tracedecay_daemon_service::DaemonNativeIntegrationRuntimeRegistrar; use tracedecay_session_runtime::session_temporal_refresh_scheduler::SessionTemporalRefreshSchedulerRegistry; +use tracedecay_store_runtime::StoreWriterGates; +pub(super) use tracedecay_store_runtime::{StoreWriterClass, WriterScope}; const BRANCH_ADMIN_TOOL_NAME: &str = "tracedecay_admin_branch"; mod project_retirement; @@ -456,7 +456,7 @@ pub(super) struct StoreAdministration { profile_host_admission_replay: Arc, profile_session_refresh_services: ProfileSessionRefreshServices, session_sync_service: Arc, - store_telemetry_sampling: super::maintenance::StoreTelemetrySamplingRegistry, + store_telemetry_sampling: tracedecay_maintenance::telemetry::StoreTelemetrySamplingRegistry, #[cfg(unix)] automation_schedulers: Arc>>, @@ -563,7 +563,8 @@ impl Default for StoreAdministration { session_sync_service: Arc::new( tracedecay_session_runtime::session_sync::DaemonSessionSyncService::default(), ), - store_telemetry_sampling: super::maintenance::StoreTelemetrySamplingRegistry::default(), + store_telemetry_sampling: + tracedecay_maintenance::telemetry::StoreTelemetrySamplingRegistry::default(), #[cfg(unix)] automation_schedulers: Arc::new(tokio::sync::Mutex::new(HashMap::new())), #[cfg(unix)] @@ -708,7 +709,7 @@ impl StoreAdministration { pub(super) fn store_telemetry_sampling( &self, - ) -> super::maintenance::StoreTelemetrySamplingRegistry { + ) -> tracedecay_maintenance::telemetry::StoreTelemetrySamplingRegistry { self.store_telemetry_sampling.clone() } diff --git a/crates/tracedecay/src/daemon/branch_admin/remote_recovery_lifecycle.rs b/crates/tracedecay/src/daemon/branch_admin/remote_recovery_lifecycle.rs index 57c87190c6..5cdd30f70f 100644 --- a/crates/tracedecay/src/daemon/branch_admin/remote_recovery_lifecycle.rs +++ b/crates/tracedecay/src/daemon/branch_admin/remote_recovery_lifecycle.rs @@ -14,11 +14,11 @@ use tracedecay_store_runtime::{ use super::{ DatabaseOwnerRegistry, StoreAdministration, StoreWriterClass, StoreWriterGates, WriterScope, }; -use crate::daemon::maintenance::StoreTelemetrySamplingRegistry; -use tracedecay_store_runtime::WriterAdmissionGuard; use tracedecay_daemon_identity::authority; use tracedecay_daemon_service::DaemonNativeIntegrationRuntimeRegistrar; use tracedecay_domain::errors::{Result, TraceDecayError}; +use tracedecay_maintenance::telemetry::StoreTelemetrySamplingRegistry; +use tracedecay_store_runtime::WriterAdmissionGuard; pub(in crate::daemon) struct RemoteRecoveryProjectLifecycleV1 { brain_id: BrainId, diff --git a/crates/tracedecay/src/daemon/doctor_kernel.rs b/crates/tracedecay/src/daemon/doctor_kernel.rs index 905dbb952a..7d552f43f7 100644 --- a/crates/tracedecay/src/daemon/doctor_kernel.rs +++ b/crates/tracedecay/src/daemon/doctor_kernel.rs @@ -35,10 +35,10 @@ use tracedecay_contracts::{ Deadline, DisclosureClass, RequestContext, now_micros, }; -use super::maintenance::GuardedStoreTelemetryPort; use tracedecay_daemon_service::{ DaemonFeedbackRuntimeRegistrar, DaemonSemanticOwnerRuntimeRegistrar, }; +use tracedecay_maintenance::telemetry::GuardedStoreTelemetryPort; const DOCTOR_REPORT_CAPABILITY: &str = "capability.application.doctor.report"; const DOCTOR_REPORT_USE_CASE: &str = "use-case.application.doctor.report"; @@ -432,7 +432,7 @@ async fn collect_over_budget_store_findings( #[hotpath::measure(label = "daemon.doctor.code_generation_retention", future = true)] pub(super) async fn collect_code_generation_retention_findings( schedulers: &tracedecay_code_index_runtime::code_index_scheduler::CodeIndexSchedulerRegistryV1, - maintenance_observations: &super::maintenance::StoreTelemetrySamplingRegistry, + maintenance_observations: &tracedecay_maintenance::telemetry::StoreTelemetrySamplingRegistry, configuration: Option< &tracedecay_application::semantic_runtime::ProductionSemanticRetrievalConfigurationStoreV1, >, @@ -458,7 +458,7 @@ pub(super) async fn collect_code_generation_retention_findings( let Some(configuration) = configuration else { return DoctorStorageFamilyReadV1::Unknown; }; - let super::maintenance::SemanticVectorRetentionReadV1::Observed { + let tracedecay_maintenance::telemetry::SemanticVectorRetentionReadV1::Observed { receipt: semantic_census, } = maintenance_observations.semantic_vector_retention_read(project_root) else { @@ -500,7 +500,9 @@ pub(super) async fn collect_code_generation_retention_findings( }; let (vector_readable_sources, retained_vector_root_count) = vector_readable_sources; let semantic_backlog = - super::maintenance::SemanticVectorRetentionBacklogV1::from_receipt(&semantic_census); + tracedecay_maintenance::telemetry::SemanticVectorRetentionBacklogV1::from_receipt( + &semantic_census, + ); if semantic_backlog.published < retained_vector_root_count { return DoctorStorageFamilyReadV1::Unknown; } @@ -794,7 +796,7 @@ pub(in crate::daemon) fn production_doctor_report_reader( diagnostic_broker: Arc>, feedback_runtimes: DaemonFeedbackRuntimeRegistrar, semantic_owner_runtime: DaemonSemanticOwnerRuntimeRegistrar, - store_telemetry_sampling: super::maintenance::StoreTelemetrySamplingRegistry, + store_telemetry_sampling: tracedecay_maintenance::telemetry::StoreTelemetrySamplingRegistry, configuration_runtime: Arc, ) -> tracedecay_dashboard_api::DoctorReportReader { Arc::new(move || { diff --git a/crates/tracedecay/src/daemon/maintenance.rs b/crates/tracedecay/src/daemon/maintenance.rs index d9f8e5a518..fac906ace3 100644 --- a/crates/tracedecay/src/daemon/maintenance.rs +++ b/crates/tracedecay/src/daemon/maintenance.rs @@ -1,1226 +1,23 @@ -use std::collections::{BTreeMap, BTreeSet, HashMap}; -use std::future::Future; +use std::collections::BTreeSet; use std::path::{Path, PathBuf}; use std::sync::Arc; -#[cfg(any(feature = "hotpath", test))] -use std::sync::atomic::AtomicUsize; -use std::sync::atomic::{AtomicBool, Ordering}; -use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; +use std::time::{Duration, Instant}; use tokio::sync::{Mutex, Notify}; use tokio::task::JoinHandle; -use tracedecay_contracts::storage::{ - StorageByteSizeV1, StorageTelemetryFuture, StorageTelemetryReadV1, StoreKeyV1, - StoreSizeSampleV1, StoreSizeTelemetryPort, TableGrowthBaselinePendingV1, TableGrowthSampleV1, - TableGrowthTelemetryReadV1, TableNameV1, +use tracedecay_maintenance::compaction_receipt::record_live_compaction_outcome; +use tracedecay_maintenance::generation::run_project_generation_maintenance; +use tracedecay_maintenance::lease::ProjectStoreMaintenanceLeaseV1; +use tracedecay_maintenance::loop_run::run_maintenance_loop; +use tracedecay_maintenance::telemetry::StoreTelemetrySamplingOutcome; +use tracedecay_maintenance::tick::{ + MaintenanceContinuation, MaintenanceTickOutcome, cursor_after_attempted_units, + select_store_window, }; -use tracedecay_contracts::{ - ApplicationContractError, CancellationContext, CapabilityGrantId, CapabilityGrantSnapshot, - Deadline, DisclosureClass, RequestAdmission, RequestContext, ResolvedScope, now_micros, -}; -use tracedecay_domain::{ManifestDigest, UtcMicros}; use super::branch_admin::StoreAdministration; -use tracedecay_contracts::request_identity::{GlobalRequestSurface, mint_global_request_id}; -use tracedecay_runtime_core::db::DatabaseStorageTelemetryHandle; - -pub(super) mod generation; -/// Upper bound on mounted session databases + project graphs a single -/// maintenance tick may process. Each store gets one writer admission, so an -/// unbounded loop over every mounted project×branch cannot monopolize the lane; -/// this budget caps total work and a round-robin cursor (`store_cursor`) -/// guarantees every store is still reached across ticks. const MAINTENANCE_STORE_PAGE_LIMIT: usize = 8; -const STORAGE_TELEMETRY_CONTEXT_HORIZON_MICROS: i64 = 30_000_000; -const STORAGE_TELEMETRY_CAPABILITY: &str = "capability.application.storage.telemetry"; -const STORAGE_TELEMETRY_USE_CASE: &str = "use-case.application.storage.telemetry.read"; -#[cfg(any(feature = "hotpath", test))] -static MAINTENANCE_FUTURES_ACTIVE: AtomicUsize = AtomicUsize::new(0); - -#[derive(Clone, Copy)] -struct TableWatermark { - bytes: StorageByteSizeV1, - observed_at: UtcMicros, -} - -#[derive(Clone, Copy, Eq, PartialEq)] -enum TableGrowthObservation { - Preview, - Advance, -} - -/// Store telemetry bound to the database's guarded read capability. -/// -/// The runtime-core handle retains the exact database client that issued it; -/// this daemon adapter must not unwrap that guard into a raw SQL handle just to -/// retain the maintenance-owned table-growth baseline. -#[derive(Clone)] -pub(super) struct GuardedStoreTelemetryPort { - handle: DatabaseStorageTelemetryHandle, - store: StoreKeyV1, - scope: ResolvedScope, - reader_wait: Duration, - table_watermarks: Arc>>>, -} - -impl GuardedStoreTelemetryPort { - fn new( - handle: DatabaseStorageTelemetryHandle, - store: StoreKeyV1, - scope: ResolvedScope, - reader_wait: Duration, - ) -> Self { - Self { - handle, - store, - scope, - reader_wait, - table_watermarks: Arc::new(std::sync::Mutex::new(None)), - } - } - - fn admits(&self, context: &RequestContext, store: &StoreKeyV1) -> bool { - context.validate().is_ok() - && context.scope() == &self.scope - && store == &self.store - && context.admission_at(now_micros()) == RequestAdmission::Admitted - } - - fn for_scope(&self, scope: ResolvedScope) -> Self { - Self { - handle: self.handle.clone(), - store: self.store.clone(), - scope, - reader_wait: self.reader_wait, - table_watermarks: Arc::clone(&self.table_watermarks), - } - } - - fn rebind(&self, handle: DatabaseStorageTelemetryHandle, scope: ResolvedScope) -> Self { - Self { - handle, - store: self.store.clone(), - scope, - reader_wait: self.reader_wait, - table_watermarks: Arc::clone(&self.table_watermarks), - } - } - - pub(super) fn preview_table_growth<'a>( - &'a self, - context: &'a RequestContext, - store: &'a StoreKeyV1, - ) -> StorageTelemetryFuture<'a, TableGrowthTelemetryReadV1> { - self.read_table_growth(context, store, TableGrowthObservation::Preview) - } - - fn read_table_growth<'a>( - &'a self, - context: &'a RequestContext, - store: &'a StoreKeyV1, - observation: TableGrowthObservation, - ) -> StorageTelemetryFuture<'a, TableGrowthTelemetryReadV1> { - Box::pin(hotpath::future!( - async move { - if !self.admits(context, store) { - return TableGrowthTelemetryReadV1::Denied { - store: store.clone(), - }; - } - let Ok(current) = self - .handle - .table_size_telemetry(self.reader_wait, || telemetry_interruption(context)) - else { - return TableGrowthTelemetryReadV1::Unknown { - store: store.clone(), - }; - }; - let observed_at = now_micros(); - let mut current_tables = BTreeMap::new(); - for sample in current { - let Ok(table) = TableNameV1::new(sample.table_name) else { - return TableGrowthTelemetryReadV1::Unknown { - store: store.clone(), - }; - }; - current_tables.insert(table, StorageByteSizeV1(sample.bytes)); - } - let mut watermarks = match self.table_watermarks.lock() { - Ok(watermarks) => watermarks, - Err(poisoned) => poisoned.into_inner(), - }; - compare_table_growth( - store, - current_tables, - observed_at, - &mut watermarks, - observation, - ) - }, - label = "daemon.maintenance.read_table_growth" - )) - } -} - -impl StoreSizeTelemetryPort for GuardedStoreTelemetryPort { - fn store_size<'a>( - &'a self, - context: &'a RequestContext, - store: &'a StoreKeyV1, - ) -> StorageTelemetryFuture<'a, StorageTelemetryReadV1> { - Box::pin(hotpath::future!( - async move { - if !self.admits(context, store) { - return StorageTelemetryReadV1::Denied { - store: store.clone(), - }; - } - let Ok(sample) = self - .handle - .store_size_telemetry(self.reader_wait, || telemetry_interruption(context)) - else { - return StorageTelemetryReadV1::Unknown { - store: store.clone(), - }; - }; - let sample = StoreSizeSampleV1 { - store: store.clone(), - page_size_bytes: sample.page_size_bytes, - page_count: sample.page_count, - freelist_pages: sample.freelist_pages, - observed_at: now_micros(), - }; - if sample.validate().is_err() { - return StorageTelemetryReadV1::Unknown { - store: store.clone(), - }; - } - StorageTelemetryReadV1::Observed { sample } - }, - label = "daemon.maintenance.read_store_size" - )) - } - - fn table_growth<'a>( - &'a self, - context: &'a RequestContext, - store: &'a StoreKeyV1, - ) -> StorageTelemetryFuture<'a, TableGrowthTelemetryReadV1> { - self.read_table_growth(context, store, TableGrowthObservation::Advance) - } -} - -#[hotpath::measure(label = "daemon.maintenance.compare_table_growth")] -fn compare_table_growth( - store: &StoreKeyV1, - current_tables: BTreeMap, - observed_at: UtcMicros, - watermarks: &mut Option>, - observation: TableGrowthObservation, -) -> TableGrowthTelemetryReadV1 { - let Some(previous_watermarks) = watermarks.as_ref() else { - if observation == TableGrowthObservation::Preview { - return TableGrowthTelemetryReadV1::Unknown { - store: store.clone(), - }; - } - let tables_observed = u64::try_from(current_tables.len()).unwrap_or(u64::MAX); - *watermarks = Some( - current_tables - .into_iter() - .map(|(table, bytes)| (table, TableWatermark { bytes, observed_at })) - .collect(), - ); - return TableGrowthTelemetryReadV1::BaselineEstablished { - store: store.clone(), - observed_at, - tables_observed, - }; - }; - - let mut growth = Vec::new(); - let mut baseline_pending = Vec::new(); - for (table, current_bytes) in ¤t_tables { - if let Some(previous) = previous_watermarks.get(table) { - let sample = TableGrowthSampleV1 { - store: store.clone(), - table: table.clone(), - previous_bytes: previous.bytes, - current_bytes: *current_bytes, - previous_observed_at: previous.observed_at, - current_observed_at: observed_at, - }; - if sample.validate().is_err() { - return TableGrowthTelemetryReadV1::Unknown { - store: store.clone(), - }; - } - growth.push(sample); - } else { - baseline_pending.push(TableGrowthBaselinePendingV1 { - store: store.clone(), - table: table.clone(), - current_bytes: *current_bytes, - observed_at, - }); - } - } - if observation == TableGrowthObservation::Advance { - *watermarks = Some( - current_tables - .into_iter() - .map(|(table, bytes)| (table, TableWatermark { bytes, observed_at })) - .collect(), - ); - } - TableGrowthTelemetryReadV1::Observed { - store: store.clone(), - samples: growth, - baseline_pending, - } -} - -fn telemetry_interruption( - context: &RequestContext, -) -> Option { - match context.admission_at(now_micros()) { - RequestAdmission::Admitted => None, - RequestAdmission::Cancelled => Some(tracedecay_store::UnavailableReasonV1::Cancelled), - RequestAdmission::TimedOut => Some(tracedecay_store::UnavailableReasonV1::DeadlineExceeded), - } -} - -#[derive(Clone)] -struct CachedStoreTelemetryPort { - scope: ResolvedScope, - store: StoreKeyV1, - port: GuardedStoreTelemetryPort, -} - -/// Daemon-owned table-growth baseline authority shared by maintenance and -/// read-only diagnostic projections. -#[derive(Clone, Default)] -pub(super) struct StoreTelemetrySamplingRegistry { - ports: Arc>>, - semantic_vector_retention: - Arc>>, - graph_replay_release: Arc>>, - graph_staging_release: - Arc>>, - /// Last by-design retention operator line per lane and project. A - /// persistent unavailable-by-design condition logs once, then counts on - /// [`daemon.git.maintenance.retention_quiet_total`]; a state change or a - /// genuine anomaly emits again. - retention_operator_log: Arc>>, - loud_retention_this_tick: Arc, -} - -/// Operator-log lane for the once-then-quiet retention pin. -#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] -pub(super) enum RetentionOperatorLogLaneV1 { - Semantic, - CodeGeneration, - Tick, -} - -#[derive(Clone, Debug, Eq, Hash, PartialEq)] -struct RetentionOperatorLogKeyV1 { - lane: RetentionOperatorLogLaneV1, - scope: PathBuf, -} - -/// Longest run of short-cadence ticks a project's graph-replay release -/// reconcile may be skipped after consecutive unhealthy attempts. At the -/// one-minute retry cadence this bounds post-recovery release latency to -/// roughly eight minutes while a wedged runtime is probed a handful of times -/// per hour instead of once per tick. -const GRAPH_REPLAY_RELEASE_BACKOFF_CAP_TICKS: u32 = 8; - -/// Per-project reconcile state for the graph-replay release queue. -/// -/// Release evidence is durable on disk, so none of this state guards -/// correctness: the cursor makes the queue walk incremental across ticks -/// (retained entries stop blocking later pages), and the backoff window -/// converts "retry a known-wedged graph runtime every tick" into a bounded -/// re-probe. Losing the state (restart, project retirement) only means the -/// next attempt starts from the front of the queue immediately. -#[derive(Clone, Debug, Default, PartialEq, Eq)] -struct GraphReplayReleaseProgressV1 { - consecutive_unhealthy: u32, - skip_remaining: u32, - cursor: Option, -} - -#[derive(Clone, Copy, Default)] -struct StoreTelemetrySamplingOutcome { - observed: u64, - unavailable: u64, -} - -#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] -pub(super) struct SemanticVectorRetentionBacklogV1 { - pub(super) pending: u64, - pub(super) ready: u64, - pub(super) published: u64, - pub(super) cancelled: u64, -} - -impl SemanticVectorRetentionBacklogV1 { - pub(super) fn from_receipt( - receipt: &tracedecay_store::SemanticVectorProjectCensusReceipt, - ) -> Self { - Self { - pending: receipt.counts.pending, - ready: receipt.counts.ready, - published: receipt.counts.published, - cancelled: receipt.counts.cancelled, - } - } -} - -/// Result of recording one semantic-vector retention census page. -/// -/// Rejected variants stay fail-closed: progress resets and no receipt is -/// accepted. `CensusCountOverflow` is the only true u64-sum overflow -/// (`receipt.validate()`); other rejects name the actual page defect. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub(super) enum SemanticVectorRetentionCensusOutcome { - Accepted, - InconsistentPage, - IncompleteTerminalPage, - CensusCountOverflow, - ReceiptIdentityMismatch, -} - -impl SemanticVectorRetentionCensusOutcome { - #[hotpath::skip] - pub(super) const fn as_failure_label(self) -> Option<&'static str> { - match self { - Self::Accepted => None, - Self::InconsistentPage => Some("inconsistent_page"), - Self::IncompleteTerminalPage => Some("incomplete_terminal_page"), - Self::CensusCountOverflow => Some("census_count_overflow"), - Self::ReceiptIdentityMismatch => Some("receipt_identity_mismatch"), - } - } -} - -// `Observed` is matched by field-destructuring across several call sites -// (doctor_kernel, git_watch/store_maintenance); boxing the receipt would -// ripple through all of them for a cold, infrequently-read maintenance -// status. -#[allow(clippy::large_enum_variant)] -#[derive(Clone, Debug, PartialEq, Eq)] -pub(super) enum SemanticVectorRetentionReadV1 { - Unknown, - /// The semantic runtime is not seated for this daemon, so no vector - /// census will ever start, let alone complete. This is the ordinary - /// default-off state, distinct from [`Self::Unknown`] (a census that has - /// not run yet or was reset by a failure or mutation). - SemanticUnseated, - Scanning, - Observed { - receipt: tracedecay_store::SemanticVectorProjectCensusReceipt, - }, -} - -#[derive(Clone, Debug, Default)] -struct SemanticVectorRetentionProgressV1 { - cursor: Option, - observed: Option, - scanning: bool, - semantic_unseated: bool, -} - -impl StoreTelemetrySamplingRegistry { - pub(super) fn register_port( - &self, - path: &Path, - scope: &ResolvedScope, - open: impl FnOnce() -> Result, - ) -> bool { - let Some(store_name) = path.file_name().and_then(std::ffi::OsStr::to_str) else { - return false; - }; - let Ok(store) = StoreKeyV1::new(store_name.to_owned()) else { - return false; - }; - let Ok(handle) = open() else { - return false; - }; - let mut ports = self - .ports - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner); - if let Some(cached) = ports.get_mut(path) { - cached.scope = scope.clone(); - cached.store = store; - cached.port = cached.port.rebind(handle, scope.clone()); - return true; - } - let port = GuardedStoreTelemetryPort::new( - handle, - store.clone(), - scope.clone(), - Duration::from_secs(5), - ); - ports.insert( - path.to_path_buf(), - CachedStoreTelemetryPort { - scope: scope.clone(), - store, - port, - }, - ); - true - } - - pub(super) fn registered_port( - &self, - path: &Path, - scope: &ResolvedScope, - ) -> Option<(StoreKeyV1, GuardedStoreTelemetryPort)> { - let ports = self - .ports - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner); - let cached = ports.get(path)?; - Some((cached.store.clone(), cached.port.for_scope(scope.clone()))) - } - - /// Release the telemetry client's exact database lease before the owning - /// project store is retired. Other project and profile sampling ports stay - /// mounted. - pub(super) fn release_retained_handle(&self, path: &Path) { - self.ports - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner) - .remove(path); - self.semantic_vector_retention - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner) - .remove(path); - self.graph_replay_release - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner) - .remove(path); - self.retention_operator_log - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner) - .retain(|key, _| key.scope != path); - } - - pub(super) fn release_retained_handles_for_shutdown(&self) { - self.ports - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner) - .clear(); - self.semantic_vector_retention - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner) - .clear(); - self.graph_replay_release - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner) - .clear(); - self.retention_operator_log - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner) - .clear(); - self.loud_retention_this_tick - .store(false, Ordering::Release); - } - - pub(super) fn semantic_vector_retention_cursor( - &self, - project_root: &Path, - ) -> Option { - self.semantic_vector_retention - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner) - .get(project_root) - .and_then(|progress| progress.cursor.clone()) - } - - fn retain_project_maintenance_state(&self, active_projects: &BTreeSet) { - self.semantic_vector_retention - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner) - .retain(|project, _| active_projects.contains(project)); - self.graph_replay_release - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner) - .retain(|project, _| active_projects.contains(project)); - self.graph_staging_release - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner) - .retain(|project, _| active_projects.contains(project)); - self.retention_operator_log - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner) - .retain(|key, _| { - key.scope.as_os_str().is_empty() || active_projects.contains(&key.scope) - }); - } - - /// Whether this tick may attempt the graph-replay release reconcile. - /// - /// Consecutive unhealthy attempts open a bounded skip window; each denied - /// tick burns one unit of it, so a wedged runtime is re-probed after at - /// most [`GRAPH_REPLAY_RELEASE_BACKOFF_CAP_TICKS`] short-cadence ticks - /// rather than being polled (and timing out) on every one. - pub(super) fn graph_replay_release_attempt_admitted(&self, project_root: &Path) -> bool { - let mut progress = self - .graph_replay_release - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner); - let Some(state) = progress.get_mut(project_root) else { - return true; - }; - if state.skip_remaining == 0 { - return true; - } - state.skip_remaining -= 1; - false - } - - /// Record a release attempt the graph runtime could not serve (deadline, - /// unavailability, or a held replay pool) and widen the skip window: - /// 1, 2, 4, then capped at [`GRAPH_REPLAY_RELEASE_BACKOFF_CAP_TICKS`]. - pub(super) fn record_graph_replay_release_unhealthy(&self, project_root: &Path) { - let mut progress = self - .graph_replay_release - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner); - let state = progress.entry(project_root.to_path_buf()).or_default(); - state.consecutive_unhealthy = state.consecutive_unhealthy.saturating_add(1); - state.skip_remaining = GRAPH_REPLAY_RELEASE_BACKOFF_CAP_TICKS - .min(1_u32 << state.consecutive_unhealthy.saturating_sub(1).min(3)); - } - - /// Record a served release attempt: close the skip window and advance the - /// durable-queue cursor to `continuation` (`None` restarts from the front - /// of the queue on the next attempt). - pub(super) fn record_graph_replay_release_served( - &self, - project_root: &Path, - continuation: Option, - ) { - let mut progress = self - .graph_replay_release - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner); - match progress.entry(project_root.to_path_buf()) { - std::collections::hash_map::Entry::Occupied(mut entry) => { - if continuation.is_none() { - entry.remove(); - } else { - *entry.get_mut() = GraphReplayReleaseProgressV1 { - consecutive_unhealthy: 0, - skip_remaining: 0, - cursor: continuation, - }; - } - } - std::collections::hash_map::Entry::Vacant(entry) => { - if continuation.is_some() { - entry.insert(GraphReplayReleaseProgressV1 { - consecutive_unhealthy: 0, - skip_remaining: 0, - cursor: continuation, - }); - } - } - } - } - - /// The release-queue cursor recorded by the last served attempt. - pub(super) fn graph_replay_release_cursor(&self, project_root: &Path) -> Option { - self.graph_replay_release - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner) - .get(project_root) - .and_then(|state| state.cursor.clone()) - } - - pub(super) fn graph_staging_release_cursor( - &self, - project_root: &Path, - ) -> Option { - self.graph_staging_release - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner) - .get(project_root) - .cloned() - } - - pub(super) fn record_graph_staging_release_cursor( - &self, - project_root: &Path, - cursor: Option, - ) { - let mut cursors = self - .graph_staging_release - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner); - if let Some(cursor) = cursor { - cursors.insert(project_root.to_path_buf(), cursor); - } else { - cursors.remove(project_root); - } - } - - /// Open a fresh per-tick loud-vs-quiet window before any retention pass - /// emits. A genuine anomaly during the tick keeps the tick line loud. - pub(super) fn begin_retention_tick_log_window(&self) { - self.loud_retention_this_tick - .store(false, Ordering::Release); - } - - /// Mark that this tick emitted a genuine retention anomaly. By-design - /// unavailable pins stay quiet; this forces the tick summary to stay loud. - pub(super) fn mark_loud_retention_log(&self) { - self.loud_retention_this_tick.store(true, Ordering::Release); - } - - /// Whether this (lane, scope, detail) pair should emit an operator line. - /// - /// Identical repeats of a persistent by-design condition increment - /// `daemon.git.maintenance.retention_quiet_total` and stay silent. A - /// changed detail logs again. - pub(super) fn admit_by_design_retention_log( - &self, - lane: RetentionOperatorLogLaneV1, - scope: &Path, - detail: &str, - ) -> bool { - let key = RetentionOperatorLogKeyV1 { - lane, - scope: scope.to_path_buf(), - }; - let mut states = self - .retention_operator_log - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner); - if states.get(&key).map(String::as_str) == Some(detail) { - hotpath::gauge!("daemon.git.maintenance.retention_quiet_total").inc(1_u64); - return false; - } - states.insert(key, detail.to_owned()); - true - } - - pub(super) fn clear_by_design_retention_log( - &self, - lane: RetentionOperatorLogLaneV1, - scope: &Path, - ) { - let key = RetentionOperatorLogKeyV1 { - lane, - scope: scope.to_path_buf(), - }; - self.retention_operator_log - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner) - .remove(&key); - } - - /// Emit `retention_degraded` once per by-design state, or every time for - /// a genuine anomaly. Repeat by-design ticks count on the quiet gauge. - pub(super) fn emit_retention_degraded( - &self, - project_root: &Path, - pass: &'static str, - failure: &str, - ) { - let lane = match pass { - "semantic_vector_generations" => RetentionOperatorLogLaneV1::Semantic, - "code_generations" => RetentionOperatorLogLaneV1::CodeGeneration, - _ => { - self.mark_loud_retention_log(); - super::log_daemon_event( - "retention_degraded", - &[("pass", pass.to_owned()), ("failure", failure.to_owned())], - ); - return; - } - }; - if retention_failure_is_by_design(lane, failure) { - if !self.admit_by_design_retention_log(lane, project_root, failure) { - return; - } - } else { - self.mark_loud_retention_log(); - self.clear_by_design_retention_log(lane, project_root); - } - super::log_daemon_event( - "retention_degraded", - &[("pass", pass.to_owned()), ("failure", failure.to_owned())], - ); - } - - /// Whether the tick summary line should be written. Repeated by-design - /// `retry` ticks stay quiet; a loud anomaly or an outcome change logs. - pub(super) fn admit_retention_tick_log(&self, outcome: MaintenanceTickOutcome) -> bool { - let detail = format!("{}:{}", outcome.succeeded(), outcome.label()); - let loud = self.loud_retention_this_tick.load(Ordering::Acquire); - if matches!(outcome, MaintenanceTickOutcome::Retry) && !loud { - return self.admit_by_design_retention_log( - RetentionOperatorLogLaneV1::Tick, - Path::new(""), - &detail, - ); - } - self.clear_by_design_retention_log(RetentionOperatorLogLaneV1::Tick, Path::new("")); - true - } - - pub(super) fn record_semantic_vector_retention_failure(&self, project_root: &Path) { - self.semantic_vector_retention - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner) - .insert( - project_root.to_path_buf(), - SemanticVectorRetentionProgressV1::default(), - ); - } - - /// Pin the project's census read to [`SemanticVectorRetentionReadV1::SemanticUnseated`]. - /// - /// The vector retention pass records this when the daemon has no seated - /// semantic runtime, so downstream passes can distinguish "no census will - /// ever exist" from a census that merely has not completed yet. - pub(super) fn record_semantic_vector_retention_unseated(&self, project_root: &Path) { - self.semantic_vector_retention - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner) - .insert( - project_root.to_path_buf(), - SemanticVectorRetentionProgressV1 { - semantic_unseated: true, - ..SemanticVectorRetentionProgressV1::default() - }, - ); - } - - pub(super) fn record_semantic_vector_retention_census( - &self, - project_root: &Path, - census: &tracedecay_graph_db::SemanticVectorRetentionCensus, - ) -> SemanticVectorRetentionCensusOutcome { - use tracedecay_graph_db::SemanticVectorRetentionAction; - - let mut retention = self - .semantic_vector_retention - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner); - let progress = retention.entry(project_root.to_path_buf()).or_default(); - // A census page can only come from a seated semantic runtime. - progress.semantic_unseated = false; - if matches!( - census.action, - SemanticVectorRetentionAction::Retired(_) - | SemanticVectorRetentionAction::Finalized(_) - | SemanticVectorRetentionAction::CancelledRemoved(_) - ) { - // The returned page describes the pre-action state. Restart from - // the beginning on the next tick instead of publishing stale sums. - *progress = SemanticVectorRetentionProgressV1::default(); - return SemanticVectorRetentionCensusOutcome::Accepted; - } - progress.cursor.clone_from(&census.continuation); - if census.continuation.is_some() { - if census.complete_receipt.is_some() { - *progress = SemanticVectorRetentionProgressV1::default(); - return SemanticVectorRetentionCensusOutcome::InconsistentPage; - } - progress.scanning = true; - progress.observed = None; - } else { - let Some(receipt) = census.complete_receipt.clone() else { - *progress = SemanticVectorRetentionProgressV1::default(); - return SemanticVectorRetentionCensusOutcome::IncompleteTerminalPage; - }; - if receipt.validate().is_err() { - *progress = SemanticVectorRetentionProgressV1::default(); - return SemanticVectorRetentionCensusOutcome::CensusCountOverflow; - } - if receipt.shard_id != census.shard_id || receipt.revision != census.revision { - *progress = SemanticVectorRetentionProgressV1::default(); - return SemanticVectorRetentionCensusOutcome::ReceiptIdentityMismatch; - } - progress.observed = Some(receipt); - progress.cursor = None; - progress.scanning = false; - } - SemanticVectorRetentionCensusOutcome::Accepted - } - - pub(super) fn semantic_vector_retention_read( - &self, - project_root: &Path, - ) -> SemanticVectorRetentionReadV1 { - let retention = self - .semantic_vector_retention - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner); - let Some(progress) = retention.get(project_root) else { - return SemanticVectorRetentionReadV1::Unknown; - }; - if progress.semantic_unseated { - return SemanticVectorRetentionReadV1::SemanticUnseated; - } - if progress.scanning { - return SemanticVectorRetentionReadV1::Scanning; - } - progress - .observed - .clone() - .map_or(SemanticVectorRetentionReadV1::Unknown, |receipt| { - SemanticVectorRetentionReadV1::Observed { receipt } - }) - } - - pub(super) fn semantic_vector_scope_collection_ready(&self, project_root: &Path) -> bool { - matches!( - self.semantic_vector_retention_read(project_root), - SemanticVectorRetentionReadV1::Observed { - receipt: tracedecay_store::SemanticVectorProjectCensusReceipt { - counts: tracedecay_store::SemanticVectorStageCensusCounts { - pending: 0, - ready: 0, - published: _, - cancelled: 0, - }, - .. - }, - } - ) - } - - #[hotpath::measure(label = "daemon.maintenance.sample_store_telemetry", future = true)] - async fn advance_registered( - &self, - active_paths: &BTreeSet, - sampled_paths: &BTreeSet, - ) -> StoreTelemetrySamplingOutcome { - let ports = { - let mut ports = self - .ports - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner); - ports.retain(|path, _| active_paths.contains(path)); - ports - .iter() - .filter(|(path, _)| sampled_paths.contains(*path)) - .map(|(_, cached)| cached.clone()) - .collect::>() - }; - let mut outcome = StoreTelemetrySamplingOutcome::default(); - for cached in ports { - let Ok(context) = storage_telemetry_request_context(cached.scope.clone()) else { - outcome.unavailable = outcome.unavailable.saturating_add(1); - continue; - }; - match cached.port.table_growth(&context, &cached.store).await { - TableGrowthTelemetryReadV1::BaselineEstablished { .. } - | TableGrowthTelemetryReadV1::Observed { .. } => { - outcome.observed = outcome.observed.saturating_add(1); - } - TableGrowthTelemetryReadV1::Unsupported { .. } - | TableGrowthTelemetryReadV1::Denied { .. } - | TableGrowthTelemetryReadV1::Unknown { .. } => { - outcome.unavailable = outcome.unavailable.saturating_add(1); - } - } - } - outcome - } -} - -/// Persistent by-design retention conditions log once, then count on gauges. -/// Corrupt, reset, denied, and cancelled failures stay loud every attempt. -pub(super) fn retention_failure_is_by_design( - lane: RetentionOperatorLogLaneV1, - failure: &str, -) -> bool { - match lane { - RetentionOperatorLogLaneV1::Semantic => { - failure == "configuration_inventory_unavailable" || failure.starts_with("unavailable:") - } - RetentionOperatorLogLaneV1::CodeGeneration => { - failure.starts_with("vector_inventory_offline:") - } - RetentionOperatorLogLaneV1::Tick => false, - } -} - -#[hotpath::measure(label = "daemon.maintenance.mint_telemetry_context")] -fn storage_telemetry_request_context( - scope: ResolvedScope, -) -> Result { - let observed_at = now_micros(); - let expires_at = tracedecay_domain::UtcMicros( - observed_at - .0 - .saturating_add(STORAGE_TELEMETRY_CONTEXT_HORIZON_MICROS), - ); - let request_id = - mint_global_request_id(GlobalRequestSurface::DaemonStorageTelemetry).map_err(|_| { - ApplicationContractError::Inconsistent { - field: "storage telemetry request identity", - } - })?; - let suffix = request_id.as_str().to_owned(); - let actor = tracedecay_domain::ActorId::new("actor.tracedecay-daemon-storage-telemetry")?; - let capability = - tracedecay_tool_catalog::CapabilityId::new(STORAGE_TELEMETRY_CAPABILITY.to_owned())?; - let use_case = tracedecay_tool_catalog::UseCaseId::new(STORAGE_TELEMETRY_USE_CASE.to_owned())?; - let manifest: ManifestDigest = tracedecay_domain::canonical_sha256(&( - "tracedecay.daemon.storage-telemetry-grant.v1", - &scope, - &capability, - &use_case, - expires_at, - ))?; - let grant = CapabilityGrantSnapshot::new( - CapabilityGrantId::new(format!("grant.daemon.storage-telemetry.{suffix}"))?, - 1, - manifest, - actor.clone(), - observed_at, - expires_at, - scope.clone(), - BTreeSet::from([capability]), - BTreeSet::from([use_case]), - DisclosureClass::Metadata, - )?; - RequestContext::new( - actor, - scope, - grant, - request_id, - Deadline::new(expires_at)?, - CancellationContext::active(format!("cancel.daemon.storage-telemetry.{suffix}"))?, - ) -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub(in crate::daemon) enum MaintenanceContinuation { - /// Resume the bounded semantic-vector phase over the normal graph window. - /// - /// This is deliberately phase-scoped rather than project-scoped: no - /// project identifier is retained in maintenance state, so mounted graphs - /// continue to receive the same bounded, round-robin service. - SemanticVectorRetention, - /// Resume bounded code-generation retention over the normal graph window: - /// a superseded-generation backlog or a partially drained graph-replay - /// release queue keeps the short cadence until it converges, instead of - /// parking multi-GiB debris behind the full maintenance interval. - /// - /// A continuation tick for this phase still runs the bounded - /// semantic-vector page first, so semantic convergence never starves - /// behind a code-generation drain. - CodeGenerationRetention, -} - -impl MaintenanceContinuation { - /// Two phases asking to continue collapse to the one whose continuation - /// tick still advances both: a code-generation continuation re-runs the - /// bounded semantic-vector page on every tick, while a semantic-only - /// continuation would starve a pending code-generation backlog. - fn combine(self, other: Self) -> Self { - match (self, other) { - (Self::CodeGenerationRetention, _) | (_, Self::CodeGenerationRetention) => { - Self::CodeGenerationRetention - } - (Self::SemanticVectorRetention, Self::SemanticVectorRetention) => { - Self::SemanticVectorRetention - } - } - } -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub(in crate::daemon) enum MaintenanceTickOutcome { - Complete, - Continue(MaintenanceContinuation), - Retry, -} - -impl MaintenanceTickOutcome { - pub(in crate::daemon) fn is_complete(self) -> bool { - self == Self::Complete - } - - fn continuation(self) -> Option { - match self { - Self::Continue(continuation) => Some(continuation), - Self::Complete | Self::Retry => None, - } - } - - fn succeeded(self) -> bool { - !matches!(self, Self::Retry) - } - - fn label(self) -> &'static str { - match self { - Self::Complete => "complete", - Self::Continue(MaintenanceContinuation::SemanticVectorRetention) => { - "semantic_vector_progress" - } - Self::Continue(MaintenanceContinuation::CodeGenerationRetention) => { - "code_generation_progress" - } - Self::Retry => "retry", - } - } - - /// A failure wins over ordinary bounded progress so the next short tick - /// retries the complete maintenance journey. The semantic census cursor is - /// durable in the graph registry, so that retry still resumes its progress - /// rather than losing the bounded semantic-vector work. - fn combine(self, other: Self) -> Self { - match (self, other) { - (Self::Retry, _) | (_, Self::Retry) => Self::Retry, - (Self::Continue(left), Self::Continue(right)) => Self::Continue(left.combine(right)), - (Self::Continue(continuation), Self::Complete) - | (Self::Complete, Self::Continue(continuation)) => Self::Continue(continuation), - (Self::Complete, Self::Complete) => Self::Complete, - } - } -} - -/// The maintenance loop parks on `tokio::time::sleep_until`, so every deadline -/// it derives must be measured on the same clock the timer wheel uses. -/// `tokio::time::Instant` is the process monotonic clock in production and the -/// runtime's virtual clock under a paused test runtime; mixing it with -/// `std::time::Instant` would leave the due check permanently in the past -/// relative to a fired timer. -type CadenceInstant = tokio::time::Instant; - -#[derive(Debug)] -pub(super) struct MaintenanceCadence { - interval: Duration, - retry_delay: Duration, - not_before: Option, - in_flight: bool, -} - -impl MaintenanceCadence { - pub(super) fn new(interval: Duration) -> Self { - Self { - interval, - retry_delay: interval.min(Duration::from_mins(1)), - not_before: None, - in_flight: false, - } - } - - pub(super) fn reserve(&mut self, now: CadenceInstant) -> bool { - if self.in_flight || self.not_before.is_some_and(|not_before| now < not_before) { - return false; - } - self.in_flight = true; - true - } - - fn finish(&mut self, now: CadenceInstant, outcome: MaintenanceTickOutcome) -> CadenceInstant { - self.in_flight = false; - let delay = match outcome { - MaintenanceTickOutcome::Complete => self.interval, - MaintenanceTickOutcome::Continue(_) | MaintenanceTickOutcome::Retry => self.retry_delay, - }; - let deadline = now + delay; - self.not_before = Some(deadline); - deadline - } -} - -struct MaintenanceLifecycleInstrumentation; - -impl MaintenanceLifecycleInstrumentation { - fn new() -> Self { - #[cfg(any(feature = "hotpath", test))] - { - let active = MAINTENANCE_FUTURES_ACTIVE.fetch_add(1, Ordering::SeqCst) + 1; - hotpath::gauge!("daemon_maintenance_futures_active").set(active); - } - Self - } - - fn record_outcome(&self, outcome: MaintenanceTickOutcome) { - match outcome { - MaintenanceTickOutcome::Complete => { - hotpath::gauge!("daemon_maintenance_outcome_complete").inc(1.0); - } - MaintenanceTickOutcome::Continue(MaintenanceContinuation::SemanticVectorRetention) => { - hotpath::gauge!("daemon_maintenance_outcome_semantic_vector_progress").inc(1.0); - } - MaintenanceTickOutcome::Continue(MaintenanceContinuation::CodeGenerationRetention) => { - hotpath::gauge!("daemon_maintenance_outcome_code_generation_progress").inc(1.0); - } - MaintenanceTickOutcome::Retry => { - hotpath::gauge!("daemon_maintenance_outcome_retry").inc(1.0); - } - } - } - - fn record_cancellation(&self) { - hotpath::gauge!("daemon_maintenance_outcome_cancelled").inc(1.0); - } -} - -impl Drop for MaintenanceLifecycleInstrumentation { - fn drop(&mut self) { - #[cfg(any(feature = "hotpath", test))] - { - let active = MAINTENANCE_FUTURES_ACTIVE - .fetch_sub(1, Ordering::SeqCst) - .saturating_sub(1); - hotpath::gauge!("daemon_maintenance_futures_active").set(active); - } - } -} - -struct MaintenancePhaseInstrumentation { - continuation: Option, -} - -impl MaintenancePhaseInstrumentation { - fn new(continuation: Option) -> Self { - match continuation { - Some(MaintenanceContinuation::SemanticVectorRetention) => { - hotpath::gauge!("daemon_maintenance_phase_semantic_vector_active").inc(1.0); - } - Some(MaintenanceContinuation::CodeGenerationRetention) => { - hotpath::gauge!("daemon_maintenance_phase_code_generation_active").inc(1.0); - } - None => { - hotpath::gauge!("daemon_maintenance_phase_full_tick_active").inc(1.0); - } - } - Self { continuation } - } -} - -impl Drop for MaintenancePhaseInstrumentation { - fn drop(&mut self) { - match self.continuation { - Some(MaintenanceContinuation::SemanticVectorRetention) => { - hotpath::gauge!("daemon_maintenance_phase_semantic_vector_active").inc(-1.0); - } - Some(MaintenanceContinuation::CodeGenerationRetention) => { - hotpath::gauge!("daemon_maintenance_phase_code_generation_active").inc(-1.0); - } - None => { - hotpath::gauge!("daemon_maintenance_phase_full_tick_active").inc(-1.0); - } - } - } -} async fn join_abandoned_maintenance_task(task: Option>, owner: &'static str) { let Some(task) = task else { @@ -1242,49 +39,6 @@ async fn join_abandoned_maintenance_task(task: Option>, owner: &' } } -async fn run_maintenance_loop( - cancellation: &tracedecay_session_memory::context::CancellationToken, - wake: &Notify, - interval: Duration, - mut run_tick: F, -) where - F: FnMut(Option) -> Fut, - Fut: Future, -{ - let _lifecycle = MaintenanceLifecycleInstrumentation::new(); - let mut cadence = MaintenanceCadence::new(interval); - let mut deadline = CadenceInstant::now() + cadence.retry_delay; - let mut continuation = None; - loop { - tokio::select! { - biased; - () = cancellation.cancelled() => { - _lifecycle.record_cancellation(); - break; - } - () = wake.notified() => {} - () = tokio::time::sleep_until(deadline) => {} - } - if cancellation.is_cancelled() { - _lifecycle.record_cancellation(); - break; - } - let now = CadenceInstant::now(); - if now < deadline || !cadence.reserve(now) { - continue; - } - let _phase = MaintenancePhaseInstrumentation::new(continuation); - let outcome = run_tick(continuation).await; - if cancellation.is_cancelled() { - _lifecycle.record_cancellation(); - break; - } - _lifecycle.record_outcome(outcome); - continuation = outcome.continuation(); - deadline = cadence.finish(CadenceInstant::now(), outcome); - } -} - async fn run_registered_store_retention( database: &tracedecay_global_db::RegisteredGlobalDb, config: &crate::config::RetentionConfig, @@ -1430,43 +184,6 @@ async fn run_profile_observability_retention( } } -pub(in crate::daemon) fn record_live_compaction_outcome( - store_name: &'static str, - outcome: tracedecay_maintenance::retention::live_compaction::LiveStoreCompactionOutcomeV1, -) -> bool { - use tracedecay_maintenance::retention::live_compaction::LiveStoreCompactionOutcomeV1; - - match outcome { - LiveStoreCompactionOutcomeV1::NotScheduled => true, - LiveStoreCompactionOutcomeV1::Compacted { - freelist_before, - freelist_after, - } => { - super::log_daemon_event( - "retention_compaction", - &[ - ("store", store_name.to_owned()), - ( - "freed_pages", - freelist_before.saturating_sub(freelist_after).to_string(), - ), - ], - ); - true - } - LiveStoreCompactionOutcomeV1::Failed(failure) => { - super::log_daemon_event( - "retention_degraded", - &[ - ("pass", "compaction".to_owned()), - ("failure", failure.as_str().to_owned()), - ], - ); - false - } - } -} - #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub(super) enum MaintenanceStoreOutcomeV1 { Processed, @@ -1569,47 +286,17 @@ impl MaintenanceStoreWork { } } -/// Pure round-robin window selection over stably-sorted store keys. -/// -/// Returns the indices to process this tick (at most `budget`, always -/// `min(budget, keys.len())`) and the cursor to resume after next tick. Sorting -/// the keys and resuming after the previous cursor guarantees that, across -/// `ceil(len / budget)` consecutive ticks, every store is processed at least -/// once — nothing that should be reclaimed is starved forever — while any -/// single tick touches no more than `budget` stores. -fn select_store_window( - keys: &[String], - after: Option<&str>, - budget: usize, -) -> (Vec, Option) { - let count = keys.len(); - if count == 0 || budget == 0 { - return (Vec::new(), after.map(str::to_owned)); - } - let start = match after { - Some(cursor) => keys.partition_point(|key| key.as_str() <= cursor) % count, - None => 0, - }; - let take = budget.min(count); - let indices = (0..take) - .map(|offset| (start + offset) % count) - .collect::>(); - let next = indices.last().map(|&index| keys[index].clone()); - (indices, next) -} - -fn cursor_after_attempted_units( - keys: &[String], - window: &[usize], - attempted: usize, - prior: Option<&str>, -) -> Option { - attempted - .checked_sub(1) - .and_then(|last| window.get(last)) - .and_then(|&index| keys.get(index)) - .cloned() - .or_else(|| prior.map(str::to_owned)) +pub(crate) fn project_store_maintenance_lease( + graph: &crate::tracedecay::TraceDecay, +) -> ProjectStoreMaintenanceLeaseV1 { + ProjectStoreMaintenanceLeaseV1::new( + graph.project_root().to_path_buf(), + graph.store_layout().clone(), + graph.db().clone(), + graph.retained_store_runtime_registry(), + std::sync::Arc::clone(graph.configuration_runtime()), + graph.profile_database().clone(), + ) } impl MaintenanceCoordinator { @@ -1848,12 +535,12 @@ impl MaintenanceCoordinator { } } MaintenanceStoreWork::Graph(graph) => { - generation::run_project_generation_maintenance( - graph, + run_project_generation_maintenance( + &project_store_maintenance_lease(graph), code_index_schedulers, &maintenance_observations, &self.cancellation, - retention, + retention.compaction.as_ref(), continuation, ) .await @@ -2163,11 +850,7 @@ pub(super) fn retention_maintenance_enabled(retention: &crate::config::Retention } pub(crate) fn now_secs_i64() -> Result { - let seconds = SystemTime::now() - .duration_since(UNIX_EPOCH) - .map_err(|_| "system_clock_before_unix_epoch")? - .as_secs(); - i64::try_from(seconds).map_err(|_| "system_clock_out_of_range") + tracedecay_maintenance::clock::now_secs_i64() } #[cfg(test)] @@ -2183,12 +866,17 @@ mod tests { use tracedecay_domain::UtcMicros; use super::{ - CadenceInstant, MAINTENANCE_FUTURES_ACTIVE, MAINTENANCE_STORE_PAGE_LIMIT, - MaintenanceCadence, MaintenanceContinuation, MaintenanceCoordinator, - MaintenanceTickOutcome, RetentionOperatorLogLaneV1, SemanticVectorRetentionCensusOutcome, + MAINTENANCE_STORE_PAGE_LIMIT, MaintenanceCoordinator, run_resident_memory_sampler_loop, + }; + use tracedecay_maintenance::loop_run::{maintenance_futures_active, run_maintenance_loop}; + use tracedecay_maintenance::telemetry::{ + RetentionOperatorLogLaneV1, SemanticVectorRetentionCensusOutcome, SemanticVectorRetentionReadV1, StoreTelemetrySamplingRegistry, TableGrowthObservation, - compare_table_growth, cursor_after_attempted_units, retention_failure_is_by_design, - run_maintenance_loop, run_resident_memory_sampler_loop, select_store_window, + compare_table_growth, retention_failure_is_by_design, + }; + use tracedecay_maintenance::tick::{ + CadenceInstant, MaintenanceCadence, MaintenanceContinuation, MaintenanceTickOutcome, + cursor_after_attempted_units, select_store_window, }; #[test] @@ -2473,7 +1161,7 @@ mod tests { let cancellation = tracedecay_session_memory::context::CancellationToken::new(); let wake = Arc::new(Notify::new()); let ticks = Arc::new(std::sync::atomic::AtomicUsize::new(0)); - let baseline = MAINTENANCE_FUTURES_ACTIVE.load(Ordering::SeqCst); + let baseline = maintenance_futures_active(); let task_cancellation = cancellation.clone(); let task_wake = Arc::clone(&wake); let task_ticks = Arc::clone(&ticks); @@ -2494,7 +1182,7 @@ mod tests { }); tokio::task::yield_now().await; assert_eq!( - MAINTENANCE_FUTURES_ACTIVE.load(Ordering::SeqCst), + maintenance_futures_active(), baseline + 1, "the loop lifecycle must become observable while the task is live" ); @@ -2520,7 +1208,7 @@ mod tests { task.await .expect("maintenance loop joins after cancellation"); assert_eq!( - MAINTENANCE_FUTURES_ACTIVE.load(Ordering::SeqCst), + maintenance_futures_active(), baseline, "cancellation must drop the lifecycle guard and clear the active gauge" ); diff --git a/crates/tracedecay/src/daemon/production_harness/generation_retention_test.rs b/crates/tracedecay/src/daemon/production_harness/generation_retention_test.rs index f61de0ce7c..682a654390 100644 --- a/crates/tracedecay/src/daemon/production_harness/generation_retention_test.rs +++ b/crates/tracedecay/src/daemon/production_harness/generation_retention_test.rs @@ -27,6 +27,7 @@ use tracedecay_semantic_contracts::{ use super::journey_test_support::git; use super::*; +use crate::daemon::maintenance::project_store_maintenance_lease; use tracedecay_application::semantic_runtime::{ ProjectSemanticActivationExt, RetainedSemanticVectorGraphV1, SemanticGraphExecutionAuthorityV1, SemanticVectorGraphScopeV1, SemanticVectorRetentionAuthorizationV1, @@ -877,33 +878,33 @@ async fn mounted_daemon_maintenance_retains_activation_lease_and_converges_after let observations = resources.store_administration.store_telemetry_sampling(); let cancellation = tracedecay_session_memory::context::CancellationToken::new(); assert!(matches!( - crate::daemon::store_maintenance::resolve_vector_retention_inventory( - graph.as_ref(), + tracedecay_maintenance::store_maintenance::resolve_vector_retention_inventory( + &project_store_maintenance_lease(graph.as_ref()), schedulers, &observations, ) .await, - crate::daemon::store_maintenance::VectorRetentionInventoryV1::Refused { .. } + tracedecay_maintenance::store_maintenance::VectorRetentionInventoryV1::Refused { .. } )); assert_eq!( - crate::daemon::store_maintenance::run_code_generation_retention( - graph.as_ref(), + tracedecay_maintenance::store_maintenance::run_code_generation_retention( + &project_store_maintenance_lease(graph.as_ref()), schedulers, &observations, &cancellation, ) .await, - crate::daemon::store_maintenance::CodeGenerationRetentionOutcomeV1::Failed, + tracedecay_maintenance::store_maintenance::CodeGenerationRetentionOutcomeV1::Failed, "an unknown census cannot discard a mounted vector provider's leases" ); assert!(first_source_file.is_file()); assert!( - !crate::daemon::maintenance::generation::run_project_generation_maintenance( - graph.as_ref(), + !tracedecay_maintenance::generation::run_project_generation_maintenance( + &project_store_maintenance_lease(graph.as_ref()), schedulers, &observations, &cancellation, - &crate::config::RetentionConfig::default(), + None, None, ) .await @@ -915,12 +916,12 @@ async fn mounted_daemon_maintenance_retains_activation_lease_and_converges_after "code deletion must not race ahead of the retained vector source" ); assert!( - crate::daemon::maintenance::generation::run_project_generation_maintenance( - graph.as_ref(), + tracedecay_maintenance::generation::run_project_generation_maintenance( + &project_store_maintenance_lease(graph.as_ref()), schedulers, &observations, &cancellation, - &crate::config::RetentionConfig::default(), + None, None, ) .await @@ -931,16 +932,17 @@ async fn mounted_daemon_maintenance_retains_activation_lease_and_converges_after first_source_file.is_file(), "exact vector-source liveness must veto the source-code deletion plan" ); - let observed_inventory = crate::daemon::store_maintenance::resolve_vector_retention_inventory( - graph.as_ref(), - schedulers, - &observations, - ) - .await; + let observed_inventory = + tracedecay_maintenance::store_maintenance::resolve_vector_retention_inventory( + &project_store_maintenance_lease(graph.as_ref()), + schedulers, + &observations, + ) + .await; assert!( matches!( observed_inventory, - crate::daemon::store_maintenance::VectorRetentionInventoryV1::Online { .. } + tracedecay_maintenance::store_maintenance::VectorRetentionInventoryV1::Online { .. } ), "a complete post-convergence census pins through the online vector inventory" ); @@ -973,16 +975,17 @@ async fn mounted_daemon_maintenance_retains_activation_lease_and_converges_after // fail-closed refusal (the offline degradation names an absent provider); // either way the pass must report it and retain every source. observations.record_semantic_vector_retention_failure(&canonical_root); - let reset_inventory = crate::daemon::store_maintenance::resolve_vector_retention_inventory( - graph.as_ref(), - schedulers, - &observations, - ) - .await; + let reset_inventory = + tracedecay_maintenance::store_maintenance::resolve_vector_retention_inventory( + &project_store_maintenance_lease(graph.as_ref()), + schedulers, + &observations, + ) + .await; assert!( matches!( reset_inventory, - crate::daemon::store_maintenance::VectorRetentionInventoryV1::Refused { .. } + tracedecay_maintenance::store_maintenance::VectorRetentionInventoryV1::Refused { .. } ), "an unreadable inventory under a mounted vector provider is a fail-closed refusal" ); @@ -992,14 +995,14 @@ async fn mounted_daemon_maintenance_retains_activation_lease_and_converges_after "the CI-facing retention_degraded event still reports pass=code_generations" ); assert_eq!( - crate::daemon::store_maintenance::run_code_generation_retention( - graph.as_ref(), + tracedecay_maintenance::store_maintenance::run_code_generation_retention( + &project_store_maintenance_lease(graph.as_ref()), schedulers, &observations, &cancellation, ) .await, - crate::daemon::store_maintenance::CodeGenerationRetentionOutcomeV1::Failed, + tracedecay_maintenance::store_maintenance::CodeGenerationRetentionOutcomeV1::Failed, "an unreadable vector inventory fails the pass instead of sweeping" ); assert!( @@ -1009,12 +1012,12 @@ async fn mounted_daemon_maintenance_retains_activation_lease_and_converges_after drop(activation_lease); assert!( - !crate::daemon::maintenance::generation::run_project_generation_maintenance( - graph.as_ref(), + !tracedecay_maintenance::generation::run_project_generation_maintenance( + &project_store_maintenance_lease(graph.as_ref()), schedulers, &observations, &cancellation, - &crate::config::RetentionConfig::default(), + None, None, ) .await @@ -1055,20 +1058,20 @@ async fn mounted_daemon_maintenance_retains_activation_lease_and_converges_after // satisfy. let mut source_released_under = None; for _ in 0..12 { - converged = crate::daemon::maintenance::generation::run_project_generation_maintenance( - restarted_graph.as_ref(), + converged = tracedecay_maintenance::generation::run_project_generation_maintenance( + &project_store_maintenance_lease(restarted_graph.as_ref()), restarted_schedulers, &restarted_observations, &restarted_cancellation, - &crate::config::RetentionConfig::default(), + None, None, ) .await .is_complete(); if source_released_under.is_none() && !first_source_file.exists() { source_released_under = Some( - crate::daemon::store_maintenance::resolve_vector_retention_inventory( - restarted_graph.as_ref(), + tracedecay_maintenance::store_maintenance::resolve_vector_retention_inventory( + &project_store_maintenance_lease(restarted_graph.as_ref()), restarted_schedulers, &restarted_observations, ) @@ -1263,12 +1266,12 @@ async fn run_generation_cadence( .expect("project server") .cg() .await; - crate::daemon::maintenance::generation::run_project_generation_maintenance( - graph.as_ref(), + tracedecay_maintenance::generation::run_project_generation_maintenance( + &project_store_maintenance_lease(graph.as_ref()), &resources.invocation.code_index_schedulers, &resources.store_administration.store_telemetry_sampling(), &tracedecay_session_memory::context::CancellationToken::new(), - &crate::config::RetentionConfig::default(), + None, None, ) .await @@ -1743,12 +1746,12 @@ async fn mounted_default_off_retention_requires_an_empty_vector_census() { .join("code-generations-v1") .join(&candidate.generation_file); assert!(source_file.is_file()); - let observations = crate::daemon::maintenance::StoreTelemetrySamplingRegistry::default(); + let observations = tracedecay_maintenance::telemetry::StoreTelemetrySamplingRegistry::default(); // This is the durable default-off observation emitted while the semantic // coordinator is unseated; the vector provider still belongs to the mount. observations.record_semantic_vector_retention_unseated(&root); - let inventory = crate::daemon::store_maintenance::resolve_vector_retention_inventory( - graph.as_ref(), + let inventory = tracedecay_maintenance::store_maintenance::resolve_vector_retention_inventory( + &project_store_maintenance_lease(graph.as_ref()), schedulers, &observations, ) @@ -1756,21 +1759,21 @@ async fn mounted_default_off_retention_requires_an_empty_vector_census() { assert!( matches!( inventory, - crate::daemon::store_maintenance::VectorRetentionInventoryV1::SemanticUnseated + tracedecay_maintenance::store_maintenance::VectorRetentionInventoryV1::SemanticUnseated ), "empty default-off inventory was refused: {:?}", inventory.degraded_reason(), ); let cancellation = tracedecay_session_memory::context::CancellationToken::new(); assert_eq!( - crate::daemon::store_maintenance::run_code_generation_retention( - graph.as_ref(), + tracedecay_maintenance::store_maintenance::run_code_generation_retention( + &project_store_maintenance_lease(graph.as_ref()), schedulers, &observations, &cancellation, ) .await, - crate::daemon::store_maintenance::CodeGenerationRetentionOutcomeV1::MoreWork, + tracedecay_maintenance::store_maintenance::CodeGenerationRetentionOutcomeV1::MoreWork, ); assert!( !source_file.exists(), @@ -1779,23 +1782,23 @@ async fn mounted_default_off_retention_requires_an_empty_vector_census() { publish_vector_generation(schedulers, &root, &latest).await; assert!(matches!( - crate::daemon::store_maintenance::resolve_vector_retention_inventory( - graph.as_ref(), + tracedecay_maintenance::store_maintenance::resolve_vector_retention_inventory( + &project_store_maintenance_lease(graph.as_ref()), schedulers, &observations, ) .await, - crate::daemon::store_maintenance::VectorRetentionInventoryV1::Refused { .. } + tracedecay_maintenance::store_maintenance::VectorRetentionInventoryV1::Refused { .. } )); assert_eq!( - crate::daemon::store_maintenance::run_code_generation_retention( - graph.as_ref(), + tracedecay_maintenance::store_maintenance::run_code_generation_retention( + &project_store_maintenance_lease(graph.as_ref()), schedulers, &observations, &cancellation, ) .await, - crate::daemon::store_maintenance::CodeGenerationRetentionOutcomeV1::Failed, + tracedecay_maintenance::store_maintenance::CodeGenerationRetentionOutcomeV1::Failed, "disabled configuration alone cannot discard published vector state", ); drop(graph); diff --git a/crates/tracedecay/src/daemon/project_composition.rs b/crates/tracedecay/src/daemon/project_composition.rs index 97fa80f50a..1303139dd2 100644 --- a/crates/tracedecay/src/daemon/project_composition.rs +++ b/crates/tracedecay/src/daemon/project_composition.rs @@ -1973,7 +1973,7 @@ fn project_dashboard_pr_autotrack_reader() /// the sampling authority. An unavailable registration is recorded and skipped, /// never fatal: telemetry must not fail an otherwise healthy project open. fn register_route_store_telemetry( - sampling: &crate::daemon::maintenance::StoreTelemetrySamplingRegistry, + sampling: &tracedecay_maintenance::telemetry::StoreTelemetrySamplingRegistry, cg: &Arc, scope: &tracedecay_contracts::ResolvedScope, session_databases: [&tracedecay_global_db::RegisteredGlobalDb; 3], diff --git a/crates/tracedecay/src/daemon/store_maintenance/mod.rs b/crates/tracedecay/src/daemon/store_maintenance/mod.rs index f5a80727a1..c2bdd66287 100644 --- a/crates/tracedecay/src/daemon/store_maintenance/mod.rs +++ b/crates/tracedecay/src/daemon/store_maintenance/mod.rs @@ -1,82 +1,18 @@ -//! Retention, compaction, and garbage-collection operations run by the daemon -//! maintenance owner. +//! Daemon sequencing for store-administration GC. //! -//! Every operation that opens or garbage-collects a store lives here so its -//! [`StoreAdministration`] lifetime is kept separate from the watcher state -//! machine. The git watcher itself never opens or mutates a store: it routes -//! exact-frontier freshness requests to the code-index scheduler and wakes the -//! maintenance owner. +//! Retention, compaction, and generation kernels live in +//! `tracedecay-maintenance`. This module keeps the pass that must hold +//! [`StoreAdministration`] for the writer-gated branch-admin action. -use std::path::{Path, PathBuf}; - -use crate::daemon::maintenance::now_secs_i64; -use crate::tracedecay::TraceDecay; -use tracedecay_application::semantic_runtime::ProjectSemanticActivationExt; use tracedecay_code_index_runtime::code_index_scheduler::CodeIndexSchedulerRegistryV1; -use tracedecay_maintenance::retention::branch_compaction::CompactionThresholdConfig; use tracedecay_runtime_core::branch::BranchAdminAction; -use tracedecay_semantic_contracts::SemanticConfig; use super::branch_admin::StoreAdministration; use super::log_daemon_event; +use crate::tracedecay::TraceDecay; -mod graph_replay; #[cfg(test)] mod vector_retention_tests; -use graph_replay::{defer_graph_replay_pool_busy, log_code_generation_retention_degraded}; - -struct ScopeRootProofInputsV1 { - live_roots: std::collections::BTreeSet, - registered_roots: - tracedecay_code_index_retention::code_index_generations::ScopeRootAuthorityReceiptV1, - git_worktrees: - tracedecay_code_index_retention::code_index_generations::ScopeRootAuthorityReceiptV1, - mounted_leases: - tracedecay_code_index_retention::code_index_generations::ScopeRootAuthorityReceiptV1, - configuration_roots: - tracedecay_code_index_retention::code_index_generations::ScopeRootAuthorityReceiptV1, - vector_census: - tracedecay_code_index_retention::code_index_generations::ScopeRootAuthorityReceiptV1, - vector_dependencies: - tracedecay_code_index_retention::code_index_generations::ScopeRootAuthorityReceiptV1, - vector_sources: std::collections::BTreeSet, -} - -impl ScopeRootProofInputsV1 { - fn bind_candidate( - &self, - scope_hash: String, - source_scope: tracedecay_store::StoreShardIdV1, - vector_revision: tracedecay_store::SemanticVectorStageCensusRevision, - ) -> Result< - tracedecay_code_index_retention::code_index_generations::ScopeRootLivenessProofV1, - &'static str, - > { - let live_scope_hashes = self - .live_roots - .iter() - .map(|root| { - tracedecay_code_index_retention::code_index_generations::code_index_scope_hash(root) - }) - .collect(); - tracedecay_code_index_retention::code_index_generations::ScopeRootLivenessProofV1::new( - live_scope_hashes, - self.registered_roots.clone(), - self.git_worktrees.clone(), - self.mounted_leases.clone(), - self.configuration_roots.clone(), - self.vector_census.clone(), - self.vector_dependencies.clone(), - tracedecay_code_index_retention::code_index_generations::ScopeRootCandidateBindingV1 { - scope_hash, - source_scope, - vector_census_revision: vector_revision.get().to_string(), - live: false, - }, - ) - .map_err(|_| "scope_liveness_proof_invalid") - } -} /// Runs branch-store GC for a project through the daemon administration /// coordinator, logging what it removed. Returns `false` when layout resolution @@ -135,1719 +71,3 @@ pub(super) async fn run_gc( } true } - -/// Advance one bounded project-wide semantic-vector retention page. -/// -/// The maintenance observation registry carries the stage cursor across ticks. -/// A mutating action resets the cursor because the returned census described -/// pre-action state; a no-action page advances it, and end-of-census publishes -/// only fixed-size aggregate counts for Doctor. -#[hotpath::measure( - label = "daemon.git.maintenance.semantic_vector_retention", - future = true -)] -pub(super) async fn run_semantic_vector_generation_retention( - graph: &TraceDecay, - schedulers: &CodeIndexSchedulerRegistryV1, - observations: &crate::daemon::maintenance::StoreTelemetrySamplingRegistry, - cancellation: &tracedecay_session_memory::context::CancellationToken, -) -> crate::daemon::maintenance::MaintenanceTickOutcome { - let root = graph.project_root(); - if cancellation.is_cancelled() { - observations.record_semantic_vector_retention_failure(root); - log_semantic_vector_retention_degraded(observations, root, "retention_cancelled"); - return crate::daemon::maintenance::MaintenanceTickOutcome::Retry; - } - let Some(configuration) = graph - .configuration_runtime() - .semantic_configuration_inventory_authority() - else { - // The activation coordinator is not seated. Whether that is the - // ordinary default-off state or a project-open overlap is decided by - // the durable semantic configuration, never by mount timing: a - // committed retrieval profile means a coordinator is expected - // imminently, so the pass stays retryable on the short cadence - // instead of pinning quiet and making the first census wait a full - // maintenance interval. - return match graph.configuration_runtime().client().current().await { - Ok(runtime_configuration) - if semantic_retrieval_profiles_disabled( - &runtime_configuration.config().semantic, - ) => - { - // Default off: no committed active or rollback retrieval - // profile, so no census will ever complete. Pin the typed - // unseated read for the code-generation pass and succeed - // quietly instead of resetting to Unknown and re-logging a - // degraded retry loop every tick. - observations.record_semantic_vector_retention_unseated(root); - crate::daemon::maintenance::MaintenanceTickOutcome::Complete - } - Ok(_) => { - observations.record_semantic_vector_retention_failure(root); - log_semantic_vector_retention_degraded( - observations, - root, - "configuration_inventory_unavailable", - ); - crate::daemon::maintenance::MaintenanceTickOutcome::Retry - } - Err(_) => { - observations.record_semantic_vector_retention_failure(root); - log_semantic_vector_retention_degraded( - observations, - root, - "runtime_configuration_unavailable", - ); - crate::daemon::maintenance::MaintenanceTickOutcome::Retry - } - }; - }; - let after = observations.semantic_vector_retention_cursor(root); - match tracedecay_code_index_runtime::code_index_scheduler::semantic_vector_graph::retire_one_project_vector_generation( - schedulers, - root, - &configuration, - after, - ) - .await - { - tracedecay_code_index_runtime::code_index_scheduler::semantic_vector_graph::ProjectSemanticVectorRetentionStep::Ready( - census, - ) => { - let convergence_pending = census.continuation.is_some() - || matches!( - census.action, - tracedecay_graph_db::SemanticVectorRetentionAction::Retired(_) - | tracedecay_graph_db::SemanticVectorRetentionAction::Finalized(_) - | tracedecay_graph_db::SemanticVectorRetentionAction::CancelledRemoved(_) - ); - if let Some(failure) = observations - .record_semantic_vector_retention_census(root, &census) - .as_failure_label() - { - log_semantic_vector_retention_degraded(observations, root, failure); - return crate::daemon::maintenance::MaintenanceTickOutcome::Retry; - } - if !matches!( - census.action, - tracedecay_graph_db::SemanticVectorRetentionAction::None - ) { - log_daemon_event( - "retention_semantic_vector_generations", - &[ - ("project", root.display().to_string()), - ("action", format!("{:?}", census.action)), - ], - ); - } - if convergence_pending { - crate::daemon::maintenance::MaintenanceTickOutcome::Continue( - crate::daemon::maintenance::MaintenanceContinuation::SemanticVectorRetention, - ) - } else { - crate::daemon::maintenance::MaintenanceTickOutcome::Complete - } - } - tracedecay_code_index_runtime::code_index_scheduler::semantic_vector_graph::ProjectSemanticVectorRetentionStep::ResetRequired( - reason, - ) => { - observations.record_semantic_vector_retention_failure(root); - log_semantic_vector_retention_degraded( - observations, - root, - &format!("reset_required:{reason}"), - ); - crate::daemon::maintenance::MaintenanceTickOutcome::Retry - } - tracedecay_code_index_runtime::code_index_scheduler::semantic_vector_graph::ProjectSemanticVectorRetentionStep::Corrupt( - reason, - ) => { - observations.record_semantic_vector_retention_failure(root); - log_semantic_vector_retention_degraded(observations, root, &format!("corrupt:{reason}")); - crate::daemon::maintenance::MaintenanceTickOutcome::Retry - } - tracedecay_code_index_runtime::code_index_scheduler::semantic_vector_graph::ProjectSemanticVectorRetentionStep::Unavailable( - reason, - ) => { - observations.record_semantic_vector_retention_failure(root); - log_semantic_vector_retention_degraded( - observations, - root, - &format!("unavailable:{reason}"), - ); - crate::daemon::maintenance::MaintenanceTickOutcome::Retry - } - tracedecay_code_index_runtime::code_index_scheduler::semantic_vector_graph::ProjectSemanticVectorRetentionStep::Denied( - reason, - ) => { - observations.record_semantic_vector_retention_failure(root); - log_semantic_vector_retention_degraded(observations, root, &format!("denied:{reason}")); - crate::daemon::maintenance::MaintenanceTickOutcome::Retry - } - } -} - -/// Semantic retrieval is genuinely disabled only when the durable -/// configuration commits neither an active nor a rollback retrieval profile. -/// A committed profile with an unseated activation coordinator is a transient -/// (or genuinely degraded) state that must stay retryable, not a quiet pin. -fn semantic_retrieval_profiles_disabled(semantic: &SemanticConfig) -> bool { - semantic.active_profile.is_none() && semantic.rollback_profile.is_none() -} - -fn log_semantic_vector_retention_degraded( - observations: &crate::daemon::maintenance::StoreTelemetrySamplingRegistry, - project_root: &Path, - failure: &str, -) { - observations.emit_retention_degraded(project_root, "semantic_vector_generations", failure); -} - -/// Vector protection inventory for one code-generation retention pass. -/// -/// `Online` carries the exact vector pin set read from the mounted code -/// graph plus the authorities needed to re-verify it under the writer freeze. -/// `SemanticUnseated` is the ordinary default-off state: no semantic runtime -/// is seated, no census will ever exist, and the pass sweeps under the -/// offline protection set without reporting a degradation. `CensusScanning` -/// is in-progress: the bounded census is still paging toward its exact pin -/// set, so the pass defers instead of planning against a mid-scan inventory. -/// `Offline` is a typed degradation for an unreadable vector inventory: the -/// live pin set is unknown, so the pass reports and retains every source -/// rather than planning against an offline protection set that cannot name -/// the sources a mounted activation lease binds. `Refused` is fail-closed -/// for the same reason: the vector authority reported reset/corrupt/denied -/// and no sweep may run. -pub(super) enum VectorRetentionInventoryV1 { - Online { - sources: std::collections::BTreeSet, - configuration: - tracedecay_application::semantic_runtime::ProductionSemanticRetrievalConfigurationStoreV1, - expected_vector_revision: tracedecay_store::SemanticVectorStageCensusRevision, - }, - SemanticUnseated, - CensusScanning, - Offline { - reason: String, - }, - Refused { - reason: String, - }, -} - -impl VectorRetentionInventoryV1 { - /// The `retention_degraded` failure the code-generation pass reports for - /// this inventory, or `None` for states that are ordinary journeys and - /// must stay quiet on every pass: an online inventory, a daemon whose - /// semantic runtime is not seated (the default-off state), and a census - /// still paging toward its exact pin set. - pub(super) fn degraded_reason(&self) -> Option { - match self { - Self::Online { .. } | Self::SemanticUnseated | Self::CensusScanning => None, - Self::Offline { reason } => Some(format!("vector_inventory_offline:{reason}")), - Self::Refused { reason } => Some(reason.clone()), - } - } -} - -pub(super) async fn resolve_vector_retention_inventory( - graph: &TraceDecay, - schedulers: &CodeIndexSchedulerRegistryV1, - observations: &crate::daemon::maintenance::StoreTelemetrySamplingRegistry, -) -> VectorRetentionInventoryV1 { - // A mounted provider can still own vector activation leases when a census - // or configuration read fails. Distinguish that refusal from an absent - // provider; neither unknown state proves its source generations are dead. - let vector_provider = schedulers - .semantic_vector_graph_provider(graph.project_root()) - .await; - let vector_provider_mounted = vector_provider.is_some(); - let unavailable = |reason: String| { - if vector_provider_mounted { - VectorRetentionInventoryV1::Refused { reason } - } else { - VectorRetentionInventoryV1::Offline { reason } - } - }; - let expected_vector_revision = match observations - .semantic_vector_retention_read(graph.project_root()) - { - crate::daemon::maintenance::SemanticVectorRetentionReadV1::Observed { receipt } => { - receipt.revision - } - crate::daemon::maintenance::SemanticVectorRetentionReadV1::SemanticUnseated => { - let Some(provider) = vector_provider.as_ref() else { - return VectorRetentionInventoryV1::SemanticUnseated; - }; - // Providers are mounted even with semantic search disabled. - // An exact empty first page proves there are no retained stages; - // a nonempty page must never be mistaken for disabled liveness. - let empty = async { - let retained = provider - .graph_for_current() - .await - .map_err(|error| error.to_string())?; - let store = tracedecay_application::store::vector_generations::GraphVectorGenerationStoreV1::read_only(&retained) - .await - .map_err(|error| error.to_string())?; - let census = store - .project_stage_census(std::sync::Arc::clone(retained.cancellation())) - .await - .map_err(|error| error.to_string())?; - Ok::<_, String>(census.records.is_empty() - && census.continuation.is_none() - && census.complete_receipt.is_some()) - } - .await; - return match empty { - Ok(true) => VectorRetentionInventoryV1::SemanticUnseated, - Ok(false) => VectorRetentionInventoryV1::Refused { - reason: "unseated_semantic_vector_stages_remain".to_owned(), - }, - Err(reason) => VectorRetentionInventoryV1::Refused { reason }, - }; - } - crate::daemon::maintenance::SemanticVectorRetentionReadV1::Scanning => { - return VectorRetentionInventoryV1::CensusScanning; - } - crate::daemon::maintenance::SemanticVectorRetentionReadV1::Unknown => { - return unavailable("vector_census_incomplete".to_owned()); - } - }; - let Some(configuration) = graph - .configuration_runtime() - .semantic_configuration_inventory_authority() - else { - return unavailable("configuration_inventory_unavailable".to_owned()); - }; - let project_root = graph.hook_store_layout().project_root.clone(); - let sources = tracedecay_code_index_runtime::code_index_scheduler::semantic_vector_graph::project_vector_readable_sources( - schedulers, - &project_root, - &configuration, - expected_vector_revision, - ) - .await; - match classify_vector_readable_sources(sources, configuration, expected_vector_revision) { - VectorRetentionInventoryV1::Offline { reason } => unavailable(reason), - inventory => inventory, - } -} - -/// Map the mounted graph's readable-source read onto the retention inventory: -/// unavailable is the typed offline degradation, while reset, corrupt, and -/// denied are refusals. Both retain every source: an inventory that cannot be -/// read cannot prove which sources a mounted activation lease binds. -fn classify_vector_readable_sources( - sources: tracedecay_code_index_runtime::code_index_scheduler::semantic_vector_graph::ProjectVectorReadableSources, - configuration: tracedecay_application::semantic_runtime::ProductionSemanticRetrievalConfigurationStoreV1, - expected_vector_revision: tracedecay_store::SemanticVectorStageCensusRevision, -) -> VectorRetentionInventoryV1 { - match sources { - tracedecay_code_index_runtime::code_index_scheduler::semantic_vector_graph::ProjectVectorReadableSources::Ready { - sources, - .. - } => VectorRetentionInventoryV1::Online { - sources, - configuration, - expected_vector_revision, - }, - tracedecay_code_index_runtime::code_index_scheduler::semantic_vector_graph::ProjectVectorReadableSources::Unavailable( - reason, - ) => VectorRetentionInventoryV1::Offline { - reason: format!("vector_graph_unavailable:{reason}"), - }, - tracedecay_code_index_runtime::code_index_scheduler::semantic_vector_graph::ProjectVectorReadableSources::ResetRequired( - reason, - ) => VectorRetentionInventoryV1::Refused { - reason: format!("vector_graph_reset_required:{reason}"), - }, - tracedecay_code_index_runtime::code_index_scheduler::semantic_vector_graph::ProjectVectorReadableSources::Corrupt( - reason, - ) => VectorRetentionInventoryV1::Refused { - reason: format!("vector_graph_corrupt:{reason}"), - }, - tracedecay_code_index_runtime::code_index_scheduler::semantic_vector_graph::ProjectVectorReadableSources::Denied( - reason, - ) => VectorRetentionInventoryV1::Refused { - reason: format!("vector_graph_denied:{reason}"), - }, - } -} - -/// Outcome of one bounded code-generation retention pass. -/// -/// `MoreWork` reports bounded progress with a remaining backlog — another -/// collectable superseded generation, or unconsumed graph-replay release -/// evidence — so the maintenance owner keeps the short cadence until the -/// store converges instead of parking multi-GiB debris behind the full -/// maintenance interval. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub(in crate::daemon) enum CodeGenerationRetentionOutcomeV1 { - Complete, - MoreWork, - Failed, -} - -/// Collect superseded code-index generations for one mounted project. -/// -/// Sealed generations are ordinary files, so no database retention or -/// compaction pass reclaims them. This runs on the ordinary maintenance cadence -/// and is independent of the semantic projection lane: the only previous caller -/// sat inside legacy vector migration, so a profile with semantic search -/// disabled never collected anything and grew without bound. -/// -/// Vector-readable source generations are pinned through the mounted code -/// graph when it is resolvable. A daemon without a seated semantic runtime -/// (the default-off state) sweeps under the offline protection set as its -/// ordinary quiet journey, and an in-progress census defers the sweep until -/// its exact pin set is complete. When the vector inventory is unreadable — -/// saturated capacity, failed activation, nothing serving, or a census reset -/// by a failure or mutation — the pass reports its degradation and collects -/// nothing: the offline protection set (active pointer head, durable pointer -/// index, rollback floor, and the serving generation) cannot name the exact -/// source generations a mounted vector activation lease binds, so sweeping -/// under it deleted a live vector source. Reset, corrupt, and denied vector -/// authorities stay fail-closed for the same reason. -#[hotpath::measure( - label = "daemon.git.maintenance.code_generation_retention", - future = true -)] -pub(in crate::daemon) async fn run_code_generation_retention( - graph: &TraceDecay, - schedulers: &CodeIndexSchedulerRegistryV1, - observations: &crate::daemon::maintenance::StoreTelemetrySamplingRegistry, - cancellation: &tracedecay_session_memory::context::CancellationToken, -) -> CodeGenerationRetentionOutcomeV1 { - if cancellation.is_cancelled() { - log_code_generation_retention_degraded( - observations, - graph.project_root(), - "retention_cancelled", - ); - return CodeGenerationRetentionOutcomeV1::Failed; - } - let layout = graph.hook_store_layout(); - let store_root = tracedecay_code_index_retention::code_index_generations::code_index_store_root( - &layout.data_root, - &layout.project_root, - ); - // A store directory that never materialized has nothing to sweep. A store - // *without* an active pointer is different: it is crash debris from a - // publish that never reached its pointer write (an OOM-killed rebuild is - // the ordinary cause), and the planner collects it as a typed unpublished - // store — before this, such orphaned partial generations were unreachable - // by every retention pass while their worktree root stayed live. - if !store_root.is_dir() { - return CodeGenerationRetentionOutcomeV1::Complete; - } - let vector_inventory = - resolve_vector_retention_inventory(graph, schedulers, observations).await; - apply_code_generation_retention( - graph, - schedulers, - observations, - vector_inventory, - cancellation, - ) - .await -} - -/// The offline protection pin: the generation the mounted scheduler is -/// currently serving, when one is mounted at all. -#[hotpath::measure( - label = "daemon.git.maintenance.serving_generation_pins", - future = true -)] -async fn serving_generation_pins( - schedulers: &CodeIndexSchedulerRegistryV1, - project_root: &Path, -) -> std::collections::BTreeSet { - let mut pins = std::collections::BTreeSet::new(); - if let Some(scope) = schedulers.serving_code_scope(project_root).await - && let Some(serving) = scope.serving_generation - { - pins.insert(serving.manifest().generation_id.clone()); - } - // A clean restart whose retained revision-7 head recovered serves through - // the text projection and never seats a second copy of its sealed - // generation, so the sealed slot alone under-reports what is live. Pin - // the level that actually serves or retention collects it out from under - // the route. - if let Some(text) = schedulers.latest_text_serving_for_root(project_root).await { - pins.insert(text.metadata().manifest().generation_id.clone()); - } - pins -} - -/// Execute one code-generation retention pass against a resolved vector -/// inventory. Emitting `retention_degraded` is decided exclusively by -/// [`VectorRetentionInventoryV1::degraded_reason`], so quiet states cannot be -/// reintroduced into the degraded log by a divergent match arm. -#[hotpath::measure( - label = "daemon.git.maintenance.code_generation_retention_apply", - future = true -)] -async fn apply_code_generation_retention( - graph: &TraceDecay, - schedulers: &CodeIndexSchedulerRegistryV1, - observations: &crate::daemon::maintenance::StoreTelemetrySamplingRegistry, - vector_inventory: VectorRetentionInventoryV1, - cancellation: &tracedecay_session_memory::context::CancellationToken, -) -> CodeGenerationRetentionOutcomeV1 { - use tracedecay_code_index_retention::code_index_generations::{ - CodeGenerationRetentionErrorV1, CodeGenerationRetentionModeV1, - DEFAULT_SUPERSEDED_GENERATION_FLOOR, execute_code_generation_retention_cancellable, - prepare_next_code_generation_retention_cancellable, - }; - let layout = graph.hook_store_layout(); - let store_root = tracedecay_code_index_retention::code_index_generations::code_index_store_root( - &layout.data_root, - &layout.project_root, - ); - // Retired generations stay reachable for graph replay through the replay - // pool; retention hard-links each one there before its release event - // becomes durable, and the replay reconciler deletes pool entries once - // the graph confirms it no longer needs them. - let graph_replay_pool_root = graph.db().database_path().with_extension("graph-replay"); - if let Some(failure) = vector_inventory.degraded_reason() { - log_code_generation_retention_degraded(observations, graph.project_root(), &failure); - } - // Published vectors live in the mounted code graph. When the graph is - // resolvable, its inventory is the exact vector pin set. Without a seated - // semantic runtime the durable configuration is canonical proof that no - // vector stage can pin a source, so that journey sweeps under the offline - // protection set (active pointer head, durable pointer index, rollback - // floor, plus the serving generation). A paging census defers: its exact - // pin set arrives when the scan completes, and the vector retention pass - // already keeps the retry cadence short while paging. - // - // An unreadable vector inventory is fail-closed. The offline protection - // set names the serving generation, never the exact source generations a - // mounted vector activation lease still binds, so planning against it - // while the inventory is unknown collected a live vector source - // (production journey cc-5583). "Unknown" is retained, not swept: the - // pass reports its degradation and collects nothing until an exact — or - // canonically empty — inventory is readable again. Reset, corrupt, and - // denied vector authorities stay fail-closed for the same reason. - let (vector_readable_sources, inventory_mode) = match &vector_inventory { - VectorRetentionInventoryV1::Online { sources, .. } => (sources.clone(), "online"), - VectorRetentionInventoryV1::SemanticUnseated => ( - serving_generation_pins(schedulers, &layout.project_root).await, - "semantic_unseated", - ), - VectorRetentionInventoryV1::CensusScanning => { - return CodeGenerationRetentionOutcomeV1::Complete; - } - VectorRetentionInventoryV1::Offline { .. } | VectorRetentionInventoryV1::Refused { .. } => { - return CodeGenerationRetentionOutcomeV1::Failed; - } - }; - // A held replay pool makes every later phase of this pass fail closed: - // the release reconcile's pool acquisition would burn its whole - // graph-operation deadline discovering the holder (the live wedge logged - // that as `graph_replay_release_failed error=DeadlineExceeded` on every - // tick), and the collection executor would then contend for the same - // lock while holding the daemon writer gate. One non-blocking probe - // defers the pass for this tick instead — before the multi-GiB - // full-digest planning below is paid — and the executor's own checked - // acquire returns `GraphReplayPoolBusy` if a publisher wins the - // probe-to-execute window, so the writer gate is never pinned on a - // blocking flock. Both paths arm the same bounded release backoff. - if graph_replay::replay_pool_is_held(&graph_replay_pool_root) { - return defer_graph_replay_pool_busy(observations, graph.project_root()); - } - // Full digest verification routinely reads several GiB. Run it before - // entering the graph transaction and preserve the daemon shutdown token - // through the blocking boundary; the planner checks it after every bounded - // read chunk and creates no journal before verification completes. - let plan_root = store_root.clone(); - let plan_sources = vector_readable_sources.clone(); - let plan_cancellation = cancellation.clone(); - let plan_pool_root = graph_replay_pool_root.clone(); - let plan = tokio::task::spawn_blocking(move || { - prepare_next_code_generation_retention_cancellable( - &plan_root, - &plan_sources, - DEFAULT_SUPERSEDED_GENERATION_FLOOR, - &|| plan_cancellation.is_cancelled(), - Some(&plan_pool_root), - ) - }) - .await; - let plan = match plan { - Ok(Ok(plan)) => plan, - Ok(Err( - tracedecay_code_index_retention::code_index_generations::CodeGenerationRetentionErrorV1::Cancelled, - )) => { - log_code_generation_retention_degraded(observations, graph.project_root(), "retention_cancelled"); - return CodeGenerationRetentionOutcomeV1::Failed; - } - Ok(Err( - tracedecay_code_index_retention::code_index_generations::CodeGenerationRetentionErrorV1::GraphReplayPoolBusy, - )) => { - return defer_graph_replay_pool_busy(observations, graph.project_root()); - } - Ok(Err(error)) => { - // The bare label proved undiagnosable on a live profile: without - // the typed error, a pointer CAS loss under rebuild churn is - // indistinguishable from unrecognized-file or storage failures. - observations.mark_loud_retention_log(); - log_daemon_event( - "retention_degraded", - &[ - ("pass", "code_generations".to_string()), - ("failure", "retention_plan_failed".to_string()), - ("error", error.to_string()), - ], - ); - return CodeGenerationRetentionOutcomeV1::Failed; - } - Err(_) => { - log_code_generation_retention_degraded(observations, graph.project_root(), "retention_task_panicked"); - return CodeGenerationRetentionOutcomeV1::Failed; - } - }; - // A failed, deferred, or retained replay reconcile keeps its durable - // release evidence for a later graph-available pass. Deleting newly - // planned files stays safe in every inventory mode — retention hard-links - // each retired generation into the replay pool before its release event - // becomes durable, so the graph can always finish its retirement later. - // The pass therefore keeps collecting instead of letting sealed - // generations and their multi-GiB text artifacts accumulate without bound - // whenever the graph is dark, wedged, or busy (a recurring - // `graph_replay_release_failed` used to abort every pass here and grew - // one store by tens of GiB in a single crash-rebuild night). A failure - // still reports degraded and fails the pass so the retry cadence stays - // short; a deferral fails the pass quietly under the bounded backoff. - let mut replay_reconcile_failed = false; - let mut release_backlog_remains = false; - let replay_reconcile_attemptable = match graph_replay::reconcile_graph_replay_releases( - graph, - &store_root, - observations, - cancellation, - ) - .await - { - graph_replay::ReconcileOutcome::Complete | graph_replay::ReconcileOutcome::Retained => true, - graph_replay::ReconcileOutcome::MoreWork => { - release_backlog_remains = true; - true - } - // A deferred or failed attempt must not be repeated by the - // post-collection reconcile below: the graph runtime already proved - // it cannot serve this tick. - graph_replay::ReconcileOutcome::Deferred | graph_replay::ReconcileOutcome::Failed => { - replay_reconcile_failed = true; - false - } - }; - if !plan.has_collectable_work() { - return if replay_reconcile_failed { - CodeGenerationRetentionOutcomeV1::Failed - } else if release_backlog_remains { - CodeGenerationRetentionOutcomeV1::MoreWork - } else { - CodeGenerationRetentionOutcomeV1::Complete - }; - } - if cancellation.is_cancelled() { - log_code_generation_retention_degraded( - observations, - graph.project_root(), - "retention_cancelled", - ); - return CodeGenerationRetentionOutcomeV1::Failed; - } - - // Freeze the vector writer, then re-read the committed active+rollback - // identities and their exact source generations. Graph head order is not - // retention authority: a newer unactivated candidate must not displace - // the configured generation from this fence. The unseated default-off - // sweep has no vector inventory to fence, so no freeze is taken there; - // every other non-online inventory already returned without collecting. - let vector_writer_freeze = if let VectorRetentionInventoryV1::Online { - configuration, - expected_vector_revision, - .. - } = &vector_inventory - { - let Some(vector_runtime) = - tracedecay_application::semantic_runtime::project_semantic_production_runtime( - &layout.project_root, - ) - else { - log_code_generation_retention_degraded( - observations, - graph.project_root(), - "vector_writer_unavailable", - ); - return CodeGenerationRetentionOutcomeV1::Failed; - }; - let vector_writer_freeze = vector_runtime.freeze_vector_mutations().await; - let pinned_vector_sources = - match tracedecay_code_index_runtime::code_index_scheduler::semantic_vector_graph::project_vector_readable_sources( - schedulers, - &layout.project_root, - configuration, - *expected_vector_revision, - ) - .await - { - tracedecay_code_index_runtime::code_index_scheduler::semantic_vector_graph::ProjectVectorReadableSources::Ready { - sources, - .. - } => sources, - tracedecay_code_index_runtime::code_index_scheduler::semantic_vector_graph::ProjectVectorReadableSources::ResetRequired( - reason, - ) => { - log_code_generation_retention_degraded(observations, graph.project_root(), &format!( - "vector_inventory_reset_required:{reason}" - )); - return CodeGenerationRetentionOutcomeV1::Failed; - } - tracedecay_code_index_runtime::code_index_scheduler::semantic_vector_graph::ProjectVectorReadableSources::Corrupt( - reason, - ) => { - log_code_generation_retention_degraded(observations, graph.project_root(), &format!( - "vector_inventory_corrupt:{reason}" - )); - return CodeGenerationRetentionOutcomeV1::Failed; - } - tracedecay_code_index_runtime::code_index_scheduler::semantic_vector_graph::ProjectVectorReadableSources::Unavailable( - reason, - ) => { - log_code_generation_retention_degraded(observations, graph.project_root(), &format!( - "vector_inventory_unavailable:{reason}" - )); - return CodeGenerationRetentionOutcomeV1::Failed; - } - tracedecay_code_index_runtime::code_index_scheduler::semantic_vector_graph::ProjectVectorReadableSources::Denied( - reason, - ) => { - log_code_generation_retention_degraded(observations, graph.project_root(), &format!( - "vector_inventory_denied:{reason}" - )); - return CodeGenerationRetentionOutcomeV1::Failed; - } - }; - if pinned_vector_sources != vector_readable_sources { - log_code_generation_retention_degraded( - observations, - graph.project_root(), - "vector_inventory_changed", - ); - return CodeGenerationRetentionOutcomeV1::Failed; - } - tracedecay_application::semantic_runtime::retain_project_semantic_code_sources( - &layout.project_root, - &pinned_vector_sources, - ); - for generation in &plan.collectable_generations { - match tracedecay_code_index_runtime::code_index_scheduler::semantic_vector_graph::project_vector_source_generation_is_live( - schedulers, - &layout.project_root, - &generation.generation_id, - *expected_vector_revision, - ) - .await - { - tracedecay_code_index_runtime::code_index_scheduler::semantic_vector_graph::ProjectSemanticVectorSourceLiveness::Ready( - true, - ) => { - // A pending, ready, published, or base-linked vector stage still - // reads this exact source. It was absent from the root-only - // planning inventory, so retain it and let vector convergence - // make the next maintenance tick eligible. - return CodeGenerationRetentionOutcomeV1::Complete; - } - tracedecay_code_index_runtime::code_index_scheduler::semantic_vector_graph::ProjectSemanticVectorSourceLiveness::Ready( - false, - ) => {} - tracedecay_code_index_runtime::code_index_scheduler::semantic_vector_graph::ProjectSemanticVectorSourceLiveness::Unavailable( - reason, - ) => { - log_code_generation_retention_degraded(observations, graph.project_root(), &format!( - "vector_source_liveness_unavailable:{reason}" - )); - return CodeGenerationRetentionOutcomeV1::Failed; - } - tracedecay_code_index_runtime::code_index_scheduler::semantic_vector_graph::ProjectSemanticVectorSourceLiveness::Denied( - reason, - ) => { - log_code_generation_retention_degraded(observations, graph.project_root(), &format!( - "vector_source_liveness_denied:{reason}" - )); - return CodeGenerationRetentionOutcomeV1::Failed; - } - tracedecay_code_index_runtime::code_index_scheduler::semantic_vector_graph::ProjectSemanticVectorSourceLiveness::ResetRequired( - reason, - ) => { - log_code_generation_retention_degraded(observations, graph.project_root(), &format!( - "vector_source_liveness_reset_required:{reason}" - )); - return CodeGenerationRetentionOutcomeV1::Failed; - } - tracedecay_code_index_runtime::code_index_scheduler::semantic_vector_graph::ProjectSemanticVectorSourceLiveness::Corrupt( - reason, - ) => { - log_code_generation_retention_degraded(observations, graph.project_root(), &format!( - "vector_source_liveness_corrupt:{reason}" - )); - return CodeGenerationRetentionOutcomeV1::Failed; - } - } - } - Some(vector_writer_freeze) - } else { - None - }; - if cancellation.is_cancelled() { - log_code_generation_retention_degraded( - observations, - graph.project_root(), - "retention_cancelled", - ); - return CodeGenerationRetentionOutcomeV1::Failed; - } - // `current_timestamp()` counts seconds; wrapping it in `UtcMicros` stamped - // every deletion receipt with a seconds value in a micros-typed field - // (live receipts read as 1970). The receipt is durable journal evidence, - // so it takes the canonical micros clock. - let completed_at = tracedecay_contracts::clock::now_micros(); - let execution_root = store_root.clone(); - let execution_pool_root = graph_replay_pool_root.clone(); - let execution_cancellation = cancellation.clone(); - let report = tokio::task::spawn_blocking(move || { - execute_code_generation_retention_cancellable( - &execution_root, - plan, - CodeGenerationRetentionModeV1::Apply, - completed_at, - Some(&execution_pool_root), - &|| execution_cancellation.is_cancelled(), - ) - }) - .await; - drop(vector_writer_freeze); - - match report { - Ok(Ok(report)) => { - let generation_reclaimed = report.receipt.as_ref().map_or_else( - || { - report - .deleted_generations - .iter() - .map(|generation| generation.size_bytes) - .sum() - }, - |receipt| receipt.reclaimed_bytes, - ); - let text_artifact_reclaimed = report.text_artifact_receipt.as_ref().map_or_else( - || { - report - .deleted_text_artifacts - .iter() - .map(|artifact| artifact.size_bytes) - .sum() - }, - |receipt| receipt.reclaimed_bytes, - ); - let reclaimed = generation_reclaimed.saturating_add(text_artifact_reclaimed); - if reclaimed > 0 { - log_daemon_event( - "retention_code_generations", - &[ - ("store", "code-index-v1".to_string()), - ("mode", inventory_mode.to_string()), - ("bytes_reclaimed", reclaimed.to_string()), - ( - "generations_collected", - report.deleted_generations.len().to_string(), - ), - ( - "text_artifacts_collected", - report.deleted_text_artifacts.len().to_string(), - ), - ], - ); - } - // The just-collected generation queued fresh release evidence; - // offer it to the graph immediately — but only when this tick's - // earlier reconcile was actually served. A deferred or failed - // runtime must not be probed twice in one tick. - let mut release_reconcile_failed = replay_reconcile_failed; - if replay_reconcile_attemptable { - match graph_replay::reconcile_graph_replay_releases( - graph, - &store_root, - observations, - cancellation, - ) - .await - { - graph_replay::ReconcileOutcome::Complete - | graph_replay::ReconcileOutcome::Retained => {} - graph_replay::ReconcileOutcome::MoreWork => { - release_backlog_remains = true; - } - graph_replay::ReconcileOutcome::Deferred - | graph_replay::ReconcileOutcome::Failed => { - release_reconcile_failed = true; - } - } - } - if release_reconcile_failed { - CodeGenerationRetentionOutcomeV1::Failed - } else if release_backlog_remains - || !report.deleted_generations.is_empty() - || !report.deleted_text_artifacts.is_empty() - { - // Something was collected, so the next bounded census may find - // another collectable unit; stay on the short cadence until a - // pass proves the store converged. A census that finds nothing - // returns Complete one tick later at metadata cost only. - CodeGenerationRetentionOutcomeV1::MoreWork - } else { - CodeGenerationRetentionOutcomeV1::Complete - } - } - Ok(Err(CodeGenerationRetentionErrorV1::Cancelled)) => { - log_code_generation_retention_degraded( - observations, - graph.project_root(), - "retention_cancelled", - ); - CodeGenerationRetentionOutcomeV1::Failed - } - Ok(Err(CodeGenerationRetentionErrorV1::GraphReplayPoolBusy)) => { - defer_graph_replay_pool_busy(observations, graph.project_root()) - } - Ok(Err(error)) => { - // Same diagnosability contract as the plan failure above: the - // apply step's typed error names the exact refusal (CAS loss, - // unsafe state, storage) instead of a bare retry label. - observations.mark_loud_retention_log(); - log_daemon_event( - "retention_degraded", - &[ - ("pass", "code_generations".to_string()), - ("failure", "retention_pass_failed".to_string()), - ("error", error.to_string()), - ], - ); - CodeGenerationRetentionOutcomeV1::Failed - } - Err(_) => { - log_code_generation_retention_degraded( - observations, - graph.project_root(), - "retention_task_panicked", - ); - CodeGenerationRetentionOutcomeV1::Failed - } - } -} - -async fn collect_scope_root_proof_inputs( - graph: &TraceDecay, - schedulers: &CodeIndexSchedulerRegistryV1, - vector_receipt: &tracedecay_store::SemanticVectorProjectCensusReceipt, -) -> Result { - let layout = graph.hook_store_layout(); - let project_id = layout - .identity - .project_id - .as_deref() - .ok_or("registered_project_identity_missing")?; - let registered = graph - .profile_database() - .registered_project_root_inventory(project_id) - .await - .map_err(|_| "registered_root_inventory_unavailable")? - .ok_or("registered_root_inventory_missing")?; - let registered_candidates = registered - .roots - .iter() - .map(PathBuf::from) - .collect::>(); - let project_id = tracedecay_domain::ProjectId::new(project_id.to_owned()) - .map_err(|_| "registered_project_identity_invalid")?; - let enrolled_roots = TraceDecay::enrolled_project_roots(registered_candidates, &project_id) - .map_err(|_| "registered_enrollment_inventory_unavailable")?; - if enrolled_roots.is_empty() { - return Err("registered_enrollment_inventory_empty"); - } - let enrolled_material = enrolled_roots - .iter() - .map(|root| root.to_string_lossy().into_owned()) - .collect::>(); - let enrolled_digest = tracedecay_domain::canonical_sha256(&( - "tracedecay.registered-enrollment-root-inventory.v1", - registered.inventory_digest.as_str(), - &enrolled_material, - )) - .map_err(|_| "registered_enrollment_inventory_digest_failed")?; - let registered_receipt = - tracedecay_code_index_retention::code_index_generations::ScopeRootAuthorityReceiptV1 { - revision: registered.inventory_digest.as_str().to_owned(), - terminal_count: u64::try_from(enrolled_roots.len()) - .map_err(|_| "registered_enrollment_count_overflow")?, - digest: enrolled_digest.as_str().to_owned(), - }; - let mut live_roots = std::collections::BTreeSet::new(); - for root in enrolled_roots { - tracedecay_code_index_retention::code_index_generations::insert_live_root_variants( - &mut live_roots, - &root, - ); - } - - let project_root = graph.project_root().to_path_buf(); - let (git_roots, git_receipt) = tokio::task::spawn_blocking(move || { - tracedecay_code_index_retention::code_index_generations::git_worktree_scope_root_inventory( - &project_root, - ) - }) - .await - .map_err(|_| "git_worktree_inventory_task_panicked")??; - live_roots.extend(git_roots); - - let mounted = schedulers.scope_retention_mounted_roots().await?; - let mounted_count = u64::try_from(mounted.len()).map_err(|_| "mounted_root_count_overflow")?; - let mounted_material = mounted - .iter() - .map(|root| root.to_string_lossy().into_owned()) - .collect::>(); - let mounted_digest = tracedecay_domain::canonical_sha256(&( - "tracedecay.mounted-code-index-root-inventory.v1", - &mounted_material, - )) - .map_err(|_| "mounted_root_inventory_digest_failed")?; - let mounted_receipt = - tracedecay_code_index_retention::code_index_generations::ScopeRootAuthorityReceiptV1 { - revision: mounted_digest.as_str().to_owned(), - terminal_count: mounted_count, - digest: mounted_digest.as_str().to_owned(), - }; - for root in mounted { - tracedecay_code_index_retention::code_index_generations::insert_live_root_variants( - &mut live_roots, - &root, - ); - } - if live_roots.is_empty() { - return Err("scope_live_root_inventory_empty"); - } - - let configuration = graph - .configuration_runtime() - .semantic_configuration_inventory_authority() - .ok_or("configuration_inventory_unavailable")?; - let ( - vector_sources, - configuration_receipt, - configured_root_receipt, - ) = match tracedecay_code_index_runtime::code_index_scheduler::semantic_vector_graph::project_vector_readable_sources( - schedulers, - graph.project_root(), - &configuration, - vector_receipt.revision, - ) - .await - { - tracedecay_code_index_runtime::code_index_scheduler::semantic_vector_graph::ProjectVectorReadableSources::Ready { - sources, - configuration_receipt, - configured_root_receipt, - } => (sources, configuration_receipt, configured_root_receipt), - tracedecay_code_index_runtime::code_index_scheduler::semantic_vector_graph::ProjectVectorReadableSources::ResetRequired( - _, - ) => return Err("scope_vector_inventory_reset_required"), - tracedecay_code_index_runtime::code_index_scheduler::semantic_vector_graph::ProjectVectorReadableSources::Corrupt( - _, - ) => return Err("scope_vector_inventory_corrupt"), - tracedecay_code_index_runtime::code_index_scheduler::semantic_vector_graph::ProjectVectorReadableSources::Unavailable( - _, - ) => return Err("scope_vector_inventory_unavailable"), - tracedecay_code_index_runtime::code_index_scheduler::semantic_vector_graph::ProjectVectorReadableSources::Denied( - _, - ) => return Err("scope_vector_inventory_denied"), - }; - let configuration_roots = - tracedecay_code_index_retention::code_index_generations::ScopeRootAuthorityReceiptV1 { - revision: configuration_receipt - .revision() - .map_or_else(|| "absent".to_owned(), |revision| revision.to_string()), - terminal_count: configuration_receipt.root_binding_count(), - digest: configuration_receipt.inventory_digest().as_str().to_owned(), - }; - let vector_dependency_digest = tracedecay_domain::canonical_sha256(&( - "tracedecay.configured-vector-dependency-inventory.v1", - configured_root_receipt.root_digest().as_str(), - &vector_sources, - )) - .map_err(|_| "vector_dependency_inventory_digest_failed")?; - let vector_dependencies = - tracedecay_code_index_retention::code_index_generations::ScopeRootAuthorityReceiptV1 { - revision: configured_root_receipt - .revision() - .map_or_else(|| "absent".to_owned(), |revision| revision.to_string()), - terminal_count: configured_root_receipt.root_count(), - digest: vector_dependency_digest.as_str().to_owned(), - }; - let vector_count = vector_receipt - .counts - .pending - .checked_add(vector_receipt.counts.ready) - .and_then(|count| count.checked_add(vector_receipt.counts.published)) - .and_then(|count| count.checked_add(vector_receipt.counts.cancelled)) - .ok_or("vector_census_count_overflow")?; - let vector_census = - tracedecay_code_index_retention::code_index_generations::ScopeRootAuthorityReceiptV1 { - revision: vector_receipt.revision.get().to_string(), - terminal_count: vector_count, - digest: vector_receipt.record_digest.as_str().to_owned(), - }; - - Ok(ScopeRootProofInputsV1 { - live_roots, - registered_roots: registered_receipt, - git_worktrees: git_receipt, - mounted_leases: mounted_receipt, - configuration_roots, - vector_census, - vector_dependencies, - vector_sources, - }) -} - -/// Reconcile whole code-index *scope roots* for one mounted repository. -/// -/// Generation retention above is scoped to a single -/// `code-index-v1//` directory, and every caller -/// derives exactly one such scope from the root it was handed. Nothing has ever -/// enumerated the siblings, so a scope whose project root is gone — a deleted -/// agent worktree is the ordinary cause — is unreachable by any retention pass -/// and uncounted by any report. One large repository carried three scope -/// directories, two of them orphaned, holding 7.2 GiB nothing could see. -/// -/// The pass is fail-closed by construction. Git proves the complete registered -/// worktree set, while the revision-pinned semantic staging authority binds -/// every candidate physical scope hash to its exact logical source shard. A -/// candidate is collected only when both authorities say it is unreferenced; -/// missing, conflicting, or stale vector evidence collects nothing. -#[hotpath::measure(label = "daemon.git.maintenance.scope_reconciliation", future = true)] -pub(super) async fn run_code_index_scope_reconciliation( - graph: &TraceDecay, - schedulers: &CodeIndexSchedulerRegistryV1, - observations: &crate::daemon::maintenance::StoreTelemetrySamplingRegistry, -) -> bool { - use tracedecay_code_index_retention::code_index_generations::{ - CodeGenerationRetentionModeV1, DEFAULT_STRANDED_SCOPE_MINIMUM_AGE_SECS, - complete_scope_root_binding_cleanup, execute_scope_root_retention, - plan_scope_root_retention, plan_scope_root_retention_with_liveness_proof, - prepare_scope_root_binding_cleanup, recover_scope_root_binding_cleanup, - recover_scope_root_retention, - }; - - let layout = graph.hook_store_layout(); - let store_root = - tracedecay_code_index_retention::code_index_generations::code_index_scope_store_root( - &layout.data_root, - ); - if !store_root.is_dir() { - return true; - } - - let recovery_root = store_root.clone(); - let pending_binding_cleanup = tokio::task::spawn_blocking(move || { - recover_scope_root_retention(&recovery_root) - .map_err(|_| "scope_reconciliation_recovery_failed")?; - recover_scope_root_binding_cleanup(&recovery_root) - .map_err(|_| "scope_binding_cleanup_recovery_failed") - }) - .await; - let pending_binding_cleanup = match pending_binding_cleanup { - Ok(Ok(pending)) => pending, - Ok(Err(failure)) => { - log_code_index_scope_reconciliation_degraded(failure); - return false; - } - Err(_) => { - log_code_index_scope_reconciliation_degraded("scope_reconciliation_task_panicked"); - return false; - } - }; - let vector_receipt = match observations.semantic_vector_retention_read(graph.project_root()) { - crate::daemon::maintenance::SemanticVectorRetentionReadV1::Observed { receipt } => receipt, - crate::daemon::maintenance::SemanticVectorRetentionReadV1::Unknown - | crate::daemon::maintenance::SemanticVectorRetentionReadV1::Scanning => { - log_code_index_scope_reconciliation_degraded("vector_census_incomplete"); - return false; - } - // Scope collection is gated on a complete post-convergence census, so - // an unseated semantic runtime can never reach this pass through the - // maintenance journey; refuse fail-closed with its own reason if a - // future caller ever does. - crate::daemon::maintenance::SemanticVectorRetentionReadV1::SemanticUnseated => { - log_code_index_scope_reconciliation_degraded("semantic_configuration_unseated"); - return false; - } - }; - if let Some(replay) = pending_binding_cleanup { - let Some(vector_runtime) = - tracedecay_application::semantic_runtime::project_semantic_production_runtime( - graph.project_root(), - ) - else { - log_code_index_scope_reconciliation_degraded("vector_writer_unavailable"); - return false; - }; - let _vector_writer = vector_runtime.freeze_vector_mutations().await; - let current_inputs = - match collect_scope_root_proof_inputs(graph, schedulers, &vector_receipt).await { - Ok(inputs) => inputs, - Err(failure) => { - log_code_index_scope_reconciliation_degraded(failure); - return false; - } - }; - match tracedecay_code_index_runtime::code_index_scheduler::semantic_vector_graph::project_vector_code_scope_is_live( - schedulers, - graph.project_root(), - &replay.scope_hash, - vector_receipt.revision, - ) - .await - { - tracedecay_code_index_runtime::code_index_scheduler::semantic_vector_graph::ProjectSemanticVectorCodeScopeLiveness::Ready { - source_scope, - live: false, - } => { - if source_scope != replay.source_scope { - log_code_index_scope_reconciliation_degraded( - "vector_scope_binding_replay_mismatch", - ); - return false; - } - let current_proof = match current_inputs.bind_candidate( - replay.scope_hash.clone(), - source_scope.clone(), - vector_receipt.revision, - ) { - Ok(proof) => proof, - Err(failure) => { - log_code_index_scope_reconciliation_degraded(failure); - return false; - } - }; - if current_proof != replay.liveness_proof { - log_code_index_scope_reconciliation_degraded( - "scope_binding_cleanup_authority_changed", - ); - return false; - } - match tracedecay_code_index_runtime::code_index_scheduler::semantic_vector_graph::remove_project_vector_code_scope_binding( - schedulers, - graph.project_root(), - &replay.scope_hash, - &source_scope, - vector_receipt.revision, - ) - .await - { - tracedecay_code_index_runtime::code_index_scheduler::semantic_vector_graph::ProjectSemanticVectorSourceLiveness::Ready(true) => {} - tracedecay_code_index_runtime::code_index_scheduler::semantic_vector_graph::ProjectSemanticVectorSourceLiveness::Ready(false) => { - log_code_index_scope_reconciliation_degraded( - "vector_scope_binding_not_removed", - ); - return false; - } - tracedecay_code_index_runtime::code_index_scheduler::semantic_vector_graph::ProjectSemanticVectorSourceLiveness::Unavailable(reason) => { - log_code_index_scope_reconciliation_degraded(&format!( - "vector_scope_binding_unavailable:{reason}" - )); - return false; - } - tracedecay_code_index_runtime::code_index_scheduler::semantic_vector_graph::ProjectSemanticVectorSourceLiveness::Denied(reason) => { - log_code_index_scope_reconciliation_degraded(&format!( - "vector_scope_binding_denied:{reason}" - )); - return false; - } - tracedecay_code_index_runtime::code_index_scheduler::semantic_vector_graph::ProjectSemanticVectorSourceLiveness::ResetRequired(reason) => { - log_code_index_scope_reconciliation_degraded(&format!( - "vector_scope_binding_reset_required:{reason}" - )); - return false; - } - tracedecay_code_index_runtime::code_index_scheduler::semantic_vector_graph::ProjectSemanticVectorSourceLiveness::Corrupt(reason) => { - log_code_index_scope_reconciliation_degraded(&format!( - "vector_scope_binding_corrupt:{reason}" - )); - return false; - } - } - } - tracedecay_code_index_runtime::code_index_scheduler::semantic_vector_graph::ProjectSemanticVectorCodeScopeLiveness::Missing => {} - tracedecay_code_index_runtime::code_index_scheduler::semantic_vector_graph::ProjectSemanticVectorCodeScopeLiveness::Ready { - live: true, - .. - } => { - log_code_index_scope_reconciliation_degraded("vector_scope_binding_still_live"); - return false; - } - tracedecay_code_index_runtime::code_index_scheduler::semantic_vector_graph::ProjectSemanticVectorCodeScopeLiveness::Unavailable( - reason, - ) => { - log_code_index_scope_reconciliation_degraded(&format!( - "vector_scope_unavailable:{reason}" - )); - return false; - } - tracedecay_code_index_runtime::code_index_scheduler::semantic_vector_graph::ProjectSemanticVectorCodeScopeLiveness::Denied( - reason, - ) => { - log_code_index_scope_reconciliation_degraded(&format!( - "vector_scope_denied:{reason}" - )); - return false; - } - tracedecay_code_index_runtime::code_index_scheduler::semantic_vector_graph::ProjectSemanticVectorCodeScopeLiveness::ResetRequired( - reason, - ) => { - log_code_index_scope_reconciliation_degraded(&format!( - "vector_scope_reset_required:{reason}" - )); - return false; - } - tracedecay_code_index_runtime::code_index_scheduler::semantic_vector_graph::ProjectSemanticVectorCodeScopeLiveness::Corrupt( - reason, - ) => { - log_code_index_scope_reconciliation_degraded(&format!( - "vector_scope_corrupt:{reason}" - )); - return false; - } - } - let completion_root = store_root.clone(); - let completion_replay = replay.clone(); - match tokio::task::spawn_blocking(move || { - complete_scope_root_binding_cleanup(&completion_root, &completion_replay) - .map_err(|_| "scope_binding_cleanup_completion_failed") - }) - .await - { - Ok(Ok(())) => {} - Ok(Err(failure)) => { - log_code_index_scope_reconciliation_degraded(failure); - return false; - } - Err(_) => { - log_code_index_scope_reconciliation_degraded("scope_reconciliation_task_panicked"); - return false; - } - } - // Removing a binding advances the vector census revision. Defer the - // next filesystem plan until maintenance has observed that revision. - return false; - } - - let now_secs = match now_secs_i64() { - Ok(now) => now, - Err(failure) => { - log_code_index_scope_reconciliation_degraded(failure); - return false; - } - }; - // Same micros-typed receipt contract as the code-generation pass above: - // `current_timestamp()` is a seconds clock and must not be stored as micros. - let completed_at = tracedecay_contracts::clock::now_micros(); - let Some(vector_runtime) = - tracedecay_application::semantic_runtime::project_semantic_production_runtime( - graph.project_root(), - ) - else { - log_code_index_scope_reconciliation_degraded("vector_writer_unavailable"); - return false; - }; - // Configuration activation, vector publication, and source-scope - // collection share this mutation fence. Root inventories are read twice - // under it and compared exactly before quarantine. - let _vector_writer = vector_runtime.freeze_vector_mutations().await; - let initial_inputs = - match collect_scope_root_proof_inputs(graph, schedulers, &vector_receipt).await { - Ok(inputs) => inputs, - Err(failure) => { - log_code_index_scope_reconciliation_degraded(failure); - return false; - } - }; - let plan_root = store_root.clone(); - let plan_live_roots = initial_inputs.live_roots.clone(); - let plan = tokio::task::spawn_blocking(move || { - recover_scope_root_retention(&plan_root) - .map_err(|_| "scope_reconciliation_recovery_failed")?; - plan_scope_root_retention( - &plan_root, - &plan_live_roots, - DEFAULT_STRANDED_SCOPE_MINIMUM_AGE_SECS, - now_secs, - ) - .map_err(|_| "scope_reconciliation_pass_failed") - }) - .await; - let plan = match plan { - Ok(Ok(plan)) => plan, - Ok(Err(failure)) => { - log_code_index_scope_reconciliation_degraded(failure); - return false; - } - Err(_) => { - log_code_index_scope_reconciliation_degraded("scope_reconciliation_task_panicked"); - return false; - } - }; - if plan.collectable_scopes.is_empty() { - return true; - } - - let candidates = plan.collectable_scopes.clone(); - let start = usize::try_from(now_secs) - .ok() - .map_or(0, |now| now % candidates.len()); - let mut selected = None; - const MAX_SCOPE_LIVENESS_CHECKS_PER_PASS: usize = 32; - for offset in 0..candidates.len().min(MAX_SCOPE_LIVENESS_CHECKS_PER_PASS) { - let candidate = &candidates[(start + offset) % candidates.len()]; - match tracedecay_code_index_runtime::code_index_scheduler::semantic_vector_graph::project_vector_code_scope_is_live( - schedulers, - graph.project_root(), - &candidate.scope_hash, - vector_receipt.revision, - ) - .await - { - tracedecay_code_index_runtime::code_index_scheduler::semantic_vector_graph::ProjectSemanticVectorCodeScopeLiveness::Ready { - source_scope, - live: false, - } => { - selected = Some((candidate.clone(), source_scope)); - break; - } - tracedecay_code_index_runtime::code_index_scheduler::semantic_vector_graph::ProjectSemanticVectorCodeScopeLiveness::Ready { - live: true, - .. - } => {} - tracedecay_code_index_runtime::code_index_scheduler::semantic_vector_graph::ProjectSemanticVectorCodeScopeLiveness::Missing => { - log_code_index_scope_reconciliation_degraded("vector_scope_binding_missing"); - return false; - } - tracedecay_code_index_runtime::code_index_scheduler::semantic_vector_graph::ProjectSemanticVectorCodeScopeLiveness::Unavailable( - reason, - ) => { - log_code_index_scope_reconciliation_degraded(&format!( - "vector_scope_unavailable:{reason}" - )); - return false; - } - tracedecay_code_index_runtime::code_index_scheduler::semantic_vector_graph::ProjectSemanticVectorCodeScopeLiveness::Denied( - reason, - ) => { - log_code_index_scope_reconciliation_degraded(&format!( - "vector_scope_denied:{reason}" - )); - return false; - } - tracedecay_code_index_runtime::code_index_scheduler::semantic_vector_graph::ProjectSemanticVectorCodeScopeLiveness::ResetRequired( - reason, - ) => { - log_code_index_scope_reconciliation_degraded(&format!( - "vector_scope_reset_required:{reason}" - )); - return false; - } - tracedecay_code_index_runtime::code_index_scheduler::semantic_vector_graph::ProjectSemanticVectorCodeScopeLiveness::Corrupt( - reason, - ) => { - log_code_index_scope_reconciliation_degraded(&format!( - "vector_scope_corrupt:{reason}" - )); - return false; - } - } - } - let Some((candidate, source_scope)) = selected else { - return true; - }; - let planned_proof = match initial_inputs.bind_candidate( - candidate.scope_hash.clone(), - source_scope.clone(), - vector_receipt.revision, - ) { - Ok(proof) => proof, - Err(failure) => { - log_code_index_scope_reconciliation_degraded(failure); - return false; - } - }; - let proof_root = store_root.clone(); - let proof_for_plan = planned_proof.clone(); - let plan = match tokio::task::spawn_blocking(move || { - plan_scope_root_retention_with_liveness_proof( - &proof_root, - proof_for_plan, - DEFAULT_STRANDED_SCOPE_MINIMUM_AGE_SECS, - now_secs, - ) - .map_err(|_| "scope_proof_bound_plan_failed") - }) - .await - { - Ok(Ok(plan)) => plan, - Ok(Err(failure)) => { - log_code_index_scope_reconciliation_degraded(failure); - return false; - } - Err(_) => { - log_code_index_scope_reconciliation_degraded("scope_reconciliation_task_panicked"); - return false; - } - }; - - // Re-read every terminal authority and the exact source binding after - // planning. This is the compare-and-swap immediately preceding quarantine. - let revalidated_inputs = - match collect_scope_root_proof_inputs(graph, schedulers, &vector_receipt).await { - Ok(inputs) => inputs, - Err(failure) => { - log_code_index_scope_reconciliation_degraded(failure); - return false; - } - }; - let revalidated_source_scope = - match tracedecay_code_index_runtime::code_index_scheduler::semantic_vector_graph::project_vector_code_scope_is_live( - schedulers, - graph.project_root(), - &candidate.scope_hash, - vector_receipt.revision, - ) - .await - { - tracedecay_code_index_runtime::code_index_scheduler::semantic_vector_graph::ProjectSemanticVectorCodeScopeLiveness::Ready { - source_scope, - live: false, - } => source_scope, - _ => { - log_code_index_scope_reconciliation_degraded( - "scope_candidate_changed_before_quarantine", - ); - return false; - } - }; - if revalidated_source_scope != source_scope { - log_code_index_scope_reconciliation_degraded( - "scope_candidate_binding_changed_before_quarantine", - ); - return false; - } - let revalidated_proof = match revalidated_inputs.bind_candidate( - candidate.scope_hash.clone(), - revalidated_source_scope, - vector_receipt.revision, - ) { - Ok(proof) => proof, - Err(failure) => { - log_code_index_scope_reconciliation_degraded(failure); - return false; - } - }; - if revalidated_proof != planned_proof { - log_code_index_scope_reconciliation_degraded( - "scope_liveness_authority_changed_before_quarantine", - ); - return false; - } - tracedecay_application::semantic_runtime::retain_project_semantic_code_sources( - graph.project_root(), - &revalidated_inputs.vector_sources, - ); - - let execute_root = - tracedecay_code_index_retention::code_index_generations::code_index_scope_store_root( - &layout.data_root, - ); - let intent_scope = candidate.scope_hash.clone(); - let intent_source_scope = source_scope.clone(); - let proof_for_execute = revalidated_proof.clone(); - let report = tokio::task::spawn_blocking(move || { - prepare_scope_root_binding_cleanup( - &execute_root, - &plan, - &intent_scope, - &intent_source_scope, - &proof_for_execute, - completed_at, - ) - .map_err(|_| "scope_binding_cleanup_prepare_failed")?; - execute_scope_root_retention( - &execute_root, - plan, - &proof_for_execute, - CodeGenerationRetentionModeV1::Apply, - now_secs, - completed_at, - ) - .map_err(|_| "scope_reconciliation_pass_failed") - }) - .await; - - match report { - Ok(Ok(report)) => { - let reclaimed = report - .receipt - .as_ref() - .map_or(0, |receipt| receipt.reclaimed_bytes); - if reclaimed > 0 || report.plan.stranded_scope_count() > 0 { - log_daemon_event( - "retention_code_index_scopes", - &[ - ("store", "code-index-v1".to_string()), - ("live_scopes", report.plan.live_scope_count.to_string()), - ( - "stranded_scopes", - report.plan.stranded_scope_count().to_string(), - ), - ( - "stranded_bytes", - report.plan.stranded_scope_bytes().to_string(), - ), - ( - "retained_immature_scopes", - report.plan.retained_immature_scopes.len().to_string(), - ), - ( - "refused_scopes", - report.plan.refused_scopes.len().to_string(), - ), - ( - "collected_scopes", - report.collected_scopes.len().to_string(), - ), - ("bytes_reclaimed", reclaimed.to_string()), - ], - ); - } - // The durable intent is deliberately completed on the next cadence. - // A daemon restart at this boundary exercises exactly the same - // replay path as an ordinary subsequent tick. - report.collected_scopes.is_empty() - } - Ok(Err(failure)) => { - log_code_index_scope_reconciliation_degraded(failure); - false - } - Err(_) => { - log_code_index_scope_reconciliation_degraded("scope_reconciliation_task_panicked"); - false - } - } -} - -/// Durable failure visibility for scope reconciliation. Every refusal names why -/// so a fail-closed pass is never mistaken for "nothing was stranded". -fn log_code_index_scope_reconciliation_degraded(failure: &str) { - log_daemon_event( - "retention_degraded", - &[ - ("pass", "code_index_scopes".to_string()), - ("failure", failure.to_string()), - ], - ); -} - -/// Runs bounded incremental-vacuum compaction over every tracked branch -/// database other than the one `cg` currently has mounted (the maintenance -/// owner compacts that store through its live-runtime authority). Best-effort -/// and independent per file: a busy or failing branch database never blocks -/// the rest, but keeps the maintenance cadence retry-eligible — see -/// `src/retention/branch_compaction.rs` for the compaction policy itself. -#[hotpath::measure(label = "daemon.git.maintenance.branch_compaction", future = true)] -pub(super) async fn run_branch_compaction( - cg: &TraceDecay, - config: &CompactionThresholdConfig, -) -> bool { - let layout = cg.store_layout(); - let Some(meta) = tracedecay_runtime_core::branch_meta::load_branch_meta(&layout.data_root) - else { - return true; - }; - let active_db_path = layout.graph_db_path.clone(); - let candidates = - tracedecay_maintenance::retention::branch_compaction::select_branch_db_candidates( - &layout.data_root, - &meta, - &active_db_path, - ); - if candidates.is_empty() { - return true; - } - let report = tracedecay_maintenance::retention::branch_compaction::compact_branch_databases( - &candidates, - config, - ); - if report.policy_invalid { - // Never silent: an out-of-range threshold disables the pass entirely - // and would otherwise be indistinguishable from "nothing to compact". - log_daemon_event( - "retention_degraded", - &[ - ("pass", "branch_compaction".to_string()), - ("failure", "invalid_compaction_policy".to_string()), - ( - "free_page_ratio_threshold", - config.free_page_ratio_threshold.to_string(), - ), - ], - ); - return false; - } - if report.compacted.is_empty() && report.skipped.is_empty() { - return true; - } - let freed_pages: u64 = report - .compacted - .iter() - .map(|outcome| outcome.freed_pages) - .sum(); - let unreclaimable = report - .skipped - .iter() - .filter(|skip| { - skip.reason - == tracedecay_maintenance::retention::branch_compaction::BranchCompactionSkipReason::IncrementalVacuumUnavailable - }) - .count(); - log_daemon_event( - "retention_branch_compaction", - &[ - ("project", cg.project_root().display().to_string()), - ("compacted", report.compacted.len().to_string()), - ("freed_pages", freed_pages.to_string()), - ("skipped", report.skipped.len().to_string()), - // Branch databases predating `auto_vacuum = INCREMENTAL`: their - // free pages need a full VACUUM this pass deliberately avoids. - ("unreclaimable", unreclaimable.to_string()), - ], - ); - branch_compaction_succeeded(&report) -} - -pub(super) fn branch_compaction_succeeded( - report: &tracedecay_maintenance::retention::branch_compaction::BranchCompactionReport, -) -> bool { - !report.policy_invalid && report.skipped.is_empty() -} diff --git a/crates/tracedecay/src/daemon/store_maintenance/vector_retention_tests.rs b/crates/tracedecay/src/daemon/store_maintenance/vector_retention_tests.rs index d43462fea0..56b06fed54 100644 --- a/crates/tracedecay/src/daemon/store_maintenance/vector_retention_tests.rs +++ b/crates/tracedecay/src/daemon/store_maintenance/vector_retention_tests.rs @@ -9,11 +9,7 @@ use std::path::{Path, PathBuf}; use sha2::{Digest, Sha256}; use tempfile::TempDir; -use crate::daemon::maintenance::{ - SemanticVectorRetentionCensusOutcome, SemanticVectorRetentionReadV1, - StoreTelemetrySamplingRegistry, -}; -use tracedecay_store_runtime::{StoreWriterGates, WriterScope}; +use crate::daemon::maintenance::project_store_maintenance_lease; use crate::tracedecay::TraceDecay; use tracedecay_application::semantic_runtime::ProjectSemanticActivationExt; use tracedecay_code_index_retention::code_index_generations::{ @@ -26,15 +22,20 @@ use tracedecay_code_index_retention::code_index_generations::{ use tracedecay_code_index_runtime::code_index_scheduler::CodeIndexSchedulerRegistryV1; use tracedecay_code_index_runtime::code_index_scheduler::semantic_vector_graph::ProjectVectorReadableSources; use tracedecay_domain::UtcMicros; -use tracedecay_semantic_contracts::{ - DEFAULT_FASTEMBED_MODEL_ID, SemanticConfig, SemanticProfileSelection, SemanticResourceCeilings, -}; - -use super::{ +use tracedecay_maintenance::store_maintenance::{ CodeGenerationRetentionOutcomeV1, VectorRetentionInventoryV1, apply_code_generation_retention, classify_vector_readable_sources, resolve_vector_retention_inventory, run_code_generation_retention, run_semantic_vector_generation_retention, + semantic_retrieval_profiles_disabled, }; +use tracedecay_maintenance::telemetry::{ + SemanticVectorRetentionCensusOutcome, SemanticVectorRetentionReadV1, + StoreTelemetrySamplingRegistry, +}; +use tracedecay_semantic_contracts::{ + DEFAULT_FASTEMBED_MODEL_ID, SemanticConfig, SemanticProfileSelection, SemanticResourceCeilings, +}; +use tracedecay_store_runtime::{StoreWriterGates, WriterScope}; const FIXTURE_GENERATION_COUNT: usize = 6; @@ -247,7 +248,7 @@ fn committed_retrieval_profiles_keep_the_unseated_state_retryable() { document_composition: tracedecay_domain::EmbeddingDocumentCompositionV1::SanitizedText, }; assert!( - super::semantic_retrieval_profiles_disabled(&disabled), + semantic_retrieval_profiles_disabled(&disabled), "no committed retrieval profile is the genuine Plan 20 default-off state" ); @@ -256,7 +257,7 @@ fn committed_retrieval_profiles_keep_the_unseated_state_retryable() { ..disabled.clone() }; assert!( - !super::semantic_retrieval_profiles_disabled(&active), + !semantic_retrieval_profiles_disabled(&active), "a committed active profile expects a seated coordinator: stay retryable" ); @@ -265,7 +266,7 @@ fn committed_retrieval_profiles_keep_the_unseated_state_retryable() { ..disabled }; assert!( - !super::semantic_retrieval_profiles_disabled(&rollback_only), + !semantic_retrieval_profiles_disabled(&rollback_only), "a committed rollback profile still pins vector machinery: stay retryable" ); } @@ -280,7 +281,7 @@ async fn unseated_semantic_runtime_sweeps_quietly_without_a_degraded_loop() { for pass in 0..2_usize { assert!( run_semantic_vector_generation_retention( - &fixture.graph, + &project_store_maintenance_lease(&fixture.graph), &fixture.schedulers, &fixture.observations, &fixture.cancellation, @@ -295,7 +296,7 @@ async fn unseated_semantic_runtime_sweeps_quietly_without_a_degraded_loop() { "pass {pass}: the census read must pin the typed unseated state" ); let inventory = resolve_vector_retention_inventory( - &fixture.graph, + &project_store_maintenance_lease(&fixture.graph), &fixture.schedulers, &fixture.observations, ) @@ -311,7 +312,7 @@ async fn unseated_semantic_runtime_sweeps_quietly_without_a_degraded_loop() { ); assert_eq!( run_code_generation_retention( - &fixture.graph, + &project_store_maintenance_lease(&fixture.graph), &fixture.schedulers, &fixture.observations, &fixture.cancellation, @@ -349,22 +350,22 @@ async fn unseated_semantic_runtime_sweeps_quietly_without_a_degraded_loop() { ticks <= FIXTURE_GENERATION_COUNT + 2, "the generation-maintenance unit must converge instead of continuing forever" ); - let outcome = crate::daemon::maintenance::generation::run_project_generation_maintenance( - &fixture.graph, + let outcome = tracedecay_maintenance::generation::run_project_generation_maintenance( + &project_store_maintenance_lease(&fixture.graph), &fixture.schedulers, &fixture.observations, &fixture.cancellation, - &crate::config::RetentionConfig::default(), + None, continuation, ) .await; match outcome { - crate::daemon::maintenance::MaintenanceTickOutcome::Complete => break, - crate::daemon::maintenance::MaintenanceTickOutcome::Continue( - crate::daemon::maintenance::MaintenanceContinuation::CodeGenerationRetention, + tracedecay_maintenance::tick::MaintenanceTickOutcome::Complete => break, + tracedecay_maintenance::tick::MaintenanceTickOutcome::Continue( + tracedecay_maintenance::tick::MaintenanceContinuation::CodeGenerationRetention, ) => { continuation = Some( - crate::daemon::maintenance::MaintenanceContinuation::CodeGenerationRetention, + tracedecay_maintenance::tick::MaintenanceContinuation::CodeGenerationRetention, ); } other => panic!("unexpected generation-maintenance outcome: {other:?}"), @@ -387,13 +388,13 @@ async fn semantic_vector_continuation_skips_code_generation_retention() { let fixture = open_unseated_graph_fixture().await; let before = sealed_generation_files(&fixture.store_root); - let outcome = crate::daemon::maintenance::generation::run_project_generation_maintenance( - &fixture.graph, + let outcome = tracedecay_maintenance::generation::run_project_generation_maintenance( + &project_store_maintenance_lease(&fixture.graph), &fixture.schedulers, &fixture.observations, &fixture.cancellation, - &crate::config::RetentionConfig::default(), - Some(crate::daemon::maintenance::MaintenanceContinuation::SemanticVectorRetention), + None, + Some(tracedecay_maintenance::tick::MaintenanceContinuation::SemanticVectorRetention), ) .await; @@ -414,7 +415,7 @@ async fn scanning_census_defers_the_sweep_without_the_degraded_reason() { record_paging_census(&fixture.observations, fixture.graph.project_root()); let inventory = resolve_vector_retention_inventory( - &fixture.graph, + &project_store_maintenance_lease(&fixture.graph), &fixture.schedulers, &fixture.observations, ) @@ -431,7 +432,7 @@ async fn scanning_census_defers_the_sweep_without_the_degraded_reason() { assert_eq!( run_code_generation_retention( - &fixture.graph, + &project_store_maintenance_lease(&fixture.graph), &fixture.schedulers, &fixture.observations, &fixture.cancellation, @@ -454,7 +455,7 @@ async fn unknown_census_reports_offline_and_retains_every_source() { // Unknown (no progress recorded at all) stays a reported degradation: // a seated runtime whose census was reset by a failure or mutation. let inventory = resolve_vector_retention_inventory( - &fixture.graph, + &project_store_maintenance_lease(&fixture.graph), &fixture.schedulers, &fixture.observations, ) @@ -476,7 +477,7 @@ async fn unknown_census_reports_offline_and_retains_every_source() { for pass in 0..2_usize { assert_eq!( run_code_generation_retention( - &fixture.graph, + &project_store_maintenance_lease(&fixture.graph), &fixture.schedulers, &fixture.observations, &fixture.cancellation, @@ -541,7 +542,7 @@ async fn reset_corrupt_and_denied_vector_authorities_refuse_the_sweep() { ); assert_eq!( apply_code_generation_retention( - &fixture.graph, + &project_store_maintenance_lease(&fixture.graph), &fixture.schedulers, &fixture.observations, inventory, @@ -624,7 +625,7 @@ async fn held_replay_pool_defers_then_backs_off_then_recovers() { // collection or release work starts; nothing blocks on the holder. assert_eq!( run_code_generation_retention( - &fixture.graph, + &project_store_maintenance_lease(&fixture.graph), &fixture.schedulers, &fixture.observations, &fixture.cancellation, @@ -652,7 +653,7 @@ async fn held_replay_pool_defers_then_backs_off_then_recovers() { // evidence durably queued. assert_eq!( run_code_generation_retention( - &fixture.graph, + &project_store_maintenance_lease(&fixture.graph), &fixture.schedulers, &fixture.observations, &fixture.cancellation, @@ -677,7 +678,7 @@ async fn held_replay_pool_defers_then_backs_off_then_recovers() { // backlog as bounded progress. assert_eq!( run_code_generation_retention( - &fixture.graph, + &project_store_maintenance_lease(&fixture.graph), &fixture.schedulers, &fixture.observations, &fixture.cancellation, From 9994ea5a3516dde520a7e987d15d3fb0a92eca47 Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Wed, 9 Sep 2026 18:04:19 +0000 Subject: [PATCH 3/6] refactor(maintenance): drop unused doctor provider alias --- crates/tracedecay/src/daemon/doctor_kernel.rs | 8 +------- crates/tracedecay/src/daemon/project_composition.rs | 2 +- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/crates/tracedecay/src/daemon/doctor_kernel.rs b/crates/tracedecay/src/daemon/doctor_kernel.rs index 7d552f43f7..22e2257dff 100644 --- a/crates/tracedecay/src/daemon/doctor_kernel.rs +++ b/crates/tracedecay/src/daemon/doctor_kernel.rs @@ -627,12 +627,6 @@ pub(super) async fn collect_code_generation_retention_findings( } } -/// Live provider of the Remote Brain operational read. Every Doctor read -/// re-observes the mounted remote authorities instead of freezing one value -/// at project-composition time. -pub(in crate::daemon) type RemoteOperationalReadProviderV1 = - Arc RemoteOperationalReadV1 + Send + Sync>; - /// Resolved kernel reads wired into the Doctor composer for one report. struct KernelDoctorSources<'a> { inputs: &'a DoctorKernelInputsV1, @@ -790,7 +784,7 @@ pub(in crate::daemon) fn production_doctor_report_reader( project_sessions: tracedecay_global_db::RegisteredGlobalDbLeaseV1, profile_root: PathBuf, host_home: Option, - remote_operational: RemoteOperationalReadProviderV1, + remote_operational: Arc RemoteOperationalReadV1 + Send + Sync>, retention: crate::config::RetentionConfig, schedulers: tracedecay_code_index_runtime::code_index_scheduler::CodeIndexSchedulerRegistryV1, diagnostic_broker: Arc>, diff --git a/crates/tracedecay/src/daemon/project_composition.rs b/crates/tracedecay/src/daemon/project_composition.rs index 1303139dd2..2ce71e6494 100644 --- a/crates/tracedecay/src/daemon/project_composition.rs +++ b/crates/tracedecay/src/daemon/project_composition.rs @@ -1293,7 +1293,7 @@ impl ProjectOpenInputs<'_> { let remote_credentials = core.graph_runtime.remote_credential_authority(); Arc::new(move || remote_credentials.operational_status()) }; - let remote_operational_read: doctor_kernel::RemoteOperationalReadProviderV1 = { + let remote_operational_read = { let remote_operational_status = Arc::clone(&remote_operational_status); Arc::new(move || remote_operational_status().doctor_read()) }; From f27db40b33c8c8ca17370ec0ff655597dab02296 Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Wed, 9 Sep 2026 18:30:21 +0000 Subject: [PATCH 4/6] style(daemon): order store-runtime imports after kernel move --- crates/tracedecay/src/daemon/bootstrap.rs | 2 +- .../tracedecay/src/daemon/branch_admin/project_retirement.rs | 2 +- crates/tracedecay/src/daemon/engine/shutdown.rs | 2 +- crates/tracedecay/src/daemon/project_server_lifecycle.rs | 4 +++- crates/tracedecay/src/daemon/shutdown_orchestration.rs | 2 +- 5 files changed, 7 insertions(+), 5 deletions(-) diff --git a/crates/tracedecay/src/daemon/bootstrap.rs b/crates/tracedecay/src/daemon/bootstrap.rs index 80a8290279..f81c3a341f 100644 --- a/crates/tracedecay/src/daemon/bootstrap.rs +++ b/crates/tracedecay/src/daemon/bootstrap.rs @@ -10,10 +10,10 @@ use tokio::task::JoinSet; #[cfg(unix)] use tracedecay_code_index_runtime::{GitWatchMaintenanceWakeV1, git_watch}; use tracedecay_daemon_control::RemoteBrainTlsConfig; -use tracedecay_store_runtime::spawn_semantic_artifact_gc_maintenance; use tracedecay_daemon_identity::authority; use tracedecay_domain::errors::{Result, TraceDecayError}; use tracedecay_runtime_core::DAEMON_SHUTDOWN_DEADLINE; +use tracedecay_store_runtime::spawn_semantic_artifact_gc_maintenance; use super::*; diff --git a/crates/tracedecay/src/daemon/branch_admin/project_retirement.rs b/crates/tracedecay/src/daemon/branch_admin/project_retirement.rs index 82945f732c..bbd51084b0 100644 --- a/crates/tracedecay/src/daemon/branch_admin/project_retirement.rs +++ b/crates/tracedecay/src/daemon/branch_admin/project_retirement.rs @@ -1,7 +1,7 @@ use std::sync::Arc; -use tracedecay_store_runtime::{ShutdownTaskOutcome, ShutdownTaskReceipt, ShutdownTaskStatus}; use super::{StoreAdministration, StoreOwnerKey}; +use tracedecay_store_runtime::{ShutdownTaskOutcome, ShutdownTaskReceipt, ShutdownTaskStatus}; pub(super) struct ProjectServerRetirement { pub(super) owner: StoreOwnerKey, diff --git a/crates/tracedecay/src/daemon/engine/shutdown.rs b/crates/tracedecay/src/daemon/engine/shutdown.rs index 131ee5ce2b..d5472f4f6e 100644 --- a/crates/tracedecay/src/daemon/engine/shutdown.rs +++ b/crates/tracedecay/src/daemon/engine/shutdown.rs @@ -23,10 +23,10 @@ use crate::daemon::shutdown_coordination::{ShutdownOwner, ShutdownStatus}; use crate::daemon::shutdown_orchestration::{ DaemonShutdownPlan, DaemonShutdownReceipt, coordinate_daemon_shutdown, }; -use tracedecay_store_runtime::ShutdownTaskReceipt; use crate::daemon::{log_daemon_event, project_open_tasks, shutdown_project_servers}; #[cfg(test)] use tracedecay_runtime_core::DAEMON_SHUTDOWN_DEADLINE; +use tracedecay_store_runtime::ShutdownTaskReceipt; impl DaemonEngine { #[hotpath::measure(label = "daemon.engine.shutdown_owner_phases", future = true)] diff --git a/crates/tracedecay/src/daemon/project_server_lifecycle.rs b/crates/tracedecay/src/daemon/project_server_lifecycle.rs index 3d6c293532..df58b1ada5 100644 --- a/crates/tracedecay/src/daemon/project_server_lifecycle.rs +++ b/crates/tracedecay/src/daemon/project_server_lifecycle.rs @@ -6,10 +6,12 @@ use super::profile_host_admission_replay::ProfileHostAdmissionBootstrapStatus; use super::shutdown_coordination::ShutdownStatus; -use tracedecay_store_runtime::{ShutdownTaskOutcome, ShutdownTaskReceipt, join_shutdown_tasks_until}; use super::*; use std::collections::HashSet; use tracedecay_daemon_identity::authority; +use tracedecay_store_runtime::{ + ShutdownTaskOutcome, ShutdownTaskReceipt, join_shutdown_tasks_until, +}; pub(super) async fn cancel_retained_session_history(store_administration: &StoreAdministration) { store_administration diff --git a/crates/tracedecay/src/daemon/shutdown_orchestration.rs b/crates/tracedecay/src/daemon/shutdown_orchestration.rs index b46be43d9e..906979d29a 100644 --- a/crates/tracedecay/src/daemon/shutdown_orchestration.rs +++ b/crates/tracedecay/src/daemon/shutdown_orchestration.rs @@ -10,13 +10,13 @@ use super::shutdown_coordination::{ DrainingGauge, ShutdownOwner, ShutdownOwnerReceipt, ShutdownReceipt, ShutdownStatus, prepare_shutdown_owner_phases, }; -use tracedecay_store_runtime::{ShutdownTaskOutcome, ShutdownTaskReceipt}; use super::{ DAEMON_BACKGROUND_DRAIN_DEADLINE, DAEMON_CLIENT_DRAIN_DEADLINE, DAEMON_PROJECT_SERVER_DRAIN_DEADLINE, DAEMON_STORE_CLOSE_RESERVE, DAEMON_TASK_ABORT_DEADLINE, DaemonLifecycle, core_lifecycle::DaemonShutdownClaim, log_daemon_event, }; use tracedecay_domain::errors::Result; +use tracedecay_store_runtime::{ShutdownTaskOutcome, ShutdownTaskReceipt}; type ProjectServerShutdownFuture = Pin + Send + 'static>>; From 75966a0dd23cd69c4848afc33ef97cc427a3363c Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Wed, 9 Sep 2026 19:09:50 +0000 Subject: [PATCH 5/6] refactor(storage): share exact repository enrollment checks --- crates/tracedecay-maintenance/src/lease.rs | 34 +------------- .../src/store_maintenance/mod.rs | 7 ++- crates/tracedecay-runtime-core/src/storage.rs | 5 +- .../src/storage/identity.rs | 37 +++++++++++++++ .../src/storage/identity_tests.rs | 46 +++++++++++++++++++ .../daemon/retained_owner/memory_target.rs | 2 +- .../src/tracedecay/lifecycle/identity.rs | 39 +--------------- 7 files changed, 94 insertions(+), 76 deletions(-) diff --git a/crates/tracedecay-maintenance/src/lease.rs b/crates/tracedecay-maintenance/src/lease.rs index 68fb8a19cf..47f9e8006b 100644 --- a/crates/tracedecay-maintenance/src/lease.rs +++ b/crates/tracedecay-maintenance/src/lease.rs @@ -7,11 +7,9 @@ use std::path::{Path, PathBuf}; use std::sync::Arc; use tracedecay_configuration::ProjectConfigurationRuntime; -use tracedecay_domain::ProjectId; -use tracedecay_domain::errors::Result; use tracedecay_global_db::RegisteredGlobalDbLeaseV1; use tracedecay_runtime_core::db::Database; -use tracedecay_runtime_core::storage::{self, StoreLayout}; +use tracedecay_runtime_core::storage::StoreLayout; use tracedecay_store_runtime::DaemonSessionRuntimeRegistryV1; /// Registered store lease for one mounted project's maintenance journey. @@ -75,33 +73,3 @@ impl ProjectStoreMaintenanceLeaseV1 { &self.profile_database } } - -/// Filter candidate roots to those whose on-disk identity names `project_id`. -pub fn enrolled_project_roots( - candidates: impl IntoIterator, - project_id: &ProjectId, -) -> Result> { - let mut candidates = candidates.into_iter().collect::>(); - candidates.sort(); - candidates.dedup(); - - let mut roots = Vec::new(); - for candidate in candidates { - let candidate = tracedecay_runtime_core::worktree::repository_identity_root(&candidate) - .unwrap_or(candidate); - let Ok(canonical) = candidate.canonicalize() else { - continue; - }; - if roots.contains(&canonical) { - continue; - } - let named_id = match storage::read_repository_identity_marker(&canonical)? { - Some(marker) => marker.project_id, - None => storage::default_profile_project_id(&canonical), - }; - if named_id == project_id.as_str() { - roots.push(canonical); - } - } - Ok(roots) -} diff --git a/crates/tracedecay-maintenance/src/store_maintenance/mod.rs b/crates/tracedecay-maintenance/src/store_maintenance/mod.rs index 1874c66140..c9bb496d15 100644 --- a/crates/tracedecay-maintenance/src/store_maintenance/mod.rs +++ b/crates/tracedecay-maintenance/src/store_maintenance/mod.rs @@ -1015,8 +1015,11 @@ async fn collect_scope_root_proof_inputs( .collect::>(); let project_id = tracedecay_domain::ProjectId::new(project_id.to_owned()) .map_err(|_| "registered_project_identity_invalid")?; - let enrolled_roots = crate::lease::enrolled_project_roots(registered_candidates, &project_id) - .map_err(|_| "registered_enrollment_inventory_unavailable")?; + let enrolled_roots = tracedecay_runtime_core::storage::enrolled_project_roots( + registered_candidates, + &project_id, + ) + .map_err(|_| "registered_enrollment_inventory_unavailable")?; if enrolled_roots.is_empty() { return Err("registered_enrollment_inventory_empty"); } diff --git a/crates/tracedecay-runtime-core/src/storage.rs b/crates/tracedecay-runtime-core/src/storage.rs index f70842fedc..75948dd7c3 100644 --- a/crates/tracedecay-runtime-core/src/storage.rs +++ b/crates/tracedecay-runtime-core/src/storage.rs @@ -435,8 +435,9 @@ mod profile_identity; #[cfg(any(test, feature = "test-helpers", feature = "test-transport"))] pub use identity::pin_fixture_repository_identity; pub use identity::{ - has_repository_identity_marker, legacy_enrollment_marker_path, read_legacy_enrollment_marker, - read_repository_identity_marker, repository_identity_path, write_repository_identity_marker, + enrolled_project_roots, has_repository_identity_marker, legacy_enrollment_marker_path, + read_legacy_enrollment_marker, read_repository_identity_marker, repository_identity_path, + write_repository_identity_marker, }; pub(crate) use layout::has_path_local_profile_store; pub use layout::{ diff --git a/crates/tracedecay-runtime-core/src/storage/identity.rs b/crates/tracedecay-runtime-core/src/storage/identity.rs index 0bd90097a0..f14d685cbc 100644 --- a/crates/tracedecay-runtime-core/src/storage/identity.rs +++ b/crates/tracedecay-runtime-core/src/storage/identity.rs @@ -2,6 +2,7 @@ use std::fs; use std::path::{Path, PathBuf}; use crate::config::TRACEDECAY_DIR; +use tracedecay_domain::ProjectId; use tracedecay_domain::errors::{Result, TraceDecayError}; use super::{ @@ -262,3 +263,39 @@ pub fn write_repository_identity_marker(project_root: &Path, project_id: &str) - })?; Ok(true) } + +/// Filters candidate roots down to the ones whose root-side evidence +/// names exactly `project_id`: a `.git/` repository identity marker with +/// that id, or (for roots without one) a deterministic path-derived +/// identity equal to it. +/// +/// This never creates or repairs a marker, so a caller that must not mount +/// a store the profile has not enrolled — a cross-project memory reader, +/// for one — can tell "not enrolled here" apart from "enrolled". +pub fn enrolled_project_roots( + candidates: impl IntoIterator, + project_id: &ProjectId, +) -> Result> { + let mut candidates = candidates.into_iter().collect::>(); + candidates.sort(); + candidates.dedup(); + + let mut roots = Vec::new(); + for candidate in candidates { + let candidate = crate::worktree::repository_identity_root(&candidate).unwrap_or(candidate); + let Ok(canonical) = candidate.canonicalize() else { + continue; + }; + if roots.contains(&canonical) { + continue; + } + let named_id = match read_repository_identity_marker(&canonical)? { + Some(marker) => marker.project_id, + None => super::default_profile_project_id(&canonical), + }; + if named_id == project_id.as_str() { + roots.push(canonical); + } + } + Ok(roots) +} diff --git a/crates/tracedecay-runtime-core/src/storage/identity_tests.rs b/crates/tracedecay-runtime-core/src/storage/identity_tests.rs index 25afcbe3f0..b62eae1351 100644 --- a/crates/tracedecay-runtime-core/src/storage/identity_tests.rs +++ b/crates/tracedecay-runtime-core/src/storage/identity_tests.rs @@ -121,4 +121,50 @@ mod identity_root_canonicalization_tests { default_profile_project_id(&primary), ); } + #[test] + fn enrolled_roots_share_exact_identity_across_linked_paths_and_refuse_foreign_markers() { + let temp = tempfile::tempdir().unwrap(); + let (primary, linked) = repository(temp.path()); + let project = tracedecay_domain::ProjectId::new("project-enrolled".to_owned()).unwrap(); + assert!(write_repository_identity_marker(&primary, project.as_str()).unwrap()); + let roots = enrolled_project_roots( + [ + linked, + primary.clone(), + primary.join("."), + temp.path().join("missing"), + ], + &project, + ) + .unwrap(); + assert_eq!(roots, vec![primary.canonicalize().unwrap()]); + let foreign = tracedecay_domain::ProjectId::new("project-foreign".to_owned()).unwrap(); + assert!( + enrolled_project_roots([primary.clone()], &foreign) + .unwrap() + .is_empty() + ); + let marker = repository_identity_path(&primary).unwrap(); + fs::write(&marker, b"invalid identity marker").unwrap(); + assert!(enrolled_project_roots([primary], &project).is_err()); + assert_eq!(fs::read(marker).unwrap(), b"invalid identity marker"); + } + + #[test] + fn enrolled_roots_allow_only_exact_path_fallback_without_creating_identity() { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().canonicalize().unwrap(); + let project = tracedecay_domain::ProjectId::new(default_profile_project_id(&root)).unwrap(); + assert_eq!( + enrolled_project_roots([root.clone()], &project).unwrap(), + vec![root.clone()] + ); + let foreign = tracedecay_domain::ProjectId::new("project-foreign".to_owned()).unwrap(); + assert!( + enrolled_project_roots([root.clone()], &foreign) + .unwrap() + .is_empty() + ); + assert!(!root.join(".git").exists()); + } } diff --git a/crates/tracedecay/src/daemon/retained_owner/memory_target.rs b/crates/tracedecay/src/daemon/retained_owner/memory_target.rs index 3efa91fc61..0f03e238f2 100644 --- a/crates/tracedecay/src/daemon/retained_owner/memory_target.rs +++ b/crates/tracedecay/src/daemon/retained_owner/memory_target.rs @@ -146,7 +146,7 @@ async fn open_selected_project_read_only<'a>( if context.project.project_id.as_str() != selected_project_id.as_str() { return denied(); } - let roots = TraceDecay::enrolled_project_roots( + let roots = tracedecay_runtime_core::storage::enrolled_project_roots( TraceDecay::registry_context_candidate_roots(&context), selected_project_id, ) diff --git a/crates/tracedecay/src/tracedecay/lifecycle/identity.rs b/crates/tracedecay/src/tracedecay/lifecycle/identity.rs index 8c0db75518..13fdd677e0 100644 --- a/crates/tracedecay/src/tracedecay/lifecycle/identity.rs +++ b/crates/tracedecay/src/tracedecay/lifecycle/identity.rs @@ -107,43 +107,6 @@ impl TraceDecay { candidates } - /// Filters candidate roots down to the ones whose root-side evidence - /// names exactly `project_id`: a `.git/` repository identity marker with - /// that id, or (for roots without one) a deterministic path-derived - /// identity equal to it. - /// - /// This never creates or repairs a marker, so a caller that must not mount - /// a store the profile has not enrolled — a cross-project memory reader, - /// for one — can tell "not enrolled here" apart from "enrolled". - pub(crate) fn enrolled_project_roots( - candidates: impl IntoIterator, - project_id: &ProjectId, - ) -> Result> { - let mut candidates = candidates.into_iter().collect::>(); - candidates.sort(); - candidates.dedup(); - - let mut roots = Vec::new(); - for candidate in candidates { - let candidate = tracedecay_runtime_core::worktree::repository_identity_root(&candidate) - .unwrap_or(candidate); - let Ok(canonical) = candidate.canonicalize() else { - continue; - }; - if roots.contains(&canonical) { - continue; - } - let named_id = match storage::read_repository_identity_marker(&canonical)? { - Some(marker) => marker.project_id, - None => storage::default_profile_project_id(&canonical), - }; - if named_id == project_id.as_str() { - roots.push(canonical); - } - } - Ok(roots) - } - #[hotpath::measure(label = "lifecycle.enrollment_roots", future = true)] pub(crate) async fn registered_enrollment_roots( project_root: &Path, @@ -162,7 +125,7 @@ impl TraceDecay { candidates.extend(Self::registry_context_candidate_roots(&context)); } - let mut roots = Self::enrolled_project_roots(candidates, project_id)?; + let mut roots = storage::enrolled_project_roots(candidates, project_id)?; // Self-heal the sanctioned `.git/`-side anchor: a session mount for a // registered project rewrites a missing repository identity marker in // place (re-adoption after loss, first mount, or a moved checkout). From 08d320ad3f50a4de5bd06d010d45df20dbf6e15b Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Wed, 9 Sep 2026 19:29:44 +0000 Subject: [PATCH 6/6] fix(maintenance): keep degraded causes visible at warn --- Cargo.lock | 1 + crates/tracedecay-maintenance/Cargo.toml | 1 + crates/tracedecay-maintenance/src/lib.rs | 62 +++++++++++++++++++++++- 3 files changed, 63 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index a08b9c52ea..ae09bb94e7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6737,6 +6737,7 @@ dependencies = [ "tracedecay-store-runtime", "tracedecay-tool-catalog", "tracing", + "tracing-subscriber", ] [[package]] diff --git a/crates/tracedecay-maintenance/Cargo.toml b/crates/tracedecay-maintenance/Cargo.toml index 71b0852d97..c98315722e 100644 --- a/crates/tracedecay-maintenance/Cargo.toml +++ b/crates/tracedecay-maintenance/Cargo.toml @@ -49,6 +49,7 @@ tracedecay-tool-catalog = { path = "../tracedecay-tool-catalog", version = "0.1. libc = "0.2" [dev-dependencies] +tracing-subscriber = { version = "0.3", default-features = false, features = ["fmt"] } filetime = "0.2" tempfile = "3" tokio = { version = "1", features = ["full", "test-util"] } diff --git a/crates/tracedecay-maintenance/src/lib.rs b/crates/tracedecay-maintenance/src/lib.rs index 5e7acf9b53..03a3b02abe 100644 --- a/crates/tracedecay-maintenance/src/lib.rs +++ b/crates/tracedecay-maintenance/src/lib.rs @@ -58,5 +58,65 @@ pub mod tick; /// Operator-log line for a maintenance kernel. Callers supply structured fields. pub fn log_maintenance_event(event: &str, fields: &[(&str, String)]) { - tracing::info!(target: "tracedecay_maintenance", event, ?fields, "maintenance event"); + if event == "retention_degraded" { + tracing::warn!(target: "tracedecay_maintenance", event, ?fields, "maintenance event"); + } else { + tracing::info!(target: "tracedecay_maintenance", event, ?fields, "maintenance event"); + } +} + +#[cfg(test)] +mod logging_tests { + use std::io::{self, Write}; + use std::sync::{Arc, Mutex}; + + #[derive(Clone)] + struct Capture(Arc>>); + + impl Write for Capture { + fn write(&mut self, bytes: &[u8]) -> io::Result { + self.0.lock().unwrap().write(bytes) + } + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } + } + + #[test] + fn degraded_retention_is_visible_at_warn_while_success_remains_informational() { + for level in [tracing::Level::WARN, tracing::Level::INFO] { + let buffer = Arc::new(Mutex::new(Vec::new())); + let writer = Capture(Arc::clone(&buffer)); + let subscriber = tracing_subscriber::fmt() + .without_time() + .with_max_level(level) + .with_writer(move || writer.clone()) + .finish(); + tracing::subscriber::with_default(subscriber, || { + super::log_maintenance_event( + "retention_degraded", + &[ + ("pass", "code_generations".to_owned()), + ( + "failure", + "registered_enrollment_inventory_unavailable".to_owned(), + ), + ], + ); + super::log_maintenance_event( + "retention_compaction", + &[("freed_pages", "12".to_owned())], + ); + }); + let output = String::from_utf8(buffer.lock().unwrap().clone()).unwrap(); + assert!(output.contains("WARN")); + assert!(output.contains("retention_degraded")); + assert!(output.contains("code_generations")); + assert!(output.contains("registered_enrollment_inventory_unavailable")); + assert_eq!( + output.contains("retention_compaction"), + level == tracing::Level::INFO + ); + } + } }