From 9b366e6e76730a29575ddac86fc52a6fbdc958fa Mon Sep 17 00:00:00 2001 From: p-diogo Date: Wed, 30 Sep 2026 18:43:34 +0000 Subject: [PATCH 01/10] feat(core): reranker pin_boost option + Candidate.pinned (PRD-04 DEP-5) Co-Authored-By: Claude Code --- rust/totalreclaw-core/src/reranker.rs | 298 +++++++++++++++++- rust/totalreclaw-core/src/search.rs | 2 + rust/totalreclaw-memory/src/backend.rs | 3 + .../tests/spec_compliance.rs | 1 + 4 files changed, 300 insertions(+), 4 deletions(-) diff --git a/rust/totalreclaw-core/src/reranker.rs b/rust/totalreclaw-core/src/reranker.rs index ba421e41..d59c54a2 100644 --- a/rust/totalreclaw-core/src/reranker.rs +++ b/rust/totalreclaw-core/src/reranker.rs @@ -18,6 +18,15 @@ //! [`LEGACY_CLAIM_FALLBACK_WEIGHT`] when the candidate has no source field). //! //! See `docs/specs/totalreclaw/retrieval-v2.md` §Tier 1. +//! +//! # PRD-04 F1: pin boost +//! +//! When [`RerankerConfig::pin_boost`] is `Some(b)` with a finite `b > 1.0` +//! and the query has an embedding, the final score of every candidate whose +//! [`Candidate::pinned`] flag is set is multiplied by `b` (after source +//! weighting, before top-k truncation). [`DEFAULT_PIN_BOOST`] is the value +//! clients enable. `None` — the default — leaves every score bit-for-bit +//! unchanged. See `docs/specs/totalreclaw/retrieval-v2.md` §Pin boost. use std::collections::HashMap; @@ -90,6 +99,17 @@ pub fn source_weight(source: MemorySource) -> f64 { .unwrap_or(LEGACY_CLAIM_FALLBACK_WEIGHT) } +/// Recall boost clients enable for pinned candidates (PRD-04 F1 / DEP-5). +/// +/// Equals [`crate::claims::PinConfig::default`]`().hard_boost`: the 1.5× +/// value signed off on 2026-04-28 (internal +/// `docs/plans/2026-04-28-f1-pin-ux-defaults.md`) as strong enough to lift +/// pinned facts about 50% above unpinned ones without making every pinned +/// fact rank first. On-wire pins are binary (`pin_status == "pinned"` / +/// `st == "p"`) and carry no pin timestamp, so the non-decaying hard-pin +/// value applies. +pub const DEFAULT_PIN_BOOST: f64 = 1.5; + /// Reranker runtime configuration. /// /// v0 callers can stay with the legacy [`rerank`] API (no source awareness). @@ -113,16 +133,27 @@ pub struct RerankerConfig { /// intent-weighted formula when `Some`. Default: `None` (adaptive). #[serde(default, skip_serializing_if = "Option::is_none")] pub vector_weight_override: Option, + /// PRD-04 F1 / DEP-5: multiplicative boost for candidates whose + /// [`Candidate::pinned`] is true, applied after source weighting and + /// before top-k truncation. `None` (default), a non-finite value or a + /// value `<= 1.0` is a no-op, and so is any call with an empty query + /// embedding (lexical-only recall has no relevance signal to separate + /// non-matching candidates, so a boost would lift every pin to the top). + /// Clients enable it with [`DEFAULT_PIN_BOOST`]. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub pin_boost: Option, } impl Default for RerankerConfig { - /// Defaults to v0-compatible behaviour (`apply_source_weights = false`) so - /// pre-v1 callers can bump the core version without ranking drift. + /// Defaults to v0-compatible behaviour (`apply_source_weights = false`, + /// no pin boost) so pre-v1 callers can bump the core version without + /// ranking drift. fn default() -> Self { RerankerConfig { apply_source_weights: false, bm25_weight_override: None, vector_weight_override: None, + pin_boost: None, } } } @@ -145,6 +176,12 @@ pub struct Candidate { /// source yields [`LEGACY_CLAIM_FALLBACK_WEIGHT`]. #[serde(default, skip_serializing_if = "Option::is_none")] pub source: Option, + /// PRD-04 F1 / DEP-5: true when the fact is pinned (clients set it from + /// `claims::is_pinned_json` on the decrypted blob). Only affects ranking + /// when [`RerankerConfig::pin_boost`] is set. JSON key `pinned`; absent + /// means false, and false is omitted on serialisation. + #[serde(default, skip_serializing_if = "is_false")] + pub pinned: bool, } /// A reranked result with scores. @@ -154,7 +191,7 @@ pub struct RankedResult { pub id: String, /// Decrypted plaintext. pub text: String, - /// Final fused score (post source-weight multiplication if enabled). + /// Final fused score (post source-weight and pin-boost multiplication if enabled). pub score: f64, /// BM25 component score. pub bm25_score: f64, @@ -172,6 +209,10 @@ fn is_one_f64(v: &f64) -> bool { (*v - 1.0).abs() < f64::EPSILON } +fn is_false(v: &bool) -> bool { + !*v +} + /// Rerank candidates using BM25 + Cosine + RRF fusion (v0-compatible). /// /// This function does NOT apply source weights — call [`rerank_with_config`] @@ -260,6 +301,10 @@ pub fn rerank_with_config( let bm25_ranks = compute_ranks(&bm25_scores); let cosine_ranks = compute_ranks(&cosine_scores); + // PRD-04 F1 / DEP-5: the pin boost needs a semantic signal (see + // `RerankerConfig::pin_boost`). + let semantic_signal = !query_embedding.is_empty(); + // Intent-weighted fusion let mut results: Vec = Vec::with_capacity(candidates.len()); for (i, candidate) in candidates.iter().enumerate() { @@ -286,7 +331,14 @@ pub fn rerank_with_config( 1.0 }; - let final_score = fused * src_weight; + // PRD-04 F1 / DEP-5 pin boost (post source weight, pre-truncation). + // `x * 1.0 == x` exactly, so the no-boost path stays bit-identical. + let pin_mult = match config.pin_boost { + Some(b) if candidate.pinned && semantic_signal && b.is_finite() && b > 1.0 => b, + _ => 1.0, + }; + + let final_score = fused * src_weight * pin_mult; results.push(RankedResult { id: candidate.id.clone(), @@ -460,6 +512,7 @@ mod tests { embedding: vec![i as f32 / 10.0; 4], timestamp: String::new(), source: None, + pinned: false, }) .collect(); @@ -505,6 +558,7 @@ mod tests { embedding, timestamp: String::new(), source, + pinned: false, } } @@ -926,6 +980,7 @@ mod tests { embedding: vec![0.1f32, 0.2], timestamp: "2026-04-17T00:00:00Z".into(), source: Some(MemorySource::User), + pinned: false, }, Candidate { id: "2".into(), @@ -933,6 +988,7 @@ mod tests { embedding: vec![0.1f32, 0.2], timestamp: String::new(), source: None, + pinned: false, }, ]; let json = serde_json::to_string(&candidates).unwrap(); @@ -1002,6 +1058,7 @@ mod tests { apply_source_weights: false, bm25_weight_override: Some(1.0), vector_weight_override: Some(0.0), + pin_boost: None, }, ) .unwrap(); @@ -1029,6 +1086,7 @@ mod tests { apply_source_weights: false, bm25_weight_override: None, vector_weight_override: None, + pin_boost: None, }, ) .unwrap(); @@ -1052,6 +1110,7 @@ mod tests { apply_source_weights: false, bm25_weight_override: Some(0.7), vector_weight_override: Some(0.3), + pin_boost: None, }; let json2 = serde_json::to_string(&cfg_with).unwrap(); assert!(json2.contains("0.7")); @@ -1061,4 +1120,235 @@ mod tests { assert_eq!(back.bm25_weight_override, Some(0.7)); assert_eq!(back.vector_weight_override, Some(0.3)); } + + // === PRD-04 F1 / DEP-5: pin boost === + + fn cand_pinned(id: &str, text: &str, embedding: Vec, pinned: bool) -> Candidate { + Candidate { + id: id.to_string(), + text: text.to_string(), + embedding, + timestamp: String::new(), + source: None, + pinned, + } + } + + /// No candidate shares a token with the query "zulu", so BM25 ties every + /// candidate and cosine alone orders them a > b > c > p > e > f. + /// Mirrored in python/tests/test_reranker_pin_boost.py and + /// mcp/tests/reranker-pin-boost.test.ts — change all three together. + fn pin_fixture(p_pinned: bool) -> Vec { + vec![ + cand_pinned("a", "alpha note", vec![1.0, 0.0, 0.0, 0.0], false), + cand_pinned("b", "bravo note", vec![0.9, 0.1, 0.0, 0.0], false), + cand_pinned("c", "charlie note", vec![0.8, 0.2, 0.0, 0.0], false), + cand_pinned("p", "papa note", vec![0.7, 0.3, 0.0, 0.0], p_pinned), + cand_pinned("e", "echo note", vec![0.6, 0.4, 0.0, 0.0], false), + cand_pinned("f", "foxtrot note", vec![0.5, 0.5, 0.0, 0.0], false), + ] + } + + const PIN_QUERY: &str = "zulu"; + const PIN_QUERY_EMBEDDING: [f32; 4] = [1.0, 0.0, 0.0, 0.0]; + + fn boosted() -> RerankerConfig { + RerankerConfig { + pin_boost: Some(DEFAULT_PIN_BOOST), + ..Default::default() + } + } + + fn ids(results: &[RankedResult]) -> Vec { + results.iter().map(|r| r.id.clone()).collect() + } + + fn assert_bit_identical(x: &[RankedResult], y: &[RankedResult]) { + assert_eq!(ids(x), ids(y)); + for (a, b) in x.iter().zip(y.iter()) { + assert_eq!(a.score.to_bits(), b.score.to_bits(), "score drift on {}", a.id); + } + } + + #[test] + fn test_default_pin_boost_is_the_signed_off_hard_pin_value() { + assert_eq!(DEFAULT_PIN_BOOST, 1.5); + assert_eq!(DEFAULT_PIN_BOOST, crate::claims::PinConfig::default().hard_boost); + } + + #[test] + fn test_reranker_config_default_has_no_pin_boost() { + assert_eq!(RerankerConfig::default().pin_boost, None); + } + + #[test] + fn test_pin_boost_lifts_pinned_candidate_just_outside_top_k_into_top_k() { + let without = rerank_with_config( + PIN_QUERY, + &PIN_QUERY_EMBEDDING, + &pin_fixture(true), + 3, + RerankerConfig::default(), + ) + .unwrap(); + assert_eq!(ids(&without), vec!["a", "b", "c"], "p must sit just outside the top 3 without a boost"); + + let with = rerank_with_config(PIN_QUERY, &PIN_QUERY_EMBEDDING, &pin_fixture(true), 3, boosted()).unwrap(); + assert_eq!(ids(&with), vec!["p", "a", "b"]); + } + + #[test] + fn test_pin_boost_leaves_unpinned_candidates_bit_identical() { + let base = rerank_with_config( + PIN_QUERY, + &PIN_QUERY_EMBEDDING, + &pin_fixture(false), + 6, + RerankerConfig::default(), + ) + .unwrap(); + let with = rerank_with_config(PIN_QUERY, &PIN_QUERY_EMBEDDING, &pin_fixture(false), 6, boosted()).unwrap(); + assert_bit_identical(&base, &with); + } + + #[test] + fn test_pin_boost_none_leaves_pinned_candidates_bit_identical() { + let unpinned = rerank_with_config( + PIN_QUERY, + &PIN_QUERY_EMBEDDING, + &pin_fixture(false), + 6, + RerankerConfig::default(), + ) + .unwrap(); + let pinned = rerank_with_config( + PIN_QUERY, + &PIN_QUERY_EMBEDDING, + &pin_fixture(true), + 6, + RerankerConfig::default(), + ) + .unwrap(); + assert_bit_identical(&unpinned, &pinned); + } + + #[test] + fn test_pin_boost_non_finite_or_not_above_one_is_a_no_op() { + let base = rerank_with_config( + PIN_QUERY, + &PIN_QUERY_EMBEDDING, + &pin_fixture(true), + 6, + RerankerConfig::default(), + ) + .unwrap(); + for b in [1.0, 0.5, 0.0, -2.0, f64::NAN, f64::INFINITY, f64::NEG_INFINITY] { + let got = rerank_with_config( + PIN_QUERY, + &PIN_QUERY_EMBEDDING, + &pin_fixture(true), + 6, + RerankerConfig { + pin_boost: Some(b), + ..Default::default() + }, + ) + .unwrap(); + assert_bit_identical(&base, &got); + } + } + + #[test] + fn test_pin_boost_is_ignored_without_query_embedding() { + // Lexical-only recall: with no query embedding every candidate ties on + // cosine and every non-matching candidate ties on BM25, so a multiplier + // would lift every pinned fact above genuine matches. Without this + // rule "p" would score 1.5 * (0.6/62 + 0.3/61) = 0.0219 > "a" 0.0148. + let base = rerank_with_config("alpha", &[], &pin_fixture(true), 6, RerankerConfig::default()).unwrap(); + let with = rerank_with_config("alpha", &[], &pin_fixture(true), 6, boosted()).unwrap(); + assert_bit_identical(&base, &with); + assert_eq!(with[0].id, "a", "the lexical match stays first"); + } + + #[test] + fn test_pin_boost_does_not_surface_an_irrelevant_pinned_fact() { + // 60 facts that match the query lexically and semantically plus one + // pinned fact that matches neither: 1.5x must not put it in the top 8. + // A pin raises rank; it does not add a fact to every recall. + let mut candidates: Vec = (0..60) + .map(|i| { + cand_pinned( + &format!("m{:02}", i), + &format!("dark mode setting {}", i + 10), + vec![1.0, i as f32 * 0.01, 0.0, 0.0], + false, + ) + }) + .collect(); + candidates.push(cand_pinned( + "pin", + "grocery list for saturday", + vec![0.0, 1.0, 0.0, 0.0], + true, + )); + let got = rerank_with_config("dark mode", &[1.0, 0.0, 0.0, 0.0], &candidates, 8, boosted()).unwrap(); + assert_eq!(got.len(), 8); + assert!( + got.iter().all(|r| r.id != "pin"), + "irrelevant pinned fact entered the top 8: {:?}", + ids(&got) + ); + } + + #[test] + fn test_pin_boost_composes_with_source_weights() { + let user = cand("u", "dark mode preference", vec![0.9, 0.1, 0.0, 0.0], Some(MemorySource::User)); + let mut assistant = cand( + "a", + "dark mode preference", + vec![0.9, 0.1, 0.0, 0.0], + Some(MemorySource::Assistant), + ); + assistant.pinned = true; + let got = rerank_with_config( + "dark mode", + &[0.9, 0.1, 0.0, 0.0], + &[user, assistant], + 2, + RerankerConfig { + apply_source_weights: true, + pin_boost: Some(DEFAULT_PIN_BOOST), + ..Default::default() + }, + ) + .unwrap(); + // assistant 0.85 x 1.5 = 1.275 beats user 1.0 on an otherwise tied base score. + assert_eq!(ids(&got), vec!["a", "u"]); + assert!((got[0].score / got[1].score - 0.85 * 1.5).abs() < 1e-9); + assert!( + (got[0].source_weight - 0.85).abs() < 1e-12, + "source_weight reports the source multiplier only" + ); + } + + #[test] + fn test_candidate_pinned_field_serde() { + let json = r#"[{"id":"x","text":"t","embedding":[0.1],"timestamp":""},{"id":"y","text":"t","embedding":[0.1],"timestamp":"","pinned":true}]"#; + let parsed: Vec = serde_json::from_str(json).unwrap(); + assert!(!parsed[0].pinned, "absent pinned must default to false"); + assert!(parsed[1].pinned); + let back = serde_json::to_string(&parsed).unwrap(); + assert_eq!(back.matches("\"pinned\"").count(), 1, "pinned=false must be omitted: {}", back); + } + + #[test] + fn test_reranker_config_pin_boost_json_key() { + let json = serde_json::to_string(&RerankerConfig::default()).unwrap(); + assert!(!json.contains("pin_boost"), "None pin_boost must be omitted: {}", json); + let cfg: RerankerConfig = + serde_json::from_str(r#"{"apply_source_weights":false,"pin_boost":1.5}"#).unwrap(); + assert_eq!(cfg.pin_boost, Some(1.5)); + let legacy: RerankerConfig = serde_json::from_str(r#"{"apply_source_weights":true}"#).unwrap(); + assert_eq!(legacy.pin_boost, None); + } } diff --git a/rust/totalreclaw-core/src/search.rs b/rust/totalreclaw-core/src/search.rs index a891c421..0b7e0530 100644 --- a/rust/totalreclaw-core/src/search.rs +++ b/rust/totalreclaw-core/src/search.rs @@ -445,6 +445,7 @@ pub fn decrypt_and_rerank( embedding: emb, timestamp: fact.timestamp.clone().unwrap_or_default(), source: None, + pinned: false, }); } @@ -486,6 +487,7 @@ pub fn decrypt_and_rerank_with_key( embedding: emb, timestamp: fact.timestamp.clone().unwrap_or_default(), source: None, + pinned: false, }); } diff --git a/rust/totalreclaw-memory/src/backend.rs b/rust/totalreclaw-memory/src/backend.rs index 1ff1b941..d4652a3e 100644 --- a/rust/totalreclaw-memory/src/backend.rs +++ b/rust/totalreclaw-memory/src/backend.rs @@ -367,6 +367,9 @@ async fn decrypt_and_prepare_candidates( embedding: emb, timestamp: fact.timestamp.clone().unwrap_or_default(), source: envelope.v1_source, + // PRD-04 DEP-5: ZeroClaw does not wire the recall pin boost yet + // (Known Gap in CLAUDE.md). + pinned: false, }); } rerank_candidates diff --git a/rust/totalreclaw-memory/tests/spec_compliance.rs b/rust/totalreclaw-memory/tests/spec_compliance.rs index 56f742f1..94d4746e 100644 --- a/rust/totalreclaw-memory/tests/spec_compliance.rs +++ b/rust/totalreclaw-memory/tests/spec_compliance.rs @@ -333,6 +333,7 @@ fn test_auto_recall_top_k_is_8() { embedding: vec![i as f32 / 20.0; 4], timestamp: String::new(), source: None, + pinned: false, }) .collect(); From b3ceddfe1f7aca48a785eef83242367544f437cd Mon Sep 17 00:00:00 2001 From: p-diogo Date: Wed, 30 Sep 2026 18:51:46 +0000 Subject: [PATCH 02/10] feat(core): expose pin_boost through PyO3 and WASM rerank_with_config (PRD-04 DEP-5) Co-Authored-By: Claude Code --- rust/totalreclaw-core/CHANGELOG.md | 10 ++++++ rust/totalreclaw-core/src/python.rs | 36 +++++++++++++++++++ .../src/python/bind_reranker.rs | 22 ++++++++++-- .../src/wasm/bind_reranker.rs | 18 ++++++++-- 4 files changed, 82 insertions(+), 4 deletions(-) diff --git a/rust/totalreclaw-core/CHANGELOG.md b/rust/totalreclaw-core/CHANGELOG.md index 044c1b82..3395aafc 100644 --- a/rust/totalreclaw-core/CHANGELOG.md +++ b/rust/totalreclaw-core/CHANGELOG.md @@ -5,6 +5,16 @@ All notable changes to `@totalreclaw/core` / `totalreclaw-core` are documented h The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [Unreleased] + +### Added + +- **[PRD-04 F1 / DEP-5] Recall pin boost.** `RerankerConfig.pin_boost: Option` (JSON `pin_boost`) multiplies the final score of candidates whose new `Candidate.pinned: bool` (JSON `pinned`, default `false`) is set, after source weighting and before top-k truncation. `None` (the default), a non-finite value, a value `<= 1.0`, or an empty query embedding is a no-op, so every existing caller ranks bit-for-bit as before and no parity vector changes. `DEFAULT_PIN_BOOST = 1.5` (= `PinConfig::default().hard_boost`, signed off 2026-04-28). PyO3: `rerank_with_config(..., pin_boost=None)`, `default_pin_boost()`. WASM: `rerankWithConfig(..., pin_boost?)`, `defaultPinBoost()`. Spec: `docs/specs/totalreclaw/retrieval-v2.md` §Pin boost. + +### Changed + +- `reranker::Candidate` gains the public field `pinned`. Rust code that builds a `Candidate` with a struct literal must add `pinned: false`; JSON callers (all WASM/PyO3 clients) are unaffected because the field defaults to `false`. In-tree callers (`search.rs`, `totalreclaw-memory`) are updated. + ## [2.6.0] - 2026-08-16 Two additive surfaces: the `derived-bundle-v1` credential bundle (Option E diff --git a/rust/totalreclaw-core/src/python.rs b/rust/totalreclaw-core/src/python.rs index 233e957e..dadb5b36 100644 --- a/rust/totalreclaw-core/src/python.rs +++ b/rust/totalreclaw-core/src/python.rs @@ -132,6 +132,7 @@ fn totalreclaw_core(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_function(wrap_pyfunction!(py_rerank_with_config, m)?)?; m.add_function(wrap_pyfunction!(py_source_weight, m)?)?; m.add_function(wrap_pyfunction!(py_legacy_claim_fallback_weight, m)?)?; + m.add_function(wrap_pyfunction!(py_default_pin_boost, m)?)?; m.add_function(wrap_pyfunction!(py_validate_memory_claim_v1, m)?)?; m.add_function(wrap_pyfunction!(py_parse_memory_type_v1, m)?)?; m.add_function(wrap_pyfunction!(py_parse_memory_source, m)?)?; @@ -774,6 +775,7 @@ mod tests { candidates, 10, true, + None, ) .unwrap(); // User must come first — the JSON array is ordered by score desc. @@ -799,6 +801,7 @@ mod tests { candidates_json, 10, false, + None, ) .unwrap(); let v0 = py_rerank( @@ -810,4 +813,37 @@ mod tests { .unwrap(); assert_eq!(off, v0, "flag OFF must equal v0 rerank output"); } + + #[test] + fn py_rerank_with_config_pin_boost_lifts_pinned_candidate() { + // Same fixture as reranker::tests::pin_fixture (PRD-04 DEP-5). + let candidates = r#"[ + {"id":"a","text":"alpha note","embedding":[1.0,0.0,0.0,0.0],"timestamp":""}, + {"id":"b","text":"bravo note","embedding":[0.9,0.1,0.0,0.0],"timestamp":""}, + {"id":"c","text":"charlie note","embedding":[0.8,0.2,0.0,0.0],"timestamp":""}, + {"id":"p","text":"papa note","embedding":[0.7,0.3,0.0,0.0],"timestamp":"","pinned":true}, + {"id":"e","text":"echo note","embedding":[0.6,0.4,0.0,0.0],"timestamp":""}, + {"id":"f","text":"foxtrot note","embedding":[0.5,0.5,0.0,0.0],"timestamp":""} + ]"#; + let ids = |s: &str| -> Vec { + serde_json::from_str::>(s) + .unwrap() + .iter() + .map(|v| v["id"].as_str().unwrap().to_string()) + .collect() + }; + let off = py_rerank_with_config("zulu", vec![1.0f32, 0.0, 0.0, 0.0], candidates, 3, false, None).unwrap(); + let on = py_rerank_with_config( + "zulu", + vec![1.0f32, 0.0, 0.0, 0.0], + candidates, + 3, + false, + Some(py_default_pin_boost()), + ) + .unwrap(); + assert_eq!(ids(&off), vec!["a", "b", "c"]); + assert_eq!(ids(&on), vec!["p", "a", "b"]); + assert_eq!(py_default_pin_boost(), 1.5); + } } diff --git a/rust/totalreclaw-core/src/python/bind_reranker.rs b/rust/totalreclaw-core/src/python/bind_reranker.rs index 972bde4b..c2b7e60d 100644 --- a/rust/totalreclaw-core/src/python/bind_reranker.rs +++ b/rust/totalreclaw-core/src/python/bind_reranker.rs @@ -35,22 +35,29 @@ pub(crate) fn py_rerank( .map_err(|e| pyo3::exceptions::PyRuntimeError::new_err(e.to_string())) } -/// Rerank candidates with a config flag (Retrieval v2 Tier 1). +/// Rerank candidates with a config flag (Retrieval v2 Tier 1) and an +/// optional pin boost (PRD-04 F1 / DEP-5). /// /// When ``apply_source_weights`` is True, each candidate's final fused score is /// multiplied by the provenance weight derived from its ``source`` field. /// Legacy candidates without ``source`` receive the v0 fallback weight. /// +/// When ``pin_boost`` is a float > 1.0, candidates carrying ``"pinned": true`` +/// are multiplied by it (ignored when ``query_embedding`` is empty). ``None`` +/// (the default) keeps the pre-DEP-5 ranking bit-for-bit. +/// /// Args: /// query: Search query text. /// query_embedding: Query embedding vector (list of floats). -/// candidates_json: JSON array of ``{ id, text, embedding, timestamp, source? }`` objects. +/// candidates_json: JSON array of ``{ id, text, embedding, timestamp, source?, pinned? }`` objects. /// top_k: Number of top results to return. /// apply_source_weights: If True, apply v1 source weighting. +/// pin_boost: Optional pin multiplier; pass :func:`default_pin_boost` to enable. /// /// Returns: /// JSON string of ranked results including ``source_weight``. #[pyfunction] +#[pyo3(signature = (query, query_embedding, candidates_json, top_k, apply_source_weights, pin_boost=None))] #[pyo3(name = "rerank_with_config")] pub(crate) fn py_rerank_with_config( query: &str, @@ -58,6 +65,7 @@ pub(crate) fn py_rerank_with_config( candidates_json: &str, top_k: usize, apply_source_weights: bool, + pin_boost: Option, ) -> PyResult { let candidates: Vec = serde_json::from_str(candidates_json).map_err(|e| PyValueError::new_err(e.to_string()))?; @@ -65,6 +73,7 @@ pub(crate) fn py_rerank_with_config( apply_source_weights, bm25_weight_override: None, vector_weight_override: None, + pin_boost, }; let results = reranker::rerank_with_config(query, &query_embedding, &candidates, top_k, config) .map_err(|e| pyo3::exceptions::PyRuntimeError::new_err(e.to_string()))?; @@ -95,6 +104,15 @@ pub(crate) fn py_legacy_claim_fallback_weight() -> f64 { reranker::LEGACY_CLAIM_FALLBACK_WEIGHT } +/// Default recall boost for pinned candidates (PRD-04 F1 / DEP-5) — 1.5. +/// +/// Pass it as ``pin_boost`` to :func:`rerank_with_config` to enable the boost. +#[pyfunction] +#[pyo3(name = "default_pin_boost")] +pub(crate) fn py_default_pin_boost() -> f64 { + reranker::DEFAULT_PIN_BOOST +} + /// Validate a Memory Taxonomy v1 claim (JSON in, canonical JSON out). /// /// Raises ``ValueError`` on any schema violation (wrong type token, missing diff --git a/rust/totalreclaw-core/src/wasm/bind_reranker.rs b/rust/totalreclaw-core/src/wasm/bind_reranker.rs index 2108d995..6d0d63ed 100644 --- a/rust/totalreclaw-core/src/wasm/bind_reranker.rs +++ b/rust/totalreclaw-core/src/wasm/bind_reranker.rs @@ -28,13 +28,18 @@ pub fn wasm_rerank( serde_wasm_bindgen::to_value(&results).map_err(|e| JsError::new(&e.to_string())) } -/// Rerank candidates with a config flag (Retrieval v2 Tier 1). +/// Rerank candidates with a config flag (Retrieval v2 Tier 1) and an optional +/// pin boost (PRD-04 F1 / DEP-5). /// /// When `apply_source_weights` is `true`, each candidate's final score is /// multiplied by the provenance weight from its `source` field (legacy /// candidates without `source` use the v0 fallback weight). /// -/// `candidates_json`: JSON array of `{ id, text, embedding, timestamp, source? }` objects. +/// When `pin_boost` is a number > 1.0, candidates carrying `"pinned": true` +/// are multiplied by it (ignored when `query_embedding` is empty). Omitted, +/// `undefined` or `null` keeps the pre-DEP-5 ranking bit-for-bit. +/// +/// `candidates_json`: JSON array of `{ id, text, embedding, timestamp, source?, pinned? }` objects. /// Returns a JsValue (array of `RankedResult` objects including `source_weight`). #[wasm_bindgen(js_name = "rerankWithConfig")] pub fn wasm_rerank_with_config( @@ -43,6 +48,7 @@ pub fn wasm_rerank_with_config( candidates_json: &str, top_k: usize, apply_source_weights: bool, + pin_boost: Option, ) -> Result { let candidates: Vec = serde_json::from_str(candidates_json) .map_err(|e| JsError::new(&format!("Invalid candidates JSON: {}", e)))?; @@ -50,6 +56,7 @@ pub fn wasm_rerank_with_config( apply_source_weights, bm25_weight_override: None, vector_weight_override: None, + pin_boost, }; let results = reranker::rerank_with_config(query, query_embedding, &candidates, top_k, config) .map_err(|e| JsError::new(&e.to_string()))?; @@ -77,6 +84,13 @@ pub fn wasm_legacy_claim_fallback_weight() -> f64 { reranker::LEGACY_CLAIM_FALLBACK_WEIGHT } +/// Default recall boost for pinned candidates (PRD-04 F1 / DEP-5) — 1.5. +/// Pass it as `pin_boost` to `rerankWithConfig` to enable the boost. +#[wasm_bindgen(js_name = "defaultPinBoost")] +pub fn wasm_default_pin_boost() -> f64 { + reranker::DEFAULT_PIN_BOOST +} + /// Validate a Memory Taxonomy v1 claim (JSON in, JSON out — canonicalised). /// /// Returns the canonical JSON encoding on success. Throws on any schema From 5cc8ea80faa07d20efaedaedd5521856b83cfb3d Mon Sep 17 00:00:00 2001 From: p-diogo Date: Wed, 30 Sep 2026 18:54:04 +0000 Subject: [PATCH 03/10] feat(python): carry pin state + entity refs through recall; enable core pin boost (PRD-04 DEP-5) Co-Authored-By: Claude Code --- python/src/totalreclaw/claims_helper.py | 55 +++++++++++++ python/src/totalreclaw/operations.py | 24 +++++- python/src/totalreclaw/reranker.py | 74 ++++++++++++++--- python/tests/test_entity_refs.py | 59 ++++++++++++++ .../test_reranker_cross_runtime_parity.py | 8 ++ python/tests/test_reranker_pin_boost.py | 74 +++++++++++++++++ python/tests/test_search_pin_state.py | 80 +++++++++++++++++++ 7 files changed, 361 insertions(+), 13 deletions(-) create mode 100644 python/tests/test_entity_refs.py create mode 100644 python/tests/test_reranker_pin_boost.py create mode 100644 python/tests/test_search_pin_state.py diff --git a/python/src/totalreclaw/claims_helper.py b/python/src/totalreclaw/claims_helper.py index 7eda5d82..8aeb71f2 100644 --- a/python/src/totalreclaw/claims_helper.py +++ b/python/src/totalreclaw/claims_helper.py @@ -583,6 +583,61 @@ def read_claim_from_blob(decrypted_json: str) -> Dict[str, Any]: return read_blob_unified(decrypted_json) +# Core ``EntityType`` tokens (rust/totalreclaw-core/src/claims.rs). Any other +# token makes core reject the whole claim, so refs outside this set are dropped +# before they reach ``totalreclaw_core.resolve_with_candidates`` (PRD-04 DEP-5). +CORE_ENTITY_TYPES = frozenset({"person", "project", "tool", "company", "concept", "place"}) + + +def to_entity_refs(raw: Any) -> List[Dict[str, str]]: + """Normalise entities to core short-key ``EntityRef`` dicts ``{"n","tp","r"?}``. + + Accepts a list of objects with ``name``/``type``/``role`` attributes + (``ExtractedEntity``), v1 dicts (``{"name","type","role"}``) or v0 dicts + (``{"n","tp","r"}``). Entries with an empty name or a type outside + :data:`CORE_ENTITY_TYPES` are dropped. Non-list input returns ``[]``. + """ + if not isinstance(raw, (list, tuple)): + return [] + out: List[Dict[str, str]] = [] + for item in raw: + if isinstance(item, dict): + name = item.get("name", item.get("n")) + etype = item.get("type", item.get("tp")) + role = item.get("role", item.get("r")) + else: + name = getattr(item, "name", None) + etype = getattr(item, "type", None) + role = getattr(item, "role", None) + if not isinstance(name, str) or not name.strip(): + continue + if not isinstance(etype, str) or etype.strip().lower() not in CORE_ENTITY_TYPES: + continue + ref: Dict[str, str] = {"n": name.strip(), "tp": etype.strip().lower()} + if isinstance(role, str) and role.strip(): + ref["r"] = role.strip() + out.append(ref) + return out + + +def entity_refs_from_blob(decrypted: str) -> List[Dict[str, str]]: + """Short-key entity refs carried by a decrypted claim blob. + + v1 blobs keep them under ``entities``; v0 short-key blobs under ``e``. + Returns ``[]`` for unparseable blobs or blobs without entities. + """ + try: + obj = json.loads(decrypted) + except (ValueError, TypeError): + return [] + if not isinstance(obj, dict): + return [] + raw = obj.get("entities") + if not isinstance(raw, list): + raw = obj.get("e") + return to_entity_refs(raw) + + def is_digest_blob(decrypted: str) -> bool: """Does this decrypted blob look like a digest claim?""" try: diff --git a/python/src/totalreclaw/operations.py b/python/src/totalreclaw/operations.py index b1321958..efa2c548 100644 --- a/python/src/totalreclaw/operations.py +++ b/python/src/totalreclaw/operations.py @@ -37,7 +37,7 @@ from .lsh import LSHHasher from .protobuf import FactPayload, encode_fact_protobuf, encode_tombstone_protobuf from .relay import RelayClient, RelayReadBlocked -from .reranker import RerankerCandidate, RerankerResult, rerank +from .reranker import RerankerCandidate, RerankerResult, default_pin_boost, rerank from .tuning_loop import maybe_write_feedback_for_pin from .userop import build_and_send_userop, build_and_send_userop_batch, MAX_BATCH_SIZE from .claims_helper import ( @@ -45,6 +45,7 @@ build_canonical_claim_v1, compute_entity_trapdoor, compute_entity_trapdoors, + entity_refs_from_blob, is_digest_blob, is_stub_blob_hex, read_blob_unified, @@ -58,6 +59,11 @@ # extracted to break. from .memory_types import V0_TO_V1_TYPE, VALID_MEMORY_TYPES +# PRD-04 F1 / DEP-5: recall pin boost (core ``reranker::DEFAULT_PIN_BOOST``, +# 1.5). ``None`` on a core wheel that predates the option -- ranking is then +# unchanged. Core ignores it for lexical-only recalls (no query embedding). +PIN_BOOST_DEFAULT: Optional[float] = default_pin_boost() + # GraphQL queries — matching TypeScript search.ts SEARCH_QUERY = """ query SearchByBlindIndex($trapdoors: [String!]!, $owner: Bytes!, $first: Int!) { @@ -939,6 +945,11 @@ async def search_facts( continue doc = read_claim_from_blob(decrypted_blob) text = doc["text"] + # PRD-04 F1 / DEP-5: pin state feeds the reranker's pin boost and + # the contradiction resolver; entity refs let the resolver find + # the shared entity it requires. + candidate_pinned = bool(_core.is_pinned_claim(decrypted_blob)) + candidate_entities = entity_refs_from_blob(decrypted_blob) or None emb: Optional[list[float]] = None encrypted_emb = fact.get("encryptedEmbedding") @@ -995,6 +1006,8 @@ async def search_facts( category=doc.get("category", "fact"), source=candidate_source, metadata=extra_meta, + pinned=candidate_pinned, + entities=candidate_entities, ) ) except Exception as e: @@ -1018,7 +1031,14 @@ async def search_facts( # Retrieval v2 Tier 1 source-weighting: OFF by default as of 2026-06-08 # (benchmark showed tie-or-worse; see APPLY_SOURCE_WEIGHTS_DEFAULT). - return rerank(query, query_embedding, candidates, top_k=top_k, apply_source_weights=APPLY_SOURCE_WEIGHTS_DEFAULT) + return rerank( + query, + query_embedding, + candidates, + top_k=top_k, + apply_source_weights=APPLY_SOURCE_WEIGHTS_DEFAULT, + pin_boost=PIN_BOOST_DEFAULT, + ) async def forget_fact( diff --git a/python/src/totalreclaw/reranker.py b/python/src/totalreclaw/reranker.py index 672fbb01..44630cfd 100644 --- a/python/src/totalreclaw/reranker.py +++ b/python/src/totalreclaw/reranker.py @@ -13,9 +13,15 @@ that benchmark E13 calibrated. The extra Python-side passes were not part of the validated baseline and were a source of cross-client divergence. +PRD-04 F1 / DEP-5 adds one core signal: a pin boost. Candidates carry +``pinned`` and ``rerank(..., pin_boost=...)`` forwards core's multiplier. +Core wheels that predate the option (``totalreclaw-core`` 2.6.x) are detected +once at import; on them ``pin_boost`` is ignored and ranking is unchanged. + Public surface kept for callers: - - ``rerank(query, query_embedding, candidates, top_k, apply_source_weights)`` + - ``rerank(query, query_embedding, candidates, top_k, apply_source_weights, pin_boost)`` -- called by ``operations.py`` + - ``default_pin_boost()`` -- core's pin multiplier, or ``None`` on an older core - ``cosine_similarity(a, b)`` -- called by ``agent/lifecycle.py`` - ``source_weight(source)`` / ``LEGACY_CLAIM_FALLBACK_WEIGHT`` - Dataclasses: ``RerankerCandidate``, ``RerankerResult`` -- ``RerankerResult`` @@ -73,6 +79,23 @@ def source_weight(source: Optional[str]) -> float: return float(totalreclaw_core.source_weight(source)) +# --------------------------------------------------------------------------- +# Pin boost (PRD-04 F1 / DEP-5, delegated to core) +# --------------------------------------------------------------------------- + +#: True when the installed core exposes ``default_pin_boost`` -- shipped in +#: the same release as the ``pin_boost`` argument of ``rerank_with_config``. +_CORE_SUPPORTS_PIN_BOOST: bool = hasattr(totalreclaw_core, "default_pin_boost") + + +def default_pin_boost() -> Optional[float]: + """Core's recall boost for pinned candidates (1.5), or ``None`` when the + installed ``totalreclaw_core`` predates the option.""" + if not _CORE_SUPPORTS_PIN_BOOST: + return None + return float(totalreclaw_core.default_pin_boost()) + + # --------------------------------------------------------------------------- # Data classes (cross-runtime stable surface) # --------------------------------------------------------------------------- @@ -93,6 +116,13 @@ class RerankerCandidate: #: session_id / subtype. Never sent to core; rejoined by id after #: ``rerank_with_config`` so read surfaces can expose provenance. metadata: Optional[dict] = None + #: PRD-04 F1 / DEP-5 — True when the decrypted blob is pinned + #: (``totalreclaw_core.is_pinned_claim``). Sent to core as ``pinned``. + pinned: bool = False + #: PRD-04 DEP-5 — short-key entity refs (``{"n","tp","r"?}``) from the + #: decrypted blob. Never sent to the reranker; rejoined by id and read by + #: ``agent/contradiction.py`` to hand core's resolver shared entities. + entities: Optional[list[dict]] = None @dataclass @@ -109,6 +139,8 @@ class RerankerResult: source: Optional[str] = None source_weight: Optional[float] = None metadata: Optional[dict] = None + pinned: bool = False + entities: Optional[list[dict]] = None # --------------------------------------------------------------------------- @@ -121,18 +153,24 @@ def rerank( candidates: list[RerankerCandidate], top_k: int = 8, apply_source_weights: bool = False, + pin_boost: Optional[float] = None, ) -> list[RerankerResult]: """Re-rank candidates by routing through ``totalreclaw_core.rerank_with_config``. Core handles intent-weighting itself based on the per-candidate cosine score. Importance / recency / MMR signals are no longer applied client-side; if you need those, contribute them to core::reranker first. + + ``pin_boost`` (PRD-04 F1 / DEP-5): multiplier for candidates with + ``pinned=True``; pass :func:`default_pin_boost`. ``None`` keeps the + pre-DEP-5 call exactly. Core ignores it when ``query_embedding`` is empty, + and this wrapper drops it when the installed core predates it. """ if not candidates: return [] # Build the core::Candidate JSON shape: - # { id, text, embedding, timestamp, source? } + # { id, text, embedding, timestamp, source?, pinned? } # ``embedding`` defaults to [] for candidates without an embedding; # core's cosine_similarity_f32 returns 0 for that case. core_candidates: list[dict] = [] @@ -145,24 +183,36 @@ def rerank( } if c.source: core_obj["source"] = c.source + if c.pinned: + core_obj["pinned"] = True core_candidates.append(core_obj) candidates_json = json.dumps(core_candidates) query_vec = [float(x) for x in (query_embedding or [])] - raw = totalreclaw_core.rerank_with_config( - query, - query_vec, - candidates_json, - top_k, - apply_source_weights, - ) + if pin_boost is not None and _CORE_SUPPORTS_PIN_BOOST: + raw = totalreclaw_core.rerank_with_config( + query, + query_vec, + candidates_json, + top_k, + apply_source_weights, + float(pin_boost), + ) + else: + raw = totalreclaw_core.rerank_with_config( + query, + query_vec, + candidates_json, + top_k, + apply_source_weights, + ) # rerank_with_config returns a JSON string of RankedResult objects. ranked: list[dict] = json.loads(raw) # Restore the optional metadata (importance / created_at / category / - # embedding / source) so consumers like agent/contradiction.py keep - # working without changes. + # embedding / source / pinned / entities) so consumers like + # agent/contradiction.py keep working without changes. by_id = {c.id: c for c in candidates} out: list[RerankerResult] = [] for r in ranked: @@ -182,6 +232,8 @@ def rerank( if apply_source_weights and "source_weight" in r else (1.0 if apply_source_weights else None), metadata=orig.metadata if orig else None, + pinned=orig.pinned if orig else False, + entities=orig.entities if orig else None, ) ) diff --git a/python/tests/test_entity_refs.py b/python/tests/test_entity_refs.py new file mode 100644 index 00000000..79599e9a --- /dev/null +++ b/python/tests/test_entity_refs.py @@ -0,0 +1,59 @@ +"""PRD-04 DEP-5 -- short-key entity refs handed to core's contradiction resolver.""" +from __future__ import annotations + +import json + +import totalreclaw_core + +from totalreclaw.agent.extraction import VALID_ENTITY_TYPES, ExtractedEntity +from totalreclaw.claims_helper import CORE_ENTITY_TYPES, entity_refs_from_blob, to_entity_refs + + +def test_core_entity_types_match_extractor_vocabulary() -> None: + assert set(CORE_ENTITY_TYPES) == set(VALID_ENTITY_TYPES) + + +def test_to_entity_refs_accepts_objects_and_both_dict_shapes() -> None: + refs = to_entity_refs([ + ExtractedEntity(name="Pedro", type="person", role="subject"), + {"name": "Lisbon", "type": "Place"}, + {"n": "TotalReclaw", "tp": "project", "r": "employer"}, + ]) + assert refs == [ + {"n": "Pedro", "tp": "person", "r": "subject"}, + {"n": "Lisbon", "tp": "place"}, + {"n": "TotalReclaw", "tp": "project", "r": "employer"}, + ] + + +def test_to_entity_refs_drops_unknown_types_and_empty_names() -> None: + assert to_entity_refs([ + {"name": "Rex", "type": "animal"}, + {"name": " ", "type": "person"}, + {"name": "Ana"}, + ]) == [] + + +def test_to_entity_refs_non_list_is_empty() -> None: + assert to_entity_refs(None) == [] + assert to_entity_refs("Pedro") == [] + + +def test_entity_refs_from_v1_and_v0_blobs() -> None: + v1 = json.dumps({ + "id": "x", "text": "t", "type": "claim", "source": "user", + "created_at": "2026-09-01T00:00:00Z", "schema_version": "1.0", + "entities": [{"name": "Pedro", "type": "person"}], + }) + v0 = json.dumps({"t": "t", "c": "fact", "cf": 0.9, "i": 8, "sa": "x", "e": [{"n": "Pedro", "tp": "person"}]}) + assert entity_refs_from_blob(v1) == [{"n": "Pedro", "tp": "person"}] + assert entity_refs_from_blob(v0) == [{"n": "Pedro", "tp": "person"}] + assert entity_refs_from_blob("not json") == [] + assert entity_refs_from_blob(json.dumps({"text": "legacy"})) == [] + + +def test_entity_refs_are_accepted_by_core_claim_parser() -> None: + refs = to_entity_refs([{"name": "Pedro", "type": "person", "role": "subject"}]) + claim = {"t": "Pedro lives in Lisbon", "c": "fact", "cf": 0.9, "i": 8, "sa": "x", "e": refs} + # Raises ValueError if core rejects the short-key claim. + totalreclaw_core.canonicalize_claim(json.dumps(claim)) diff --git a/python/tests/test_reranker_cross_runtime_parity.py b/python/tests/test_reranker_cross_runtime_parity.py index 5036886d..ea8fec9f 100644 --- a/python/tests/test_reranker_cross_runtime_parity.py +++ b/python/tests/test_reranker_cross_runtime_parity.py @@ -106,3 +106,11 @@ def test_score_changes_when_source_weights_toggle() -> None: sum_on = sum(r.rrf_score for r in sw_on) sum_off = sum(r.rrf_score for r in sw_off) assert sum_on != sum_off, "Source weighting toggle had no effect" + + +def test_top_k_unchanged_when_pin_boost_on_and_nothing_pinned() -> None: + """PRD-04 DEP-5: turning the pin boost on must not move this parity + fixture (nothing in it is pinned). Mirrored in + mcp/tests/reranker-cross-runtime-parity.test.ts.""" + results = rerank(QUERY, QUERY_EMBEDDING, FIXTURE, top_k=8, apply_source_weights=True, pin_boost=1.5) + assert [r.id for r in results] == EXPECTED_TOP8 diff --git a/python/tests/test_reranker_pin_boost.py b/python/tests/test_reranker_pin_boost.py new file mode 100644 index 00000000..f67f0171 --- /dev/null +++ b/python/tests/test_reranker_pin_boost.py @@ -0,0 +1,74 @@ +"""PRD-04 F1 / DEP-5 -- recall pin boost through the Python reranker wrapper. + +Fixture mirrors rust/totalreclaw-core/src/reranker.rs (``pin_fixture``) and +mcp/tests/reranker-pin-boost.test.ts. Change all three together. +""" +from __future__ import annotations + +import json + +import totalreclaw_core + +from totalreclaw import reranker as rr +from totalreclaw.reranker import RerankerCandidate, rerank + +QUERY = "zulu" +QUERY_EMBEDDING = [1.0, 0.0, 0.0, 0.0] + + +def _fixture(p_pinned: bool) -> list[RerankerCandidate]: + return [ + RerankerCandidate(id="a", text="alpha note", embedding=[1.0, 0.0, 0.0, 0.0]), + RerankerCandidate(id="b", text="bravo note", embedding=[0.9, 0.1, 0.0, 0.0]), + RerankerCandidate(id="c", text="charlie note", embedding=[0.8, 0.2, 0.0, 0.0]), + RerankerCandidate(id="p", text="papa note", embedding=[0.7, 0.3, 0.0, 0.0], pinned=p_pinned), + RerankerCandidate(id="e", text="echo note", embedding=[0.6, 0.4, 0.0, 0.0]), + RerankerCandidate(id="f", text="foxtrot note", embedding=[0.5, 0.5, 0.0, 0.0]), + ] + + +def test_core_default_pin_boost_is_1_5() -> None: + assert rr.default_pin_boost() == 1.5 + + +def test_pinned_candidate_just_outside_top_k_enters_top_k() -> None: + without = rerank(QUERY, QUERY_EMBEDDING, _fixture(True), top_k=3) + assert [r.id for r in without] == ["a", "b", "c"] + boosted = rerank(QUERY, QUERY_EMBEDDING, _fixture(True), top_k=3, pin_boost=rr.default_pin_boost()) + assert [r.id for r in boosted] == ["p", "a", "b"] + assert boosted[0].pinned is True + assert boosted[1].pinned is False + + +def test_pin_boost_with_nothing_pinned_changes_nothing() -> None: + base = rerank(QUERY, QUERY_EMBEDDING, _fixture(False), top_k=6) + boosted = rerank(QUERY, QUERY_EMBEDDING, _fixture(False), top_k=6, pin_boost=1.5) + assert [r.id for r in base] == [r.id for r in boosted] + assert [r.rrf_score for r in base] == [r.rrf_score for r in boosted] + + +def test_entities_round_trip_through_rerank() -> None: + cands = _fixture(True) + cands[0].entities = [{"n": "Pedro", "tp": "person"}] + out = {r.id: r for r in rerank(QUERY, QUERY_EMBEDDING, cands, top_k=6)} + assert out["a"].entities == [{"n": "Pedro", "tp": "person"}] + assert out["b"].entities is None + + +def test_old_core_without_pin_support_ignores_pin_boost(monkeypatch) -> None: + calls: list[tuple] = [] + real = totalreclaw_core.rerank_with_config + + def spy(*args): + calls.append(args) + return real(*args[:5]) + + monkeypatch.setattr(rr, "_CORE_SUPPORTS_PIN_BOOST", False) + monkeypatch.setattr(totalreclaw_core, "rerank_with_config", spy) + assert rr.default_pin_boost() is None + out = rerank(QUERY, QUERY_EMBEDDING, _fixture(True), top_k=3, pin_boost=1.5) + assert [r.id for r in out] == ["a", "b", "c"] + assert calls and all(len(c) == 5 for c in calls), "pin_boost must not reach an old core" + sent = json.loads(calls[0][2]) + assert sent[3]["pinned"] is True + assert "pinned" not in sent[0] diff --git a/python/tests/test_search_pin_state.py b/python/tests/test_search_pin_state.py new file mode 100644 index 00000000..0f07a948 --- /dev/null +++ b/python/tests/test_search_pin_state.py @@ -0,0 +1,80 @@ +"""PRD-04 F1 / DEP-5 -- search_facts surfaces pin state + entity refs and +passes core's pin boost to the reranker.""" +from __future__ import annotations + +import base64 +import json +from unittest import mock + +from totalreclaw import operations +from totalreclaw.crypto import encrypt + +KEY = bytes(range(32)) + + +def _blob_hex(obj: dict) -> str: + return "0x" + base64.b64decode(encrypt(json.dumps(obj), KEY)).hex() + + +def _fact(fact_id: str, text: str, pinned: bool, entities: list[dict]) -> dict: + blob = { + "id": fact_id, "text": text, "type": "claim", "source": "user", + "created_at": "2026-09-01T00:00:00Z", "schema_version": "1.0", "importance": 8, + "entities": entities, + } + if pinned: + blob["pin_status"] = "pinned" + return { + "id": fact_id, "encryptedBlob": _blob_hex(blob), "encryptedEmbedding": None, + "decayScore": "0.9", "timestamp": "", "isActive": True, + } + + +class _FakeRelay: + def __init__(self, facts: list[dict]) -> None: + self._facts = facts + + async def query_subgraph(self, gql: str, variables: dict) -> dict: + return { + "data": { + "blindIndex": [{"id": f"bi-{f['id']}", "fact": f} for f in self._facts], + "facts": list(self._facts), + } + } + + +def _keys(): + keys = mock.Mock() + keys.encryption_key = KEY + return keys + + +async def test_search_facts_marks_pinned_and_carries_entity_refs() -> None: + relay = _FakeRelay([ + _fact("f-pinned", "Pedro lives in Lisbon", True, [{"name": "Pedro", "type": "person"}]), + # "animal" is outside core's EntityType enum -> dropped from the refs. + _fact("f-plain", "Pedro likes coffee", False, + [{"name": "Pedro", "type": "person"}, {"name": "Rex", "type": "animal"}]), + ]) + with mock.patch.object(operations, "generate_blind_indices", return_value=["td"]): + results = await operations.search_facts( + query="where does Pedro live", keys=_keys(), owner="0x" + "ab" * 20, + relay=relay, max_candidates=10, top_k=8, + ) + by_id = {r.id: r for r in results} + assert by_id["f-pinned"].pinned is True + assert by_id["f-plain"].pinned is False + assert by_id["f-pinned"].entities == [{"n": "Pedro", "tp": "person"}] + assert by_id["f-plain"].entities == [{"n": "Pedro", "tp": "person"}] + + +async def test_search_facts_passes_core_pin_boost_to_rerank() -> None: + relay = _FakeRelay([_fact("f-pinned", "Pedro lives in Lisbon", True, [])]) + with mock.patch.object(operations, "generate_blind_indices", return_value=["td"]), \ + mock.patch.object(operations, "rerank", wraps=operations.rerank) as spy: + await operations.search_facts( + query="where does Pedro live", keys=_keys(), owner="0x" + "ab" * 20, + relay=relay, max_candidates=10, top_k=8, + ) + assert operations.PIN_BOOST_DEFAULT == 1.5 + assert spy.call_args.kwargs["pin_boost"] == operations.PIN_BOOST_DEFAULT From 49a81dedbd64320154e2aa5ae7a651b4de9a68ce Mon Sep 17 00:00:00 2001 From: p-diogo Date: Wed, 30 Sep 2026 18:55:23 +0000 Subject: [PATCH 04/10] feat(python): get_fact_pin_status on operations + client (PRD-04 DEP-5) Co-Authored-By: Claude Code --- python/src/totalreclaw/client.py | 16 ++++ python/src/totalreclaw/operations.py | 53 ++++++++++++ python/tests/test_fact_pin_status.py | 119 +++++++++++++++++++++++++++ 3 files changed, 188 insertions(+) create mode 100644 python/tests/test_fact_pin_status.py diff --git a/python/src/totalreclaw/client.py b/python/src/totalreclaw/client.py index 91f2f4fe..09228774 100644 --- a/python/src/totalreclaw/client.py +++ b/python/src/totalreclaw/client.py @@ -42,6 +42,7 @@ export_facts, pin_fact, unpin_fact, + get_fact_pin_status, find_existing_content_fps, # internal#448 — byte-capped batching + halve-on-simfail hoisted into the # shared write path so EVERY remember_batch caller inherits it. @@ -1185,6 +1186,21 @@ async def unpin_fact(self, fact_id: str) -> dict: data_edge_address=self._data_edge_address, ) + async def get_fact_pin_status(self, fact_id: str) -> bool: + """Whether ``fact_id`` is pinned (PRD-04 F1 / DEP-5). + + Delegates to :func:`totalreclaw.operations.get_fact_pin_status`. + Raises when the state cannot be read -- callers fail closed. + """ + await self._ensure_address() + await self._ensure_registered() + return await get_fact_pin_status( + fact_id=fact_id, + keys=self._keys, + owner=self._wallet_address, + relay=self._relay, + ) + async def retype(self, fact_id: str, new_type: str) -> dict: """Re-type an existing memory (claim → preference, etc.). diff --git a/python/src/totalreclaw/operations.py b/python/src/totalreclaw/operations.py index efa2c548..a9fc58fd 100644 --- a/python/src/totalreclaw/operations.py +++ b/python/src/totalreclaw/operations.py @@ -1151,6 +1151,59 @@ async def _fetch_fact_by_id( return data.get("data", {}).get("fact") +def _has_raw_pin_sentinel(decrypted: str) -> bool: + """Pin sentinel check on raw JSON, for blobs core cannot parse as a claim. + + Fail-closed fallback for the pin guard only (PRD-04 DEP-5): a pinned blob + written by a newer client (e.g. an enum value this core does not know) + must still count as pinned. Recall ranking uses core alone. + """ + try: + obj = _json.loads(decrypted) + except (ValueError, TypeError): + return False + if not isinstance(obj, dict): + return False + return obj.get("pin_status") == "pinned" or obj.get("st") == "p" + + +async def get_fact_pin_status( + fact_id: str, + keys: DerivedKeys, + owner: str, + relay: RelayClient, +) -> bool: + """Is on-chain fact ``fact_id`` pinned right now? (PRD-04 F1 / DEP-5) + + ``True`` iff the fact exists, is active, and its decrypted blob is pinned + per ``totalreclaw_core.is_pinned_claim`` (v1.1 ``pin_status == "pinned"`` + or v0 ``st == "p"``) or, for a blob core cannot parse, carries one of + those sentinels. A missing fact, an inactive fact and a tombstone stub + are all ``False`` -- tombstoning them again cannot hurt a pin. + + Raises on any relay error (including :class:`RelayReadBlocked`) and on + decrypt failure. Callers MUST treat a raise as "pin state unknown" and + fail closed (see ``agent/pin_guard.py``). + """ + if not isinstance(fact_id, str) or not fact_id.strip(): + raise ValueError("fact_id must be a non-empty string") + fact = await _fetch_fact_by_id(fact_id.strip(), owner, relay) + if not fact: + return False + if fact.get("isActive") is False: + return False + blob_hex = fact.get("encryptedBlob", "") or "" + if is_stub_blob_hex(blob_hex): + return False + if blob_hex.startswith(("0x", "0X")): + blob_hex = blob_hex[2:] + blob_b64 = base64.b64encode(bytes.fromhex(blob_hex)).decode("ascii") + decrypted = decrypt(blob_b64, keys.encryption_key) + if _core.is_pinned_claim(decrypted): + return True + return _has_raw_pin_sentinel(decrypted) + + def _decrypt_and_parse_claim( fact: dict, keys: DerivedKeys, diff --git a/python/tests/test_fact_pin_status.py b/python/tests/test_fact_pin_status.py new file mode 100644 index 00000000..2951efc0 --- /dev/null +++ b/python/tests/test_fact_pin_status.py @@ -0,0 +1,119 @@ +"""PRD-04 DEP-5 -- reading a fact's pin state from the subgraph (the pin +guard's source of truth).""" +from __future__ import annotations + +import base64 +import json +from unittest import mock +from unittest.mock import AsyncMock + +import pytest + +from totalreclaw import operations +from totalreclaw.crypto import encrypt + +KEY = bytes(range(32)) +OTHER_KEY = bytes(range(1, 33)) +FACT_ID = "44444444-4444-4444-8444-444444444444" + +_V1_BASE = { + "id": FACT_ID, "text": "Pedro lives in Lisbon", "type": "claim", "source": "user", + "created_at": "2026-09-01T00:00:00Z", "schema_version": "1.0", "importance": 8, +} +V1_PINNED = json.dumps({**_V1_BASE, "pin_status": "pinned"}) +V1_PLAIN = json.dumps(_V1_BASE) +V0_PINNED = json.dumps({"t": "Pedro lives in Lisbon", "c": "fact", "cf": 0.9, "i": 8, "sa": "x", "st": "p"}) +# A pinned blob core cannot parse (unknown ``type`` token, e.g. written by a +# future client): core says "not pinned", the guard must still see the pin. +V1_UNPARSEABLE_PINNED = json.dumps({**_V1_BASE, "type": "memo", "pin_status": "pinned"}) + + +def _keys(key: bytes = KEY): + keys = mock.Mock() + keys.encryption_key = key + return keys + + +def _hex(plaintext: str, key: bytes = KEY) -> str: + return "0x" + base64.b64decode(encrypt(plaintext, key)).hex() + + +def _fact(blob_hex: str, active: bool = True) -> dict: + return {"id": FACT_ID, "owner": "0xabc", "encryptedBlob": blob_hex, "isActive": active} + + +class _Relay: + def __init__(self, fact=None, exc: Exception | None = None) -> None: + self.fact = fact + self.exc = exc + self.calls: list[dict] = [] + + async def query_subgraph(self, gql: str, variables: dict) -> dict: + self.calls.append(variables) + if self.exc is not None: + raise self.exc + return {"data": {"fact": self.fact}} + + +@pytest.mark.parametrize( + "plaintext, expected", + [(V1_PINNED, True), (V1_PLAIN, False), (V0_PINNED, True)], +) +async def test_pin_state_from_decrypted_blob(plaintext: str, expected: bool) -> None: + relay = _Relay(_fact(_hex(plaintext))) + assert await operations.get_fact_pin_status(FACT_ID, _keys(), "0xabc", relay) is expected + assert relay.calls == [{"id": FACT_ID}] + + +async def test_unparseable_blob_with_pin_sentinel_counts_as_pinned() -> None: + import totalreclaw_core + + assert totalreclaw_core.is_pinned_claim(V1_UNPARSEABLE_PINNED) is False + relay = _Relay(_fact(_hex(V1_UNPARSEABLE_PINNED))) + assert await operations.get_fact_pin_status(FACT_ID, _keys(), "0xabc", relay) is True + + +async def test_missing_fact_is_not_pinned() -> None: + assert await operations.get_fact_pin_status(FACT_ID, _keys(), "0xabc", _Relay(None)) is False + + +async def test_inactive_fact_is_not_pinned() -> None: + relay = _Relay(_fact(_hex(V1_PINNED), active=False)) + assert await operations.get_fact_pin_status(FACT_ID, _keys(), "0xabc", relay) is False + + +async def test_tombstone_stub_is_not_pinned() -> None: + relay = _Relay(_fact("0x00")) + assert await operations.get_fact_pin_status(FACT_ID, _keys(), "0xabc", relay) is False + + +async def test_relay_error_propagates() -> None: + with pytest.raises(RuntimeError): + await operations.get_fact_pin_status(FACT_ID, _keys(), "0xabc", _Relay(exc=RuntimeError("503"))) + + +async def test_wrong_key_propagates() -> None: + relay = _Relay(_fact(_hex(V1_PINNED))) + with pytest.raises(Exception): + await operations.get_fact_pin_status(FACT_ID, _keys(OTHER_KEY), "0xabc", relay) + + +async def test_empty_fact_id_rejected() -> None: + with pytest.raises(ValueError): + await operations.get_fact_pin_status(" ", _keys(), "0xabc", _Relay(None)) + + +async def test_client_get_fact_pin_status_delegates(monkeypatch) -> None: + from totalreclaw import client as client_mod + from totalreclaw.client import TotalReclaw + + c = TotalReclaw.__new__(TotalReclaw) + c._ensure_address = AsyncMock() + c._ensure_registered = AsyncMock() + c._keys = object() + c._wallet_address = "0xabc" + c._relay = object() + spy = AsyncMock(return_value=True) + monkeypatch.setattr(client_mod, "get_fact_pin_status", spy) + assert await c.get_fact_pin_status("f1") is True + spy.assert_awaited_once_with(fact_id="f1", keys=c._keys, owner="0xabc", relay=c._relay) From 69a29148011e4378eab8937d494a460a6843cf10 Mon Sep 17 00:00:00 2001 From: p-diogo Date: Wed, 30 Sep 2026 18:56:40 +0000 Subject: [PATCH 05/10] fix(python): pin guard for LLM UPDATE/DELETE in auto-extraction (PRD-04 DEP-5) Co-Authored-By: Claude Code --- python/src/totalreclaw/agent/lifecycle.py | 17 +++ python/src/totalreclaw/agent/pin_guard.py | 121 ++++++++++++++++++++++ python/tests/test_hermes_plugin.py | 6 ++ python/tests/test_lifecycle_pin_guard.py | 94 +++++++++++++++++ python/tests/test_pin_guard.py | 116 +++++++++++++++++++++ 5 files changed, 354 insertions(+) create mode 100644 python/src/totalreclaw/agent/pin_guard.py create mode 100644 python/tests/test_lifecycle_pin_guard.py create mode 100644 python/tests/test_pin_guard.py diff --git a/python/src/totalreclaw/agent/lifecycle.py b/python/src/totalreclaw/agent/lifecycle.py index 98b09135..d73778a2 100644 --- a/python/src/totalreclaw/agent/lifecycle.py +++ b/python/src/totalreclaw/agent/lifecycle.py @@ -22,6 +22,7 @@ from .extraction import ExtractedFact, extract_facts_llm, extract_facts_heuristic from .contradiction import detect_and_resolve_contradictions +from .pin_guard import apply_pin_guard, fail_closed_actions from .debrief import generate_crystal from .loop_runner import ( run_sync, @@ -351,6 +352,22 @@ def _auto_extract_inner( raise InterpreterShutdownError(str(exc)) from exc logger.debug("Contradiction detection failed (proceeding with all facts): %s", exc) + # Pin contract (PRD-04 F1 / DEP-5): an LLM UPDATE/DELETE never tombstones + # a pinned fact. Must run before the action loop below, which is the only + # place auto-extraction issues tombstones. + try: + facts = run_sync(apply_pin_guard(facts, client, logger)) + except InterpreterShutdownError: + raise + except Exception as exc: + if is_interpreter_shutdown_error(exc): + raise InterpreterShutdownError(str(exc)) from exc + logger.warning( + "Pin guard failed (%s); no UPDATE/DELETE tombstones this cycle", + type(exc).__name__, + ) + facts = fail_closed_actions(facts, logger) + # ----------------------------------------------------------------------- # Two-pass approach: # Pass 1 — handle NOOP / DELETE actions individually (no batch needed). diff --git a/python/src/totalreclaw/agent/pin_guard.py b/python/src/totalreclaw/agent/pin_guard.py new file mode 100644 index 00000000..e2a1bd05 --- /dev/null +++ b/python/src/totalreclaw/agent/pin_guard.py @@ -0,0 +1,121 @@ +"""Pin contract for LLM-guided UPDATE / DELETE actions (PRD-04 F1, leaf DEP-5). + +Auto-extraction's LLM may answer UPDATE (store a replacement, then tombstone +``existing_fact_id``) or DELETE (tombstone ``existing_fact_id``). A pinned fact +must never be tombstoned by an automatic process +(``docs/specs/totalreclaw/memory-taxonomy-v1.md`` §Pin semantics), so before +``_auto_extract_inner`` applies any action this guard reads the target's pin +state with ``client.get_fact_pin_status`` and: + +* target pinned -> drop the action entirely (UPDATE and DELETE). This is the + same outcome core's contradiction resolver gives a new claim that + contradicts a pinned one (``SkipNew { reason: ExistingPinned }``). +* pin state unknown (the read raised) -> fail closed on the destructive half: + DELETE is dropped, UPDATE is kept as a plain ADD (no tombstone). +* target not pinned -> action unchanged. + +Log lines carry the action and a reason category only -- never fact text or +fact ids. No decision-log row is written (PRD-04 DEP-5 plan, decision D2). +""" +from __future__ import annotations + +import dataclasses +import logging +from typing import TYPE_CHECKING, Any, Optional + +from .loop_runner import InterpreterShutdownError, is_interpreter_shutdown_error + +if TYPE_CHECKING: + from .extraction import ExtractedFact + +logger = logging.getLogger(__name__) + +REASON_EXISTING_PINNED = "existing_pinned" +REASON_PIN_STATE_UNKNOWN = "pin_state_unknown" + +_GUARDED_ACTIONS = ("UPDATE", "DELETE") + + +def _is_guarded(fact: "ExtractedFact") -> bool: + return fact.action in _GUARDED_ACTIONS and bool(fact.existing_fact_id) + + +def _as_add(fact: "ExtractedFact") -> "ExtractedFact": + return dataclasses.replace(fact, action="ADD", existing_fact_id=None) + + +def fail_closed_actions( + facts: list["ExtractedFact"], + log: Optional[Any] = None, +) -> list["ExtractedFact"]: + """Apply the unknown-pin-state rule to every guarded action. + + DELETE actions are dropped and UPDATE actions become ADD, so nothing is + tombstoned. Used per fact when its pin lookup failed, and for the whole + batch when the guard itself could not run. + """ + if log is None: + log = logger + out: list["ExtractedFact"] = [] + for fact in facts: + if not _is_guarded(fact): + out.append(fact) + elif fact.action == "DELETE": + log.info("pin_guard: skipped DELETE (reason=%s)", REASON_PIN_STATE_UNKNOWN) + else: + log.info( + "pin_guard: UPDATE stored as ADD without tombstone (reason=%s)", + REASON_PIN_STATE_UNKNOWN, + ) + out.append(_as_add(fact)) + return out + + +async def apply_pin_guard( + facts: list["ExtractedFact"], + client: Any, + log: Optional[Any] = None, +) -> list["ExtractedFact"]: + """Return ``facts`` with every UPDATE/DELETE that targets a pinned fact + removed (see module docstring for the full rule). One pin-state lookup + per distinct target id. Re-raises ``InterpreterShutdownError``; every + other lookup failure is treated as "pin state unknown".""" + if log is None: + log = logger + + targets: list[str] = [] + for fact in facts: + if _is_guarded(fact) and fact.existing_fact_id not in targets: + targets.append(fact.existing_fact_id) + if not targets: + return list(facts) + + pinned_state: dict[str, Optional[bool]] = {} + for fact_id in targets: + try: + pinned_state[fact_id] = bool(await client.get_fact_pin_status(fact_id)) + except InterpreterShutdownError: + raise + except Exception as exc: + if is_interpreter_shutdown_error(exc): + raise InterpreterShutdownError(str(exc)) from exc + log.debug("pin_guard: pin-state lookup failed (%s)", type(exc).__name__) + pinned_state[fact_id] = None + + out: list["ExtractedFact"] = [] + for fact in facts: + if not _is_guarded(fact): + out.append(fact) + continue + state = pinned_state.get(fact.existing_fact_id) + if state is False: + out.append(fact) + elif state is True: + log.info( + "pin_guard: skipped %s on a pinned fact (reason=%s)", + fact.action, + REASON_EXISTING_PINNED, + ) + else: + out.extend(fail_closed_actions([fact], log)) + return out diff --git a/python/tests/test_hermes_plugin.py b/python/tests/test_hermes_plugin.py index d9ee71bd..d2742739 100644 --- a/python/tests/test_hermes_plugin.py +++ b/python/tests/test_hermes_plugin.py @@ -620,6 +620,12 @@ async def mock_forget(fact_id): mock_client.recall = mock_recall mock_client.forget = mock_forget + # PRD-04 DEP-5: the pin guard reads the UPDATE target's pin state. + async def mock_get_fact_pin_status(fact_id): + return False + + mock_client.get_fact_pin_status = mock_get_fact_pin_status + state.add_message("user", "I like dark mode") state.add_message("assistant", "OK") diff --git a/python/tests/test_lifecycle_pin_guard.py b/python/tests/test_lifecycle_pin_guard.py new file mode 100644 index 00000000..0bf3d3ed --- /dev/null +++ b/python/tests/test_lifecycle_pin_guard.py @@ -0,0 +1,94 @@ +"""PRD-04 F1 / DEP-5 -- _auto_extract_inner applies the pin guard before any +tombstone is issued. Helpers mirror tests/test_auto_extract_uses_batch.py.""" +from __future__ import annotations + +import os +from pathlib import Path +from unittest.mock import AsyncMock, MagicMock, patch + +from totalreclaw.agent.extraction import ExtractedFact + +PINNED_ID = "11111111-1111-4111-8111-111111111111" +OTHER_ID = "33333333-3333-4333-8333-333333333333" + + +def _make_state_and_client(pin_status): + from totalreclaw.hermes.state import PluginState + + with patch.dict(os.environ, {}, clear=True): + with patch.object(Path, "exists", return_value=False): + state = PluginState() + + client = MagicMock() + client.recall = AsyncMock(return_value=[]) + client.forget = AsyncMock(return_value=True) + client.get_fact_pin_status = AsyncMock(side_effect=pin_status) + + async def _batch(facts, source="python-client"): + return [f"new-{i}" for i in range(len(facts))] + + client.remember_batch = AsyncMock(side_effect=_batch) + state._client = client + return state, client + + +def _run(state, facts): + from totalreclaw.agent.lifecycle import auto_extract + + async def fake_extract(*args, **kwargs): + return facts + + async def passthrough(fs, *args, **kwargs): + return fs + + with patch("totalreclaw.agent.lifecycle.extract_facts_llm", new=fake_extract), \ + patch("totalreclaw.agent.lifecycle.detect_and_resolve_contradictions", new=passthrough), \ + patch("totalreclaw.embedding.get_embedding", return_value=None): + state.add_message("user", "My home city changed") + state.add_message("assistant", "Noted") + return auto_extract(state) + + +def test_update_and_delete_targeting_a_pinned_fact_are_not_applied() -> None: + state, client = _make_state_and_client(lambda fact_id: fact_id == PINNED_ID) + facts = [ + ExtractedFact(text="Home city is Porto", type="claim", importance=8, + action="UPDATE", existing_fact_id=PINNED_ID, source="user"), + ExtractedFact(text="Home city is Lisbon", type="claim", importance=8, + action="DELETE", existing_fact_id=PINNED_ID, source="user"), + ExtractedFact(text="Likes espresso", type="preference", importance=7, + action="ADD", source="user"), + ] + stored = _run(state, facts) + client.forget.assert_not_awaited() + assert stored == ["Likes espresso"] + assert [d["text"] for d in client.remember_batch.call_args.args[0]] == ["Likes espresso"] + + +def test_update_on_unpinned_fact_still_tombstones_after_store() -> None: + state, client = _make_state_and_client(lambda fact_id: False) + facts = [ + ExtractedFact(text="Home city is Porto", type="claim", importance=8, + action="UPDATE", existing_fact_id=OTHER_ID, source="user"), + ] + stored = _run(state, facts) + assert stored == ["Home city is Porto"] + client.forget.assert_awaited_once_with(OTHER_ID) + + +def test_guard_crash_fails_closed() -> None: + state, client = _make_state_and_client(lambda fact_id: False) + + async def boom(*args, **kwargs): + raise RuntimeError("guard exploded") + + facts = [ + ExtractedFact(text="Home city is Porto", type="claim", importance=8, + action="UPDATE", existing_fact_id=OTHER_ID, source="user"), + ExtractedFact(text="Home city is Lisbon", type="claim", importance=8, + action="DELETE", existing_fact_id=OTHER_ID, source="user"), + ] + with patch("totalreclaw.agent.lifecycle.apply_pin_guard", new=boom): + stored = _run(state, facts) + client.forget.assert_not_awaited() + assert stored == ["Home city is Porto"] diff --git a/python/tests/test_pin_guard.py b/python/tests/test_pin_guard.py new file mode 100644 index 00000000..16ee7d9b --- /dev/null +++ b/python/tests/test_pin_guard.py @@ -0,0 +1,116 @@ +"""PRD-04 F1 / DEP-5 -- every skip path of the LLM UPDATE/DELETE pin guard.""" +from __future__ import annotations + +import logging +from unittest.mock import AsyncMock + +import pytest + +from totalreclaw.agent.extraction import ExtractedFact +from totalreclaw.agent.loop_runner import InterpreterShutdownError +from totalreclaw.agent.pin_guard import ( + REASON_EXISTING_PINNED, + REASON_PIN_STATE_UNKNOWN, + apply_pin_guard, + fail_closed_actions, +) + +SECRET_TEXT = "Pedro's home city is Porto" +PINNED_ID = "11111111-1111-4111-8111-111111111111" +PLAIN_ID = "22222222-2222-4222-8222-222222222222" +BROKEN_ID = "33333333-3333-4333-8333-333333333333" +LOGGER = "totalreclaw.agent.pin_guard" + + +def _fact(action: str, target: str | None = None) -> ExtractedFact: + return ExtractedFact( + text=SECRET_TEXT, type="claim", importance=8, action=action, + existing_fact_id=target, source="user", + ) + + +def _client() -> AsyncMock: + def lookup(fact_id: str) -> bool: + if fact_id == BROKEN_ID: + raise RuntimeError("relay 503") + return fact_id == PINNED_ID + + client = AsyncMock() + client.get_fact_pin_status = AsyncMock(side_effect=lookup) + return client + + +async def test_update_on_pinned_target_is_dropped(caplog) -> None: + with caplog.at_level(logging.INFO, logger=LOGGER): + out = await apply_pin_guard([_fact("UPDATE", PINNED_ID)], _client()) + assert out == [] + assert f"reason={REASON_EXISTING_PINNED}" in caplog.text + + +async def test_delete_on_pinned_target_is_dropped() -> None: + assert await apply_pin_guard([_fact("DELETE", PINNED_ID)], _client()) == [] + + +async def test_update_on_unpinned_target_passes_unchanged() -> None: + fact = _fact("UPDATE", PLAIN_ID) + out = await apply_pin_guard([fact], _client()) + assert out == [fact] and out[0] is fact + + +async def test_delete_on_unpinned_target_passes_unchanged() -> None: + fact = _fact("DELETE", PLAIN_ID) + assert await apply_pin_guard([fact], _client()) == [fact] + + +async def test_update_with_unknown_pin_state_becomes_add(caplog) -> None: + with caplog.at_level(logging.INFO, logger=LOGGER): + out = await apply_pin_guard([_fact("UPDATE", BROKEN_ID)], _client()) + assert len(out) == 1 + assert out[0].action == "ADD" + assert out[0].existing_fact_id is None + assert out[0].text == SECRET_TEXT + assert f"reason={REASON_PIN_STATE_UNKNOWN}" in caplog.text + + +async def test_delete_with_unknown_pin_state_is_dropped() -> None: + assert await apply_pin_guard([_fact("DELETE", BROKEN_ID)], _client()) == [] + + +async def test_add_and_noop_never_trigger_a_lookup() -> None: + client = _client() + facts = [_fact("ADD"), _fact("NOOP"), _fact("UPDATE"), _fact("DELETE")] + assert await apply_pin_guard(facts, client) == facts + client.get_fact_pin_status.assert_not_awaited() + + +async def test_one_lookup_per_distinct_target() -> None: + client = _client() + await apply_pin_guard([_fact("UPDATE", PLAIN_ID), _fact("DELETE", PLAIN_ID), _fact("UPDATE", PINNED_ID)], client) + assert [c.args[0] for c in client.get_fact_pin_status.await_args_list] == [PLAIN_ID, PINNED_ID] + + +async def test_interpreter_shutdown_propagates() -> None: + client = AsyncMock() + client.get_fact_pin_status = AsyncMock(side_effect=InterpreterShutdownError("shutting down")) + with pytest.raises(InterpreterShutdownError): + await apply_pin_guard([_fact("DELETE", PLAIN_ID)], client) + + +async def test_logs_carry_category_only(caplog) -> None: + with caplog.at_level(logging.DEBUG, logger=LOGGER): + await apply_pin_guard( + [_fact("UPDATE", PINNED_ID), _fact("DELETE", BROKEN_ID), _fact("UPDATE", BROKEN_ID)], + _client(), + ) + assert caplog.records, "expected skip log lines" + assert SECRET_TEXT not in caplog.text + for fact_id in (PINNED_ID, BROKEN_ID): + assert fact_id not in caplog.text + + +def test_fail_closed_actions_rule() -> None: + add, update, delete = _fact("ADD"), _fact("UPDATE", PLAIN_ID), _fact("DELETE", PLAIN_ID) + out = fail_closed_actions([add, update, delete]) + assert [f.action for f in out] == ["ADD", "ADD"] + assert out[0] is add + assert out[1].existing_fact_id is None From 9ed15490c9471dc81b1f2ed5d5264fa74a9549af Mon Sep 17 00:00:00 2001 From: p-diogo Date: Wed, 30 Sep 2026 18:57:50 +0000 Subject: [PATCH 06/10] fix(python): feed entity refs + pin state to the contradiction resolver (PRD-04 DEP-5) Co-Authored-By: Claude Code --- python/src/totalreclaw/agent/contradiction.py | 183 ++++++++++-------- python/tests/test_contradiction_pin_state.py | 111 +++++++++++ 2 files changed, 212 insertions(+), 82 deletions(-) create mode 100644 python/tests/test_contradiction_pin_state.py diff --git a/python/src/totalreclaw/agent/contradiction.py b/python/src/totalreclaw/agent/contradiction.py index 1b0dd4d4..9dd3d2ab 100644 --- a/python/src/totalreclaw/agent/contradiction.py +++ b/python/src/totalreclaw/agent/contradiction.py @@ -4,11 +4,21 @@ For each new fact with entities, recalls existing facts from the vault that share an entity, then delegates to ``totalreclaw_core.resolve_with_candidates()`` to check for semantic contradictions (cosine similarity in the [0.3, 0.85) -band). Facts where an existing claim wins (SkipNew) are filtered out. +band). + +Pin contract (PRD-04 F1 / DEP-5): the new claim and every candidate reach core +with their entity refs (short-key ``e``) and, for pinned candidates, the v0 +``st: "p"`` sentinel. Core compares only claims that share an entity and reads +pin state from ``st`` on this shape, so both are required for its pin check +(``SkipNew { reason: ExistingPinned }``) to fire. A new fact that contradicts +a pinned fact is dropped. Every other resolver outcome (``skip_new`` with +``existing_wins``, ``supersede_existing``, ``tie_leave_both``) is logged and +NOT acted on -- applying those is PRD-04 DEP-12. Falls back to "store everything" if ``totalreclaw_core`` is not installed or -any error occurs — contradiction detection is best-effort and must never block -the store pipeline. +any error occurs -- contradiction detection is best-effort and must never block +the store pipeline. Log lines carry categories and counts only, never fact +text. """ from __future__ import annotations @@ -31,27 +41,78 @@ CONTRADICTION_THRESHOLD_LOWER = 0.30 CONTRADICTION_THRESHOLD_UPPER = 0.85 +#: Resolver ``skip_new`` reason for a new claim that contradicts a pinned claim. +SKIP_REASON_EXISTING_PINNED = "existing_pinned" + +# v1 memory type -> short-key category accepted by core's ``ClaimCategory``. +_V1_TO_SHORT_CATEGORY = { + "claim": "fact", + "preference": "pref", + "directive": "rule", + "commitment": "goal", + "episode": "epi", + "summary": "sum", +} + + +def _short_key_claim_for_resolver( + *, + text: str, + fact_type: str, + importance: int, + confidence: float, + source_agent: str, + created_at: str, + entities: Optional[List[dict]] = None, + pinned: bool = False, +) -> dict: + """Build a short-key canonical claim for core.resolve_with_candidates(). + + The resolver accepts the v0 short-key format (``{t, c, cf, i, sa, ea}``) + -- the v1 claim shape is NOT accepted by core's resolver. We emit short + keys only for the resolver's transient input; the actual on-chain write + goes through ``build_canonical_claim_v1`` which emits a v1 JSON blob. + + ``entities`` are short-key refs (``{"n", "tp", "r"?}``, see + :func:`totalreclaw.claims_helper.to_entity_refs`) emitted as ``e``. + ``pinned`` emits ``st: "p"`` -- the pin sentinel core's + ``is_pinned_claim`` reads on this shape. + """ + claim: dict = { + "t": text, + "c": _V1_TO_SHORT_CATEGORY.get(fact_type, "fact"), + "cf": confidence, + "i": importance, + "sa": source_agent, + "ea": created_at, + } + if entities: + claim["e"] = list(entities) + if pinned: + claim["st"] = "p" + return claim + async def detect_and_resolve_contradictions( new_facts: List["ExtractedFact"], client: "TotalReclaw", log: Optional[Any] = None, ) -> List["ExtractedFact"]: - """Filter out new facts that lose to existing vault claims. + """Filter out new facts that contradict a pinned vault claim. For each new fact that has entities and an embedding, this function: - 1. Computes entity trapdoors and recalls existing claims from the vault - that share at least one entity. - 2. Decrypts the candidates and pairs them with their embeddings. - 3. Calls ``totalreclaw_core.resolve_with_candidates()`` to run the P2-3 - contradiction formula. - 4. If any resolution action is ``skip_new``, the fact is dropped. + 1. Recalls existing claims from the vault that share its entities. + 2. Projects the new fact and each candidate to short-key claims carrying + entity refs and (candidates only) pin state. + 3. Calls ``totalreclaw_core.resolve_with_candidates()``. + 4. Drops the fact if any action is ``skip_new`` with reason + ``existing_pinned``. Other actions are logged, not applied (DEP-12). Returns the subset of ``new_facts`` that should proceed to storage. On any error (missing core, subgraph issues, decrypt failures), returns - the full ``new_facts`` list unchanged — contradiction detection is + the full ``new_facts`` list unchanged -- contradiction detection is best-effort. Parameters @@ -76,46 +137,13 @@ async def detect_and_resolve_contradictions( try: from totalreclaw.embedding import get_embedding from totalreclaw.claims_helper import ( - TYPE_TO_CATEGORY_V1, compute_entity_trapdoor, + to_entity_refs, ) except ImportError: log.debug("Required modules not available — skipping contradiction detection") return list(new_facts) - def _short_key_claim_for_resolver( - *, text: str, fact_type: str, importance: int, confidence: float, - source_agent: str, created_at: str, - ) -> dict: - """Build a short-key canonical claim for core.resolve_with_candidates(). - - The resolver accepts the v0 short-key format (``{t, c, cf, i, sa, - ea}``) — the v1 claim shape is NOT accepted by core 2.0.0's resolver - as of the plugin-v3 pairing. We emit short keys only for the - resolver's transient input; the actual on-chain write goes through - ``build_canonical_claim_v1`` which emits a v1 JSON blob. - """ - # Map v1 type → short-key category expected by the resolver. - # ``TYPE_TO_CATEGORY_V1`` gives v1 type → display short tag; we need - # a category the Rust ``ClaimCategory`` enum accepts. - v1_to_short = { - "claim": "fact", - "preference": "pref", - "directive": "rule", - "commitment": "goal", - "episode": "epi", - "summary": "sum", - } - c_key = v1_to_short.get(fact_type, "fact") - return { - "t": text, - "c": c_key, - "cf": confidence, - "i": importance, - "sa": source_agent, - "ea": created_at, - } - # Load default resolution weights once try: weights_json = totalreclaw_core.default_resolution_weights() @@ -130,6 +158,7 @@ def _short_key_claim_for_resolver( now_unix = int(time.time()) kept: List["ExtractedFact"] = [] + pinned_conflicts = 0 for fact_idx, fact in enumerate(new_facts): # Only run contradiction detection on facts with entities @@ -195,14 +224,11 @@ def _short_key_claim_for_resolver( kept.append(fact) continue - # Build the new claim JSON for the resolver + # Build the new claim JSON for the resolver (short-key shape, with + # entity refs so core's shared-entity detection can run). importance_int = max(1, min(10, int(round( fact.importance if fact.importance > 1 else fact.importance * 10 )))) - # Build a short-key claim for the resolver only. The on-chain - # write pipeline still emits v1 JSON; this transient short-key - # shape is the format ``core.resolve_with_candidates`` accepts - # as of core 2.0.0. new_claim_json_obj = _short_key_claim_for_resolver( text=fact.text, fact_type=fact.type, @@ -210,6 +236,7 @@ def _short_key_claim_for_resolver( confidence=fact.confidence, source_agent="hermes-auto", created_at=time.strftime("%Y-%m-%dT%H:%M:%S.000Z", time.gmtime()), + entities=to_entity_refs(fact.entities), ) new_claim_json = json.dumps(new_claim_json_obj, ensure_ascii=False, separators=(",", ":")) new_claim_id = f"pending-{id(fact)}" @@ -220,13 +247,11 @@ def _short_key_claim_for_resolver( for result in existing_results: if not result.embedding or not result.text: continue - # Parse the result text back into a claim-like structure. - # The results come from the reranker as RerankerResult with - # .text, .id, .embedding, .importance, .category try: existing_importance = max(1, min(10, int(round( result.importance * 10 if result.importance <= 1 else result.importance )))) + result_entities = getattr(result, "entities", None) existing_short = _short_key_claim_for_resolver( text=result.text, fact_type=result.category or "claim", @@ -238,6 +263,8 @@ def _short_key_claim_for_resolver( ) if getattr(result, "created_at", None) else time.strftime( "%Y-%m-%dT%H:%M:%S.000Z", time.gmtime() ), + entities=result_entities if isinstance(result_entities, list) else None, + pinned=getattr(result, "pinned", False) is True, ) candidates.append({ "claim": existing_short, @@ -245,7 +272,7 @@ def _short_key_claim_for_resolver( "embedding": list(result.embedding), }) except Exception as exc: - log.debug("Failed to build candidate from result %s: %s", result.id, exc) + log.debug("Failed to build a contradiction candidate (%s)", type(exc).__name__) continue if not candidates: @@ -267,47 +294,39 @@ def _short_key_claim_for_resolver( actions = json.loads(actions_json) - # Check if any action tells us to skip the new fact - skip = False - for action in actions: - action_type = action.get("type", "") - if action_type == "skip_new": - reason = action.get("reason", "unknown") - existing_id = action.get("existing_id", "?") - log.info( - "Contradiction: skipping new fact %r — existing %s wins (reason=%s)", - fact.text[:60], - existing_id, - reason, - ) - skip = True - break - - if skip: + if any( + action.get("type") == "skip_new" + and action.get("reason") == SKIP_REASON_EXISTING_PINNED + for action in actions + ): + pinned_conflicts += 1 + log.info( + "Contradiction: new fact not stored, it contradicts a pinned fact (reason=%s)", + SKIP_REASON_EXISTING_PINNED, + ) continue - # Log supersede actions (informational — the new fact proceeds, - # but we don't tombstone the old one from here; that's a future - # enhancement matching the plugin's full flow) + # Every other outcome is observed only; PRD-04 DEP-12 owns applying + # skip_new/existing_wins and supersede_existing. for action in actions: - if action.get("type") == "supersede_existing": - log.info( - "Contradiction: new fact %r supersedes existing %s (not tombstoned yet)", - fact.text[:60], - action.get("existing_id", "?"), - ) + log.debug( + "Contradiction: observed %s (reason=%s), not applied (PRD-04 DEP-12)", + action.get("type", "?"), + action.get("reason", "-"), + ) kept.append(fact) except Exception as exc: # Any per-fact error: keep the fact and continue - log.debug("Contradiction check failed for fact %r: %s", fact.text[:60], exc) + log.debug("Contradiction check failed for a fact (%s); keeping it", type(exc).__name__) kept.append(fact) log.info( - "Contradiction detection: %d/%d facts passed (removed %d)", + "Contradiction detection: %d/%d facts passed (removed %d, pinned conflicts %d)", len(kept), len(new_facts), len(new_facts) - len(kept), + pinned_conflicts, ) return kept diff --git a/python/tests/test_contradiction_pin_state.py b/python/tests/test_contradiction_pin_state.py new file mode 100644 index 00000000..1a47c236 --- /dev/null +++ b/python/tests/test_contradiction_pin_state.py @@ -0,0 +1,111 @@ +"""PRD-04 F1 / DEP-5 -- the contradiction resolver receives entity refs + pin +state, and only a conflict with a pinned fact drops the new fact. + +Uses the REAL totalreclaw_core resolver except where noted.""" +from __future__ import annotations + +import json +import logging +import time +from unittest.mock import AsyncMock, patch + +import totalreclaw_core + +from totalreclaw.agent.contradiction import ( + SKIP_REASON_EXISTING_PINNED, + _short_key_claim_for_resolver, + detect_and_resolve_contradictions, +) +from totalreclaw.agent.extraction import ExtractedEntity, ExtractedFact +from totalreclaw.reranker import RerankerResult + +NEW_TEXT = "Pedro's home city is Porto" +PINNED_TEXT = "Pedro's home city is Lisbon" +# cosine([0.6, 0.8, 0, 0], [1, 0, 0, 0]) = 0.6 -- inside the [0.3, 0.85) band. +NEW_EMBEDDING = [0.6, 0.8, 0.0, 0.0] +EXISTING_EMBEDDING = [1.0, 0.0, 0.0, 0.0] + + +def _new_fact() -> ExtractedFact: + return ExtractedFact( + text=NEW_TEXT, type="claim", importance=8, action="ADD", source="user", + confidence=0.9, entities=[ExtractedEntity(name="Pedro", type="person")], + ) + + +def _existing(pinned: bool, entities=None) -> RerankerResult: + return RerankerResult( + id="existing-1", text=PINNED_TEXT, embedding=list(EXISTING_EMBEDDING), + importance=0.9, created_at=time.time() - 86400, category="claim", + pinned=pinned, + entities=[{"n": "Pedro", "tp": "person"}] if entities is None else entities, + ) + + +def _client(results) -> AsyncMock: + client = AsyncMock() + client.recall = AsyncMock(return_value=results) + return client + + +def test_short_key_claim_carries_pin_sentinel_and_entities() -> None: + claim = _short_key_claim_for_resolver( + text=PINNED_TEXT, fact_type="claim", importance=8, confidence=0.9, + source_agent="unknown", created_at="2026-09-01T00:00:00.000Z", + entities=[{"n": "Pedro", "tp": "person"}], pinned=True, + ) + assert claim["st"] == "p" + assert claim["e"] == [{"n": "Pedro", "tp": "person"}] + encoded = json.dumps(claim) + assert totalreclaw_core.is_pinned_claim(encoded) is True + totalreclaw_core.canonicalize_claim(encoded) # raises if core rejects it + + +def test_short_key_claim_unpinned_omits_st_and_empty_entities() -> None: + claim = _short_key_claim_for_resolver( + text="x", fact_type="preference", importance=5, confidence=0.9, + source_agent="hermes-auto", created_at="2026-09-01T00:00:00.000Z", + ) + assert "st" not in claim and "e" not in claim + assert claim["c"] == "pref" + + +async def test_new_fact_contradicting_a_pinned_fact_is_dropped() -> None: + with patch("totalreclaw.embedding.get_embedding", return_value=list(NEW_EMBEDDING)): + kept = await detect_and_resolve_contradictions([_new_fact()], _client([_existing(pinned=True)])) + assert kept == [] + + +async def test_same_conflict_with_an_unpinned_fact_is_kept() -> None: + with patch("totalreclaw.embedding.get_embedding", return_value=list(NEW_EMBEDDING)): + kept = await detect_and_resolve_contradictions([_new_fact()], _client([_existing(pinned=False)])) + assert [f.text for f in kept] == [NEW_TEXT] + + +async def test_pinned_fact_without_shared_entity_cannot_block() -> None: + # Core only compares claims that share an entity. + with patch("totalreclaw.embedding.get_embedding", return_value=list(NEW_EMBEDDING)): + kept = await detect_and_resolve_contradictions( + [_new_fact()], _client([_existing(pinned=True, entities=[])]), + ) + assert [f.text for f in kept] == [NEW_TEXT] + + +async def test_existing_wins_is_not_acted_on() -> None: + fake_actions = json.dumps([ + {"type": "skip_new", "reason": "existing_wins", "existing_id": "existing-1", "new_id": "n"}, + ]) + with patch("totalreclaw.embedding.get_embedding", return_value=list(NEW_EMBEDDING)), \ + patch.object(totalreclaw_core, "resolve_with_candidates", return_value=fake_actions): + kept = await detect_and_resolve_contradictions([_new_fact()], _client([_existing(pinned=False)])) + assert [f.text for f in kept] == [NEW_TEXT] + + +async def test_pinned_skip_log_has_no_fact_text(caplog) -> None: + with caplog.at_level(logging.DEBUG, logger="totalreclaw.agent.contradiction"), \ + patch("totalreclaw.embedding.get_embedding", return_value=list(NEW_EMBEDDING)): + await detect_and_resolve_contradictions([_new_fact()], _client([_existing(pinned=True)])) + assert f"reason={SKIP_REASON_EXISTING_PINNED}" in caplog.text + assert NEW_TEXT not in caplog.text + assert PINNED_TEXT not in caplog.text + assert "existing-1" not in caplog.text From 22788eb3cfec44789898461e470fda23bcaf0d74 Mon Sep 17 00:00:00 2001 From: p-diogo Date: Wed, 30 Sep 2026 19:00:18 +0000 Subject: [PATCH 07/10] test(python): pinned fact survives three contradicting extractions (PRD-04 G2) Co-Authored-By: Claude Code --- python/tests/test_pin_contract_integration.py | 137 ++++++++++++++++++ 1 file changed, 137 insertions(+) create mode 100644 python/tests/test_pin_contract_integration.py diff --git a/python/tests/test_pin_contract_integration.py b/python/tests/test_pin_contract_integration.py new file mode 100644 index 00000000..ede6a127 --- /dev/null +++ b/python/tests/test_pin_contract_integration.py @@ -0,0 +1,137 @@ +"""PRD-04 G2 (in-process): a pinned fact survives three contradicting extractions. + +Runs the real ``auto_extract`` pipeline three times against an in-memory vault. +Real: ``apply_pin_guard``, ``detect_and_resolve_contradictions`` and +``totalreclaw_core.resolve_with_candidates`` (core is NOT mocked). Stubbed: +the extraction LLM (one scripted fact per round) and embeddings (fixed +vectors). The dedup-context fetch runs for real against the in-memory vault. +""" +from __future__ import annotations + +import os +import time +from pathlib import Path +from unittest.mock import patch + +from totalreclaw.agent.extraction import ExtractedEntity, ExtractedFact +from totalreclaw.reranker import RerankerResult + +PINNED_ID = "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa" +PINNED_TEXT = "Pedro's home city is Lisbon" +PINNED_EMBEDDING = [1.0, 0.0, 0.0, 0.0] +# cosine 0.6 against PINNED_EMBEDDING: inside core's contradiction band [0.3, 0.85) +NEW_FACT_EMBEDDING = [0.6, 0.8, 0.0, 0.0] + + +class InMemoryVault: + """The slice of ``TotalReclaw`` the auto-extraction pipeline calls.""" + + def __init__(self, pinned: bool) -> None: + self.facts: dict[str, dict] = { + PINNED_ID: { + "text": PINNED_TEXT, + "embedding": list(PINNED_EMBEDDING), + "pinned": pinned, + "active": True, + "entities": [{"n": "Pedro", "tp": "person"}], + } + } + self.forgotten: list[str] = [] + self.stored_texts: list[str] = [] + self.resolved_chain_id = 100 + self._relay = None + self._next = 0 + + async def recall(self, query, query_embedding=None, top_k=8, max_candidates=250): + out = [] + for fact_id, f in self.facts.items(): + if not f["active"]: + continue + out.append(RerankerResult( + id=fact_id, text=f["text"], embedding=list(f["embedding"]), + importance=0.9, created_at=time.time() - 86400, category="claim", + pinned=f["pinned"], entities=list(f["entities"]), + )) + return out[:top_k] + + async def get_fact_pin_status(self, fact_id): + f = self.facts.get(fact_id) + return bool(f and f["active"] and f["pinned"]) + + async def forget(self, fact_id): + self.forgotten.append(fact_id) + if fact_id in self.facts: + self.facts[fact_id]["active"] = False + return True + + async def remember_batch(self, facts, source="python-client"): + ids = [] + for d in facts: + self._next += 1 + fact_id = f"new-{self._next}" + self.facts[fact_id] = { + "text": d["text"], "embedding": list(d.get("embedding") or []), + "pinned": False, "active": True, "entities": [], + } + self.stored_texts.append(d["text"]) + ids.append(fact_id) + return ids + + +def _state_with(vault: InMemoryVault): + from totalreclaw.hermes.state import PluginState + + with patch.dict(os.environ, {}, clear=True): + with patch.object(Path, "exists", return_value=False): + state = PluginState() + state._client = vault + return state + + +def _extract_round(state, fact: ExtractedFact): + from totalreclaw.agent.lifecycle import auto_extract + + async def scripted_llm(*args, **kwargs): + return [fact] + + with patch("totalreclaw.agent.lifecycle.extract_facts_llm", new=scripted_llm), \ + patch("totalreclaw.embedding.get_embedding", return_value=list(NEW_FACT_EMBEDDING)): + state.add_message("user", "An update about where I live") + state.add_message("assistant", "Got it") + return auto_extract(state) + + +CONTRADICTIONS = [ + # 1. LLM rewrites the pinned fact (no entities -> the resolver skips it, + # the pin guard must stop it). + ExtractedFact(text="Pedro's home city is Porto", type="claim", importance=8, + action="UPDATE", existing_fact_id=PINNED_ID, source="user"), + # 2. LLM deletes the pinned fact (pin guard). + ExtractedFact(text=PINNED_TEXT, type="claim", importance=8, + action="DELETE", existing_fact_id=PINNED_ID, source="user"), + # 3. LLM adds a contradicting fact sharing the "Pedro" entity (core resolver + # with pin state: SkipNew { ExistingPinned }). + ExtractedFact(text="Pedro's home city is Madrid", type="claim", importance=8, + action="ADD", source="user", + entities=[ExtractedEntity(name="Pedro", type="person")]), +] + + +def test_pinned_fact_survives_three_contradicting_extractions() -> None: + vault = InMemoryVault(pinned=True) + state = _state_with(vault) + for fact in CONTRADICTIONS: + _extract_round(state, fact) + assert vault.forgotten == [] + assert vault.facts[PINNED_ID]["active"] is True + assert vault.stored_texts == [] + + +def test_control_unpinned_fact_is_updated() -> None: + """Same first round with the pin removed: the harness must observe the + tombstone, so the pinned test above cannot pass vacuously.""" + vault = InMemoryVault(pinned=False) + state = _state_with(vault) + _extract_round(state, CONTRADICTIONS[0]) + assert vault.forgotten == [PINNED_ID] + assert vault.stored_texts == ["Pedro's home city is Porto"] From ff096f75e6b2c0dfde566da76941f2e3da63a9f2 Mon Sep 17 00:00:00 2001 From: p-diogo Date: Wed, 30 Sep 2026 19:02:05 +0000 Subject: [PATCH 08/10] docs: pin contract + recall pin boost (PRD-04 DEP-5) Co-Authored-By: Claude Code --- CLAUDE.md | 9 +++++++-- docs/specs/totalreclaw/memory-taxonomy-v1.md | 3 +++ docs/specs/totalreclaw/retrieval-v2.md | 20 ++++++++++++++++++++ python/CHANGELOG.md | 10 ++++++++++ python/src/totalreclaw/hermes/SKILL.md | 2 +- python/tests/test_skill_md_pin_semantics.py | 17 +++++++++++++++++ 6 files changed, 58 insertions(+), 3 deletions(-) create mode 100644 python/tests/test_skill_md_pin_semantics.py diff --git a/CLAUDE.md b/CLAUDE.md index 4e5fc865..e0bef96c 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -195,8 +195,10 @@ Features across OpenClaw plugin (`skill/plugin/`), MCP server (`mcp/`), NanoClaw | **Knowledge Graph (Core Hoist Tier 1)** | | | | | | | All via `@totalreclaw/core` 1.5.0 WASM/PyO3 with local fallbacks | | Store-time dedup (best-match) | Yes (core) | Yes (core) | Yes (via MCP) | Yes (core) | Yes (via MCP) | Yes (core) | `find_best_near_duplicate` — returns highest-similarity match, not first | | Bulk clustering | Yes (core) | Yes (core) | Yes (via MCP) | -- | Yes (via MCP) | -- | `cluster_facts` — greedy single-pass for consolidation tool | -| Pin status semantics | Yes (core) | -- | -- | -- | -- | Yes (core) | `is_pinned_claim`, `respect_pin_in_resolution` | -| Contradiction detection orchestration | Yes (core) | -- | -- | Yes (core) | -- | Yes (core) | `resolve_with_candidates` — full pipeline: detect → pin check → resolve → tie-zone | +| Pin status semantics | Yes (core) | -- | -- | Yes (core) | -- | Yes (core) | `is_pinned_claim`, `respect_pin_in_resolution`. Hermes (PRD-04 DEP-5): pin guard on LLM UPDATE/DELETE + pin state fed to the contradiction resolver. | +| Contradiction detection orchestration | Yes (core) | -- | -- | Yes (core) | -- | Yes (core) | `resolve_with_candidates` — full pipeline: detect → pin check → resolve → tie-zone. Hermes passes entity refs + pin state since PRD-04 DEP-5 and applies `skip_new`/`existing_pinned` only (see Known Gaps). | +| Pin guard on LLM UPDATE/DELETE | -- | n/a | -- | Yes | n/a | -- | PRD-04 F1 / DEP-5 — `python/src/totalreclaw/agent/pin_guard.py`: an auto-extraction UPDATE/DELETE whose target is pinned (core `is_pinned_claim` on the decrypted blob) is dropped; unreadable pin state fails closed (DELETE dropped, UPDATE stored as ADD, nothing tombstoned). MCP has no LLM UPDATE/DELETE path. OpenClaw plugin (parked): see Known Gaps. | +| Recall pin boost (core `pin_boost`, 1.5×) | -- | Pending | Pending | Yes (core) | Pending | -- | PRD-04 F1 / DEP-5 — core `RerankerConfig.pin_boost` + `Candidate.pinned`; `DEFAULT_PIN_BOOST` = 1.5; off by default (bit-identical ranking) and ignored without a query embedding. Hermes: every recall through `search_facts` (auto-recall is lexical-only until PRD-04 DEP-8, so it gets the boost with DEP-8). MCP: DEP-5 Part B, after the core release. A pin raises rank; it does not add a fact to every recall. | | Decision log types | Yes (core) | -- | -- | -- | -- | Yes (core) | `DecisionLogEntry`, `find_loser_claim_in_decision_log` — enables pin-on-tombstone recovery | | Shadow mode filtering | Yes (core) | -- | -- | -- | -- | Yes (core) | `filter_shadow_mode` — observer-only validation mode | | Importance rubric (1-10 anchored) | Yes | Yes (via prompt) | Yes | Yes | Yes (via prompt) | Yes | Phase 2.2.6 — explicit band definitions in extraction prompt | @@ -326,6 +328,9 @@ Managed Service tier model (single-chain Gnosis — ops-1 shipped, #283 closed 2 | Provisioned signing key is not revocable | MEDIUM | The bundle ships `signing.kind: "owner-eoa"` — the Smart Account owner key. It carries full account authority and cannot be revoked, because canonical `SimpleAccount` has no module dispatch and no owner setter. A compromised host still requires vault-identity rotation. The `session-key` variant (scoped to DataEdge `execute`/`executeBatch`, `valueMax = 0`, on-chain revocable) is specified in the schema but requires the signing-delegation phase. | | Phrase remains on env-var and external-provider installs | LOW | Auto-migration covers `credentials.json` states only. `TOTALRECLAW_RECOVERY_PHRASE` and operator-authored external-provider payloads are read-only from the daemon's perspective and must be replaced by the operator. See `docs/guides/headless-deployment.md`. | | Phrase not recoverable from a migrated host | LOW | After migration the host holds only derived material; the phrase cannot be reconstructed from it. Users must have their own copy. A one-cycle `credentials.json.bak` is the recovery path. | +| Pin guard on LLM UPDATE/DELETE — Hermes only | MEDIUM | PRD-04 DEP-5 guards Hermes auto-extraction. The parked OpenClaw plugin's LLM UPDATE/DELETE branches (`LLM dedup: UPDATE` / `LLM dedup: DELETE` in `skill/plugin/index.ts`) still tombstone `existingFactId` with no pin check, so a plugin auto-extraction can tombstone a fact pinned from any client. Port on un-park: check core `isPinnedClaim` on the target's decrypted blob before queueing the tombstone, mirroring `python/src/totalreclaw/agent/pin_guard.py`. | +| Recall pin boost — partial client coverage | LOW | Wired: Hermes (explicit recall and contradiction candidates; auto-recall once PRD-04 DEP-8 adds a query embedding). Pending: MCP managed recall (DEP-5 Part B, needs the core release). Not wired: OpenClaw plugin (parked), ZeroClaw (`rust/totalreclaw-memory` passes `pinned: false`), MCP self-hosted HTTP recall (`mcp/src/tools/recall.ts`, ranks via `@totalreclaw/client`). | +| Hermes contradiction resolver applies pin skips only | LOW | Since PRD-04 DEP-5 Hermes hands core's resolver entity refs + pin state, so detection runs; only `skip_new`/`existing_pinned` is applied. `skip_new`/`existing_wins`, `supersede_existing` and `tie_leave_both` are logged, not applied — PRD-04 DEP-12 owns applying them (bench-gated). | --- diff --git a/docs/specs/totalreclaw/memory-taxonomy-v1.md b/docs/specs/totalreclaw/memory-taxonomy-v1.md index 408dcb7d..6d3cdda3 100644 --- a/docs/specs/totalreclaw/memory-taxonomy-v1.md +++ b/docs/specs/totalreclaw/memory-taxonomy-v1.md @@ -162,6 +162,9 @@ Every v1.1-compliant client MUST: 3. **Write on pin**: pinning a fact produces a NEW claim with `pin_status == "pinned"` that supersedes the original via the standard tombstone + new-fact pattern (same pattern as `retype` and `set_scope`). The new claim is wrapped in the v1 outer protobuf (`version = 4`) with `schema_version = "1.0"` inner JSON. 4. **Write on unpin**: unpinning produces a NEW claim with `pin_status == "unpinned"` (or omits the field — both are equivalent) and supersedes the pinned one. 5. **Round-trip**: readers at any compliance level MUST preserve the `pin_status` field on round-trip (read → write) so a fact pinned by one client does not silently lose its pinned status when surfaced by another. +6. **Guard LLM-guided rewrites** (PRD-04 F1 / DEP-5): an automatic UPDATE or DELETE produced by an extraction LLM MUST NOT tombstone a pinned claim. Before applying it, the client reads the target's pin state (`is_pinned_json` on the decrypted blob). Target pinned → drop the whole action; for UPDATE the replacement is not stored either, the same outcome as `SkipNew { reason: ExistingPinned }`. Pin state unreadable → fail closed on the destructive half: DELETE is dropped and UPDATE is stored as a plain ADD with no tombstone. Explicit user tools (`totalreclaw_forget`, `totalreclaw_unpin`) are user-initiated and unaffected. Reference: `python/src/totalreclaw/agent/pin_guard.py`. +7. **Feed pin state to the resolver**: a client that projects claims into the short-key `Claim` shape for `resolve_with_candidates` MUST carry `st: "p"` for pinned claims and the claim's entity refs (`e`). The resolver only compares claims that share an entity, and reads pin state from `st` on this shape. +8. **Recall boost (recommended)**: recall MAY rank pinned claims higher by setting `pinned: true` on the reranker candidate and `pin_boost` (core `DEFAULT_PIN_BOOST` = 1.5) in `RerankerConfig`. A pin raises rank; it does not add a claim to every recall. See `retrieval-v2.md` §Pin boost. ### Cross-client compatibility diff --git a/docs/specs/totalreclaw/retrieval-v2.md b/docs/specs/totalreclaw/retrieval-v2.md index 71d3fbfb..6e459edb 100644 --- a/docs/specs/totalreclaw/retrieval-v2.md +++ b/docs/specs/totalreclaw/retrieval-v2.md @@ -149,6 +149,26 @@ assistant-source facts surface when the user query targets them. --- +## Pin boost (PRD-04 F1 / DEP-5) + +**What:** `RerankerConfig.pin_boost: Option` (JSON `pin_boost`) multiplies the final score of candidates whose `Candidate.pinned` is `true` (JSON `pinned`, default `false`): + + final_score = fused × source_weight × (pinned ? pin_boost : 1.0) + +The multiplication happens after Tier 1 source weighting and before top-k truncation. `None` (the default), a non-finite value, a value ≤ 1.0, or an **empty query embedding** is a no-op. Callers that do not opt in therefore rank bit-for-bit as before, and no parity vector changed. + +**Enabled value:** `DEFAULT_PIN_BOOST = 1.5` (core `reranker.rs`; equal to `PinConfig::default().hard_boost`). Pedro signed off 1.5× on 2026-04-28 (internal F1 pin-UX defaults note) as strong enough to lift pinned facts about 50% above unpinned ones without making every pin rank first. On-wire pins are binary (`pin_status == "pinned"`) and carry no pin timestamp, so the non-decaying hard-pin value applies. Soft-pin decay (`claims::pin_boost`) needs a pin timestamp on the wire first. + +**Magnitude:** RRF-fused scores span only about 2× between rank 1 and rank 60, so 1.5× moves a candidate from combined rank r to roughly (60 + r) / 1.5 − 60. A relevant pinned fact just outside the top k moves into it. A pinned fact that matches neither signal stays out (core tests `test_pin_boost_lifts_pinned_candidate_just_outside_top_k_into_top_k`, `test_pin_boost_does_not_surface_an_irrelevant_pinned_fact`). + +**Why no boost without a query embedding:** with no embedding, every candidate ties on cosine, and every candidate without a lexical match ties on one BM25 rank. Any multiplier would then lift every pinned fact above genuine matches. Hermes auto-recall is lexical-only until PRD-04 DEP-8 adds a query embedding, so its boost turns on with DEP-8. Explicit recall, contradiction-candidate recall and MCP recall all embed the query. + +**Pin state source:** clients set `pinned` from core `is_pinned_claim` / `isPinnedClaim` on the decrypted blob (v1.1 `pin_status` or v0 `st == "p"`). + +**Status:** Hermes: every recall through `search_facts`. MCP managed recall: DEP-5 Part B. Not wired: OpenClaw plugin (parked), ZeroClaw, MCP self-hosted HTTP recall. + +--- + ## Tier 2 — Scope pre-filter (OPTIONAL, post-v1) **Rationale:** If user query carries a scope hint ("at work…", "for my health…"), pre-filtering candidates by scope improves precision. Otherwise, scope is ignored at retrieval time (as today). diff --git a/python/CHANGELOG.md b/python/CHANGELOG.md index a401d5b7..b6e390ad 100644 --- a/python/CHANGELOG.md +++ b/python/CHANGELOG.md @@ -8,6 +8,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Fixed + +- **[PRD-04 F1 / DEP-5] Pinned facts are protected from auto-extraction.** An auto-extraction UPDATE or DELETE whose target is pinned is now dropped (`agent/pin_guard.py`). When the pin state cannot be read, DELETE is dropped and UPDATE is stored as a plain ADD, so nothing is tombstoned. The contradiction resolver now receives entity refs and pin state, so it actually runs, and a new fact that contradicts a pinned fact is not stored (core `SkipNew { reason: existing_pinned }`). Other resolver outcomes stay log-only until DEP-12. Log lines carry the category only, never fact text or ids. +- **SKILL.md** no longer claims pinned facts surface in every recall. They rank higher when they match the query. + +### Added + +- **Recall pin boost.** Recall ranks pinned facts 1.5× higher through core `rerank_with_config(..., pin_boost=)` (`operations.PIN_BOOST_DEFAULT`). The option is feature-detected: a `totalreclaw-core` wheel without it keeps the old ranking. Core applies no boost when the query has no embedding (Hermes auto-recall until DEP-8). +- `TotalReclaw.get_fact_pin_status(fact_id)`. + ## [2.5.2] — 2026-09-26 ### Fixed diff --git a/python/src/totalreclaw/hermes/SKILL.md b/python/src/totalreclaw/hermes/SKILL.md index 7c3177a7..fb32b0fe 100644 --- a/python/src/totalreclaw/hermes/SKILL.md +++ b/python/src/totalreclaw/hermes/SKILL.md @@ -79,7 +79,7 @@ Hermes provides `session_search` for the local session DB, which has limited ret Pattern is always **recall first → mutate second**, because the mutation tools need `fact_id`: - `forget X` → `totalreclaw_recall("X")` → pick the right `fact_id` → `totalreclaw_forget(fact_id)`. Tombstones the fact (still retained for audit; recall filters it out). -- `pin X as canonical` → recall → `totalreclaw_pin(fact_id)`. Pinned facts surface in every subsequent recall regardless of query similarity. +- `pin X as canonical` → recall → `totalreclaw_pin(fact_id)`. Automatic extraction never updates or deletes a pinned fact, and pinned facts rank higher in recall when they match the query — pinning does not add a fact to every recall. - `unpin X` → recall → `totalreclaw_unpin(fact_id)`. - `change type of X to ` → recall → `totalreclaw_retype(fact_id, type)`. Use when the extractor misclassified. - `change scope of X to ` → recall → `totalreclaw_set_scope(fact_id, scope)`. diff --git a/python/tests/test_skill_md_pin_semantics.py b/python/tests/test_skill_md_pin_semantics.py new file mode 100644 index 00000000..2a26c898 --- /dev/null +++ b/python/tests/test_skill_md_pin_semantics.py @@ -0,0 +1,17 @@ +"""PRD-04 F1 / DEP-5 (+ F11): SKILL.md must describe pin behaviour truthfully. + +Pinned facts are protected from automatic updates/deletes and rank higher in +recall when relevant; they are NOT injected into every recall (signed-off +F1 design: no "always rank first").""" +from __future__ import annotations + +from pathlib import Path + +_REPO_ROOT = Path(__file__).resolve().parents[2] +SKILL_MD = _REPO_ROOT / "python" / "src" / "totalreclaw" / "hermes" / "SKILL.md" + + +def test_skill_md_does_not_claim_pinned_facts_surface_in_every_recall() -> None: + text = SKILL_MD.read_text(encoding="utf-8") + assert "surface in every subsequent recall" not in text + assert "Automatic extraction never updates or deletes a pinned fact" in text From ea8f77f80dab017a6409a670d05f30e48309a87c Mon Sep 17 00:00:00 2001 From: p-diogo Date: Wed, 30 Sep 2026 19:07:23 +0000 Subject: [PATCH 09/10] test(python): staging E2E for the pin contract (PRD-04 DEP-5) Co-Authored-By: Claude Code --- python/tests/test_staging_pin_contract.py | 77 +++++++++++++++++++++++ 1 file changed, 77 insertions(+) create mode 100644 python/tests/test_staging_pin_contract.py diff --git a/python/tests/test_staging_pin_contract.py b/python/tests/test_staging_pin_contract.py new file mode 100644 index 00000000..aa624a0b --- /dev/null +++ b/python/tests/test_staging_pin_contract.py @@ -0,0 +1,77 @@ +"""Staging E2E for PRD-04 F1 / DEP-5: pin contract + recall pin state. + +STAGING ONLY. Skipped unless TOTALRECLAW_RECOVERY_PHRASE is set to a throwaway +staging vault supplied through the environment (never written to any file). +Asserts the client resolved the STAGING DataEdge before anything is written. + +Run: + TOTALRECLAW_RECOVERY_PHRASE=... python -m pytest tests/test_staging_pin_contract.py -v -s +""" +from __future__ import annotations + +import asyncio +import os +import time + +import pytest + +from totalreclaw.agent.extraction import ExtractedFact +from totalreclaw.agent.pin_guard import apply_pin_guard +from totalreclaw.client import TotalReclaw + +MNEMONIC = os.environ.get("TOTALRECLAW_RECOVERY_PHRASE", "") +STAGING_URL = "https://api-staging.totalreclaw.xyz" +STAGING_DATA_EDGE = "0xe7a4d2677b686e13775ba9092631089e35f0bb91" +PROD_DATA_EDGE = "0xc445af1d4eb9fce4e1e61fe96ea7b8febf03c5ca" +INDEX_WAIT_S = 30 + +pytestmark = pytest.mark.skipif(not MNEMONIC, reason="TOTALRECLAW_RECOVERY_PHRASE not set") + + +@pytest.fixture +async def client(): + c = TotalReclaw(mnemonic=MNEMONIC, relay_url=STAGING_URL, is_test=True) + await c.resolve_address() + await c.resolve_chain_id() + data_edge = (c._data_edge_address or "").lower() + assert data_edge == STAGING_DATA_EDGE, f"refusing to run: DataEdge {data_edge!r} is not staging" + assert data_edge != PROD_DATA_EDGE + yield c + await c.close() + + +async def test_pin_contract_on_staging(client) -> None: + unique = f"dep5pin{int(time.time())}" + original_id = await client.remember(f"Pedro {unique} home city is Lisbon", importance=0.9) + await asyncio.sleep(INDEX_WAIT_S) + + pin = await client.pin_fact(original_id) + assert pin["success"] is True + pinned_id = pin["new_fact_id"] + await asyncio.sleep(INDEX_WAIT_S) + assert await client.get_fact_pin_status(pinned_id) is True + + contradictions = [ + ExtractedFact(text=f"Pedro {unique} home city is Porto", type="claim", importance=8, + action="UPDATE", existing_fact_id=pinned_id, source="user"), + ExtractedFact(text=f"Pedro {unique} home city is Madrid", type="claim", importance=8, + action="UPDATE", existing_fact_id=pinned_id, source="user"), + ExtractedFact(text=f"Pedro {unique} home city is Lisbon", type="claim", importance=8, + action="DELETE", existing_fact_id=pinned_id, source="user"), + ] + assert await apply_pin_guard(contradictions, client) == [] + + # Still pinned and active on the staging subgraph. + assert await client.get_fact_pin_status(pinned_id) is True + + # Recall (with a query embedding, so the core pin boost path runs) returns + # the fact flagged as pinned. + from totalreclaw.embedding import get_embedding + + query = f"Pedro {unique} home city" + results = await client.recall(query, query_embedding=get_embedding(query), top_k=8) + hits = [r for r in results if r.id == pinned_id] + assert hits and hits[0].pinned is True + + # Tidy the throwaway vault (explicit forget is user-initiated, not guarded). + await client.forget(pinned_id) From cc36fbb2dcfdd7bc9c7e62d5b57237164cbf4003 Mon Sep 17 00:00:00 2001 From: p-diogo Date: Wed, 30 Sep 2026 22:07:21 +0000 Subject: [PATCH 10/10] fix(python): get_fact_pin_status fails closed on subgraph error bodies (DEP-5) relay.query_subgraph returns the parsed body for any HTTP 2xx, so a GraphQL error body ({"errors": [...]}) carries no "data" key; the lenient .get chain collapsed it to None, get_fact_pin_status returned False, and the pin guard let an LLM UPDATE/DELETE tombstone a pinned fact during a subgraph hiccup. _fetch_fact_by_id gains a keyword-only strict mode, used only by get_fact_pin_status: a non-empty "errors" key, or a body whose "data" is not a dict carrying "fact", raises RuntimeError (pin state unknown -> guard fails closed). False is returned only for an explicit data.fact is None. Other callers keep the lenient shape unchanged. Generated with [Claude Code](https://claude.ai/code) via [Happy](https://happy.engineering) Co-Authored-By: Claude Code Co-Authored-By: Happy --- python/src/totalreclaw/operations.py | 36 ++++++++++++++++++++++++--- python/tests/test_fact_pin_status.py | 37 +++++++++++++++++++++++++++- 2 files changed, 68 insertions(+), 5 deletions(-) diff --git a/python/src/totalreclaw/operations.py b/python/src/totalreclaw/operations.py index a9fc58fd..946bffd6 100644 --- a/python/src/totalreclaw/operations.py +++ b/python/src/totalreclaw/operations.py @@ -1140,14 +1140,40 @@ async def _fetch_fact_by_id( fact_id: str, owner: str, relay: RelayClient, + *, + strict: bool = False, ) -> Optional[dict]: """Fetch a single fact's on-chain record via the subgraph. Returns the raw ``fact`` subgraph object or ``None`` if not found. The caller is responsible for checking ``owner`` matches the expected smart-account address. + + With ``strict=True`` (PRD-04 F1 / DEP-5, review fix-up) a 2xx body that + does not carry a ``data`` dict with a ``fact`` key raises ``RuntimeError`` + instead of collapsing to ``None``: ``relay.query_subgraph`` returns the + parsed body for any HTTP 2xx, so an indexer error body (``{"errors": + [...]}``) has no ``data`` key and the lenient ``.get`` chain would make + "pin state unreadable" look like "fact absent". The pin guard must fail + closed on that difference, so ``get_fact_pin_status`` is the strict + caller; other callers keep the lenient shape. """ data = await relay.query_subgraph(FACT_BY_ID_QUERY, {"id": fact_id}) + if strict: + if isinstance(data, dict) and data.get("errors"): + # Categorical message only — never echo the body, which may carry + # indexer diagnostics (pin-guard log hygiene, PRD-04 DEP-5). + raise RuntimeError( + "fact-by-id subgraph query returned errors " + f"({len(data['errors'])} item(s))" + ) + inner = data.get("data") if isinstance(data, dict) else None + if not isinstance(inner, dict) or "fact" not in inner: + raise RuntimeError( + "fact-by-id subgraph response has no data.fact " + f"(body type {type(data).__name__})" + ) + return inner["fact"] return data.get("data", {}).get("fact") @@ -1181,13 +1207,15 @@ async def get_fact_pin_status( those sentinels. A missing fact, an inactive fact and a tombstone stub are all ``False`` -- tombstoning them again cannot hurt a pin. - Raises on any relay error (including :class:`RelayReadBlocked`) and on - decrypt failure. Callers MUST treat a raise as "pin state unknown" and - fail closed (see ``agent/pin_guard.py``). + Raises on any relay error (including :class:`RelayReadBlocked`), on a 2xx + subgraph body that does not carry ``data.fact`` (a GraphQL error body — + never mistake an unreadable state for an absent fact), and on decrypt + failure. Callers MUST treat a raise as "pin state unknown" and fail closed + (see ``agent/pin_guard.py``). """ if not isinstance(fact_id, str) or not fact_id.strip(): raise ValueError("fact_id must be a non-empty string") - fact = await _fetch_fact_by_id(fact_id.strip(), owner, relay) + fact = await _fetch_fact_by_id(fact_id.strip(), owner, relay, strict=True) if not fact: return False if fact.get("isActive") is False: diff --git a/python/tests/test_fact_pin_status.py b/python/tests/test_fact_pin_status.py index 2951efc0..c2582ada 100644 --- a/python/tests/test_fact_pin_status.py +++ b/python/tests/test_fact_pin_status.py @@ -43,15 +43,18 @@ def _fact(blob_hex: str, active: bool = True) -> dict: class _Relay: - def __init__(self, fact=None, exc: Exception | None = None) -> None: + def __init__(self, fact=None, exc: Exception | None = None, body: dict | None = None) -> None: self.fact = fact self.exc = exc + self.body = body self.calls: list[dict] = [] async def query_subgraph(self, gql: str, variables: dict) -> dict: self.calls.append(variables) if self.exc is not None: raise self.exc + if self.body is not None: + return self.body return {"data": {"fact": self.fact}} @@ -92,6 +95,38 @@ async def test_relay_error_propagates() -> None: await operations.get_fact_pin_status(FACT_ID, _keys(), "0xabc", _Relay(exc=RuntimeError("503"))) +async def test_subgraph_error_body_raises() -> None: + """A 2xx GraphQL error body is "pin state unknown", not "fact absent". + + ``relay.query_subgraph`` returns the parsed body for any HTTP 2xx, so an + indexer error arrives as ``{"errors": [...]}`` with no ``data`` key. The + lenient ``.get`` chain collapsed that to ``None`` → ``False`` and the pin + guard let an UPDATE/DELETE through (review fix-up, PRD-04 DEP-5). + """ + with pytest.raises(RuntimeError): + await operations.get_fact_pin_status( + FACT_ID, _keys(), "0xabc", + _Relay(body={"errors": [{"message": "indexer error"}]}), + ) + + +@pytest.mark.parametrize("body", [{}, {"data": {}}, {"data": None}]) +async def test_body_without_data_fact_raises(body: dict) -> None: + """Any 2xx body that does not carry ``data.fact`` must raise, not return + ``False`` -- the pin guard cannot distinguish a malformed response from a + missing fact, so it must fail closed.""" + with pytest.raises(RuntimeError): + await operations.get_fact_pin_status(FACT_ID, _keys(), "0xabc", _Relay(body=body)) + + +async def test_explicitly_absent_fact_is_still_false() -> None: + """Only ``data.fact is None`` (the subgraph answering "no such fact") + counts as a genuinely absent, unpinned fact.""" + assert await operations.get_fact_pin_status( + FACT_ID, _keys(), "0xabc", _Relay(body={"data": {"fact": None}}) + ) is False + + async def test_wrong_key_propagates() -> None: relay = _Relay(_fact(_hex(V1_PINNED))) with pytest.raises(Exception):