diff --git a/Cargo.lock b/Cargo.lock index 8916f17..9e50cf3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1890,6 +1890,7 @@ dependencies = [ "tinymemory-api", "tinymemory-conformance", "tinymemory-core", + "tinymemory-documents", "tinymemory-remote", "tinymemory-sources", "tinymemory-sync", @@ -1973,6 +1974,20 @@ dependencies = [ "wiremock", ] +[[package]] +name = "tinymemory-documents" +version = "0.1.0" +dependencies = [ + "async-trait", + "chrono", + "reqwest", + "serde", + "serde_json", + "tinymemory-api", + "tinymemory-sources", + "tokio", +] + [[package]] name = "tinymemory-remote" version = "0.1.0" @@ -2033,6 +2048,7 @@ dependencies = [ "serde_json", "tinymemory", "tinymemory-api", + "tinymemory-documents", "tinymemory-remote", "tinymemory-tinycortex", "tokio", diff --git a/README.md b/README.md index 4458862..4769102 100644 --- a/README.md +++ b/README.md @@ -43,6 +43,10 @@ crates/ │ them ├── tinymemory-sources/ memory-source contracts and readers — local folders │ always, GitHub/RSS/web pages behind `network` +├── tinymemory-documents/ document and URL intake: sniff a format, convert it +│ to markdown, and write it into whichever engine is +│ bound. The URL half is behind `network` and reuses the +│ source readers' SSRF guard rather than growing a second ├── tinymemory-tinycortex/ the TinyCortex engine seen through the contract ├── tinymemory-remote/ native HTTP dialects for Supermemory, Mem0, and Cognee ├── tinymemory-conformance/ the behavioural suite every driver must pass @@ -79,6 +83,8 @@ composition — no storage engine, no HTTP stack, no native library. | `sync` | `tinymemory::sync` — the Composio normalisers | | `sources` | `tinymemory::sources` — source contracts and local readers | | `sources-network` | `sources`, plus the GitHub/RSS/web-page readers | +| `documents` | `tinymemory::documents` — document intake and markdown conversion | +| `documents-network` | `documents`, plus the URL fetch path | | `conformance` | `tinymemory::conformance` — the driver contract suite | | `memory-git` | git-backed diff snapshots (implies `tinycortex`; links libgit2) | | `contacts` | the macOS address-book seeding path (implies `core`) | diff --git a/crates/tinymemory-api/src/lib.rs b/crates/tinymemory-api/src/lib.rs index a8eff52..8c38bd0 100644 --- a/crates/tinymemory-api/src/lib.rs +++ b/crates/tinymemory-api/src/lib.rs @@ -70,6 +70,13 @@ //! - [`traits`]: the [`traits::Memory`] storage-backend trait. //! - [`chunks`]: the persisted chunk model ([`chunks::Chunk`], [`chunks::Metadata`], //! [`chunks::SourceRef`], …) and the deterministic [`chunks::chunk_id`]. +//! - [`graph`]: the bounded graph-view model ([`graph::GraphView`], +//! [`graph::GraphViewQuery`], [`graph::GraphNode`], [`graph::GraphEdge`]) — +//! the graph counterpart of [`tree`], and what +//! [`provider::MemoryGraph::graph_view`] returns. +//! - [`namespace`]: the `
:` namespace convention +//! ([`namespace::Namespace`], [`namespace::MemorySection`]) and its +//! validator. //! - [`tree`]: the markdown summary-tree node model ([`tree::TreeNode`], //! [`tree::NodeLevel`], [`tree::TreeStatus`], …). //! - [`tool_memory`]: tool-scoped rule contracts ([`tool_memory::ToolMemoryRule`], …). @@ -99,7 +106,8 @@ pub mod host; // point: a second definition would need a conversion at the module seam that // nothing type-checks. pub use tinymemory_bus::{ - capabilities, chunks, error, goals, health, recall, tool_memory, tree, types, version, wire, + capabilities, chunks, error, goals, graph, health, namespace, recall, tool_memory, tree, types, + version, wire, }; /// The mandatory-family composition: wrap any [`traits::Memory`] backend as a /// complete [`provider::MemoryProvider`]. diff --git a/crates/tinymemory-api/src/provider/knowledge.rs b/crates/tinymemory-api/src/provider/knowledge.rs index a1ae366..cc979cd 100644 --- a/crates/tinymemory-api/src/provider/knowledge.rs +++ b/crates/tinymemory-api/src/provider/knowledge.rs @@ -10,12 +10,26 @@ //! extraction models, hotness decay curves, and snapshot retention are driver //! concerns and appear in none of these signatures. +use std::collections::BTreeSet; + use async_trait::async_trait; use crate::error::MemoryError; +use crate::graph::{GraphEdge, GraphNode, GraphView, GraphViewQuery}; use crate::provider::types::{DiffReport, EntityHit, SnapshotRef}; use crate::types::{GraphRelationRecord, MemoryKvRecord}; +/// How many edges the default [`MemoryGraph::graph_view`] traversal scans per +/// predicate when it has to resolve *inbound* edges. +/// +/// [`MemoryGraph::relations`] filters by subject and predicate but not by +/// object, so inbound expansion has no indexed form in this contract and the +/// default traversal falls back to a bounded scan. The bound exists so a graph +/// larger than memory cannot be pulled into a view; hitting it sets +/// [`GraphView::truncated`]. A driver whose store indexes the object column +/// should override `graph_view` and skip this path entirely. +pub const INBOUND_SCAN_LIMIT: usize = 4_096; + /// The entity index: who and what the stored memory is about. #[async_trait] pub trait MemoryEntities: Send + Sync { @@ -140,6 +154,238 @@ pub trait MemoryGraph: Send + Sync { /// [`MemoryError::Invalid`] for a malformed edge, otherwise backend /// failures. async fn put_relation(&self, relation: GraphRelationRecord) -> Result<(), MemoryError>; + + /// Assemble a bounded, renderable slice of the graph. + /// + /// This is to [`Self::relations`] what + /// [`crate::provider::MemoryTree::drill_down`] is to a raw node read: one + /// call returns a node *together with its surroundings*, already joined + /// into a node set and an edge set, so navigating a graph is a sequence of + /// view calls rather than a client-side reassembly that every caller would + /// write differently. + /// + /// # The default implementation + /// + /// Provided, not required: it breadth-first expands + /// [`GraphViewQuery::seeds`] using [`Self::relations`] alone, so every + /// existing driver gains a graph view without writing one, and a driver + /// that advertises no [`crate::capabilities::Capability::Graph`] family + /// still surfaces the same [`MemoryError::Unsupported`] its `relations` + /// returns. + /// + /// It costs one `relations` call per node visited, including one final + /// round at the outermost hop that adds no nodes and exists only to close + /// edges *between* nodes already in the view — without it the outer ring + /// renders as a star rather than as the graph it is. A driver with a native + /// multi-hop traversal should override this and use it. + /// + /// Inbound expansion has no indexed form here — `relations` cannot filter + /// by object — so [`crate::graph::GraphDirection::In`] and + /// [`crate::graph::GraphDirection::Both`] fall back to a scan capped at + /// [`INBOUND_SCAN_LIMIT`] per predicate. + /// + /// # Errors + /// + /// Whatever [`Self::relations`] returns. Bounds are never an error: a + /// traversal that hits one returns the partial view with + /// [`GraphView::truncated`] set. + async fn graph_view(&self, query: &GraphViewQuery) -> Result { + let namespace = query.namespace.as_deref(); + let mut view = GraphView { + namespace: query.namespace.clone(), + seeds: query.seeds.clone(), + ..GraphView::default() + }; + + // Each predicate needs its own call: `relations` takes one, not a set. + // An empty filter becomes the single unfiltered call rather than a + // special case further down. + let predicates: Vec> = if query.predicates.is_empty() { + vec![None] + } else { + query.predicates.iter().map(|p| Some(p.as_str())).collect() + }; + + // Nodes reached but never expanded, either because a bound was hit or + // because they sit one hop past the requested depth. A set, not a + // counter: the same boundary node is commonly reached from several + // directions, and counting it twice would overstate what is left. + let mut unexpanded: BTreeSet = BTreeSet::new(); + + // Unseeded: an overview, not a traversal. One bounded scan of the + // slice, and the node set is whatever the returned edges touch. + if query.seeds.is_empty() { + for predicate in &predicates { + // One past the ceiling: a call that asked for exactly + // `max_edges` and got them cannot tell a full slice from a + // truncated one. + let records = self + .relations( + namespace, + None, + *predicate, + query.max_edges.saturating_add(1), + ) + .await?; + for record in records { + push_view_edge(&mut view, record, &mut unexpanded, query, 0); + } + } + view.stats.frontier_remaining = unexpanded.len(); + view.recompute_stats(); + return Ok(view); + } + + // Inbound edges are resolved from one scan per predicate rather than + // one per node: the scan is the expensive part, and repeating it for + // every node visited would multiply it by `max_nodes`. + let mut inbound: Vec = Vec::new(); + if query.direction.follows_in() { + for predicate in &predicates { + let records = self + .relations(namespace, None, *predicate, INBOUND_SCAN_LIMIT) + .await?; + if records.len() >= INBOUND_SCAN_LIMIT { + view.truncated = true; + } + inbound.extend(records); + } + } + + let mut frontier: Vec = Vec::new(); + for seed in &query.seeds { + if view.nodes.iter().any(|n| &n.id == seed) { + continue; + } + if view.nodes.len() >= query.max_nodes { + unexpanded.insert(seed.clone()); + view.truncated = true; + continue; + } + view.nodes.push(GraphNode::bare(seed.clone(), 0)); + frontier.push(seed.clone()); + } + + for hop in 0..=query.depth { + if frontier.is_empty() { + break; + } + let mut next: Vec = Vec::new(); + for node_id in &frontier { + let mut incident: Vec = Vec::new(); + if query.direction.follows_out() { + for predicate in &predicates { + incident.extend( + self.relations( + namespace, + Some(node_id), + *predicate, + query.max_edges.saturating_add(1), + ) + .await?, + ); + } + } + if query.direction.follows_in() { + incident.extend( + inbound + .iter() + .filter(|record| &record.object == node_id) + .cloned(), + ); + } + + for record in incident { + if !query.accepts_predicate(&record.predicate) { + continue; + } + let other = if &record.subject == node_id { + record.object.clone() + } else { + record.subject.clone() + }; + let known = view.nodes.iter().any(|n| n.id == other); + if !known { + // An edge to a node the view will not hold would + // dangle, so it is dropped either way — but *why* it + // was dropped matters. Reaching the requested depth is + // the caller getting what they asked for; hitting the + // node ceiling is not, and only the second makes the + // view truncated. Conflating them would set the flag on + // every finite traversal of a connected graph and leave + // it saying nothing. + if hop >= query.depth { + unexpanded.insert(other); + continue; + } + if view.nodes.len() >= query.max_nodes { + unexpanded.insert(other); + view.truncated = true; + continue; + } + view.nodes.push(GraphNode::bare(other.clone(), hop + 1)); + next.push(other); + } + push_view_edge(&mut view, record, &mut unexpanded, query, hop); + } + } + frontier = next; + } + + view.stats.frontier_remaining = unexpanded.len(); + view.recompute_stats(); + Ok(view) + } +} + +/// Add one relation to a view, deduplicating by triple and honouring +/// [`GraphViewQuery::max_edges`]. +/// +/// A separate function rather than a closure so the borrow of `view` ends +/// between calls, which the traversal above needs while it is also pushing +/// nodes. +fn push_view_edge( + view: &mut GraphView, + record: GraphRelationRecord, + unexpanded: &mut BTreeSet, + query: &GraphViewQuery, + depth: u32, +) { + if !query.accepts_predicate(&record.predicate) { + return; + } + let triple = ( + record.subject.clone(), + record.predicate.clone(), + record.object.clone(), + ); + if view + .edges + .iter() + .any(|e| e.key() == (&triple.0, &triple.1, &triple.2)) + { + return; + } + if view.edges.len() >= query.max_edges { + unexpanded.insert(triple.0); + unexpanded.insert(triple.2); + view.truncated = true; + return; + } + // The unseeded overview derives its node set from the edges it found; the + // seeded traversal has already placed both endpoints. + for id in [triple.0.clone(), triple.2.clone()] { + if view.nodes.iter().any(|n| n.id == id) { + continue; + } + if view.nodes.len() >= query.max_nodes { + unexpanded.insert(id); + view.truncated = true; + return; + } + view.nodes.push(GraphNode::bare(id, depth)); + } + view.edges.push(GraphEdge::from(record)); } /// Snapshot capture and change computation over synced sources. diff --git a/crates/tinymemory-api/src/provider/mod.rs b/crates/tinymemory-api/src/provider/mod.rs index 74e7bb7..32d2c86 100644 --- a/crates/tinymemory-api/src/provider/mod.rs +++ b/crates/tinymemory-api/src/provider/mod.rs @@ -78,7 +78,7 @@ pub use chunks::{ChunkDetail, ChunkEmbedding, ChunkQuery, MemoryChunks}; pub use content::{MemoryDocuments, MemoryIngest, MemoryTree}; pub use driver::MemoryProvider; pub use episodic::{ConversationSegment, EpisodicTurn, MemoryEpisodic}; -pub use knowledge::{MemoryDiff, MemoryEntities, MemoryGraph}; +pub use knowledge::{MemoryDiff, MemoryEntities, MemoryGraph, INBOUND_SCAN_LIMIT}; pub use mandatory::{MemoryCore, MemoryPortability, MemoryRecall}; pub use people::{ AddressBookSeedOutcome, MemoryPeople, PersonHandle, PersonInteraction, PersonRecord, PersonRef, diff --git a/crates/tinymemory-api/tests/graph_view.rs b/crates/tinymemory-api/tests/graph_view.rs new file mode 100644 index 0000000..e2ebe05 --- /dev/null +++ b/crates/tinymemory-api/tests/graph_view.rs @@ -0,0 +1,433 @@ +//! Behavioural tests for the default [`MemoryGraph::graph_view`] traversal. +//! +//! Driven through a fixed in-memory edge list rather than a real engine: the +//! point under test is the traversal the contract provides for free, and a +//! store that answers `relations` from a `Vec` is the smallest thing that can +//! exercise it deterministically. + +use async_trait::async_trait; +use tinymemory_api::error::MemoryError; +use tinymemory_api::graph::{GraphDirection, GraphViewQuery}; +use tinymemory_api::provider::MemoryGraph; +use tinymemory_api::types::{GraphRelationRecord, MemoryKvRecord}; + +/// A `MemoryGraph` whose relation tier is a fixed edge list. +/// +/// Only `relations` is real; the key/value half is out of scope for the +/// traversal and reports `Unsupported`, exactly as a driver without one would. +struct EdgeList { + edges: Vec, +} + +impl EdgeList { + fn new(edges: &[(&str, &str, &str)]) -> Self { + Self { + edges: edges + .iter() + .map(|(subject, predicate, object)| GraphRelationRecord { + namespace: None, + subject: (*subject).to_string(), + predicate: (*predicate).to_string(), + object: (*object).to_string(), + attrs: serde_json::Value::Null, + updated_at: 0.0, + evidence_count: 1, + order_index: None, + document_ids: Vec::new(), + chunk_ids: Vec::new(), + }) + .collect(), + } + } + + /// A path `n0 -> n1 -> … -> n{len}`, for depth-bound tests. + fn chain(len: usize) -> Self { + let names: Vec = (0..=len).map(|i| format!("n{i}")).collect(); + let pairs: Vec<(&str, &str, &str)> = (0..len) + .map(|i| (names[i].as_str(), "next", names[i + 1].as_str())) + .collect(); + Self::new(&pairs) + } +} + +#[async_trait] +impl MemoryGraph for EdgeList { + async fn kv_get( + &self, + _namespace: Option<&str>, + _key: &str, + ) -> Result, MemoryError> { + Err(MemoryError::unsupported( + tinymemory_api::capabilities::Capability::Graph, + )) + } + + async fn kv_put( + &self, + _namespace: Option<&str>, + _key: &str, + _value: serde_json::Value, + ) -> Result<(), MemoryError> { + Err(MemoryError::unsupported( + tinymemory_api::capabilities::Capability::Graph, + )) + } + + async fn kv_delete(&self, _namespace: Option<&str>, _key: &str) -> Result { + Err(MemoryError::unsupported( + tinymemory_api::capabilities::Capability::Graph, + )) + } + + async fn kv_list( + &self, + _namespace: Option<&str>, + _prefix: Option<&str>, + _limit: usize, + ) -> Result, MemoryError> { + Err(MemoryError::unsupported( + tinymemory_api::capabilities::Capability::Graph, + )) + } + + async fn relations( + &self, + _namespace: Option<&str>, + subject: Option<&str>, + predicate: Option<&str>, + limit: usize, + ) -> Result, MemoryError> { + Ok(self + .edges + .iter() + .filter(|e| subject.is_none_or(|s| e.subject == s)) + .filter(|e| predicate.is_none_or(|p| e.predicate == p)) + .take(limit) + .cloned() + .collect()) + } + + async fn put_relation(&self, _relation: GraphRelationRecord) -> Result<(), MemoryError> { + Err(MemoryError::unsupported( + tinymemory_api::capabilities::Capability::Graph, + )) + } +} + +/// A graph that refuses every relation query, standing in for a driver with no +/// graph family at all. +struct NoGraph; + +#[async_trait] +impl MemoryGraph for NoGraph { + async fn kv_get( + &self, + _namespace: Option<&str>, + _key: &str, + ) -> Result, MemoryError> { + Err(MemoryError::unsupported( + tinymemory_api::capabilities::Capability::Graph, + )) + } + + async fn kv_put( + &self, + _namespace: Option<&str>, + _key: &str, + _value: serde_json::Value, + ) -> Result<(), MemoryError> { + Err(MemoryError::unsupported( + tinymemory_api::capabilities::Capability::Graph, + )) + } + + async fn kv_delete(&self, _namespace: Option<&str>, _key: &str) -> Result { + Err(MemoryError::unsupported( + tinymemory_api::capabilities::Capability::Graph, + )) + } + + async fn kv_list( + &self, + _namespace: Option<&str>, + _prefix: Option<&str>, + _limit: usize, + ) -> Result, MemoryError> { + Err(MemoryError::unsupported( + tinymemory_api::capabilities::Capability::Graph, + )) + } + + async fn relations( + &self, + _namespace: Option<&str>, + _subject: Option<&str>, + _predicate: Option<&str>, + _limit: usize, + ) -> Result, MemoryError> { + Err(MemoryError::unsupported( + tinymemory_api::capabilities::Capability::Graph, + )) + } + + async fn put_relation(&self, _relation: GraphRelationRecord) -> Result<(), MemoryError> { + Err(MemoryError::unsupported( + tinymemory_api::capabilities::Capability::Graph, + )) + } +} + +fn ids(view: &tinymemory_api::graph::GraphView) -> Vec<&str> { + let mut ids: Vec<&str> = view.nodes.iter().map(|n| n.id.as_str()).collect(); + ids.sort_unstable(); + ids +} + +#[tokio::test] +async fn a_one_hop_view_returns_the_seed_and_its_neighbours() { + let store = EdgeList::new(&[ + ("ada", "works_with", "charles"), + ("ada", "wrote", "notes"), + ("charles", "designed", "engine"), + ]); + let view = store + .graph_view(&GraphViewQuery::around("ada")) + .await + .unwrap(); + + assert_eq!(ids(&view), vec!["ada", "charles", "notes"]); + assert_eq!(view.edges.len(), 2); + assert!(!view.truncated); + assert_eq!(view.stats.max_depth, 1); + assert_eq!(view.seeds, vec!["ada".to_string()]); +} + +#[tokio::test] +async fn a_seed_with_no_edges_is_still_a_node() { + let store = EdgeList::new(&[("charles", "designed", "engine")]); + let view = store + .graph_view(&GraphViewQuery::around("ada")) + .await + .unwrap(); + + assert_eq!(ids(&view), vec!["ada"]); + assert!(view.edges.is_empty()); + assert_eq!(view.nodes[0].degree, 0); +} + +#[tokio::test] +async fn depth_zero_returns_only_edges_between_the_seeds() { + let store = EdgeList::new(&[ + ("ada", "works_with", "charles"), + ("ada", "wrote", "notes"), + ("charles", "designed", "engine"), + ]); + let query = GraphViewQuery { + seeds: vec!["ada".into(), "charles".into()], + depth: 0, + ..GraphViewQuery::default() + }; + let view = store.graph_view(&query).await.unwrap(); + + assert_eq!(ids(&view), vec!["ada", "charles"]); + assert_eq!(view.edges.len(), 1); + assert_eq!(view.edges[0].key(), ("ada", "works_with", "charles")); + // `notes` and `engine` sit past the requested depth. That is the caller + // getting what they asked for, so the view is not truncated — but it does + // say the graph continues there. + assert!(!view.truncated); + assert_eq!(view.stats.frontier_remaining, 2); +} + +#[tokio::test] +async fn the_outermost_hop_closes_edges_between_nodes_already_in_the_view() { + // A triangle: at depth 1 from `ada` both `b` and `c` are in the view, and + // the b -> c edge must be drawn even though it adds no node. + let store = EdgeList::new(&[("ada", "e", "b"), ("ada", "e", "c"), ("b", "e", "c")]); + let view = store + .graph_view(&GraphViewQuery::around("ada")) + .await + .unwrap(); + + assert_eq!(ids(&view), vec!["ada", "b", "c"]); + assert_eq!(view.edges.len(), 3); + assert!(view.edges.iter().any(|e| e.key() == ("b", "e", "c"))); +} + +#[tokio::test] +async fn depth_bounds_the_traversal() { + let store = EdgeList::chain(5); + for depth in 0..=4 { + let view = store + .graph_view(&GraphViewQuery::around("n0").with_depth(depth)) + .await + .unwrap(); + assert_eq!( + view.nodes.len(), + depth as usize + 1, + "depth {depth} should reach {} nodes", + depth + 1 + ); + assert_eq!(view.stats.max_depth, depth); + } +} + +#[tokio::test] +async fn a_predicate_filter_excludes_other_relation_types() { + let store = EdgeList::new(&[ + ("ada", "works_with", "charles"), + ("ada", "wrote", "notes"), + ("ada", "wrote", "letters"), + ]); + let query = GraphViewQuery::around("ada").with_predicates(vec!["wrote".into()]); + let view = store.graph_view(&query).await.unwrap(); + + assert_eq!(ids(&view), vec!["ada", "letters", "notes"]); + assert!(view.edges.iter().all(|e| e.predicate == "wrote")); +} + +#[tokio::test] +async fn several_predicates_are_unioned() { + let store = EdgeList::new(&[ + ("ada", "works_with", "charles"), + ("ada", "wrote", "notes"), + ("ada", "read", "papers"), + ]); + let query = GraphViewQuery::around("ada").with_predicates(vec!["wrote".into(), "read".into()]); + let view = store.graph_view(&query).await.unwrap(); + + assert_eq!(ids(&view), vec!["ada", "notes", "papers"]); + assert_eq!(view.edges.len(), 2); +} + +#[tokio::test] +async fn inbound_expansion_follows_edges_the_seed_is_the_object_of() { + let store = EdgeList::new(&[("charles", "cites", "ada"), ("ada", "cites", "babbage")]); + let view = store + .graph_view(&GraphViewQuery::around("ada").with_direction(GraphDirection::In)) + .await + .unwrap(); + + assert_eq!(ids(&view), vec!["ada", "charles"]); + assert_eq!(view.edges[0].key(), ("charles", "cites", "ada")); +} + +#[tokio::test] +async fn both_directions_reach_either_side() { + let store = EdgeList::new(&[("charles", "cites", "ada"), ("ada", "cites", "babbage")]); + let view = store + .graph_view(&GraphViewQuery::around("ada").with_direction(GraphDirection::Both)) + .await + .unwrap(); + + assert_eq!(ids(&view), vec!["ada", "babbage", "charles"]); + assert_eq!(view.edges.len(), 2); + assert_eq!(view.nodes.iter().find(|n| n.id == "ada").unwrap().degree, 2); +} + +#[tokio::test] +async fn a_cycle_terminates_and_visits_each_node_once() { + let store = EdgeList::new(&[("a", "e", "b"), ("b", "e", "c"), ("c", "e", "a")]); + let view = store + .graph_view(&GraphViewQuery::around("a").with_depth(10)) + .await + .unwrap(); + + assert_eq!(ids(&view), vec!["a", "b", "c"]); + assert_eq!(view.edges.len(), 3); +} + +#[tokio::test] +async fn the_node_ceiling_truncates_rather_than_erroring() { + let store = EdgeList::new(&[ + ("hub", "e", "a"), + ("hub", "e", "b"), + ("hub", "e", "c"), + ("hub", "e", "d"), + ]); + let query = GraphViewQuery::around("hub").with_bounds(3, 512); + let view = store.graph_view(&query).await.unwrap(); + + assert_eq!(view.nodes.len(), 3); + assert!(view.truncated); + assert!(view.stats.frontier_remaining > 0); +} + +#[tokio::test] +async fn the_edge_ceiling_truncates_rather_than_erroring() { + let store = EdgeList::new(&[("hub", "e", "a"), ("hub", "e", "b"), ("hub", "e", "c")]); + let query = GraphViewQuery::around("hub").with_bounds(256, 2); + let view = store.graph_view(&query).await.unwrap(); + + assert_eq!(view.edges.len(), 2); + assert!(view.truncated); +} + +#[tokio::test] +async fn every_edge_in_a_view_has_both_endpoints_in_its_node_set() { + let store = EdgeList::new(&[ + ("hub", "e", "a"), + ("hub", "e", "b"), + ("hub", "e", "c"), + ("a", "e", "deep"), + ]); + for bound in 1..=5 { + let query = GraphViewQuery::around("hub") + .with_depth(2) + .with_bounds(bound, bound); + let mut view = store.graph_view(&query).await.unwrap(); + assert_eq!( + view.prune_dangling_edges(), + 0, + "view bounded at {bound} emitted a dangling edge" + ); + } +} + +#[tokio::test] +async fn an_unseeded_query_returns_an_overview_of_the_slice() { + let store = EdgeList::new(&[("ada", "works_with", "charles"), ("charles", "e", "engine")]); + let view = store + .graph_view(&GraphViewQuery::overview("learning:history")) + .await + .unwrap(); + + assert_eq!(view.namespace.as_deref(), Some("learning:history")); + assert!(view.seeds.is_empty()); + assert_eq!(ids(&view), vec!["ada", "charles", "engine"]); + assert_eq!(view.edges.len(), 2); + assert!(view.nodes.iter().all(|n| n.depth == 0)); +} + +#[tokio::test] +async fn an_unseeded_query_honours_its_predicate_filter() { + let store = EdgeList::new(&[("ada", "works_with", "charles"), ("charles", "e", "engine")]); + let query = + GraphViewQuery::overview("learning:history").with_predicates(vec!["works_with".into()]); + let view = store.graph_view(&query).await.unwrap(); + + assert_eq!(ids(&view), vec!["ada", "charles"]); + assert_eq!(view.edges.len(), 1); +} + +#[tokio::test] +async fn a_driver_without_a_graph_family_reports_unsupported_not_an_empty_view() { + let error = NoGraph + .graph_view(&GraphViewQuery::around("ada")) + .await + .unwrap_err(); + assert!( + matches!(error, MemoryError::Unsupported { .. }), + "expected Unsupported, got {error:?}" + ); +} + +#[tokio::test] +async fn a_view_is_reachable_through_a_trait_object() { + let store: Box = Box::new(EdgeList::new(&[("ada", "e", "charles")])); + let view = store + .graph_view(&GraphViewQuery::around("ada")) + .await + .unwrap(); + assert_eq!(view.nodes.len(), 2); +} diff --git a/crates/tinymemory-bus/src/chunks.rs b/crates/tinymemory-bus/src/chunks.rs index 2c141b7..5ae50dc 100644 --- a/crates/tinymemory-bus/src/chunks.rs +++ b/crates/tinymemory-bus/src/chunks.rs @@ -89,6 +89,16 @@ pub enum DataSource { MeetingNotes, /// Google Drive document. Feeds [`SourceKind::Document`]. DriveDocs, + /// A file a user handed to the memory layer directly — a PDF, a `.docx`, + /// an HTML export. Feeds [`SourceKind::Document`]. + /// + /// Distinct from the connector variants above because there is no upstream + /// provider to re-read it from: the bytes arrived once and the memory layer + /// is now the only copy, which is exactly what a re-sync path must not + /// assume it can refetch. + Upload, + /// A page fetched from a URL. Feeds [`SourceKind::Document`]. + WebPage, } impl DataSource { @@ -99,7 +109,9 @@ impl DataSource { SourceKind::Chat } Self::Gmail | Self::OtherEmail => SourceKind::Email, - Self::Notion | Self::MeetingNotes | Self::DriveDocs => SourceKind::Document, + Self::Notion | Self::MeetingNotes | Self::DriveDocs | Self::Upload | Self::WebPage => { + SourceKind::Document + } } } @@ -115,6 +127,8 @@ impl DataSource { Self::Notion => "notion", Self::MeetingNotes => "meeting_notes", Self::DriveDocs => "drive_docs", + Self::Upload => "upload", + Self::WebPage => "web_page", } } @@ -134,6 +148,8 @@ impl DataSource { "notion" => Ok(Self::Notion), "meeting_notes" => Ok(Self::MeetingNotes), "drive_docs" => Ok(Self::DriveDocs), + "upload" => Ok(Self::Upload), + "web_page" => Ok(Self::WebPage), other => Err(format!("unknown data source: {other}")), } } @@ -151,6 +167,8 @@ impl DataSource { Self::Notion, Self::MeetingNotes, Self::DriveDocs, + Self::Upload, + Self::WebPage, ] } } diff --git a/crates/tinymemory-bus/src/chunks_tests.rs b/crates/tinymemory-bus/src/chunks_tests.rs index 49d8487..4aef7c8 100644 --- a/crates/tinymemory-bus/src/chunks_tests.rs +++ b/crates/tinymemory-bus/src/chunks_tests.rs @@ -96,7 +96,7 @@ fn data_source_round_trip() { #[test] fn data_source_has_all_variants() { - assert_eq!(DataSource::all().len(), 9); + assert_eq!(DataSource::all().len(), 11); } #[test] @@ -108,7 +108,7 @@ fn data_source_kind_mapping() { for ds in [Gmail, OtherEmail] { assert_eq!(ds.kind(), SourceKind::Email); } - for ds in [Notion, MeetingNotes, DriveDocs] { + for ds in [Notion, MeetingNotes, DriveDocs, Upload, WebPage] { assert_eq!(ds.kind(), SourceKind::Document); } } diff --git a/crates/tinymemory-bus/src/graph.rs b/crates/tinymemory-bus/src/graph.rs new file mode 100644 index 0000000..3cdd2ae --- /dev/null +++ b/crates/tinymemory-bus/src/graph.rs @@ -0,0 +1,420 @@ +//! Domain types for the **graph view**: a bounded, renderable slice of the +//! relation graph. +//! +//! The traits that produce these types live in `tinymemory-api`, which this +//! crate sits underneath and therefore cannot name — the references to +//! `MemoryGraph` and `MemoryTree` below are deliberately unlinked for that +//! reason, not by oversight. +//! +//! `MemoryGraph::relations` answers "which edges match this filter" and returns +//! a flat list. That is the right shape for a query and the wrong shape for a +//! *view*: a caller that wants to draw a graph, or hand one to an agent, needs +//! the node set as well as the edge set, needs to know how far each node sits +//! from where it started, and needs the answer bounded so an over-connected hub +//! cannot return the whole store. +//! +//! This module is the graph counterpart of [`crate::tree`], and +//! `MemoryGraph::graph_view` is the counterpart of `MemoryTree::drill_down`: +//! one call returns a node together with its surroundings, already assembled, +//! so navigation is a sequence of view calls rather than a client-side join. +//! +//! ## What is a driver concern and what is not +//! +//! Traversal *strategy* is a driver concern — an engine with a native +//! multi-hop traversal should use it. Traversal *bounds* are not: they are on +//! [`GraphViewQuery`], because the caller is the only party that knows how big +//! an answer it can render. A driver must honour them and must set +//! [`GraphView::truncated`] when it drops anything. + +use serde::{Deserialize, Serialize}; + +use crate::types::GraphRelationRecord; + +/// Which direction a traversal follows out of a node. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum GraphDirection { + /// Follow edges where the node is the subject. Wire string `"out"`. + #[default] + Out, + /// Follow edges where the node is the object. Wire string `"in"`. + In, + /// Follow edges in both directions. Wire string `"both"`. + Both, +} + +impl GraphDirection { + /// Whether outbound edges are followed. + pub fn follows_out(self) -> bool { + matches!(self, Self::Out | Self::Both) + } + + /// Whether inbound edges are followed. + pub fn follows_in(self) -> bool { + matches!(self, Self::In | Self::Both) + } +} + +/// What a node in a view stands for. +/// +/// The default traversal cannot infer this — it only ever sees edge endpoint +/// strings — so it reports [`GraphNodeKind::Unknown`]. A driver whose store +/// knows the answer should populate it, because a renderer that has to guess +/// from the id guesses differently from every other renderer. +#[derive(Debug, Clone, PartialEq, Eq, Default, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum GraphNodeKind { + /// Kind not reported by the driver. Wire string `"unknown"`. + #[default] + Unknown, + /// An extracted entity — a person, place, organisation, concept. Wire + /// string `"entity"`. + Entity, + /// A whole stored document. Wire string `"document"`. + Document, + /// A single chunk of a document. Wire string `"chunk"`. + Chunk, + /// A summary-tree node. Wire string `"tree_node"`. + TreeNode, + /// A key/value record. Wire string `"kv"`. + Kv, + /// A driver-specific kind, carried verbatim. + Other(String), +} + +/// One node in a rendered [`GraphView`]. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct GraphNode { + /// Stable node identifier; matches the `subject`/`object` strings on + /// [`GraphEdge`]. + pub id: String, + /// Human-readable label. Falls back to [`Self::id`] when the driver has + /// nothing better. + pub label: String, + /// What the node stands for, when the driver knows. + #[serde(default)] + pub kind: GraphNodeKind, + /// Hops from the nearest seed. Seeds are `0`. + pub depth: u32, + /// Edges incident to this node **within this view**. Deliberately not the + /// node's degree in the whole store: a bounded view cannot see that, and + /// reporting a number that changes with the bounds would be worse than + /// reporting a number that is honestly local. + pub degree: u32, + /// Arbitrary structured attributes attached to the node. + #[serde(default)] + pub attrs: serde_json::Value, +} + +impl GraphNode { + /// A node with no attributes, no known kind, and its id as its label. + pub fn bare(id: impl Into, depth: u32) -> Self { + let id = id.into(); + Self { + label: id.clone(), + id, + kind: GraphNodeKind::Unknown, + depth, + degree: 0, + attrs: serde_json::Value::Null, + } + } +} + +/// One edge in a rendered [`GraphView`]. +/// +/// A projection of [`GraphRelationRecord`] rather than the record itself: a +/// view repeats the namespace once on the [`GraphView`] instead of once per +/// edge, and carries a derived [`Self::weight`] a renderer can size a line by. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct GraphEdge { + /// Edge subject (head node id). + pub subject: String, + /// Relation type linking subject to object. + pub predicate: String, + /// Edge object (tail node id). + pub object: String, + /// Number of independent observations supporting this edge. + pub evidence_count: u32, + /// Relative confidence in `0.0..=1.0`, derived from + /// [`Self::evidence_count`] by [`edge_weight`]. + pub weight: f64, + /// Last-update time as a Unix timestamp (seconds). + pub updated_at: f64, + /// Documents that contributed evidence for this edge. + #[serde(default)] + pub document_ids: Vec, + /// Chunks that contributed evidence for this edge. + #[serde(default)] + pub chunk_ids: Vec, + /// Arbitrary structured attributes attached to the edge. + #[serde(default)] + pub attrs: serde_json::Value, +} + +impl From for GraphEdge { + fn from(record: GraphRelationRecord) -> Self { + Self { + weight: edge_weight(record.evidence_count), + subject: record.subject, + predicate: record.predicate, + object: record.object, + evidence_count: record.evidence_count, + updated_at: record.updated_at, + document_ids: record.document_ids, + chunk_ids: record.chunk_ids, + attrs: record.attrs, + } + } +} + +impl GraphEdge { + /// The `(subject, predicate, object)` triple that identifies this edge. + /// + /// The same key `MemoryGraph::put_relation` upserts by, + /// so deduplicating a view by it cannot merge two edges the store holds + /// separately. + pub fn key(&self) -> (&str, &str, &str) { + (&self.subject, &self.predicate, &self.object) + } +} + +/// Map an observation count onto a `0.0..=1.0` weight. +/// +/// Saturating rather than linear: the difference between one observation and +/// five is worth more than the difference between fifty and fifty-four, and a +/// linear scale would make every edge in a well-observed graph look identical. +/// An edge with no evidence at all still gets a non-zero weight, because it is +/// in the store and a renderer that drew it at zero width would hide it. +pub fn edge_weight(evidence_count: u32) -> f64 { + let n = f64::from(evidence_count); + (n / (n + 3.0)).mul_add(0.9, 0.1) +} + +/// Counters describing what a traversal actually did. +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub struct GraphViewStats { + /// Nodes in [`GraphView::nodes`]. + pub node_count: usize, + /// Edges in [`GraphView::edges`]. + pub edge_count: usize, + /// Greatest [`GraphNode::depth`] present, or `0` for an empty view. + pub max_depth: u32, + /// Distinct nodes that were reached but never expanded — either because + /// they sit one hop past [`GraphViewQuery::depth`] or because a bound was + /// hit. + /// + /// Non-zero does **not** imply [`GraphView::truncated`]: a traversal that + /// stops exactly where it was told to stop is complete, not truncated. + /// Read this as "the graph continues here" and `truncated` as "we could + /// not fit what you asked for". + pub frontier_remaining: usize, +} + +/// What a `MemoryGraph::graph_view` call asks for. +/// +/// Every bound has a default, so the cheapest useful call is +/// `GraphViewQuery::around("ada")` — the one-hop neighbourhood, capped. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct GraphViewQuery { + /// Namespace to read, or `None` for the global, namespace-less slice. + #[serde(default)] + pub namespace: Option, + /// Node ids to start from. Empty means "no particular starting point": + /// the driver returns a representative slice of the namespace instead of + /// traversing, which is what an overview screen wants. + #[serde(default)] + pub seeds: Vec, + /// How many hops to expand out of the seeds. `0` returns the seeds and the + /// edges directly between them. + #[serde(default = "default_depth")] + pub depth: u32, + /// Hard ceiling on [`GraphView::nodes`]. + #[serde(default = "default_max_nodes")] + pub max_nodes: usize, + /// Hard ceiling on [`GraphView::edges`]. + #[serde(default = "default_max_edges")] + pub max_edges: usize, + /// Restrict to these relation types. Empty means every predicate. + #[serde(default)] + pub predicates: Vec, + /// Which way to follow edges out of a node. + #[serde(default)] + pub direction: GraphDirection, +} + +fn default_depth() -> u32 { + 1 +} + +fn default_max_nodes() -> usize { + 256 +} + +fn default_max_edges() -> usize { + 512 +} + +impl Default for GraphViewQuery { + fn default() -> Self { + Self { + namespace: None, + seeds: Vec::new(), + depth: default_depth(), + max_nodes: default_max_nodes(), + max_edges: default_max_edges(), + predicates: Vec::new(), + direction: GraphDirection::default(), + } + } +} + +impl GraphViewQuery { + /// The one-hop neighbourhood of a single node, with default bounds. + pub fn around(seed: impl Into) -> Self { + Self { + seeds: vec![seed.into()], + ..Self::default() + } + } + + /// An unseeded overview of one namespace, with default bounds. + pub fn overview(namespace: impl Into) -> Self { + Self { + namespace: Some(namespace.into()), + ..Self::default() + } + } + + /// Scope this query to `namespace`. + #[must_use] + pub fn in_namespace(mut self, namespace: impl Into) -> Self { + self.namespace = Some(namespace.into()); + self + } + + /// Expand `depth` hops out of the seeds. + #[must_use] + pub fn with_depth(mut self, depth: u32) -> Self { + self.depth = depth; + self + } + + /// Follow edges in `direction`. + #[must_use] + pub fn with_direction(mut self, direction: GraphDirection) -> Self { + self.direction = direction; + self + } + + /// Restrict the traversal to these relation types. + #[must_use] + pub fn with_predicates(mut self, predicates: Vec) -> Self { + self.predicates = predicates; + self + } + + /// Cap the view at `max_nodes` nodes and `max_edges` edges. + #[must_use] + pub fn with_bounds(mut self, max_nodes: usize, max_edges: usize) -> Self { + self.max_nodes = max_nodes; + self.max_edges = max_edges; + self + } + + /// Whether `predicate` passes this query's predicate filter. + pub fn accepts_predicate(&self, predicate: &str) -> bool { + self.predicates.is_empty() || self.predicates.iter().any(|p| p == predicate) + } +} + +/// A bounded, self-contained slice of the relation graph. +/// +/// Self-contained in the sense that matters to a renderer: every id named by +/// an edge in [`Self::edges`] is present in [`Self::nodes`]. A driver that +/// cannot honour that must drop the edge rather than emit a dangling one. +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub struct GraphView { + /// Namespace the view was read from, or `None` for the global slice. + #[serde(default)] + pub namespace: Option, + /// The seeds the traversal started from, echoed back verbatim. + /// + /// Echoed rather than filtered to the ones that exist: "this id has no + /// edges" and "this id is not in the store" are different facts, and a + /// traversal over an edge list cannot tell them apart. A seed that is + /// absent from the store still appears in [`Self::nodes`] with a degree of + /// zero, so a renderer draws the question the caller asked. + #[serde(default)] + pub seeds: Vec, + /// Every node reachable within the query's bounds. + pub nodes: Vec, + /// Every edge between two nodes in [`Self::nodes`]. + pub edges: Vec, + /// True when a bound was hit and the store holds more than is shown. + /// + /// Load-bearing: without it an empty-looking neighbourhood is + /// indistinguishable from a truncated one, and a caller would stop paging. + #[serde(default)] + pub truncated: bool, + /// Counters describing what the traversal did. + #[serde(default)] + pub stats: GraphViewStats, +} + +impl GraphView { + /// An empty view of one namespace. + pub fn empty(namespace: Option) -> Self { + Self { + namespace, + ..Self::default() + } + } + + /// Recompute [`Self::stats`] and every [`GraphNode::degree`] from the + /// current node and edge sets. + /// + /// Call this after assembling a view by hand; the default traversal already + /// does. + pub fn recompute_stats(&mut self) { + for node in &mut self.nodes { + node.degree = 0; + } + for edge in &self.edges { + for node in &mut self.nodes { + if node.id == edge.subject || node.id == edge.object { + node.degree = node.degree.saturating_add(1); + } + } + } + self.stats.node_count = self.nodes.len(); + self.stats.edge_count = self.edges.len(); + self.stats.max_depth = self.nodes.iter().map(|n| n.depth).max().unwrap_or(0); + } + + /// Drop every edge whose endpoints are not both in [`Self::nodes`]. + /// + /// The invariant this type promises, enforced. Returns how many edges were + /// dropped so a caller can decide whether that counts as truncation. + pub fn prune_dangling_edges(&mut self) -> usize { + let before = self.edges.len(); + let ids: std::collections::HashSet<&str> = + self.nodes.iter().map(|n| n.id.as_str()).collect(); + let keep: Vec = self + .edges + .iter() + .map(|e| ids.contains(e.subject.as_str()) && ids.contains(e.object.as_str())) + .collect(); + let mut index = 0; + self.edges.retain(|_| { + let keep = keep[index]; + index += 1; + keep + }); + before - self.edges.len() + } +} + +#[cfg(test)] +#[path = "graph_tests.rs"] +mod tests; diff --git a/crates/tinymemory-bus/src/graph_tests.rs b/crates/tinymemory-bus/src/graph_tests.rs new file mode 100644 index 0000000..8e1f735 --- /dev/null +++ b/crates/tinymemory-bus/src/graph_tests.rs @@ -0,0 +1,237 @@ +//! Tests for the bounded graph-view model. + +// A failed assertion in a test is a panic either way; `unwrap`/`expect` here say +// what the invariant was. Same allowance every other test module in this crate +// takes. +#![allow(clippy::expect_used, clippy::unwrap_used, clippy::panic)] + +use super::*; + +fn edge(subject: &str, predicate: &str, object: &str) -> GraphEdge { + GraphEdge { + subject: subject.to_string(), + predicate: predicate.to_string(), + object: object.to_string(), + evidence_count: 1, + weight: edge_weight(1), + updated_at: 0.0, + document_ids: Vec::new(), + chunk_ids: Vec::new(), + attrs: serde_json::Value::Null, + } +} + +#[test] +fn direction_follows_the_sides_it_names() { + assert!(GraphDirection::Out.follows_out()); + assert!(!GraphDirection::Out.follows_in()); + assert!(GraphDirection::In.follows_in()); + assert!(!GraphDirection::In.follows_out()); + assert!(GraphDirection::Both.follows_out()); + assert!(GraphDirection::Both.follows_in()); +} + +#[test] +fn direction_defaults_to_outbound() { + assert_eq!(GraphDirection::default(), GraphDirection::Out); +} + +#[test] +fn edge_weight_saturates_and_never_reaches_zero() { + assert!(edge_weight(0) > 0.0); + assert!(edge_weight(1) > edge_weight(0)); + assert!(edge_weight(50) < 1.0); + // Saturating, not linear: the first observations are worth far more than + // the fiftieth. + assert!(edge_weight(1) - edge_weight(0) > edge_weight(50) - edge_weight(49)); +} + +#[test] +fn edge_projects_a_relation_record_and_derives_its_weight() { + let record = GraphRelationRecord { + namespace: Some("conversation:thread-1".into()), + subject: "ada".into(), + predicate: "works_with".into(), + object: "charles".into(), + attrs: serde_json::json!({ "since": 1843 }), + updated_at: 12.5, + evidence_count: 3, + order_index: None, + document_ids: vec!["doc-1".into()], + chunk_ids: vec!["chunk-1".into()], + }; + let edge = GraphEdge::from(record); + assert_eq!(edge.key(), ("ada", "works_with", "charles")); + assert_eq!(edge.evidence_count, 3); + assert!((edge.weight - edge_weight(3)).abs() < f64::EPSILON); + assert_eq!(edge.document_ids, vec!["doc-1".to_string()]); +} + +#[test] +fn query_defaults_are_the_cheapest_useful_call() { + let query = GraphViewQuery::around("ada"); + assert_eq!(query.seeds, vec!["ada".to_string()]); + assert_eq!(query.depth, 1); + assert_eq!(query.direction, GraphDirection::Out); + assert!(query.namespace.is_none()); + assert!(query.predicates.is_empty()); +} + +#[test] +fn query_builders_compose() { + let query = GraphViewQuery::around("ada") + .in_namespace("document:papers") + .with_depth(3) + .with_direction(GraphDirection::Both) + .with_predicates(vec!["cites".into()]) + .with_bounds(10, 20); + assert_eq!(query.namespace.as_deref(), Some("document:papers")); + assert_eq!(query.depth, 3); + assert_eq!(query.direction, GraphDirection::Both); + assert_eq!(query.max_nodes, 10); + assert_eq!(query.max_edges, 20); +} + +#[test] +fn an_empty_predicate_filter_accepts_everything() { + let query = GraphViewQuery::default(); + assert!(query.accepts_predicate("cites")); + assert!(query.accepts_predicate("anything")); +} + +#[test] +fn a_predicate_filter_rejects_what_it_does_not_name() { + let query = GraphViewQuery::default().with_predicates(vec!["cites".into()]); + assert!(query.accepts_predicate("cites")); + assert!(!query.accepts_predicate("works_with")); +} + +#[test] +fn overview_scopes_to_a_namespace_without_seeding() { + let query = GraphViewQuery::overview("learning:rust"); + assert_eq!(query.namespace.as_deref(), Some("learning:rust")); + assert!(query.seeds.is_empty()); +} + +#[test] +fn recompute_stats_counts_degrees_within_the_view() { + let mut view = GraphView { + nodes: vec![ + GraphNode::bare("ada", 0), + GraphNode::bare("charles", 1), + GraphNode::bare("lovelace", 1), + ], + edges: vec![ + edge("ada", "works_with", "charles"), + edge("ada", "known_as", "lovelace"), + ], + ..GraphView::default() + }; + view.recompute_stats(); + assert_eq!(view.stats.node_count, 3); + assert_eq!(view.stats.edge_count, 2); + assert_eq!(view.stats.max_depth, 1); + assert_eq!(view.nodes[0].degree, 2); + assert_eq!(view.nodes[1].degree, 1); + assert_eq!(view.nodes[2].degree, 1); +} + +#[test] +fn recompute_stats_on_an_empty_view_reports_zero_depth() { + let mut view = GraphView::empty(Some("conversation:thread-1".into())); + view.recompute_stats(); + assert_eq!(view.stats.max_depth, 0); + assert_eq!(view.stats.node_count, 0); + assert_eq!(view.namespace.as_deref(), Some("conversation:thread-1")); +} + +#[test] +fn prune_dangling_edges_enforces_the_self_contained_invariant() { + let mut view = GraphView { + nodes: vec![GraphNode::bare("ada", 0), GraphNode::bare("charles", 1)], + edges: vec![ + edge("ada", "works_with", "charles"), + edge("ada", "cites", "absent"), + edge("absent", "cites", "ada"), + ], + ..GraphView::default() + }; + assert_eq!(view.prune_dangling_edges(), 2); + assert_eq!(view.edges.len(), 1); + assert_eq!(view.edges[0].key(), ("ada", "works_with", "charles")); +} + +#[test] +fn prune_dangling_edges_keeps_a_clean_view_untouched() { + let mut view = GraphView { + nodes: vec![GraphNode::bare("ada", 0), GraphNode::bare("charles", 1)], + edges: vec![edge("ada", "works_with", "charles")], + ..GraphView::default() + }; + assert_eq!(view.prune_dangling_edges(), 0); + assert_eq!(view.edges.len(), 1); +} + +#[test] +fn a_bare_node_labels_itself_by_its_id() { + let node = GraphNode::bare("ada", 2); + assert_eq!(node.label, "ada"); + assert_eq!(node.depth, 2); + assert_eq!(node.degree, 0); + assert_eq!(node.kind, GraphNodeKind::Unknown); +} + +#[test] +fn node_kind_round_trips_through_its_wire_strings() { + for (kind, wire) in [ + (GraphNodeKind::Unknown, "\"unknown\""), + (GraphNodeKind::Entity, "\"entity\""), + (GraphNodeKind::Document, "\"document\""), + (GraphNodeKind::Chunk, "\"chunk\""), + (GraphNodeKind::TreeNode, "\"tree_node\""), + (GraphNodeKind::Kv, "\"kv\""), + ] { + assert_eq!(serde_json::to_string(&kind).unwrap(), wire); + assert_eq!( + serde_json::from_str::(wire).unwrap(), + kind, + "round trip for {wire}" + ); + } +} + +#[test] +fn a_driver_specific_node_kind_is_carried_verbatim() { + let kind = GraphNodeKind::Other("commit".into()); + let wire = serde_json::to_string(&kind).unwrap(); + assert_eq!(serde_json::from_str::(&wire).unwrap(), kind); +} + +#[test] +fn a_query_deserializes_from_its_bounds_alone() { + let query: GraphViewQuery = serde_json::from_str(r#"{"seeds":["ada"]}"#).unwrap(); + assert_eq!(query.depth, 1); + assert_eq!(query.max_nodes, 256); + assert_eq!(query.max_edges, 512); + assert_eq!(query.direction, GraphDirection::Out); +} + +#[test] +fn a_view_round_trips_through_json() { + let mut view = GraphView { + namespace: Some("learning:rust".into()), + seeds: vec!["ada".into()], + nodes: vec![GraphNode::bare("ada", 0), GraphNode::bare("charles", 1)], + edges: vec![edge("ada", "works_with", "charles")], + truncated: true, + ..GraphView::default() + }; + view.recompute_stats(); + let wire = serde_json::to_string(&view).unwrap(); + let decoded: GraphView = serde_json::from_str(&wire).unwrap(); + assert_eq!(decoded.nodes.len(), 2); + assert_eq!(decoded.edges.len(), 1); + assert!(decoded.truncated); + assert_eq!(decoded.stats.edge_count, 1); + assert_eq!(decoded.seeds, vec!["ada".to_string()]); +} diff --git a/crates/tinymemory-bus/src/lib.rs b/crates/tinymemory-bus/src/lib.rs index 445f5e2..6e85297 100644 --- a/crates/tinymemory-bus/src/lib.rs +++ b/crates/tinymemory-bus/src/lib.rs @@ -12,6 +12,11 @@ //! - [`names`] — the bus name, the object path, and one constant per member. //! - [`types`], [`chunks`], [`recall`], [`tree`], [`goals`], [`tool_memory`], //! [`health`], [`capabilities`], [`evidence`] — the value vocabulary. +//! - [`graph`] — the bounded graph-view model ([`graph::GraphView`], +//! [`graph::GraphViewQuery`]), the graph counterpart of [`tree`]. +//! - [`namespace`] — the `
:` namespace convention +//! ([`namespace::Namespace`], [`namespace::MemorySection`]) and its +//! validator. //! - [`provider`] — the value types the capability families exchange. //! - [`error`] and [`wire`] — [`error::MemoryError`] and the name table it //! round-trips through when a driver is reached over a wire. @@ -71,8 +76,10 @@ pub mod chunks; pub mod error; pub mod evidence; pub mod goals; +pub mod graph; pub mod health; pub mod names; +pub mod namespace; pub mod provider; pub mod recall; pub mod tool_memory; diff --git a/crates/tinymemory-bus/src/namespace.rs b/crates/tinymemory-bus/src/namespace.rs new file mode 100644 index 0000000..04c942e --- /dev/null +++ b/crates/tinymemory-bus/src/namespace.rs @@ -0,0 +1,470 @@ +//! The namespace naming convention: `
:`. +//! +//! Namespaces are the only partitioning primitive this contract has, and every +//! family takes them as a bare `&str`. That is deliberate — a driver's +//! container vocabulary is its own, and a typed namespace threaded through +//! eighteen trait families would force every engine to agree on a shape none of +//! them share. What was missing was not a type in the signatures but a *shared +//! convention* for what goes in the string, so that "conversational memory", +//! "document memory", and "learnings" mean the same thing to every caller and +//! every engine instead of being three ad-hoc prefixes per host. +//! +//! ## The convention +//! +//! ```text +//! conversation:thread-8f21 a single conversation +//! document:handbook a document collection +//! learning:rust-async a topic the agent has learned about +//! entity:people an entity index slice +//! profile:default user-state facets +//! tool:github tool-scoped rules and outcomes +//! research-notes unsectioned — legacy, still valid +//! ``` +//! +//! The section is a closed vocabulary ([`MemorySection`]) plus an escape hatch +//! ([`MemorySection::Custom`]); the scope is free-form within the character +//! rules below. Splitting happens at the **first** colon, so a scope may +//! contain colons of its own and still round-trip. +//! +//! ## Unsectioned names stay valid +//! +//! A name with no recognised prefix parses as an unsectioned namespace and +//! renders back byte-for-byte. Every namespace written before this convention +//! existed keeps working, and nothing here silently rewrites a caller's string +//! — [`Namespace::parse`] is the only thing that interprets it, and it is a +//! caller's choice to run it. +//! +//! ## What this is not +//! +//! Not a permission boundary, and not a mapping table. A driver that cannot +//! store a colon should render a namespace with [`Namespace::flatten`] at its +//! own boundary; the contract does not decide that for it, because only the +//! driver knows what its store accepts. + +use std::fmt; + +use serde::{Deserialize, Serialize}; + +use crate::error::MemoryError; + +/// Longest namespace string this convention accepts, in bytes. +/// +/// Chosen to sit under the shortest limit among the engines this workspace +/// adapts rather than at any one engine's ceiling: a name that validates here +/// must be storable everywhere, otherwise validation would pass and the write +/// would fail, which is the worst of both. +pub const MAX_NAMESPACE_LEN: usize = 200; + +/// What kind of memory a namespace holds. +/// +/// A closed vocabulary so that two hosts, two engines, and an agent tool +/// description all name the same thing the same way — plus +/// [`MemorySection::Custom`], because a closed vocabulary with no escape hatch +/// gets worked around with prefixes nobody agrees on, which is the problem this +/// type exists to solve. +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum MemorySection { + /// Turn-by-turn conversational memory. Wire prefix `conversation`. + Conversation, + /// Whole documents and the chunks they were split into. Wire prefix + /// `document`. + Document, + /// Durable conclusions the agent drew and expects to reuse. Wire prefix + /// `learning`. + Learning, + /// The entity index — who and what the memory is about. Wire prefix + /// `entity`. + Entity, + /// User-state facets and preferences. Wire prefix `profile`. + Profile, + /// Tool-scoped rules and remembered outcomes. Wire prefix `tool`. + Tool, + /// Content pulled in from an external source. Wire prefix `source`. + Source, + /// A section this vocabulary does not name, carried verbatim. + Custom(String), +} + +impl MemorySection { + /// The wire prefix for this section. + pub fn as_str(&self) -> &str { + match self { + Self::Conversation => "conversation", + Self::Document => "document", + Self::Learning => "learning", + Self::Entity => "entity", + Self::Profile => "profile", + Self::Tool => "tool", + Self::Source => "source", + Self::Custom(name) => name, + } + } + + /// Every section in the closed vocabulary, in declaration order. + /// + /// [`MemorySection::Custom`] is absent by construction: it has no fixed + /// spelling to list. + pub fn known() -> [MemorySection; 7] { + [ + Self::Conversation, + Self::Document, + Self::Learning, + Self::Entity, + Self::Profile, + Self::Tool, + Self::Source, + ] + } + + /// Whether this section is one the vocabulary names. + pub fn is_known(&self) -> bool { + !matches!(self, Self::Custom(_)) + } + + /// Map a prefix onto a section, falling back to + /// [`MemorySection::Custom`]. + /// + /// Never fails: an unrecognised prefix is a custom section, not an error, + /// because a host that invents one is doing exactly what the escape hatch + /// is for. + pub fn from_prefix(prefix: &str) -> Self { + match prefix { + "conversation" => Self::Conversation, + "document" => Self::Document, + "learning" => Self::Learning, + "entity" => Self::Entity, + "profile" => Self::Profile, + "tool" => Self::Tool, + "source" => Self::Source, + other => Self::Custom(other.to_string()), + } + } +} + +impl fmt::Display for MemorySection { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(self.as_str()) + } +} + +/// A validated namespace name, optionally carrying a [`MemorySection`]. +/// +/// Construct one with a section helper ([`Namespace::conversation`], …) or by +/// parsing an existing string, then hand [`Namespace::as_str`] to any contract +/// method that takes a namespace. +/// +/// # Examples +/// +/// ``` +/// use tinymemory_bus::namespace::{MemorySection, Namespace}; +/// +/// let ns = Namespace::conversation("thread-8f21")?; +/// assert_eq!(ns.as_str(), "conversation:thread-8f21"); +/// assert_eq!(ns.section(), Some(&MemorySection::Conversation)); +/// assert_eq!(ns.scope(), "thread-8f21"); +/// +/// // A name written before the convention existed still parses, and renders +/// // back byte-for-byte. +/// let legacy = Namespace::parse("research-notes")?; +/// assert!(legacy.section().is_none()); +/// assert_eq!(legacy.as_str(), "research-notes"); +/// # Ok::<(), tinymemory_bus::error::MemoryError>(()) +/// ``` +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +#[serde(try_from = "String", into = "String")] +pub struct Namespace { + section: Option, + scope: String, + rendered: String, +} + +impl Namespace { + /// Parse and validate a namespace string. + /// + /// Splits at the first colon. A name with no colon, or whose prefix fails + /// the section character rules, is an unsectioned namespace rather than an + /// error. + /// + /// # Errors + /// + /// [`MemoryError::Invalid`] when the name is empty, longer than + /// [`MAX_NAMESPACE_LEN`], contains a character outside the allowed set, or + /// contains a `..` path-traversal segment. + pub fn parse(raw: &str) -> Result { + validate_name(raw)?; + match raw.split_once(':') { + Some((prefix, scope)) if is_valid_section(prefix) && !scope.is_empty() => Ok(Self { + section: Some(MemorySection::from_prefix(prefix)), + scope: scope.to_string(), + rendered: raw.to_string(), + }), + _ => Ok(Self { + section: None, + scope: raw.to_string(), + rendered: raw.to_string(), + }), + } + } + + /// Build a namespace in `section` with `scope`. + /// + /// # Errors + /// + /// [`MemoryError::Invalid`] when the section prefix or the resulting name + /// fails validation. + pub fn new(section: MemorySection, scope: impl Into) -> Result { + let scope = scope.into(); + // A `Custom` section that spells a known prefix must not become a + // second representation of the same rendered name: `from_prefix` maps + // it onto the matching known variant so `PartialEq`/`Hash` agree with + // `Namespace::parse` on the same string. + let section = MemorySection::from_prefix(section.as_str()); + if !is_valid_section(section.as_str()) { + return Err(MemoryError::Invalid(format!( + "namespace section {:?} must be lowercase letters, digits, '-' or '_'", + section.as_str() + ))); + } + if scope.is_empty() { + return Err(MemoryError::Invalid( + "namespace scope must not be empty".to_string(), + )); + } + let rendered = format!("{}:{scope}", section.as_str()); + validate_name(&rendered)?; + Ok(Self { + section: Some(section), + scope, + rendered, + }) + } + + /// A namespace with no section prefix. + /// + /// # Errors + /// + /// [`MemoryError::Invalid`] as [`Namespace::parse`]; additionally rejects a + /// name that *would* parse as sectioned, because silently accepting one + /// here would make `unsectioned("document:x").section()` return `Some`. + pub fn unsectioned(raw: impl Into) -> Result { + let raw = raw.into(); + let parsed = Self::parse(&raw)?; + if parsed.section.is_some() { + return Err(MemoryError::Invalid(format!( + "namespace {raw:?} carries a section prefix; use Namespace::parse" + ))); + } + Ok(parsed) + } + + /// Turn-by-turn conversational memory for one conversation. + /// + /// # Errors + /// + /// As [`Namespace::new`]. + pub fn conversation(scope: impl Into) -> Result { + Self::new(MemorySection::Conversation, scope) + } + + /// A document collection. + /// + /// # Errors + /// + /// As [`Namespace::new`]. + pub fn document(scope: impl Into) -> Result { + Self::new(MemorySection::Document, scope) + } + + /// Durable conclusions about one topic. + /// + /// # Errors + /// + /// As [`Namespace::new`]. + pub fn learning(scope: impl Into) -> Result { + Self::new(MemorySection::Learning, scope) + } + + /// A slice of the entity index. + /// + /// # Errors + /// + /// As [`Namespace::new`]. + pub fn entity(scope: impl Into) -> Result { + Self::new(MemorySection::Entity, scope) + } + + /// User-state facets. + /// + /// # Errors + /// + /// As [`Namespace::new`]. + pub fn profile(scope: impl Into) -> Result { + Self::new(MemorySection::Profile, scope) + } + + /// Tool-scoped rules and outcomes. + /// + /// # Errors + /// + /// As [`Namespace::new`]. + pub fn tool(scope: impl Into) -> Result { + Self::new(MemorySection::Tool, scope) + } + + /// Content pulled in from one external source. + /// + /// # Errors + /// + /// As [`Namespace::new`]. + pub fn source(scope: impl Into) -> Result { + Self::new(MemorySection::Source, scope) + } + + /// The section this namespace belongs to, or `None` when unsectioned. + pub fn section(&self) -> Option<&MemorySection> { + self.section.as_ref() + } + + /// The part after the section prefix — or the whole name when unsectioned. + pub fn scope(&self) -> &str { + &self.scope + } + + /// The canonical `
:` string to pass to a contract method. + pub fn as_str(&self) -> &str { + &self.rendered + } + + /// Whether this namespace carries a section prefix. + pub fn is_sectioned(&self) -> bool { + self.section.is_some() + } + + /// Whether this namespace is in `section`. + pub fn is_in(&self, section: &MemorySection) -> bool { + self.section.as_ref() == Some(section) + } + + /// Render with `separator` in place of the colon. + /// + /// For a store that cannot hold a colon in a container name. The result is + /// **not** parseable back into a [`Namespace`] unless `separator` is `":"` + /// — it is an output format for a driver boundary, not a second canonical + /// form. + /// + /// # Examples + /// + /// ``` + /// use tinymemory_bus::namespace::Namespace; + /// + /// let ns = Namespace::document("handbook")?; + /// assert_eq!(ns.flatten("__"), "document__handbook"); + /// # Ok::<(), tinymemory_bus::error::MemoryError>(()) + /// ``` + pub fn flatten(&self, separator: &str) -> String { + match &self.section { + Some(section) => format!("{}{separator}{}", section.as_str(), self.scope), + None => self.scope.clone(), + } + } +} + +impl fmt::Display for Namespace { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(&self.rendered) + } +} + +impl AsRef for Namespace { + fn as_ref(&self) -> &str { + &self.rendered + } +} + +impl std::str::FromStr for Namespace { + type Err = MemoryError; + + fn from_str(raw: &str) -> Result { + Self::parse(raw) + } +} + +impl TryFrom for Namespace { + type Error = MemoryError; + + fn try_from(raw: String) -> Result { + Self::parse(&raw) + } +} + +impl From for String { + fn from(namespace: Namespace) -> Self { + namespace.rendered + } +} + +/// Whether `prefix` may act as a section label. +/// +/// Deliberately narrower than the scope rules: a section is a vocabulary word, +/// so `Document` and `document` must not be two sections, and a prefix +/// containing a slash or a dot is far more likely to be the first segment of an +/// unsectioned path-shaped name than a section anyone meant. +fn is_valid_section(prefix: &str) -> bool { + !prefix.is_empty() + && prefix + .chars() + .all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || c == '-' || c == '_') +} + +/// Validate a whole namespace string against the character and length rules. +/// +/// # Errors +/// +/// [`MemoryError::Invalid`] naming the specific rule that failed, so a caller +/// can show the user which one rather than "invalid namespace". +pub fn validate_name(raw: &str) -> Result<(), MemoryError> { + if raw.is_empty() { + return Err(MemoryError::Invalid( + "namespace must not be empty".to_string(), + )); + } + if raw.len() > MAX_NAMESPACE_LEN { + return Err(MemoryError::Invalid(format!( + "namespace is {} bytes, over the {MAX_NAMESPACE_LEN}-byte limit", + raw.len() + ))); + } + if let Some(bad) = raw.chars().find(|c| !is_allowed_char(*c)) { + return Err(MemoryError::Invalid(format!( + "namespace contains disallowed character {bad:?}" + ))); + } + // Namespaces reach engines that map them onto directories. A traversal + // segment is rejected here rather than sanitised, because sanitising would + // silently change which container a write lands in. + if raw.split('/').any(|segment| segment == "..") { + return Err(MemoryError::Invalid( + "namespace must not contain a '..' segment".to_string(), + )); + } + if raw.starts_with('/') || raw.ends_with('/') { + return Err(MemoryError::Invalid( + "namespace must not start or end with '/'".to_string(), + )); + } + Ok(()) +} + +/// Whether `c` may appear anywhere in a namespace. +/// +/// ASCII only, and no whitespace: a namespace is a key that ends up in URLs, +/// file paths, and SQL parameters across several engines, and every character +/// outside this set is one of those engines' escaping problem. +fn is_allowed_char(c: char) -> bool { + c.is_ascii_alphanumeric() || matches!(c, '-' | '_' | '.' | '/' | ':' | '@' | '+') +} + +#[cfg(test)] +#[path = "namespace_tests.rs"] +mod tests; diff --git a/crates/tinymemory-bus/src/namespace_tests.rs b/crates/tinymemory-bus/src/namespace_tests.rs new file mode 100644 index 0000000..5de63f5 --- /dev/null +++ b/crates/tinymemory-bus/src/namespace_tests.rs @@ -0,0 +1,259 @@ +//! Tests for the `
:` namespace convention and its validator. + +// A failed assertion in a test is a panic either way; `unwrap`/`expect` here say +// what the invariant was. Same allowance every other test module in this crate +// takes. +#![allow(clippy::expect_used, clippy::unwrap_used, clippy::panic)] + +use super::*; + +#[test] +fn a_section_helper_renders_the_canonical_form() { + let ns = Namespace::conversation("thread-8f21").unwrap(); + assert_eq!(ns.as_str(), "conversation:thread-8f21"); + assert_eq!(ns.section(), Some(&MemorySection::Conversation)); + assert_eq!(ns.scope(), "thread-8f21"); + assert!(ns.is_sectioned()); +} + +#[test] +fn every_section_helper_uses_its_own_prefix() { + for (rendered, section) in [ + ( + Namespace::conversation("x").unwrap(), + MemorySection::Conversation, + ), + (Namespace::document("x").unwrap(), MemorySection::Document), + (Namespace::learning("x").unwrap(), MemorySection::Learning), + (Namespace::entity("x").unwrap(), MemorySection::Entity), + (Namespace::profile("x").unwrap(), MemorySection::Profile), + (Namespace::tool("x").unwrap(), MemorySection::Tool), + (Namespace::source("x").unwrap(), MemorySection::Source), + ] { + assert_eq!(rendered.as_str(), format!("{}:x", section.as_str())); + assert!(rendered.is_in(§ion)); + } +} + +#[test] +fn parsing_recovers_the_section_and_scope() { + let ns = Namespace::parse("learning:rust-async").unwrap(); + assert_eq!(ns.section(), Some(&MemorySection::Learning)); + assert_eq!(ns.scope(), "rust-async"); +} + +#[test] +fn a_scope_may_contain_colons_and_still_round_trips() { + let ns = Namespace::parse("document:acme:handbook:v2").unwrap(); + assert_eq!(ns.section(), Some(&MemorySection::Document)); + assert_eq!(ns.scope(), "acme:handbook:v2"); + assert_eq!( + Namespace::parse(ns.as_str()).unwrap().scope(), + "acme:handbook:v2" + ); +} + +#[test] +fn an_unrecognised_prefix_becomes_a_custom_section() { + let ns = Namespace::parse("audit:2026-q1").unwrap(); + assert_eq!( + ns.section(), + Some(&MemorySection::Custom("audit".to_string())) + ); + assert_eq!(ns.scope(), "2026-q1"); + assert!(!ns.section().unwrap().is_known()); +} + +#[test] +fn a_bare_name_parses_as_unsectioned_and_renders_verbatim() { + let ns = Namespace::parse("research-notes").unwrap(); + assert!(ns.section().is_none()); + assert!(!ns.is_sectioned()); + assert_eq!(ns.scope(), "research-notes"); + assert_eq!(ns.as_str(), "research-notes"); +} + +#[test] +fn a_path_shaped_legacy_name_stays_unsectioned() { + // The prefix rules exclude '/' and '.', so the first segment of a + // path-shaped name is not mistaken for a section. + let ns = Namespace::parse("projects/acme/notes").unwrap(); + assert!(ns.section().is_none()); + assert_eq!(ns.as_str(), "projects/acme/notes"); +} + +#[test] +fn an_uppercase_prefix_is_not_a_section() { + let ns = Namespace::parse("Document:handbook").unwrap(); + assert!( + ns.section().is_none(), + "a section vocabulary with two spellings is not a vocabulary" + ); + assert_eq!(ns.as_str(), "Document:handbook"); +} + +#[test] +fn a_trailing_colon_with_no_scope_is_not_a_section() { + let ns = Namespace::parse("document:").unwrap(); + assert!(ns.section().is_none()); + assert_eq!(ns.as_str(), "document:"); +} + +#[test] +fn unsectioned_rejects_a_name_that_carries_a_prefix() { + let error = Namespace::unsectioned("document:handbook").unwrap_err(); + assert!(matches!(error, MemoryError::Invalid(_)), "got {error:?}"); +} + +#[test] +fn unsectioned_accepts_a_bare_name() { + assert_eq!( + Namespace::unsectioned("research-notes").unwrap().as_str(), + "research-notes" + ); +} + +#[test] +fn an_empty_namespace_is_rejected() { + assert!(matches!( + Namespace::parse("").unwrap_err(), + MemoryError::Invalid(_) + )); +} + +#[test] +fn an_empty_scope_is_rejected_by_the_builders() { + assert!(matches!( + Namespace::conversation("").unwrap_err(), + MemoryError::Invalid(_) + )); +} + +#[test] +fn an_overlong_namespace_is_rejected() { + let long = "a".repeat(MAX_NAMESPACE_LEN + 1); + let error = Namespace::parse(&long).unwrap_err(); + assert!(error.to_string().contains("limit"), "got {error}"); +} + +#[test] +fn a_namespace_of_exactly_the_limit_is_accepted() { + let at_limit = "a".repeat(MAX_NAMESPACE_LEN); + assert!(Namespace::parse(&at_limit).is_ok()); +} + +#[test] +fn whitespace_and_control_characters_are_rejected() { + for bad in ["has space", "tab\there", "new\nline", "nul\0byte"] { + assert!( + matches!(Namespace::parse(bad), Err(MemoryError::Invalid(_))), + "{bad:?} should be rejected" + ); + } +} + +#[test] +fn a_traversal_segment_is_rejected_rather_than_sanitised() { + for bad in ["../etc", "a/../b", "document:a/../../b"] { + let error = Namespace::parse(bad).unwrap_err(); + assert!(error.to_string().contains(".."), "{bad:?} gave {error}"); + } +} + +#[test] +fn a_dotted_segment_that_is_not_traversal_is_allowed() { + assert!(Namespace::parse("document:v1.2.3").is_ok()); + assert!(Namespace::parse("a/..b/c").is_ok()); +} + +#[test] +fn a_leading_or_trailing_slash_is_rejected() { + assert!(Namespace::parse("/absolute").is_err()); + assert!(Namespace::parse("trailing/").is_err()); +} + +#[test] +fn a_custom_section_can_be_built_and_round_trips() { + let ns = Namespace::new(MemorySection::Custom("audit".into()), "2026-q1").unwrap(); + assert_eq!(ns.as_str(), "audit:2026-q1"); + assert_eq!(Namespace::parse(ns.as_str()).unwrap(), ns); +} + +#[test] +fn a_custom_section_with_an_invalid_prefix_is_rejected() { + let error = Namespace::new(MemorySection::Custom("Audit Log".into()), "x").unwrap_err(); + assert!(matches!(error, MemoryError::Invalid(_)), "got {error:?}"); +} + +#[test] +fn flatten_swaps_the_separator_for_a_store_that_cannot_hold_a_colon() { + let ns = Namespace::document("handbook").unwrap(); + assert_eq!(ns.flatten("__"), "document__handbook"); + assert_eq!(ns.flatten("/"), "document/handbook"); + assert_eq!(ns.flatten(":"), ns.as_str()); +} + +#[test] +fn flatten_leaves_an_unsectioned_name_alone() { + let ns = Namespace::unsectioned("research-notes").unwrap(); + assert_eq!(ns.flatten("__"), "research-notes"); +} + +#[test] +fn known_lists_the_closed_vocabulary_and_excludes_custom() { + let known = MemorySection::known(); + assert_eq!(known.len(), 7); + assert!(known.iter().all(MemorySection::is_known)); + assert!(known.contains(&MemorySection::Conversation)); + assert!(known.contains(&MemorySection::Learning)); +} + +#[test] +fn every_known_prefix_maps_back_to_its_own_section() { + for section in MemorySection::known() { + assert_eq!(MemorySection::from_prefix(section.as_str()), section); + } +} + +#[test] +fn a_namespace_serializes_as_its_canonical_string() { + let ns = Namespace::learning("rust-async").unwrap(); + assert_eq!( + serde_json::to_string(&ns).unwrap(), + "\"learning:rust-async\"" + ); + let decoded: Namespace = serde_json::from_str("\"learning:rust-async\"").unwrap(); + assert_eq!(decoded, ns); +} + +#[test] +fn deserializing_an_invalid_namespace_fails() { + assert!(serde_json::from_str::("\"has space\"").is_err()); +} + +#[test] +fn a_namespace_parses_through_from_str_and_displays_back() { + let ns: Namespace = "document:handbook".parse().unwrap(); + assert_eq!(ns.to_string(), "document:handbook"); + assert_eq!(ns.as_ref(), "document:handbook"); +} + +#[test] +fn is_in_distinguishes_sections() { + let ns = Namespace::document("handbook").unwrap(); + assert!(ns.is_in(&MemorySection::Document)); + assert!(!ns.is_in(&MemorySection::Conversation)); +} + +#[test] +fn validate_name_accepts_the_characters_engines_actually_need() { + for good in [ + "conversation:thread-8f21", + "user@example.com", + "a+b", + "projects/acme/notes", + "v1.2.3", + ] { + assert!(validate_name(good).is_ok(), "{good:?} should be allowed"); + } +} diff --git a/crates/tinymemory-documents/Cargo.toml b/crates/tinymemory-documents/Cargo.toml new file mode 100644 index 0000000..6ff2ab3 --- /dev/null +++ b/crates/tinymemory-documents/Cargo.toml @@ -0,0 +1,65 @@ +[package] +name = "tinymemory-documents" +version = "0.1.0" +edition = "2021" +rust-version = "1.96" +license = "MIT" +repository = "https://github.com/tinyhumansai/tinymemory" +description = "Document and URL intake for TinyMemory: sniff a format, convert it to markdown, put it in whichever engine is bound" +publish = false + +[dependencies] +# The contract. Everything this crate produces is handed to a `MemoryProvider`, +# and every error it returns is a `MemoryError`, so intake speaks the same +# language as the drivers it feeds. +tinymemory-api = { path = "../tinymemory-api" } +# `DocumentConverter` is an object-safe async trait: a host swaps the converter +# without this crate knowing which one it got. +async-trait = "0.1" +# `ConvertedDocument::metadata` is an open `Value`, and `MemoryDocuments` +# returns its listings as one. +serde_json = "1" +# The intake types are persisted by whichever engine receives them and cross +# the testing harness's HTTP boundary. +serde = { version = "1", features = ["derive"] } +# Ingested content carries an event time for tree placement. +chrono = { version = "0.4", features = ["serde"] } +# The URL path reuses the source readers' SSRF guard rather than growing a +# second one — see `fetch`. Optional because a host that only accepts uploads +# links no HTTP stack. +tinymemory-sources = { path = "../tinymemory-sources", optional = true } +# The fetch path needs the response's status, content type and body. Same +# default-features-off rustls configuration as `tinymemory-sources`, so the two +# do not pull in two TLS backends. +reqwest = { version = "0.12", default-features = false, features = ["rustls-tls"], optional = true } + +[dev-dependencies] +# The converter and ingest paths are async. +tokio = { version = "1", features = ["macros", "rt", "rt-multi-thread"] } + +[features] +# Nothing by default: uploads and conversion need no network. +default = [] +# The URL intake path, and with it the SSRF guard and an HTTP client. +network = ["dep:tinymemory-sources", "tinymemory-sources/network", "dep:reqwest"] + +[lints.rust] +unsafe_code = "forbid" +missing_docs = "warn" +missing_debug_implementations = "warn" +unreachable_pub = "warn" +rust_2018_idioms = { level = "warn", priority = -1 } + +[lints.clippy] +all = { level = "warn", priority = -1 } +unwrap_used = "warn" +expect_used = "warn" +panic = "warn" +todo = "warn" +unimplemented = "warn" +missing_errors_doc = "warn" +missing_panics_doc = "warn" + +[lints.rustdoc] +broken_intra_doc_links = "warn" +private_intra_doc_links = "warn" diff --git a/crates/tinymemory-documents/README.md b/crates/tinymemory-documents/README.md new file mode 100644 index 0000000..1c1eccc --- /dev/null +++ b/crates/tinymemory-documents/README.md @@ -0,0 +1,92 @@ +# tinymemory-documents + +Document and URL intake for TinyMemory: work out what a file is, turn it into +markdown, and put it in whichever engine is bound. + +## Why this is a crate and not a function + +Because it is three separable decisions, and only the host can make two of +them. + +**What the file is** is a detection problem with three unreliable signals. +`DocumentFormat::sniff` reads magic bytes first (the only signal a caller +cannot get wrong), then the declared MIME type, then the filename, then falls +back to looking at the bytes. A browser that sends +`application/octet-stream` for a PDF still gets a PDF. + +**What it becomes** is markdown, always. It is the one representation that +survives every hop the content makes afterwards: chunkers split on its +headings, embedders read it as prose, and a human can read the stored copy +without a renderer. Converting to plain text would throw away the structure a +chunker needs; keeping the original bytes would push the problem onto every +engine separately. + +**Where it lands** depends on the driver, and the contract offers three +answers. `DocumentIntake` picks the best one the bound driver actually +implements and reports which it used — so the same upload behaves the same way +against TinyCortex, Mem0 and a mandatory-only driver, and a host can see when a +document did *not* get chunked. + +## Public surface + +| Item | What it is | +| --- | --- | +| `DocumentFormat` | markdown / plain text / HTML / PDF / DOCX / unknown, and `sniff` | +| `RawDocument` | bytes plus filename, declared MIME, and origin | +| `ConvertedDocument` | markdown plus title, source format, and converter metadata | +| `DocumentConverter` | the conversion seam — object-safe and async | +| `NativeConverter` | text, markdown and HTML, with no dependencies | +| `ConverterChain` | converters in priority order; first claim wins | +| `DocumentIntake` | conversion plus the write, against a bound `MemoryProvider` | +| `IntakeRequest` / `IntakeReceipt` | where a document should go, and what happened | +| `fetch::fetch_url` | one URL, once, behind the shared SSRF guard (`network`) | +| `html::to_markdown` | the structural HTML converter, usable on its own | + +## PDF and DOCX + +Not handled here. Both need a real extractor, and which one a deployment uses +is its own decision — an in-process crate, a TinyBus module, a service. So +conversion is a trait a host binds: + +```rust,ignore +let chain = ConverterChain::default().prepend(Box::new(MyPdfConverter)); +``` + +A format nothing in the chain claims is rejected with an error naming the +format and listing what the build *can* convert. It is never a silent empty +document — storing an empty body loses the upload while looking like a success. + +## Routing rules + +| Driver implements | Route | What happens | +| --- | --- | --- | +| `MemoryIngest` | `ingest` | the driver chunks and embeds the markdown | +| `MemoryDocuments` | `documents` | stored whole, queryable by the document tier | +| neither | `core` | one entry through the mandatory family | + +`DocumentIntake::route()` answers this without performing a write, so a host can +tell a user what will happen before it happens. + +## Operational constraints + +- **Taint is passed through, never assigned.** The contract is explicit that + the host stamps provenance. `IntakeRequest` defaults to `ExternalSync` — the + closed default — and a host that knows better sets it. +- **Size is capped before conversion.** `MAX_DOCUMENT_BYTES` (32 MiB) is + checked on the raw bytes, because a document that would not fit is one this + process should never finish decoding. +- **Keys are derived and stable.** The same URL or filename always produces the + same key, so re-ingesting a document upserts instead of storing a second copy. + URLs lose their scheme first, so `http://` and `https://` fetches of one page + do not diverge. +- **The namespace is validated first.** Against the `tinymemory_api::namespace` + convention, before any write, so a malformed namespace fails at the boundary + rather than inside an engine. +- **URL fetches reuse the source readers' SSRF guard.** Two SSRF + implementations in one workspace means one of them is the weaker, and nobody + knows which. + +## Features + +- `network` — `fetch::fetch_url`. Off by default; a host that only accepts + uploads links no HTTP stack. diff --git a/crates/tinymemory-documents/src/convert/mod.rs b/crates/tinymemory-documents/src/convert/mod.rs new file mode 100644 index 0000000..7345a7c --- /dev/null +++ b/crates/tinymemory-documents/src/convert/mod.rs @@ -0,0 +1,239 @@ +//! The converter seam: bytes in, markdown out. +//! +//! Markdown is the intermediate form for everything the memory layer ingests, +//! because it is the one format that survives every hop the content makes — +//! chunkers split on its headings, embedders read it as prose, agents are +//! trained on it, and a human can read the stored copy without a renderer. +//! +//! ## Why this is a trait +//! +//! Text, markdown and HTML convert with no dependencies, and this crate does +//! them ([`NativeConverter`]). PDF and DOCX do not: they need a real extractor, +//! and which extractor a deployment uses is its own decision — an in-process +//! crate, a TinyBus module, a service. So conversion is a trait a host binds +//! rather than a fixed table, and [`ConverterChain`] composes the native +//! converter with whatever the host brings. +//! +//! A format with no converter is [`MemoryError::Invalid`] naming the format, +//! never a silent empty document. + +mod types; + +use async_trait::async_trait; + +use tinymemory_api::error::MemoryError; + +use crate::error::Result; +use crate::format::DocumentFormat; +use crate::html; + +pub use types::{ConvertedDocument, RawDocument, MAX_DOCUMENT_BYTES}; + +/// Turns a document of some format into markdown. +/// +/// Object-safe and async: a converter that shells out to a bus module or an +/// HTTP service is as bindable as one that runs in-process. +#[async_trait] +pub trait DocumentConverter: Send + Sync { + /// A short name for this converter, for diagnostics and metadata. + fn name(&self) -> &str; + + /// Whether this converter handles `format`. + /// + /// Consulted before [`Self::convert`] so a chain can skip a converter + /// without paying for a failed attempt. + fn supports(&self, format: DocumentFormat) -> bool; + + /// Convert `document` to markdown. + /// + /// # Errors + /// + /// [`MemoryError::Invalid`] for a format this converter does not handle or + /// a document it cannot decode, [`MemoryError::BudgetExceeded`] for one + /// over [`MAX_DOCUMENT_BYTES`]. + async fn convert(&self, document: &RawDocument) -> Result; +} + +/// Reject a document that is empty or over the size cap. +/// +/// Every converter should call this first. Free-standing rather than a default +/// method so a converter that overrides nothing else still cannot forget it by +/// implementing `convert` from scratch — the check is one call, and a missing +/// call is visible in review. +/// +/// # Errors +/// +/// [`MemoryError::Invalid`] for an empty body, [`MemoryError::BudgetExceeded`] +/// for one over [`MAX_DOCUMENT_BYTES`]. +pub fn check_size(document: &RawDocument) -> Result<()> { + if document.bytes.is_empty() { + return Err(MemoryError::Invalid("document body is empty".to_string())); + } + if document.bytes.len() > MAX_DOCUMENT_BYTES { + return Err(MemoryError::BudgetExceeded(format!( + "document is {} bytes, over the {MAX_DOCUMENT_BYTES}-byte intake limit", + document.bytes.len() + ))); + } + Ok(()) +} + +/// The formats this crate converts without help: markdown, plain text, HTML. +/// +/// Everything it handles is already text, so the whole implementation is +/// decoding plus, for HTML, [`crate::html::to_markdown`]. PDF and DOCX are +/// deliberately absent — see the module docs. +#[derive(Debug, Default, Clone, Copy)] +pub struct NativeConverter; + +#[async_trait] +impl DocumentConverter for NativeConverter { + fn name(&self) -> &str { + "native" + } + + fn supports(&self, format: DocumentFormat) -> bool { + format.is_textual() + } + + async fn convert(&self, document: &RawDocument) -> Result { + check_size(document)?; + let format = document.format(); + if !self.supports(format) { + return Err(MemoryError::Invalid(format!( + "the native converter does not handle {format}; bind a converter that does" + ))); + } + let text = std::str::from_utf8(&document.bytes).map_err(|error| { + MemoryError::Invalid(format!("document is not valid utf-8: {error}")) + })?; + + let (markdown, title) = match format { + DocumentFormat::Html => (html::to_markdown(text), html::extract_title(text)), + // Plain text is valid markdown. Rewriting it — escaping, wrapping, + // guessing at headings — would change the user's words, which is + // worse than storing prose that happens to lack markup. + DocumentFormat::Markdown | DocumentFormat::PlainText => (text.to_string(), None), + other => { + return Err(MemoryError::Invalid(format!( + "the native converter does not handle {other}" + ))) + } + }; + + if markdown.trim().is_empty() { + return Err(MemoryError::Invalid(format!( + "converting {format} produced no text" + ))); + } + + Ok( + ConvertedDocument::new(markdown, format, document.bytes.len()) + .with_title(title) + .with_metadata(serde_json::json!({ "converter": self.name() })), + ) + } +} + +/// Tries each converter in order and uses the first that claims the format. +/// +/// Order is priority: a host that wants its own HTML handling puts it before +/// [`NativeConverter`]. The chain does not fall through on failure — a +/// converter that claims a format and then fails has found a real problem, and +/// retrying it against a converter that already declined would turn a precise +/// error into a vague one. +pub struct ConverterChain { + converters: Vec>, +} + +impl std::fmt::Debug for ConverterChain { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ConverterChain") + .field( + "converters", + &self.converters.iter().map(|c| c.name()).collect::>(), + ) + .finish() + } +} + +impl Default for ConverterChain { + /// A chain holding only [`NativeConverter`] — text, markdown and HTML, and + /// a clear error for anything else. + fn default() -> Self { + Self::new(vec![Box::new(NativeConverter)]) + } +} + +impl ConverterChain { + /// Build a chain from converters in priority order. + pub fn new(converters: Vec>) -> Self { + Self { converters } + } + + /// Put `converter` ahead of everything already in the chain. + #[must_use] + pub fn prepend(mut self, converter: Box) -> Self { + self.converters.insert(0, converter); + self + } + + /// Put `converter` behind everything already in the chain. + #[must_use] + pub fn push(mut self, converter: Box) -> Self { + self.converters.push(converter); + self + } + + /// Every format some converter in this chain claims. + pub fn supported_formats(&self) -> Vec { + [ + DocumentFormat::Markdown, + DocumentFormat::PlainText, + DocumentFormat::Html, + DocumentFormat::Pdf, + DocumentFormat::Docx, + ] + .into_iter() + .filter(|format| self.supports(*format)) + .collect() + } +} + +#[async_trait] +impl DocumentConverter for ConverterChain { + fn name(&self) -> &str { + "chain" + } + + fn supports(&self, format: DocumentFormat) -> bool { + self.converters.iter().any(|c| c.supports(format)) + } + + async fn convert(&self, document: &RawDocument) -> Result { + check_size(document)?; + let format = document.format(); + match self.converters.iter().find(|c| c.supports(format)) { + Some(converter) => converter.convert(document).await, + None => Err(MemoryError::Invalid(format!( + "no converter handles {format}; this build converts {}", + describe(&self.supported_formats()) + ))), + } + } +} + +/// Render a format list for an error message. +fn describe(formats: &[DocumentFormat]) -> String { + if formats.is_empty() { + return "nothing".to_string(); + } + formats + .iter() + .map(DocumentFormat::to_string) + .collect::>() + .join(", ") +} + +#[cfg(test)] +mod test; diff --git a/crates/tinymemory-documents/src/convert/test.rs b/crates/tinymemory-documents/src/convert/test.rs new file mode 100644 index 0000000..239fe4f --- /dev/null +++ b/crates/tinymemory-documents/src/convert/test.rs @@ -0,0 +1,302 @@ +//! Tests for the converter seam. + +use super::*; + +fn raw(bytes: &str, mime: &str) -> RawDocument { + RawDocument::new(bytes).with_mime(mime) +} + +/// A converter that claims one format and always succeeds, for chain-ordering +/// tests. +struct Stub { + name: &'static str, + format: DocumentFormat, +} + +#[async_trait] +impl DocumentConverter for Stub { + fn name(&self) -> &str { + self.name + } + + fn supports(&self, format: DocumentFormat) -> bool { + format == self.format + } + + async fn convert(&self, document: &RawDocument) -> Result { + check_size(document)?; + Ok(ConvertedDocument::new( + format!("from {}", self.name), + self.format, + document.bytes.len(), + ) + .with_title(Some("Stubbed".to_string()))) + } +} + +/// A converter that claims a format and then fails, to prove a chain does not +/// silently fall through to a converter that already declined. +struct Failing; + +#[async_trait] +impl DocumentConverter for Failing { + fn name(&self) -> &str { + "failing" + } + + fn supports(&self, format: DocumentFormat) -> bool { + format == DocumentFormat::Pdf + } + + async fn convert(&self, _document: &RawDocument) -> Result { + Err(MemoryError::Backend("extractor crashed".to_string())) + } +} + +#[tokio::test] +async fn markdown_passes_through_untouched() { + let source = "# Title\n\nSome *prose*.\n"; + let converted = NativeConverter + .convert(&raw(source, "text/markdown")) + .await + .unwrap(); + assert_eq!(converted.markdown, source); + assert_eq!(converted.format, DocumentFormat::Markdown); + assert_eq!(converted.source_bytes, source.len()); +} + +#[tokio::test] +async fn plain_text_is_stored_as_written_rather_than_reformatted() { + let source = "line one\nline two\n indented"; + let converted = NativeConverter + .convert(&raw(source, "text/plain")) + .await + .unwrap(); + assert_eq!(converted.markdown, source); + assert_eq!(converted.format, DocumentFormat::PlainText); +} + +#[tokio::test] +async fn html_is_converted_and_its_title_recovered() { + let source = + "Notes

Heading

Body.

"; + let converted = NativeConverter + .convert(&raw(source, "text/html")) + .await + .unwrap(); + assert_eq!(converted.markdown, "# Heading\n\nBody."); + assert_eq!(converted.title.as_deref(), Some("Notes")); + assert_eq!(converted.format, DocumentFormat::Html); +} + +#[tokio::test] +async fn the_converter_records_its_own_name_in_metadata() { + let converted = NativeConverter + .convert(&raw("text", "text/plain")) + .await + .unwrap(); + assert_eq!(converted.metadata["converter"], "native"); +} + +#[tokio::test] +async fn a_pdf_is_refused_with_an_error_that_says_what_is_missing() { + let pdf = RawDocument::new(b"%PDF-1.7\ncontent".to_vec()); + let error = NativeConverter.convert(&pdf).await.unwrap_err(); + assert!(matches!(error, MemoryError::Invalid(_)), "got {error:?}"); + assert!(error.to_string().contains("pdf"), "got {error}"); +} + +#[tokio::test] +async fn an_empty_document_is_rejected() { + let error = NativeConverter + .convert(&RawDocument::new(Vec::new())) + .await + .unwrap_err(); + assert!(error.to_string().contains("empty"), "got {error}"); +} + +#[tokio::test] +async fn a_document_over_the_cap_is_a_budget_error_not_a_validation_one() { + let oversized = RawDocument::new(vec![b'a'; MAX_DOCUMENT_BYTES + 1]).with_mime("text/plain"); + let error = NativeConverter.convert(&oversized).await.unwrap_err(); + assert!( + matches!(error, MemoryError::BudgetExceeded(_)), + "got {error:?}" + ); +} + +#[tokio::test] +async fn a_document_of_exactly_the_cap_is_accepted() { + let at_cap = RawDocument::new(vec![b'a'; MAX_DOCUMENT_BYTES]).with_mime("text/plain"); + assert!(NativeConverter.convert(&at_cap).await.is_ok()); +} + +#[tokio::test] +async fn invalid_utf8_in_a_textual_format_is_rejected() { + let bad = RawDocument::new(vec![b'h', b'i', 0xFF]).with_mime("text/plain"); + let error = NativeConverter.convert(&bad).await.unwrap_err(); + assert!(error.to_string().contains("utf-8"), "got {error}"); +} + +#[tokio::test] +async fn html_that_converts_to_nothing_is_an_error_not_an_empty_document() { + let empty = raw( + "", + "text/html", + ); + let error = NativeConverter.convert(&empty).await.unwrap_err(); + assert!(error.to_string().contains("no text"), "got {error}"); +} + +#[tokio::test] +async fn the_default_chain_converts_the_three_native_formats_and_nothing_else() { + let chain = ConverterChain::default(); + assert_eq!( + chain.supported_formats(), + vec![ + DocumentFormat::Markdown, + DocumentFormat::PlainText, + DocumentFormat::Html + ] + ); + assert!(!chain.supports(DocumentFormat::Pdf)); +} + +#[tokio::test] +async fn a_chain_uses_the_first_converter_that_claims_the_format() { + let chain = ConverterChain::new(vec![ + Box::new(Stub { + name: "first", + format: DocumentFormat::Html, + }), + Box::new(Stub { + name: "second", + format: DocumentFormat::Html, + }), + ]); + let converted = chain.convert(&raw("

x

", "text/html")).await.unwrap(); + assert_eq!(converted.markdown, "from first"); +} + +#[tokio::test] +async fn prepending_a_converter_puts_it_ahead_of_the_native_one() { + let chain = ConverterChain::default().prepend(Box::new(Stub { + name: "custom", + format: DocumentFormat::Html, + })); + let converted = chain + .convert(&raw("

real

", "text/html")) + .await + .unwrap(); + assert_eq!(converted.markdown, "from custom"); +} + +#[tokio::test] +async fn appending_a_converter_extends_what_the_chain_handles() { + let chain = ConverterChain::default().push(Box::new(Stub { + name: "pdf", + format: DocumentFormat::Pdf, + })); + assert!(chain.supports(DocumentFormat::Pdf)); + let converted = chain + .convert(&RawDocument::new(b"%PDF-1.7\nx".to_vec())) + .await + .unwrap(); + assert_eq!(converted.markdown, "from pdf"); + // The native converter still owns the formats it already handled. + let html = chain + .convert(&raw("

real

", "text/html")) + .await + .unwrap(); + assert_eq!(html.markdown, "# real"); +} + +#[tokio::test] +async fn a_chain_does_not_fall_through_when_its_chosen_converter_fails() { + let chain = ConverterChain::new(vec![ + Box::new(Failing), + Box::new(Stub { + name: "fallback", + format: DocumentFormat::Pdf, + }), + ]); + let error = chain + .convert(&RawDocument::new(b"%PDF-1.7\nx".to_vec())) + .await + .unwrap_err(); + assert!(matches!(error, MemoryError::Backend(_)), "got {error:?}"); +} + +#[tokio::test] +async fn an_unhandled_format_names_what_the_build_can_convert() { + let chain = ConverterChain::default(); + let error = chain + .convert(&RawDocument::new(b"%PDF-1.7\nx".to_vec())) + .await + .unwrap_err(); + let message = error.to_string(); + assert!(message.contains("pdf"), "{message}"); + assert!(message.contains("markdown"), "{message}"); +} + +#[tokio::test] +async fn an_empty_chain_says_it_converts_nothing() { + let chain = ConverterChain::new(Vec::new()); + let error = chain.convert(&raw("text", "text/plain")).await.unwrap_err(); + assert!(error.to_string().contains("nothing"), "got {error}"); +} + +#[test] +fn a_raw_document_detects_its_own_format_from_what_it_carries() { + assert_eq!( + RawDocument::new("x").with_mime("text/html").format(), + DocumentFormat::Html + ); + assert_eq!( + RawDocument::new("x").with_filename("a.md").format(), + DocumentFormat::Markdown + ); +} + +#[test] +fn a_display_name_prefers_the_filename_then_the_origin() { + let named = RawDocument::new("x") + .with_filename("report.pdf") + .with_origin("https://example.com/report.pdf"); + assert_eq!(named.display_name(), "report.pdf"); + + let fetched = RawDocument::new("x").with_origin("https://example.com/page"); + assert_eq!(fetched.display_name(), "https://example.com/page"); + + let anonymous = RawDocument::new("plain text"); + assert_eq!(anonymous.display_name(), "document.txt"); +} + +#[test] +fn title_or_falls_back_to_the_first_heading_before_the_supplied_default() { + let converted = ConvertedDocument::new("# Real Title\n\nbody", DocumentFormat::Markdown, 20); + assert_eq!(converted.title_or("upload.md"), "Real Title"); + + let untitled = ConvertedDocument::new("just body text", DocumentFormat::PlainText, 14); + assert_eq!(untitled.title_or("upload.txt"), "upload.txt"); +} + +#[test] +fn an_explicit_title_wins_over_a_heading() { + let converted = ConvertedDocument::new("# Heading", DocumentFormat::Markdown, 9) + .with_title(Some("Explicit".to_string())); + assert_eq!(converted.title_or("fallback"), "Explicit"); +} + +#[test] +fn a_blank_title_is_treated_as_no_title() { + let converted = ConvertedDocument::new("body", DocumentFormat::PlainText, 4) + .with_title(Some(" ".to_string())); + assert_eq!(converted.title, None); +} + +#[test] +fn an_empty_heading_is_not_mistaken_for_a_title() { + let converted = ConvertedDocument::new("#\n\nbody", DocumentFormat::Markdown, 7); + assert_eq!(converted.title_or("fallback"), "fallback"); +} diff --git a/crates/tinymemory-documents/src/convert/types.rs b/crates/tinymemory-documents/src/convert/types.rs new file mode 100644 index 0000000..e278458 --- /dev/null +++ b/crates/tinymemory-documents/src/convert/types.rs @@ -0,0 +1,147 @@ +//! The two values every conversion moves between: [`RawDocument`] in, +//! [`ConvertedDocument`] out. + +use serde::{Deserialize, Serialize}; + +use crate::format::DocumentFormat; + +/// Largest document intake will accept, in bytes. +/// +/// A ceiling on what one call may hold in memory, not a judgement about what is +/// worth remembering. It is enforced before conversion rather than after, +/// because a 200 MB PDF costs the same to reject early and far more to decode +/// first. +pub const MAX_DOCUMENT_BYTES: usize = 32 * 1024 * 1024; + +/// A document as it arrived, before anything has interpreted it. +/// +/// Carries the three signals format detection needs plus the origin, so a +/// converter never has to be told separately where the bytes came from. +#[derive(Debug, Clone)] +pub struct RawDocument { + /// The document body, exactly as received. + pub bytes: Vec, + /// Original filename, when the caller had one. + pub filename: Option, + /// MIME type the caller declared. Advisory: detection may overrule it. + pub declared_mime: Option, + /// Where the bytes came from — a URL for a fetch, `None` for an upload. + pub origin: Option, +} + +impl RawDocument { + /// A document from an upload, with no filename or declared type. + pub fn new(bytes: impl Into>) -> Self { + Self { + bytes: bytes.into(), + filename: None, + declared_mime: None, + origin: None, + } + } + + /// Attach the original filename. + #[must_use] + pub fn with_filename(mut self, filename: impl Into) -> Self { + self.filename = Some(filename.into()); + self + } + + /// Attach the caller-declared MIME type. + #[must_use] + pub fn with_mime(mut self, mime: impl Into) -> Self { + self.declared_mime = Some(mime.into()); + self + } + + /// Attach the URL the bytes were fetched from. + #[must_use] + pub fn with_origin(mut self, origin: impl Into) -> Self { + self.origin = Some(origin.into()); + self + } + + /// Detect this document's format from every signal it carries. + pub fn format(&self) -> DocumentFormat { + DocumentFormat::sniff( + &self.bytes, + self.filename.as_deref(), + self.declared_mime.as_deref(), + ) + } + + /// A display name for this document: its filename, else its origin, else a + /// generated name based on the detected format. + pub fn display_name(&self) -> String { + self.filename + .clone() + .or_else(|| self.origin.clone()) + .unwrap_or_else(|| format!("document.{}", self.format().extension())) + } +} + +/// A document after conversion: markdown, plus what was learned on the way. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ConvertedDocument { + /// The document body as markdown. Never empty — a conversion that produced + /// nothing is an error, not an empty success, because storing an empty + /// document silently loses the upload. + pub markdown: String, + /// Document title, when one could be recovered. + pub title: Option, + /// Format the source was detected as. + pub format: DocumentFormat, + /// Size of the source document in bytes, before conversion. + pub source_bytes: usize, + /// Anything else the converter learned — page counts, author, the + /// converter's own name. Open on purpose: this crate cannot know what a + /// host's converter will find worth keeping. + #[serde(default)] + pub metadata: serde_json::Value, +} + +impl ConvertedDocument { + /// A converted document with no title and no metadata. + pub fn new(markdown: impl Into, format: DocumentFormat, source_bytes: usize) -> Self { + Self { + markdown: markdown.into(), + title: None, + format, + source_bytes, + metadata: serde_json::Value::Null, + } + } + + /// Attach a title. + #[must_use] + pub fn with_title(mut self, title: Option) -> Self { + self.title = title.filter(|t| !t.trim().is_empty()); + self + } + + /// Attach converter metadata. + #[must_use] + pub fn with_metadata(mut self, metadata: serde_json::Value) -> Self { + self.metadata = metadata; + self + } + + /// The title if there is one, otherwise the first markdown heading, + /// otherwise `fallback`. + /// + /// Documents that carry no title metadata almost always open with their + /// title as a heading, and a stored document named `upload.pdf` is one + /// nobody finds again. + pub fn title_or(&self, fallback: &str) -> String { + if let Some(title) = &self.title { + return title.clone(); + } + self.markdown + .lines() + .find_map(|line| { + let heading = line.trim_start_matches('#').trim(); + (line.starts_with('#') && !heading.is_empty()).then(|| heading.to_string()) + }) + .unwrap_or_else(|| fallback.to_string()) + } +} diff --git a/crates/tinymemory-documents/src/error/mod.rs b/crates/tinymemory-documents/src/error/mod.rs new file mode 100644 index 0000000..2fff917 --- /dev/null +++ b/crates/tinymemory-documents/src/error/mod.rs @@ -0,0 +1,23 @@ +//! The crate-wide result alias. +//! +//! There is deliberately no `tinymemory_documents::Error`. Everything this +//! crate produces is on its way into a [`tinymemory_api::provider::MemoryProvider`], +//! and every failure it can have — a format nothing can convert, a body over +//! the size cap, a URL the guard refuses, a backend that rejected the write — +//! already has a name in [`MemoryError`]. A second enum would mean every caller +//! converting between two vocabularies for the same failures, and the +//! conversion would lose the variant a retry policy keys on. +//! +//! Which variant means what here: +//! +//! - [`MemoryError::Invalid`] — the caller's input: an empty body, a format no +//! converter handles, a namespace that fails validation. +//! - [`MemoryError::BudgetExceeded`] — a document larger than the cap. +//! - [`MemoryError::Unsupported`] — the *bound driver* cannot accept content at +//! all, which is a deployment fact rather than a bad request. +//! - [`MemoryError::Unreachable`] / [`MemoryError::Backend`] — the URL fetch. + +use tinymemory_api::error::MemoryError; + +/// Result alias for this crate's fallible operations. +pub type Result = std::result::Result; diff --git a/crates/tinymemory-documents/src/fetch/mod.rs b/crates/tinymemory-documents/src/fetch/mod.rs new file mode 100644 index 0000000..18f76a1 --- /dev/null +++ b/crates/tinymemory-documents/src/fetch/mod.rs @@ -0,0 +1,111 @@ +//! Fetching a URL into a [`RawDocument`]. +//! +//! ## Why this reuses the source readers' guard +//! +//! A URL a user types is an SSRF vector: `http://169.254.169.254/` is a cloud +//! metadata endpoint, `http://localhost:6379/` is somebody's Redis, and a +//! hostname that resolves publicly on the first lookup can resolve to a private +//! address on the second. `tinymemory-sources` already solved this for the RSS +//! and web-page readers — a scheme and host policy plus a resolver that pins +//! connections to globally routable addresses — and this module uses that +//! guard rather than growing a second one. Two SSRF implementations in one +//! workspace means one of them is the weaker, and nobody knows which. +//! +//! ## What it does not do +//! +//! No scheduling, no retries, no credentials, no robots.txt. Those are host +//! policy, and the same rule that keeps them out of a driver keeps them out of +//! here: this fetches one URL, once, when asked. + +use tinymemory_api::error::MemoryError; +use tinymemory_sources::readers::ssrf::{build_client, is_url_allowed, read_body_capped}; + +use crate::convert::{RawDocument, MAX_DOCUMENT_BYTES}; +use crate::error::Result; + +/// Fetch `url` and return its body as a [`RawDocument`]. +/// +/// The response's `Content-Type` becomes the document's declared MIME type and +/// the URL becomes its origin, so format detection and key derivation both have +/// what they need without the caller repeating itself. +/// +/// # Errors +/// +/// - [`MemoryError::Invalid`] for a malformed URL, or one the SSRF guard +/// refuses. +/// - [`MemoryError::Unreachable`] when the request never completed. +/// - [`MemoryError::Backend`] for a non-success status. +/// - [`MemoryError::BudgetExceeded`] for a body over +/// [`MAX_DOCUMENT_BYTES`]. +pub async fn fetch_url(url: &str) -> Result { + let parsed = reqwest::Url::parse(url) + .map_err(|error| MemoryError::Invalid(format!("invalid url {url:?}: {error}")))?; + if !is_url_allowed(&parsed) { + return Err(MemoryError::Invalid(format!( + "url {url:?} is not an allowed fetch target" + ))); + } + + let client = build_client().map_err(MemoryError::Backend)?; + let response = client + .get(parsed.clone()) + .send() + .await + .map_err(|error| MemoryError::Unreachable(format!("fetching {url:?}: {error}")))?; + + let status = response.status(); + if !status.is_success() { + return Err(MemoryError::Backend(format!( + "fetching {url:?} answered {status}" + ))); + } + + let content_type = response + .headers() + .get(reqwest::header::CONTENT_TYPE) + .and_then(|value| value.to_str().ok()) + .map(str::to_string); + + // The cap is applied while reading, not after: a body that would not fit is + // one this process should never have finished buffering. + let bytes = read_body_capped(response, MAX_DOCUMENT_BYTES as u64) + .await + .map_err(|error| read_error(url, &error))?; + + if bytes.is_empty() { + return Err(MemoryError::Invalid(format!("{url:?} returned no body"))); + } + + let mut document = RawDocument::new(bytes).with_origin(parsed.to_string()); + if let Some(content_type) = content_type { + document = document.with_mime(content_type); + } + // A URL's last path segment is often the only filename there is, and format + // detection falls back to it when the server sent no useful type. + if let Some(name) = parsed + .path_segments() + .and_then(|mut segments| segments.next_back()) + .filter(|name| !name.is_empty() && name.contains('.')) + { + document = document.with_filename(name.to_string()); + } + Ok(document) +} + +/// Turn a `read_body_capped` failure into the right [`MemoryError`] variant. +/// +/// `read_body_capped` collapses two different failures into one `String`: a +/// body over the size cap, and a stream that failed mid-read. Those need +/// different retry policies from a caller, so this tells them apart by the +/// message `read_body_capped` always uses for the size case, rather than +/// reporting every failure as a budget overrun. +fn read_error(url: &str, error: &str) -> MemoryError { + if error.contains("exceeds") && error.contains("-byte limit") { + MemoryError::BudgetExceeded(format!("reading {url:?}: {error}")) + } else { + MemoryError::Unreachable(format!("reading {url:?}: {error}")) + } +} + +#[cfg(test)] +mod test; diff --git a/crates/tinymemory-documents/src/fetch/test.rs b/crates/tinymemory-documents/src/fetch/test.rs new file mode 100644 index 0000000..f21eae6 --- /dev/null +++ b/crates/tinymemory-documents/src/fetch/test.rs @@ -0,0 +1,70 @@ +//! Tests for URL intake. +//! +//! Only the guard and the argument handling are exercised here. Anything that +//! would actually reach the network is out of scope by the repository's testing +//! rules — the fetch itself is covered by `tinymemory-sources`' own reader +//! tests, which own the client this module borrows. + +use super::*; + +#[tokio::test] +async fn a_malformed_url_is_rejected_before_anything_is_fetched() { + let error = fetch_url("not a url").await.unwrap_err(); + assert!(matches!(error, MemoryError::Invalid(_)), "got {error:?}"); + assert!(error.to_string().contains("invalid url"), "got {error}"); +} + +#[tokio::test] +async fn loopback_and_link_local_targets_are_refused() { + for url in [ + "http://127.0.0.1/", + "http://localhost:6379/", + "http://169.254.169.254/latest/meta-data/", + "http://[::1]/", + ] { + let error = fetch_url(url).await.unwrap_err(); + assert!( + error.to_string().contains("not an allowed fetch target"), + "{url} gave {error}" + ); + } +} + +#[tokio::test] +async fn a_non_http_scheme_is_refused() { + for url in [ + "file:///etc/passwd", + "ftp://example.com/x", + "gopher://example.com/", + ] { + let error = fetch_url(url).await.unwrap_err(); + assert!( + matches!(error, MemoryError::Invalid(_)), + "{url} gave {error:?}" + ); + } +} + +#[test] +fn a_size_limit_failure_is_reported_as_budget_exceeded() { + let error = read_error( + "https://example.com/", + "response body exceeds 8-byte limit (Content-Length=9)", + ); + assert!( + matches!(error, MemoryError::BudgetExceeded(_)), + "got {error:?}" + ); +} + +#[test] +fn an_interrupted_read_is_reported_as_unreachable_not_budget_exceeded() { + let error = read_error( + "https://example.com/", + "failed to read response body: connection reset", + ); + assert!( + matches!(error, MemoryError::Unreachable(_)), + "got {error:?}" + ); +} diff --git a/crates/tinymemory-documents/src/format/mod.rs b/crates/tinymemory-documents/src/format/mod.rs new file mode 100644 index 0000000..10baf79 --- /dev/null +++ b/crates/tinymemory-documents/src/format/mod.rs @@ -0,0 +1,196 @@ +//! Document format detection. +//! +//! Intake gets a byte buffer and, if it is lucky, a filename and a MIME type. +//! None of the three is reliable on its own: browsers send +//! `application/octet-stream` for files they cannot place, a `.txt` extension +//! says nothing about what is inside, and a buffer alone cannot distinguish +//! markdown from plain text. So [`DocumentFormat::sniff`] consults all three in +//! order of trustworthiness — magic bytes first, because they are the only +//! signal a caller cannot get wrong. + +use std::fmt; + +use serde::{Deserialize, Serialize}; + +/// A document format intake can recognise. +/// +/// Deliberately short. This is the set that has a defined conversion, not a +/// catalogue of everything that exists: a format nobody converts would be a +/// variant that only ever appears in an error message. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum DocumentFormat { + /// Markdown. Already the target format; conversion is a passthrough. + Markdown, + /// Plain text. Wrapped into markdown without interpretation. + PlainText, + /// HTML. Converted structurally — headings, lists, links, code. + Html, + /// PDF. Needs a real extractor; see [`crate::convert::DocumentConverter`]. + Pdf, + /// Office Open XML word processing (`.docx`). Needs a real extractor. + Docx, + /// A format detection could not place. + Unknown, +} + +impl DocumentFormat { + /// The canonical MIME type for this format. + pub fn mime(self) -> &'static str { + match self { + Self::Markdown => "text/markdown", + Self::PlainText => "text/plain", + Self::Html => "text/html", + Self::Pdf => "application/pdf", + Self::Docx => "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + Self::Unknown => "application/octet-stream", + } + } + + /// The usual file extension, without a dot. + pub fn extension(self) -> &'static str { + match self { + Self::Markdown => "md", + Self::PlainText => "txt", + Self::Html => "html", + Self::Pdf => "pdf", + Self::Docx => "docx", + Self::Unknown => "bin", + } + } + + /// Whether the bytes of this format are text a human could read directly. + /// + /// The line that decides whether intake can decode a buffer itself or has + /// to hand it to an extractor. + pub fn is_textual(self) -> bool { + matches!(self, Self::Markdown | Self::PlainText | Self::Html) + } + + /// Detect the format from every signal available. + /// + /// Magic bytes win when present, because they are the one signal a caller + /// cannot get wrong. A declared MIME type comes next, then the filename, + /// and a textual buffer with no other evidence is plain text. + pub fn sniff(bytes: &[u8], filename: Option<&str>, mime: Option<&str>) -> Self { + if let Some(format) = Self::from_magic(bytes) { + return format; + } + if let Some(format) = mime.and_then(Self::from_mime) { + return format; + } + if let Some(format) = filename.and_then(Self::from_filename) { + return format; + } + // An HTML document served without a type or an extension is common + // enough — and cheap enough to spot — to be worth one more look. + if looks_like_html(bytes) { + return Self::Html; + } + if is_probably_text(bytes) { + Self::PlainText + } else { + Self::Unknown + } + } + + /// Detect from leading magic bytes alone. + /// + /// Returns `None` rather than [`DocumentFormat::Unknown`]: "no magic bytes" + /// and "magic bytes that match nothing" both mean *keep looking*, and a + /// caller that got `Unknown` here would stop. + pub fn from_magic(bytes: &[u8]) -> Option { + if bytes.starts_with(b"%PDF-") { + return Some(Self::Pdf); + } + // Every OOXML file is a zip. Which OOXML it is lives in the archive, + // which needs a zip reader intake does not have — so this reports the + // container and lets the extractor disagree. + if bytes.starts_with(b"PK\x03\x04") { + return Some(Self::Docx); + } + None + } + + /// Map a MIME type onto a format. + /// + /// Parameters (`; charset=utf-8`) are stripped, and the type is compared + /// case-insensitively, because both vary by client and neither carries + /// meaning here. + pub fn from_mime(mime: &str) -> Option { + let essence = mime + .split(';') + .next() + .unwrap_or(mime) + .trim() + .to_ascii_lowercase(); + match essence.as_str() { + "text/markdown" | "text/x-markdown" => Some(Self::Markdown), + "text/plain" => Some(Self::PlainText), + "text/html" | "application/xhtml+xml" => Some(Self::Html), + "application/pdf" => Some(Self::Pdf), + // Deliberately excludes `application/msword`: that MIME type + // names the legacy binary `.doc` format, not the Open XML `.docx` + // package this variant's converter targets. Claiming `Docx` for + // it would hand a bound DOCX extractor input it cannot read. + "application/vnd.openxmlformats-officedocument.wordprocessingml.document" => { + Some(Self::Docx) + } + _ => None, + } + } + + /// Map a filename or path onto a format by its extension. + pub fn from_filename(filename: &str) -> Option { + let extension = filename.rsplit_once('.')?.1.to_ascii_lowercase(); + match extension.as_str() { + "md" | "markdown" | "mdown" => Some(Self::Markdown), + "txt" | "text" | "log" => Some(Self::PlainText), + "html" | "htm" | "xhtml" => Some(Self::Html), + "pdf" => Some(Self::Pdf), + // `.doc` is the legacy binary Word format, not Open XML `.docx`; + // see the `application/msword` note in `from_mime`. + "docx" => Some(Self::Docx), + _ => None, + } + } +} + +impl fmt::Display for DocumentFormat { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(match self { + Self::Markdown => "markdown", + Self::PlainText => "plain_text", + Self::Html => "html", + Self::Pdf => "pdf", + Self::Docx => "docx", + Self::Unknown => "unknown", + }) + } +} + +/// Whether a buffer opens with something only HTML opens with. +/// +/// Only the first bytes are examined, and only for the two openings that are +/// unambiguous. A page whose first tag is a `
` is not worth guessing at: +/// it will have arrived with a content type. +fn looks_like_html(bytes: &[u8]) -> bool { + let head = &bytes[..bytes.len().min(512)]; + let Ok(text) = std::str::from_utf8(head) else { + return false; + }; + let lower = text.trim_start().to_ascii_lowercase(); + lower.starts_with(" bool { + !bytes.is_empty() && !bytes.contains(&0) && std::str::from_utf8(bytes).is_ok() +} + +#[cfg(test)] +mod test; diff --git a/crates/tinymemory-documents/src/format/test.rs b/crates/tinymemory-documents/src/format/test.rs new file mode 100644 index 0000000..29c892f --- /dev/null +++ b/crates/tinymemory-documents/src/format/test.rs @@ -0,0 +1,200 @@ +//! Tests for document format detection. + +use super::*; + +#[test] +fn magic_bytes_beat_a_wrong_mime_type_and_a_wrong_extension() { + let pdf = b"%PDF-1.7\n%\xE2\xE3\xCF\xD3\n"; + assert_eq!( + DocumentFormat::sniff(pdf, Some("notes.txt"), Some("text/plain")), + DocumentFormat::Pdf + ); +} + +#[test] +fn a_zip_container_is_reported_as_docx() { + // Every OOXML file is a zip; telling docx from xlsx needs a zip reader + // intake does not have, so the container is what gets reported. + let zip = b"PK\x03\x04\x14\x00\x06\x00"; + assert_eq!(DocumentFormat::from_magic(zip), Some(DocumentFormat::Docx)); +} + +#[test] +fn from_magic_says_keep_looking_rather_than_unknown() { + assert_eq!(DocumentFormat::from_magic(b"# A heading"), None); + assert_eq!(DocumentFormat::from_magic(b""), None); +} + +#[test] +fn a_declared_mime_type_beats_the_filename() { + assert_eq!( + DocumentFormat::sniff(b"hello", Some("notes.txt"), Some("text/markdown")), + DocumentFormat::Markdown + ); +} + +#[test] +fn mime_parameters_and_casing_are_ignored() { + assert_eq!( + DocumentFormat::from_mime("Text/HTML; charset=UTF-8"), + Some(DocumentFormat::Html) + ); + assert_eq!( + DocumentFormat::from_mime("text/plain ; charset=utf-8"), + Some(DocumentFormat::PlainText) + ); +} + +#[test] +fn an_octet_stream_mime_falls_through_to_the_filename() { + assert_eq!( + DocumentFormat::sniff( + b"hello there", + Some("notes.md"), + Some("application/octet-stream") + ), + DocumentFormat::Markdown + ); +} + +#[test] +fn every_recognised_extension_maps_to_a_format() { + for (filename, expected) in [ + ("a.md", DocumentFormat::Markdown), + ("a.markdown", DocumentFormat::Markdown), + ("a.txt", DocumentFormat::PlainText), + ("a.log", DocumentFormat::PlainText), + ("a.html", DocumentFormat::Html), + ("a.htm", DocumentFormat::Html), + ("a.pdf", DocumentFormat::Pdf), + ("a.docx", DocumentFormat::Docx), + ("path/to/report.PDF", DocumentFormat::Pdf), + ] { + assert_eq!( + DocumentFormat::from_filename(filename), + Some(expected), + "{filename}" + ); + } +} + +#[test] +fn a_filename_with_no_extension_maps_to_nothing() { + assert_eq!(DocumentFormat::from_filename("README"), None); + assert_eq!(DocumentFormat::from_filename(""), None); +} + +#[test] +fn legacy_doc_is_not_claimed_as_docx() { + // `.doc` and `application/msword` name the legacy binary Word format, not + // the Open XML `.docx` package the `Docx` converter targets. + assert_eq!(DocumentFormat::from_filename("report.doc"), None); + assert_eq!(DocumentFormat::from_mime("application/msword"), None); +} + +#[test] +fn html_is_recognised_from_its_opening_alone() { + assert_eq!( + DocumentFormat::sniff(b"hi", None, None), + DocumentFormat::Html + ); + assert_eq!( + DocumentFormat::sniff(b" hi", None, None), + DocumentFormat::Html + ); +} + +#[test] +fn unlabelled_text_is_plain_text() { + assert_eq!( + DocumentFormat::sniff(b"just some prose", None, None), + DocumentFormat::PlainText + ); +} + +#[test] +fn unlabelled_binary_is_unknown() { + assert_eq!( + DocumentFormat::sniff(&[0x00, 0x01, 0x02, 0xFF], None, None), + DocumentFormat::Unknown + ); +} + +#[test] +fn an_empty_buffer_is_unknown() { + assert_eq!( + DocumentFormat::sniff(b"", None, None), + DocumentFormat::Unknown + ); +} + +#[test] +fn textual_formats_are_the_ones_intake_can_decode_itself() { + assert!(DocumentFormat::Markdown.is_textual()); + assert!(DocumentFormat::PlainText.is_textual()); + assert!(DocumentFormat::Html.is_textual()); + assert!(!DocumentFormat::Pdf.is_textual()); + assert!(!DocumentFormat::Docx.is_textual()); + assert!(!DocumentFormat::Unknown.is_textual()); +} + +#[test] +fn a_canonical_mime_round_trips_back_to_its_format() { + for format in [ + DocumentFormat::Markdown, + DocumentFormat::PlainText, + DocumentFormat::Html, + DocumentFormat::Pdf, + DocumentFormat::Docx, + ] { + assert_eq!(DocumentFormat::from_mime(format.mime()), Some(format)); + } +} + +#[test] +fn a_canonical_extension_round_trips_back_to_its_format() { + for format in [ + DocumentFormat::Markdown, + DocumentFormat::PlainText, + DocumentFormat::Html, + DocumentFormat::Pdf, + DocumentFormat::Docx, + ] { + assert_eq!( + DocumentFormat::from_filename(&format!("file.{}", format.extension())), + Some(format) + ); + } +} + +#[test] +fn a_format_round_trips_through_json() { + for format in [ + DocumentFormat::Markdown, + DocumentFormat::PlainText, + DocumentFormat::Html, + DocumentFormat::Pdf, + DocumentFormat::Docx, + DocumentFormat::Unknown, + ] { + let wire = serde_json::to_string(&format).unwrap(); + assert_eq!( + serde_json::from_str::(&wire).unwrap(), + format + ); + } +} + +#[test] +fn display_matches_the_wire_spelling() { + assert_eq!(DocumentFormat::PlainText.to_string(), "plain_text"); + assert_eq!(DocumentFormat::Html.to_string(), "html"); +} + +#[test] +fn invalid_utf8_without_magic_bytes_is_unknown_not_text() { + assert_eq!( + DocumentFormat::sniff(&[0xFF, 0xFE, 0xFD], None, None), + DocumentFormat::Unknown + ); +} diff --git a/crates/tinymemory-documents/src/html/entity.rs b/crates/tinymemory-documents/src/html/entity.rs new file mode 100644 index 0000000..1f6bd60 --- /dev/null +++ b/crates/tinymemory-documents/src/html/entity.rs @@ -0,0 +1,79 @@ +//! HTML entity decoding. +//! +//! Covers the named entities that actually appear in prose plus the numeric +//! forms, and leaves anything else alone. An unrecognised entity is passed +//! through verbatim rather than dropped: a literal `&foo;` in the output is a +//! visible, fixable wart, whereas a silently deleted one is a hole in the text +//! nobody notices. + +/// Decode HTML entities in `text`. +pub(super) fn decode_entities(text: &str) -> String { + if !text.contains('&') { + return text.to_string(); + } + let mut out = String::with_capacity(text.len()); + let mut rest = text; + while let Some(at) = rest.find('&') { + out.push_str(&rest[..at]); + let tail = &rest[at..]; + // An entity is short; a '&' with no ';' within that window is a + // literal ampersand, which is far more common than a malformed entity. + // The window has to land on a char boundary, or slicing panics on a + // multibyte character sitting across the 12-byte mark. + let mut window = tail.len().min(12); + while window > 0 && !tail.is_char_boundary(window) { + window -= 1; + } + let Some(end) = tail[..window].find(';') else { + out.push('&'); + rest = &tail[1..]; + continue; + }; + match decode_one(&tail[1..end]) { + Some(decoded) => out.push_str(&decoded), + None => out.push_str(&tail[..=end]), + } + rest = &tail[end + 1..]; + } + out.push_str(rest); + out +} + +/// Decode the body of a single entity — what sits between `&` and `;`. +fn decode_one(body: &str) -> Option { + if let Some(digits) = body.strip_prefix("#x").or_else(|| body.strip_prefix("#X")) { + let code = u32::from_str_radix(digits, 16).ok()?; + return char::from_u32(code).map(String::from); + } + if let Some(digits) = body.strip_prefix('#') { + let code = digits.parse::().ok()?; + return char::from_u32(code).map(String::from); + } + let literal = match body { + "amp" => "&", + "lt" => "<", + "gt" => ">", + "quot" => "\"", + "apos" | "#39" => "'", + "nbsp" => " ", + "hellip" => "…", + "mdash" => "—", + "ndash" => "–", + "lsquo" => "\u{2018}", + "rsquo" => "\u{2019}", + "ldquo" => "\u{201C}", + "rdquo" => "\u{201D}", + "copy" => "©", + "reg" => "®", + "trade" => "™", + "deg" => "°", + "middot" => "·", + "bull" => "•", + _ => return None, + }; + Some(literal.to_string()) +} + +#[cfg(test)] +#[path = "entity_test.rs"] +mod test; diff --git a/crates/tinymemory-documents/src/html/entity_test.rs b/crates/tinymemory-documents/src/html/entity_test.rs new file mode 100644 index 0000000..cd1aa2c --- /dev/null +++ b/crates/tinymemory-documents/src/html/entity_test.rs @@ -0,0 +1,68 @@ +//! Tests for HTML entity decoding. + +use super::*; + +#[test] +fn text_without_an_ampersand_is_returned_unchanged() { + assert_eq!(decode_entities("plain prose"), "plain prose"); +} + +#[test] +fn the_named_entities_that_appear_in_prose_are_decoded() { + assert_eq!(decode_entities("a & b"), "a & b"); + assert_eq!(decode_entities("<tag>"), ""); + assert_eq!(decode_entities(""quoted""), "\"quoted\""); + assert_eq!(decode_entities("it's"), "it's"); + assert_eq!(decode_entities("wait…"), "wait…"); + assert_eq!(decode_entities("a—b"), "a—b"); +} + +#[test] +fn a_non_breaking_space_becomes_an_ordinary_one() { + assert_eq!(decode_entities("a b"), "a b"); +} + +#[test] +fn decimal_and_hex_numeric_entities_are_decoded() { + assert_eq!(decode_entities("AB"), "AB"); + assert_eq!(decode_entities("AB"), "AB"); + assert_eq!(decode_entities("…"), "…"); +} + +#[test] +fn an_unrecognised_entity_is_passed_through_verbatim() { + assert_eq!(decode_entities("&nosuch;"), "&nosuch;"); +} + +#[test] +fn an_out_of_range_numeric_entity_is_passed_through() { + assert_eq!(decode_entities("�"), "�"); +} + +#[test] +fn a_bare_ampersand_survives() { + assert_eq!(decode_entities("Tom & Jerry"), "Tom & Jerry"); + assert_eq!(decode_entities("ends with &"), "ends with &"); +} + +#[test] +fn a_long_run_after_an_ampersand_is_not_treated_as_an_entity() { + let input = "a & this is a long sentence; not an entity"; + assert_eq!(decode_entities(input), input); +} + +#[test] +fn several_entities_in_one_string_are_all_decoded() { + assert_eq!( + decode_entities("<a href="x">A & B</a>"), + "A & B" + ); +} + +#[test] +fn a_multibyte_run_after_an_ampersand_does_not_panic() { + // Each `€` is 3 bytes, so the 12-byte scan window lands mid-character + // unless the scan snaps back to a char boundary first. + let input = "&€€€€;"; + assert_eq!(decode_entities(input), input); +} diff --git a/crates/tinymemory-documents/src/html/mod.rs b/crates/tinymemory-documents/src/html/mod.rs new file mode 100644 index 0000000..26e6b62 --- /dev/null +++ b/crates/tinymemory-documents/src/html/mod.rs @@ -0,0 +1,405 @@ +//! HTML to markdown. +//! +//! A small, dependency-free structural converter, not a browser. It walks the +//! tag stream once and keeps the structure that survives being stored as +//! memory — headings, paragraphs, lists, links, code, block quotes, emphasis — +//! and discards the rest. +//! +//! ## Why not a real HTML parser +//! +//! Because the output is prose for a language model to read, and the failure +//! modes of a tag-stream walk are all cosmetic: a malformed nesting produces +//! slightly wrong emphasis, never wrong text. Pulling in a full DOM parser +//! would cost this crate its "no heavy dependencies" position for output +//! nobody renders. If a host needs fidelity beyond this, it supplies its own +//! [`crate::convert::DocumentConverter`]. +//! +//! Script and style bodies are removed before anything else, so their contents +//! can never reach the output as text. `` goes with them: it is document +//! metadata, [`extract_title`] reads it from the original source, and leaving it +//! in would open every converted page with its own title as a stray line of +//! prose. + +mod entity; + +use entity::decode_entities; + +/// Convert an HTML document to markdown. +/// +/// Never fails: HTML has no error state this converter can be pushed into, and +/// malformed input degrades to slightly worse markdown rather than to an error +/// a caller would have to handle. +pub fn to_markdown(html: &str) -> String { + let cleaned = strip_raw_text_elements(html); + let mut out = Renderer::default(); + let mut rest = cleaned.as_str(); + + while let Some(open) = rest.find('<') { + out.text(&rest[..open]); + let after = &rest[open + 1..]; + let Some(close) = after.find('>') else { + // An unterminated '<' is literal text, not a tag. + out.text(&rest[open..]); + rest = ""; + break; + }; + out.tag(&after[..close]); + rest = &after[close + 1..]; + } + out.text(rest); + out.finish() +} + +/// Extract the contents of `<title>`, if the document has one. +pub fn extract_title(html: &str) -> Option<String> { + let lower = html.to_ascii_lowercase(); + let start = lower.find("<title")?; + let content_start = lower[start..].find('>')? + start + 1; + let end = lower[content_start..].find("")? + content_start; + let title = decode_entities(html.get(content_start..end)?) + .trim() + .to_string(); + (!title.is_empty()).then_some(title) +} + +/// Remove `

real

"#; + let markdown = to_markdown(html); + assert_eq!(markdown, "real"); +} + +#[test] +fn an_unclosed_script_swallows_the_rest_of_the_document() { + let markdown = to_markdown("

before