diff --git a/Cargo.toml b/Cargo.toml index fd07be03..8a316582 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -24,7 +24,7 @@ thiserror = "2" tracing = "0.1" crossbeam-channel = "0.5" parking_lot = "0.12" -rusqlite = { version = "0.40", features = ["bundled"] } +rusqlite = { version = "0.40", features = ["bundled", "functions"] } blake3 = "1" xxhash-rust = { version = "0.8", features = ["xxh3"] } unicode-normalization = "0.1" diff --git a/crates/cp-core/src/lib.rs b/crates/cp-core/src/lib.rs index 16ede115..3d4fac55 100644 --- a/crates/cp-core/src/lib.rs +++ b/crates/cp-core/src/lib.rs @@ -19,6 +19,7 @@ pub mod stamp; pub mod thumbnail; pub mod token; pub mod trouble; +pub mod typo; pub mod watch; pub mod watching; diff --git a/crates/cp-core/src/search.rs b/crates/cp-core/src/search.rs index d7249072..ed6979a4 100644 --- a/crates/cp-core/src/search.rs +++ b/crates/cp-core/src/search.rs @@ -39,14 +39,38 @@ fn fold_each(input: &str, mut push: impl FnMut(char, usize)) { } } -pub fn terms_of(query: &str) -> Vec { - fold(query) - .split_whitespace() - .filter(|word| word.chars().any(char::is_alphanumeric)) - .map(str::to_owned) +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct Term { + pub text: String, + pub whole: bool, + pub alike: Vec, +} + +pub fn terms_of(query: &str) -> Vec { + let folded = fold(query); + let typing = !folded.ends_with(char::is_whitespace); + let words: Vec<&str> = folded.split_whitespace().collect(); + let last = words.len().saturating_sub(1); + words + .into_iter() + .enumerate() + .filter(|(_, word)| word.chars().any(char::is_alphanumeric)) + .map(|(at, word)| Term { + text: word.to_owned(), + whole: at < last || !typing, + alike: Vec::new(), + }) .collect() } +pub fn holds(text: &str, terms: &[Term]) -> bool { + let chars: Vec = fold(text).chars().collect(); + let tokens = tokens_of(&chars); + terms + .iter() + .all(|term| !spots_of(&chars, &tokens, term).is_empty()) +} + pub const EXCERPT_CHARS: usize = 160; #[derive(Debug, Clone, PartialEq, Eq)] @@ -66,12 +90,12 @@ impl Excerpt { } } -pub fn excerpt(text: &str, terms: &[String], width: usize) -> Option { +pub fn excerpt(text: &str, terms: &[Term], width: usize) -> Option { let folded = fold_mapped(text); let tokens = tokens_of(&folded.chars); let mut hits: Vec<(usize, usize)> = terms .iter() - .flat_map(|term| matches_of(&folded.chars, &tokens, term)) + .flat_map(|term| spots_of(&folded.chars, &tokens, term)) .map(|(from, to)| { let last = folded.origin[to - 1]; ( @@ -170,6 +194,47 @@ fn tokens_of(chars: &[char]) -> Vec<(usize, usize)> { tokens } +fn spots_of(chars: &[char], tokens: &[(usize, usize)], term: &Term) -> Vec<(usize, usize)> { + std::iter::once(&term.text) + .chain(&term.alike) + .flat_map(|word| { + if term.whole { + alone_of(chars, word) + } else { + matches_of(chars, tokens, word) + } + }) + .collect() +} + +fn alone_of(chars: &[char], word: &str) -> Vec<(usize, usize)> { + let wanted: Vec = word + .trim_matches(|c: char| !c.is_alphanumeric()) + .chars() + .collect(); + if wanted.is_empty() { + return Vec::new(); + } + let mut found = Vec::new(); + let mut at = 0; + while at < chars.len() { + let start = at; + while at < chars.len() && !chars[at].is_whitespace() { + at += 1; + } + let chunk = &chars[start..at]; + if let (Some(from), Some(to)) = ( + chunk.iter().position(|c| c.is_alphanumeric()), + chunk.iter().rposition(|c| c.is_alphanumeric()), + ) && chunk[from..=to] == wanted[..] + { + found.push((start + from, start + to + 1)); + } + at += 1; + } + found +} + fn matches_of(chars: &[char], tokens: &[(usize, usize)], term: &str) -> Vec<(usize, usize)> { let wanted: Vec> = term .split(|c: char| !c.is_alphanumeric()) diff --git a/crates/cp-core/src/search_test.rs b/crates/cp-core/src/search_test.rs index 09d554e2..5142206b 100644 --- a/crates/cp-core/src/search_test.rs +++ b/crates/cp-core/src/search_test.rs @@ -1,4 +1,4 @@ -use super::{EXCERPT_CHARS, Segment, excerpt, fold, fold_mapped, terms_of}; +use super::{EXCERPT_CHARS, Segment, Term, excerpt, fold, fold_mapped, holds, terms_of}; fn plain(text: &str) -> Segment { Segment { @@ -14,8 +14,82 @@ fn hit(text: &str) -> Segment { } } -fn terms(words: &[&str]) -> Vec { - words.iter().map(|word| (*word).to_owned()).collect() +fn terms(words: &[&str]) -> Vec { + words + .iter() + .map(|word| Term { + text: (*word).to_owned(), + ..Term::default() + }) + .collect() +} + +fn texts(query: &str) -> Vec { + terms_of(query).into_iter().map(|term| term.text).collect() +} + +fn whole(word: &str) -> Term { + Term { + text: word.into(), + whole: true, + ..Term::default() + } +} + +#[test] +fn a_word_followed_by_a_space_is_whole_and_the_one_being_typed_is_not() { + let wholes = |query| { + terms_of(query) + .into_iter() + .map(|term| term.whole) + .collect::>() + }; + assert_eq!(wholes("jira"), vec![false]); + assert_eq!(wholes("jira "), vec![true]); + assert_eq!(wholes("jira abr"), vec![true, false]); + assert_eq!(wholes("jira !!"), vec![true]); +} + +#[test] +fn a_whole_word_skips_links_paths_mail_and_longer_words() { + let jira = [whole("jira")]; + assert!(!holds("https://jira.empresa.com/browse/AB-1", &jira)); + assert!(!holds("C:\\tools\\jira\\notes.txt", &jira)); + assert!(!holds("jira@empresa.com", &jira)); + assert!(!holds("las jiras del equipo", &jira)); + assert!(holds("abre el ticket en Jira mañana", &jira)); + assert!(holds("lo vimos en jira.", &jira)); + assert!(holds("(Jira) y ¿jira?", &jira)); + assert!(holds("jira", &jira)); +} + +#[test] +fn a_word_being_typed_still_finds_links_and_prefixes() { + assert!(holds("https://jira.empresa.com", &terms(&["jir"]))); + assert!(holds("las jiras", &terms(&["jira"]))); +} + +#[test] +fn every_term_must_hold() { + let both = [ + whole("jira"), + Term { + text: "abr".into(), + ..Term::default() + }, + ]; + assert!(holds("abrir jira hoy", &both)); + assert!(!holds("abrir jira.com", &both)); + assert!(!holds("jira hoy", &both)); +} + +#[test] +fn a_whole_word_marks_only_the_standalone_one() { + let found = excerpt("ve jira.com o jira, da igual", &[whole("jira")], 100).expect("a hit"); + assert_eq!( + found.segments, + vec![plain("ve jira.com o "), hit("jira"), plain(", da igual")] + ); } #[test] @@ -192,7 +266,7 @@ fn nothing_to_look_for_is_nothing_found() { #[test] fn the_terms_are_the_words_the_index_would_look_for() { - assert_eq!(terms_of("Café AB-4417 ... !!"), vec!["cafe", "ab-4417"]); + assert_eq!(texts("Café AB-4417 ... !!"), vec!["cafe", "ab-4417"]); assert!(terms_of(" ").is_empty()); assert!(terms_of("!!! ...").is_empty()); } @@ -291,28 +365,43 @@ fn nfc_and_nfd_spellings_of_the_same_letter_fold_the_same() { #[test] fn terms_of_an_empty_or_symbol_only_query_is_empty() { - assert_eq!(terms_of(""), Vec::::new()); - assert_eq!(terms_of(" "), Vec::::new()); - assert_eq!(terms_of("!?#"), Vec::::new()); + assert!(terms_of("").is_empty()); + assert!(terms_of(" ").is_empty()); + assert!(terms_of("!?#").is_empty()); } #[test] fn a_leading_dash_stays_glued_to_its_word() { - assert_eq!(terms_of("gato -perro"), vec!["gato", "-perro"]); + assert_eq!(texts("gato -perro"), vec!["gato", "-perro"]); } #[test] fn quotes_are_not_a_token_boundary() { - assert_eq!(terms_of("\"hola mundo\""), vec!["\"hola", "mundo\""]); + assert_eq!(texts("\"hola mundo\""), vec!["\"hola", "mundo\""]); } #[test] fn a_term_longer_than_the_text_finds_nothing_and_does_not_panic() { - assert_eq!(excerpt("ab", &["abcdef".to_owned()], 100), None); + assert_eq!(excerpt("ab", &terms(&["abcdef"]), 100), None); } #[test] fn a_width_of_zero_still_shows_something() { - let found = excerpt("a match in here", &["match".to_owned()], 0).expect("a hit"); + let found = excerpt("a match in here", &terms(&["match"]), 0).expect("a hit"); assert!(found.segments.iter().any(|one| one.matched)); } + +#[test] +fn a_guessed_spelling_is_marked_and_found_like_the_typed_one() { + let guessed = [Term { + text: "cormanillo".into(), + whole: true, + alike: vec!["cormanilo".into()], + }]; + assert!(holds("si es desde Porter sera cormanilo", &guessed)); + let found = excerpt("sera cormanilo.", &guessed, 100).expect("a hit"); + assert_eq!( + found.segments, + vec![plain("sera "), hit("cormanilo"), plain(".")] + ); +} diff --git a/crates/cp-core/src/typo.rs b/crates/cp-core/src/typo.rs new file mode 100644 index 00000000..58435679 --- /dev/null +++ b/crates/cp-core/src/typo.rs @@ -0,0 +1,54 @@ +pub fn tolerance(typed: &str) -> usize { + match typed.chars().count() { + 0..=3 => 0, + 4..=7 => 1, + _ => 2, + } +} + +pub fn distance(typed: &str, word: &str, prefix: bool) -> Option { + let limit = tolerance(typed); + if limit == 0 { + return None; + } + let a: Vec = typed.chars().collect(); + let b: Vec = word.chars().collect(); + let reach = if prefix { + b.len().min(a.len() + limit) + } else { + b.len() + }; + if reach + limit < a.len() || reach > a.len() + limit { + return None; + } + let b = &b[..reach]; + let mut rows = vec![vec![0usize; b.len() + 1]; a.len() + 1]; + for (j, cell) in rows[0].iter_mut().enumerate() { + *cell = j; + } + for i in 1..=a.len() { + rows[i][0] = i; + for j in 1..=b.len() { + let cost = usize::from(a[i - 1] != b[j - 1]); + let mut best = (rows[i - 1][j] + 1) + .min(rows[i][j - 1] + 1) + .min(rows[i - 1][j - 1] + cost); + if i > 1 && j > 1 && a[i - 1] == b[j - 2] && a[i - 2] == b[j - 1] { + best = best.min(rows[i - 2][j - 2] + 1); + } + rows[i][j] = best; + } + } + let last = &rows[a.len()]; + let found = if prefix { + let from = a.len().saturating_sub(limit).min(b.len()); + last[from..].iter().copied().min()? + } else { + last[b.len()] + }; + (found <= limit).then_some(found) +} + +#[cfg(test)] +#[path = "typo_test.rs"] +mod tests; diff --git a/crates/cp-core/src/typo_test.rs b/crates/cp-core/src/typo_test.rs new file mode 100644 index 00000000..80a858cc --- /dev/null +++ b/crates/cp-core/src/typo_test.rs @@ -0,0 +1,40 @@ +use super::{distance, tolerance}; + +#[test] +fn short_words_are_never_guessed() { + assert_eq!(tolerance("sop"), 0); + assert_eq!(distance("sop", "sopa", false), None); + assert_eq!(distance("sop", "sopa", true), None); +} + +#[test] +fn the_longer_the_word_the_more_mistakes_it_forgives() { + assert_eq!(tolerance("sopa"), 1); + assert_eq!(tolerance("cormanil"), 2); + assert_eq!(distance("sopas", "sopa", false), Some(1)); + assert_eq!(distance("sopxyz", "sopa", false), None); + assert_eq!(distance("cormanillo", "cormanilo", false), Some(1)); + assert_eq!(distance("cromanilo", "cormanilo", false), Some(1)); + assert_eq!(distance("kormanyllo", "cormanilo", false), None); +} + +#[test] +fn a_swap_of_two_letters_is_one_mistake() { + assert_eq!(distance("piza", "pzia", false), Some(1)); + assert_eq!(distance("reunoin", "reunion", false), Some(1)); +} + +#[test] +fn a_word_being_typed_is_compared_with_the_start_of_the_other() { + assert_eq!(distance("cormanill", "cormanilo", true), Some(1)); + assert_eq!(distance("cormanil", "cormanilo", true), Some(0)); + assert_eq!(distance("cromani", "cormanilo", true), Some(1)); + assert_eq!(distance("cromani", "cormanilo", false), None); + assert_eq!(distance("zzzzz", "cormanilo", true), None); +} + +#[test] +fn letters_beyond_the_basic_alphabet_count_as_one() { + assert_eq!(distance("ñandú", "ñandu", false), Some(1)); + assert_eq!(distance("日本語です", "日本語でず", false), Some(1)); +} diff --git a/crates/cp-panel/src/view.rs b/crates/cp-panel/src/view.rs index 2ac7056d..149d7fce 100644 --- a/crates/cp-panel/src/view.rs +++ b/crates/cp-panel/src/view.rs @@ -606,7 +606,8 @@ pub fn harvest(query: &str) -> (Vec, String) { } pub fn sweeten(query: &str) -> String { - query + let spaced = query.ends_with(char::is_whitespace); + let sweet = query .split_whitespace() .map(|token| { let (sign, rest) = match token.strip_prefix('-') { @@ -619,7 +620,12 @@ pub fn sweeten(query: &str) -> String { } }) .collect::>() - .join(" ") + .join(" "); + if spaced && !sweet.is_empty() { + sweet + " " + } else { + sweet + } } pub fn label_of_form(form: Form) -> &'static str { diff --git a/crates/cp-panel/src/view_test.rs b/crates/cp-panel/src/view_test.rs index 243a78bf..20d8bf81 100644 --- a/crates/cp-panel/src/view_test.rs +++ b/crates/cp-panel/src/view_test.rs @@ -1050,3 +1050,11 @@ fn a_long_text_card_shows_the_lines_the_chosen_height_allows() { let shut = |most| face_for(&long, false, most).shut_px(); assert!(shut(3) > shut(2) && shut(2) > shut(1)); } + +#[test] +fn the_space_after_the_last_word_reaches_the_search() { + assert_eq!(sweeten("jira "), "jira "); + assert_eq!(sweeten("jira"), "jira"); + assert_eq!(sweeten("#imagen jira "), "k:image jira "); + assert_eq!(sweeten(" "), ""); +} diff --git a/crates/cp-store/examples/budget.rs b/crates/cp-store/examples/budget.rs index e131010a..330f626f 100644 --- a/crates/cp-store/examples/budget.rs +++ b/crates/cp-store/examples/budget.rs @@ -119,6 +119,24 @@ fn main() -> std::process::ExitCode { .expect("queried"); }, ); + over += measure( + Budget { + what: "a whole word leaving links out", + ceiling: Duration::from_millis(20), + }, + || { + store + .list( + &Filter { + query: Some("example ".into()), + ..Default::default() + }, + 100, + None, + ) + .expect("queried"); + }, + ); over += measure( Budget { what: "worst case: matches the whole history", @@ -151,7 +169,7 @@ fn main() -> std::process::ExitCode { over += measure( Budget { what: "page ten by cursor", - ceiling: Duration::from_millis(20), + ceiling: Duration::from_millis(60), }, || { let filter = Filter { diff --git a/crates/cp-store/src/lib.rs b/crates/cp-store/src/lib.rs index 4f21ada2..7049e5c6 100644 --- a/crates/cp-store/src/lib.rs +++ b/crates/cp-store/src/lib.rs @@ -5,6 +5,7 @@ pub mod legacy; pub mod query; pub mod recover; pub mod schema; +mod seek; pub mod store; pub use backup::{Brought, Made, Taken}; diff --git a/crates/cp-store/src/query.rs b/crates/cp-store/src/query.rs index 5fa340f8..e4d0ff75 100644 --- a/crates/cp-store/src/query.rs +++ b/crates/cp-store/src/query.rs @@ -27,7 +27,10 @@ pub fn parse(input: &str, clock: &Clock) -> Filter { let mut filter = Filter::default(); let mut words: Vec = Vec::new(); let mut label: Vec = Vec::new(); - for token in tokens(input) { + let tokens = tokens(input); + let count = tokens.len(); + let mut typing = false; + for (at, token) in tokens.into_iter().enumerate() { match operator(&token, clock) { Some(Op::Kinds(kinds, false)) => filter.kinds.extend(kinds), Some(Op::Kinds(kinds, true)) => filter.exclude_kinds.extend(kinds), @@ -39,11 +42,20 @@ pub fn parse(input: &str, clock: &Clock) -> Filter { Some(Op::OnlyBroken) => filter.broken = Broken::Only, Some(Op::Label(text)) => label.push(text), Some(Op::Order(order)) => filter.order = order, - None if token.chars().any(char::is_alphanumeric) => words.push(token), + None if token.chars().any(char::is_alphanumeric) => { + typing = at + 1 == count && !input.ends_with(char::is_whitespace); + words.push(token); + } None => {} } } - filter.query = (!words.is_empty()).then(|| words.join(" ")); + filter.query = (!words.is_empty()).then(|| { + let mut query = words.join(" "); + if !typing { + query.push(' '); + } + query + }); filter.label_query = (!label.is_empty()).then(|| label.join(" ")); filter } diff --git a/crates/cp-store/src/query_test.rs b/crates/cp-store/src/query_test.rs index e32734c9..19ef6dcb 100644 --- a/crates/cp-store/src/query_test.rs +++ b/crates/cp-store/src/query_test.rs @@ -222,7 +222,7 @@ fn a_word_with_nothing_to_search_for_does_not_blank_the_list() { "«{typed}» is the whole history" ); } - assert_eq!(parsed("hello ...").query.as_deref(), Some("hello")); + assert_eq!(parsed("hello ...").query.as_deref(), Some("hello ")); } #[test] @@ -242,7 +242,7 @@ fn the_example_from_the_design_note() { assert_eq!( filter, Filter { - query: Some("token".into()), + query: Some("token ".into()), apps: vec!["code".into()], since: Some(CLOCK.now - HOUR), ..Default::default() @@ -253,7 +253,7 @@ fn the_example_from_the_design_note() { #[test] fn everything_at_once() { let filter = parsed(" request k:json,text -@safari #red ~7d is:pinned l:may sort:used "); - assert_eq!(filter.query.as_deref(), Some("request")); + assert_eq!(filter.query.as_deref(), Some("request ")); assert_eq!(filter.kinds, vec![Kind::Json, Kind::Text]); assert_eq!(filter.exclude_apps, vec!["safari"]); assert_eq!(filter.colors, vec![1]); @@ -340,3 +340,15 @@ fn a_quoted_app_with_a_comma_is_split_into_two_apps_not_kept_whole() { can ever match the real application name" ); } + +#[test] +fn a_space_after_a_word_says_it_is_finished() { + assert_eq!(parsed("jira").query.as_deref(), Some("jira")); + assert_eq!(parsed("jira ").query.as_deref(), Some("jira ")); + assert_eq!(parsed("jira abr").query.as_deref(), Some("jira abr")); + assert_eq!( + parsed("jira k:text").query.as_deref(), + Some("jira "), + "a word followed by a filter was finished too" + ); +} diff --git a/crates/cp-store/src/seek.rs b/crates/cp-store/src/seek.rs new file mode 100644 index 00000000..322073f1 --- /dev/null +++ b/crates/cp-store/src/seek.rs @@ -0,0 +1,151 @@ +use crate::Result; +use crate::store::{Clauses, Filter, Order}; +use cp_core::search::{Term, holds, terms_of}; +use cp_core::typo::{distance, tolerance}; +use rusqlite::functions::FunctionFlags; +use rusqlite::{Connection, params_from_iter}; + +pub(crate) const EVERYWHERE: &str = "items.search_text || ' ' || items.search_label || ' ' \ + || items.search_app || ' ' || items.search_ocr"; + +const WORDLESS: &str = + "(items.search_label = '' AND (items.kind IS 'image' OR items.search_text = ''))"; + +const WEIGHED: &str = "bm25(items_fts, 1.0, 2.0, 0.3, 0.5)"; + +const GUESSES: usize = 8; + +pub(crate) fn teach(db: &Connection) -> Result<()> { + db.create_scalar_function( + "cp_holds", + 2, + FunctionFlags::SQLITE_UTF8 | FunctionFlags::SQLITE_DETERMINISTIC, + |ctx| { + let text = ctx.get::(0)?; + let terms = ctx.get::(1)?; + Ok(holds(&text, &decode(&terms))) + }, + )?; + db.execute_batch( + "CREATE VIRTUAL TABLE IF NOT EXISTS temp.items_vocab + USING fts5vocab(main, items_fts, row);", + )?; + Ok(()) +} + +pub(crate) fn fts_expression(terms: &[Term]) -> Option { + let quoted = |word: &str| format!("\"{}\"", word.replace('"', "\"\"")); + let groups: Vec = terms + .iter() + .map(|term| { + let mut ways = vec![quoted(&term.text)]; + if !term.whole { + ways.push(quoted(&term.text) + "*"); + } + ways.extend(term.alike.iter().map(|word| quoted(word))); + if ways.len() == 1 { + ways.remove(0) + } else { + format!("({})", ways.join(" OR ")) + } + }) + .collect(); + (!groups.is_empty()).then(|| groups.join(" AND ")) +} + +pub(crate) fn encode(terms: &[Term]) -> String { + terms + .iter() + .map(|term| { + let mark = if term.whole { "w" } else { "o" }; + [mark, term.text.as_str()] + .into_iter() + .chain(term.alike.iter().map(String::as_str)) + .collect::>() + .join(" ") + }) + .collect::>() + .join("\n") +} + +fn decode(text: &str) -> Vec { + text.lines() + .filter_map(|line| { + let mut words = line.split(' '); + let whole = words.next()? == "w"; + let text = words.next()?.to_owned(); + Some(Term { + text, + whole, + alike: words.map(str::to_owned).collect(), + }) + }) + .collect() +} + +pub(crate) fn rank_of(filter: &Filter, terms: &[Term]) -> Option { + if terms.is_empty() { + return None; + } + Some(if filter.order == Order::Recent { + format!("(CASE WHEN {WORDLESS} THEN 1000000.0 ELSE 0.0 END + {WEIGHED})") + } else { + WORDLESS.into() + }) +} + +pub(crate) fn terms_for(db: &Connection, filter: &Filter) -> Result>> { + let typed = filter.query.as_deref().map(terms_of).unwrap_or_default(); + if typed.iter().all(|term| tolerance(&term.text) == 0) { + return Ok(Some(typed)); + } + let Some(clauses) = Clauses::of(filter, &typed, false, true) else { + return Ok(Some(typed)); + }; + let sql = format!("SELECT EXISTS (SELECT 1 {})", clauses.source()); + let found: bool = db.query_row(&sql, params_from_iter(clauses.bound.iter()), |row| { + row.get(0) + })?; + if found { + return Ok(Some(typed)); + } + guessed(db, typed) +} + +fn guessed(db: &Connection, mut terms: Vec) -> Result>> { + let shortest = terms + .iter() + .filter(|term| tolerance(&term.text) > 0) + .map(|term| term.text.chars().count() - tolerance(&term.text)) + .min() + .unwrap_or(0); + let mut near: Vec> = vec![Vec::new(); terms.len()]; + let mut stmt = db.prepare("SELECT term, doc FROM temp.items_vocab WHERE length(term) >= ?")?; + let rows = stmt.query_map([shortest as i64], |row| { + Ok((row.get::<_, String>(0)?, row.get::<_, i64>(1)?)) + })?; + for row in rows { + let (word, docs) = row?; + for (term, found) in terms.iter().zip(near.iter_mut()) { + if let Some(off) = distance(&term.text, &word, !term.whole) + && off > 0 + { + found.push((off, -docs, word.clone())); + } + } + } + for (term, mut found) in terms.iter_mut().zip(near) { + found.sort(); + term.alike = found + .into_iter() + .take(GUESSES) + .map(|(_, _, word)| word) + .collect(); + } + let guessed = terms.iter().any(|term| !term.alike.is_empty()); + Ok(guessed.then_some(terms)) +} + +#[cfg(test)] +#[path = "seek_test.rs"] +mod tests; diff --git a/crates/cp-store/src/seek_test.rs b/crates/cp-store/src/seek_test.rs new file mode 100644 index 00000000..dcb49476 --- /dev/null +++ b/crates/cp-store/src/seek_test.rs @@ -0,0 +1,39 @@ +use super::{decode, encode, fts_expression}; +use cp_core::search::{Term, terms_of}; + +#[test] +fn the_terms_travel_to_the_database_and_back_unchanged() { + let mut terms = terms_of("jira cormanillo abr"); + terms[1].alike = vec!["cormanilo".into(), "cormanito".into()]; + assert_eq!(decode(&encode(&terms)), terms); + assert!(decode("").is_empty()); +} + +#[test] +fn a_word_being_typed_weighs_its_exact_spelling_above_its_prefix() { + assert_eq!( + fts_expression(&terms_of("jira abr")).as_deref(), + Some(r#""jira" AND ("abr" OR "abr"*)"#) + ); +} + +#[test] +fn a_guess_joins_the_word_it_stands_for() { + let terms = [Term { + text: "cormanillo".into(), + whole: true, + alike: vec!["cormanilo".into()], + }]; + assert_eq!( + fts_expression(&terms).as_deref(), + Some(r#"("cormanillo" OR "cormanilo")"#) + ); +} + +#[test] +fn a_quote_inside_a_word_is_escaped() { + assert_eq!( + fts_expression(&terms_of("it's\"x ")).as_deref(), + Some(r#""it's""x""#) + ); +} diff --git a/crates/cp-store/src/store.rs b/crates/cp-store/src/store.rs index 1caa541d..2ac04a71 100644 --- a/crates/cp-store/src/store.rs +++ b/crates/cp-store/src/store.rs @@ -1,18 +1,11 @@ +use crate::seek::{EVERYWHERE, encode, fts_expression, rank_of, teach, terms_for}; use crate::{Error, Result}; use cp_core::item::{Item, Payload}; use cp_core::kind::Kind; -use cp_core::search::{EXCERPT_CHARS, Excerpt, excerpt, fold, terms_of}; +use cp_core::search::{EXCERPT_CHARS, Excerpt, Term, excerpt, fold, terms_of}; use rusqlite::types::ToSql; use rusqlite::{Connection, OptionalExtension, params, params_from_iter}; -fn fts_expression(query: &str) -> Option { - let terms: Vec = terms_of(query) - .iter() - .map(|word| format!("\"{}\"*", word.replace('"', "\"\""))) - .collect(); - (!terms.is_empty()).then(|| terms.join(" ")) -} - #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum FoundIn { Text, @@ -49,24 +42,34 @@ pub struct Listed { #[derive(Debug, Clone, PartialEq, Eq)] pub struct Cursor { order: Order, + rank: String, key: String, id: i64, } impl Cursor { pub fn encode(&self) -> String { - format!("{}:{}:{}", self.order.as_str(), self.key, self.id) + format!( + "{}:{}:{}:{}", + self.order.as_str(), + self.rank, + self.key, + self.id + ) } pub fn decode(text: &str) -> Option { let (name, rest) = text.split_once(':')?; let order = Order::from_name(name)?; + let (rank, rest) = rest.split_once(':')?; + rank.parse::().ok()?; let (key, id) = rest.rsplit_once(':')?; if order != Order::ByGroup && key.parse::().is_err() { return None; } Some(Cursor { order, + rank: rank.to_owned(), key: key.to_owned(), id: id.parse().ok()?, }) @@ -156,25 +159,43 @@ pub struct AppCount { pub count: i64, } -struct Clauses { +pub(crate) struct Clauses { joins_index: bool, conditions: Vec, - bound: Vec>, + pub(crate) bound: Vec>, } impl Clauses { - fn of(filter: &Filter, with_kinds: bool, with_apps: bool) -> Option { + pub(crate) fn of( + filter: &Filter, + terms: &[Term], + with_kinds: bool, + with_apps: bool, + ) -> Option { let mut clauses = Self { joins_index: false, conditions: vec!["items.deleted_at IS NULL".into()], bound: Vec::new(), }; let mut index = Vec::new(); - if let Some(text) = &filter.query { - index.push(fts_expression(text)?); + if filter.query.is_some() { + index.push(fts_expression(terms)?); + if terms.iter().any(|term| term.whole) { + clauses + .conditions + .push(format!("cp_holds({EVERYWHERE}, ?)")); + clauses.bound.push(Box::new(encode(terms))); + } } if let Some(label) = &filter.label_query { - index.push(format!("search_label : ({})", fts_expression(label)?)); + let typed: Vec = terms_of(label) + .into_iter() + .map(|term| Term { + whole: false, + ..term + }) + .collect(); + index.push(format!("search_label : ({})", fts_expression(&typed)?)); } if !index.is_empty() { clauses.joins_index = true; @@ -245,7 +266,7 @@ impl Clauses { } } - fn source(&self) -> String { + pub(crate) fn source(&self) -> String { let conditions = self.conditions.join(" AND "); if self.joins_index { format!( @@ -288,6 +309,7 @@ impl Store { pub fn in_memory() -> Result { let db = Connection::open_in_memory()?; crate::schema::create(&db)?; + teach(&db)?; Ok(Self { db, blobs: None, @@ -304,6 +326,7 @@ impl Store { crate::schema::refuse_future(&db)?; crate::schema::create(&db)?; crate::schema::migrate(&db)?; + teach(&db)?; let exposure = restrict(path, 0o600)?; for side in sidecars(path) { if side.exists() { @@ -960,7 +983,10 @@ impl Store { } pub fn count_matching(&self, filter: &Filter) -> Result { - let Some(clauses) = Clauses::of(filter, true, true) else { + let Some(terms) = terms_for(&self.db, filter)? else { + return Ok(0); + }; + let Some(clauses) = Clauses::of(filter, &terms, true, true) else { return Ok(0); }; let sql = format!("SELECT COUNT(*) {}", clauses.source()); @@ -972,11 +998,15 @@ impl Store { } pub fn list(&self, filter: &Filter, limit: usize, after: Option) -> Result { - let Some(mut clauses) = Clauses::of(filter, true, true) else { + let Some(terms) = terms_for(&self.db, filter)? else { + return Ok(Page::default()); + }; + let Some(mut clauses) = Clauses::of(filter, &terms, true, true) else { return Ok(Page::default()); }; let key = filter.order.key(); - let terms = filter.query.as_deref().map(terms_of).unwrap_or_default(); + let rank = rank_of(filter, &terms); + let resumed = after.is_some(); if let Some(cursor) = after { if cursor.order != filter.order { return Err(Error::WrongCursor { @@ -984,14 +1014,23 @@ impl Store { order: filter.order.as_str(), }); } - clauses - .conditions - .push(format!("({key} < ? OR ({key} = ? AND items.id < ?))")); + if rank.is_some() { + let at: f64 = cursor.rank.parse().unwrap_or(f64::MIN); + clauses.bound.push(Box::new(at)); + clauses.bound.push(Box::new(at)); + } else { + clauses + .conditions + .push(format!("({key} < ? OR ({key} = ? AND items.id < ?))")); + } clauses.bound.push(bound_key(filter.order, &cursor.key)); clauses.bound.push(bound_key(filter.order, &cursor.key)); clauses.bound.push(Box::new(cursor.id)); } - let sql = page_sql(&clauses, key, !terms.is_empty()); + let sql = match &rank { + Some(rank) => ranked_sql(&clauses, key, rank, resumed), + None => page_sql(&clauses, key), + }; let fetch = i64::try_from(limit).map_or(i64::MAX, |limit| limit.saturating_add(1)); clauses.bound.push(Box::new(fetch)); @@ -1025,27 +1064,31 @@ impl Store { row.get::<_, i64>(14)?.to_string() } }; - Ok((listed, key)) + Ok((listed, row.get::<_, f64>(17)?, key)) })?; - let mut keyed: Vec<(Listed, String)> = rows.collect::>()?; + let mut keyed: Vec<(Listed, f64, String)> = rows.collect::>()?; let more = keyed.len() > limit; keyed.truncate(limit); let next = match keyed.last() { - Some((last, key)) if more => Some(Cursor { + Some((last, rank, key)) if more => Some(Cursor { order: filter.order, + rank: rank.to_string(), key: key.clone(), id: last.id, }), _ => None, }; Ok(Page { - rows: keyed.into_iter().map(|(listed, _)| listed).collect(), + rows: keyed.into_iter().map(|(listed, _, _)| listed).collect(), next, }) } pub fn facets(&self, filter: &Filter) -> Result> { - let Some(mut clauses) = Clauses::of(filter, false, true) else { + let Some(terms) = terms_for(&self.db, filter)? else { + return Ok(Vec::new()); + }; + let Some(mut clauses) = Clauses::of(filter, &terms, false, true) else { return Ok(Vec::new()); }; clauses.conditions.push("items.kind IS NOT NULL".into()); @@ -1069,7 +1112,10 @@ impl Store { } pub fn distinct_apps(&self, filter: &Filter) -> Result> { - let Some(mut clauses) = Clauses::of(filter, true, false) else { + let Some(terms) = terms_for(&self.db, filter)? else { + return Ok(Vec::new()); + }; + let Some(mut clauses) = Clauses::of(filter, &terms, true, false) else { return Ok(Vec::new()); }; clauses @@ -1218,33 +1264,47 @@ fn bound_key(order: Order, key: &str) -> Box { } } -fn page_sql(clauses: &Clauses, key: &str, with_query: bool) -> String { +fn page_columns() -> String { format!( - "SELECT items.id, items.modified_at, items.created_at, items.kind, - SUBSTR(items.preview_text, 1, {preview}), items.app_source, items.label, - items.card_color, items.thumb_path, items.paste_count, items.last_used_at, - items.broken_since, items.pinned, {ocr}, page.key, {whole}, - items.group_key + "items.id, items.modified_at, items.created_at, items.kind, + SUBSTR(items.preview_text, 1, {PREVIEW_CHARS}), items.app_source, items.label, + items.card_color, items.thumb_path, items.paste_count, items.last_used_at, + items.broken_since, items.pinned" + ) +} + +fn page_sql(clauses: &Clauses, key: &str) -> String { + format!( + "SELECT {}, '', page.key, '', items.group_key, 0 FROM (SELECT items.id AS id, {key} AS key {} ORDER BY {key} DESC, items.id DESC LIMIT ?) AS page JOIN items ON items.id = page.id ORDER BY page.key DESC, page.id DESC", + page_columns(), + clauses.source() + ) +} + +fn ranked_sql(clauses: &Clauses, key: &str, rank: &str, resumed: bool) -> String { + let past = if resumed { + "WHERE rank > ? OR (rank = ? AND (key < ? OR (key = ? AND id < ?)))" + } else { + "" + }; + format!( + "WITH found AS MATERIALIZED (SELECT items.id AS id, {key} AS key, {rank} AS rank {}) + SELECT {}, COALESCE(items.ocr_text, items.search_ocr), page.key, + items.preview_text, items.group_key, page.rank + FROM (SELECT id, key, rank FROM found {past} + ORDER BY rank, key DESC, id DESC LIMIT ?) AS page + JOIN items ON items.id = page.id + ORDER BY page.rank, page.key DESC, page.id DESC", clauses.source(), - preview = PREVIEW_CHARS, - ocr = if with_query { - "COALESCE(items.ocr_text, items.search_ocr)" - } else { - "''" - }, - whole = if with_query { - "items.preview_text" - } else { - "''" - } + page_columns() ) } -fn snippet_of(listed: &Listed, whole: &str, ocr: &str, terms: &[String]) -> Option { +fn snippet_of(listed: &Listed, whole: &str, ocr: &str, terms: &[Term]) -> Option { if terms.is_empty() { return None; } diff --git a/crates/cp-store/src/store_listing_test.rs b/crates/cp-store/src/store_listing_test.rs index 07629cfa..467fd8ef 100644 --- a/crates/cp-store/src/store_listing_test.rs +++ b/crates/cp-store/src/store_listing_test.rs @@ -531,8 +531,8 @@ fn no_order_sorts_the_history_in_memory() { ..Default::default() }; let clauses = - Clauses::of(&filter, true, true).expect("with no term there are still clauses"); - let sql = page_sql(&clauses, order.key(), false); + Clauses::of(&filter, &[], true, true).expect("with no term there are still clauses"); + let sql = page_sql(&clauses, order.key()); let plan: Vec = store .db .prepare(&format!("EXPLAIN QUERY PLAN {sql}")) @@ -734,16 +734,16 @@ fn a_cursor_from_another_order_is_refused_not_misread() { "it goes out and comes back as text" ); assert_eq!(Cursor::decode("recent:1"), None); - assert_eq!(Cursor::decode("sideways:1:2"), None); - assert_eq!(Cursor::decode("recent:1:2:3"), None); + assert_eq!(Cursor::decode("sideways:0:1:2"), None); + assert_eq!(Cursor::decode("recent:x:1:2"), None); assert_eq!( - Cursor::decode("recent:x:2"), + Cursor::decode("recent:0:x:2"), None, "a numeric order wants a number" ); - let grouped = Cursor::decode("by-group:ejemplo.test:7").expect("a group cursor"); + let grouped = Cursor::decode("by-group:0:ejemplo.test:7").expect("a group cursor"); assert_eq!(Cursor::decode(&grouped.encode()), Some(grouped)); - let colons = Cursor::decode("by-group:carpeta:de:red:7").expect("a key may hold colons"); + let colons = Cursor::decode("by-group:1:carpeta:de:red:7").expect("a key may hold colons"); assert_eq!(Cursor::decode(&colons.encode()), Some(colons)); } @@ -1368,3 +1368,256 @@ fn every_order_including_by_group_pages_without_repeating_or_skipping_when_most_ ids.dedup(); assert_eq!(ids.len(), 30, "ByGroup repeated rows"); } + +fn jira_history() -> Store { + let store = Store::in_memory().expect("schema"); + let note = "abre el ticket en Jira mañana"; + store + .insert_item("uuid-note", &text_item(note, Kind::Text), note, 10) + .expect("note"); + let link = "https://jira.empresa.test/browse/AB-1"; + store + .insert_item("uuid-link", &text_item(link, Kind::Link), link, 20) + .expect("link"); + let image = Item { + kind: Some(Kind::Image), + formats: vec![Format { + id: "public.png".into(), + payload: Payload::Inline(vec![1]), + }], + }; + let id = store + .insert_item("uuid-shot", &image, "", 30) + .expect("image"); + store.set_ocr_text(id, "Jira board", 31).expect("ocr"); + store +} + +fn searched(store: &Store, query: &str) -> Vec> { + let filter = Filter { + query: Some(query.into()), + ..Default::default() + }; + all(store, &filter).iter().map(|one| one.kind).collect() +} + +#[test] +fn what_matched_in_the_words_comes_before_what_was_read_in_an_image() { + let store = jira_history(); + assert_eq!( + searched(&store, "jira"), + vec![Some(Kind::Text), Some(Kind::Link), Some(Kind::Image)], + "the newest image still waits behind the texts" + ); +} + +#[test] +fn a_word_followed_by_a_space_leaves_the_links_out() { + let store = jira_history(); + assert_eq!( + searched(&store, "jira "), + vec![Some(Kind::Text), Some(Kind::Image)] + ); + assert_eq!(searched(&store, "jir"), searched(&store, "jira")); + assert!( + searched(&store, "jir ").is_empty(), + "a whole word is not a prefix" + ); + let filter = Filter { + query: Some("jira ".into()), + ..Default::default() + }; + assert_eq!(store.count_matching(&filter).expect("counted"), 2); +} + +#[test] +fn paging_walks_from_the_words_into_the_images_without_repeating() { + let store = jira_history(); + let filter = Filter { + query: Some("jira".into()), + ..Default::default() + }; + let mut seen = Vec::new(); + let mut after = None; + loop { + let page = store.list(&filter, 1, after).expect("a page"); + seen.extend(page.rows.iter().map(|one| one.kind)); + match page.next { + Some(next) => after = Some(next), + None => break, + } + } + assert_eq!( + seen, + vec![Some(Kind::Text), Some(Kind::Link), Some(Kind::Image)] + ); +} + +#[test] +fn a_quote_in_the_query_does_not_break_the_ranking() { + let store = Store::in_memory().expect("schema"); + let text = "it's done"; + store + .insert_item("uuid-q", &text_item(text, Kind::Text), text, 1) + .expect("insert"); + assert_eq!(searched(&store, "it's"), vec![Some(Kind::Text)]); + assert_eq!(searched(&store, "it's "), vec![Some(Kind::Text)]); +} + +fn notes(rows: &[(&str, i64)]) -> Store { + let store = Store::in_memory().expect("schema"); + for (at, (text, when)) in rows.iter().enumerate() { + store + .insert_item( + &format!("uuid-{at}"), + &text_item(text, Kind::Text), + text, + *when, + ) + .expect("insert"); + } + store +} + +fn found(store: &Store, filter: &Filter) -> Vec { + all(store, filter) + .into_iter() + .map(|one| one.preview) + .collect() +} + +fn asked(query: &str) -> Filter { + Filter { + query: Some(query.into()), + ..Default::default() + } +} + +#[test] +fn the_most_relevant_text_comes_first_and_the_date_only_breaks_ties() { + let store = notes(&[ + ("sopa cormanilo, revisar", 1), + ( + "entonces en la sopa, se activaran los pagos relacionados con vg team y meat, \ + pero si es desde Porter sera cormanilo y lo demás queda igual que siempre", + 5, + ), + ("algo sin nada que ver", 9), + ]); + assert_eq!( + found(&store, &asked("sopa cormanilo")), + vec![ + "sopa cormanilo, revisar".to_owned(), + all(&store, &asked("porter"))[0].preview.clone() + ], + "the short note about both words beats the newer long text" + ); +} + +#[test] +fn a_word_spelled_out_beats_a_word_that_only_starts_the_same() { + let store = notes(&[("reunión del lunes", 1), ("reuniones del lunes", 2)]); + assert_eq!( + found(&store, &asked("reunion")), + vec!["reunión del lunes", "reuniones del lunes"] + ); +} + +#[test] +fn an_order_chosen_by_hand_is_kept_while_searching() { + let store = notes(&[ + ("sopa", 1), + ("sopa y mucho más texto alrededor de la sopa", 2), + ]); + let mut filter = asked("sopa"); + filter.order = Order::LastUsed; + let first = all(&store, &filter)[0].id; + store.record_paste(first, 50).expect("pasted"); + assert_eq!(all(&store, &filter)[0].id, first); + filter.order = Order::MostPasted; + assert_eq!(all(&store, &filter)[0].id, first); +} + +#[test] +fn a_typo_finds_the_word_only_when_nothing_matches_exactly() { + let store = notes(&[ + ("si es desde Porter sera cormanilo", 1), + ("la sopa de hoy", 2), + ]); + assert_eq!( + found(&store, &asked("cormanillo")), + vec!["si es desde Porter sera cormanilo"] + ); + assert_eq!( + found(&store, &asked("cromanilo ")), + vec!["si es desde Porter sera cormanilo"] + ); + assert_eq!( + found(&store, &asked("sopa cormanillo")), + Vec::::new(), + "every word still has to be there" + ); + assert_eq!( + store.count_matching(&asked("cormanillo")).expect("counted"), + 1 + ); + let rows = all(&store, &asked("porter cormanillo")); + let marked: Vec<&str> = rows[0] + .snippet + .as_ref() + .expect("a snippet") + .excerpt + .segments + .iter() + .filter(|one| one.matched) + .map(|one| one.text.as_str()) + .collect(); + assert_eq!( + marked, + vec!["Porter", "cormanilo"], + "the guessed word is marked" + ); +} + +#[test] +fn a_typo_is_not_guessed_when_the_word_exists() { + let store = notes(&[("la sopa de hoy", 1), ("la ropa de hoy", 2)]); + assert_eq!(found(&store, &asked("sopa")), vec!["la sopa de hoy"]); + assert!(found(&store, &asked("zzzz")).is_empty()); + assert!( + found(&store, &asked("sop ")).is_empty(), + "three letters are too few to guess" + ); +} + +#[test] +fn a_typo_while_still_typing_finds_the_word_it_starts() { + let store = notes(&[("si es desde Porter sera cormanilo", 1)]); + assert_eq!( + found(&store, &asked("cromani")), + vec!["si es desde Porter sera cormanilo"] + ); +} + +#[test] +fn paging_by_relevance_neither_repeats_nor_skips() { + let rows: Vec<(String, i64)> = (0..25) + .map(|at| (format!("sopa {}", "y más ".repeat(at % 7)), at as i64)) + .collect(); + let borrowed: Vec<(&str, i64)> = rows.iter().map(|(t, at)| (t.as_str(), *at)).collect(); + let store = notes(&borrowed); + let filter = asked("sopa"); + let whole: Vec = all(&store, &filter).iter().map(|one| one.id).collect(); + let mut walked = Vec::new(); + let mut after = None; + loop { + let page = store.list(&filter, 4, after).expect("a page"); + walked.extend(page.rows.iter().map(|one| one.id)); + match page.next { + Some(next) => after = Some(next), + None => break, + } + } + assert_eq!(walked, whole); + assert_eq!(walked.len(), 25); +} diff --git a/crates/cp-store/src/store_test.rs b/crates/cp-store/src/store_test.rs index 6a2b105a..6142b584 100644 --- a/crates/cp-store/src/store_test.rs +++ b/crates/cp-store/src/store_test.rs @@ -16,7 +16,8 @@ fn a_pinned_search_lets_the_index_drive_and_never_scans_the_fts_per_row() { ..Default::default() }, ] { - let clauses = Clauses::of(&filter, true, true).expect("clauses"); + let terms = terms_of("r"); + let clauses = Clauses::of(&filter, &terms, true, true).expect("clauses"); let sql = format!("EXPLAIN QUERY PLAN SELECT COUNT(*) {}", clauses.source()); let mut stmt = store.db.prepare(&sql).expect("a plan"); let steps: Vec = stmt