feat(search): whole words, relevance ranking and typo-tolerant search

Cargo.toml

@@ -24,7 +24,7 @@ thiserror = "2"
 tracing = "0.1"
 crossbeam-channel = "0.5"
 parking_lot = "0.12"
-rusqlite = { version = "0.40", features = ["bundled"] }
+rusqlite = { version = "0.40", features = ["bundled", "functions"] }
 blake3 = "1"
 xxhash-rust = { version = "0.8", features = ["xxh3"] }
 unicode-normalization = "0.1"

crates/cp-core/src/lib.rs

@@ -19,6 +19,7 @@ pub mod stamp;
 pub mod thumbnail;
 pub mod token;
 pub mod trouble;
+pub mod typo;
 pub mod watch;
 pub mod watching;
 

crates/cp-core/src/search.rs

@@ -39,14 +39,38 @@
     }
 }
 
-pub fn terms_of(query: &str) -> Vec<String> {
-    fold(query)
-        .split_whitespace()
-        .filter(|word| word.chars().any(char::is_alphanumeric))
-        .map(str::to_owned)
+#[derive(Debug, Clone, Default, PartialEq, Eq)]
+pub struct Term {
+    pub text: String,
+    pub whole: bool,
+    pub alike: Vec<String>,
+}
+
+pub fn terms_of(query: &str) -> Vec<Term> {
+    let folded = fold(query);
+    let typing = !folded.ends_with(char::is_whitespace);
+    let words: Vec<&str> = folded.split_whitespace().collect();
+    let last = words.len().saturating_sub(1);
+    words
+        .into_iter()
+        .enumerate()
+        .filter(|(_, word)| word.chars().any(char::is_alphanumeric))
+        .map(|(at, word)| Term {
+            text: word.to_owned(),
+            whole: at < last || !typing,
+            alike: Vec::new(),
+        })
         .collect()
 }
 
+pub fn holds(text: &str, terms: &[Term]) -> bool {
+    let chars: Vec<char> = fold(text).chars().collect();
+    let tokens = tokens_of(&chars);
+    terms
+        .iter()
+        .all(|term| !spots_of(&chars, &tokens, term).is_empty())
+}
+
 pub const EXCERPT_CHARS: usize = 160;
 
 #[derive(Debug, Clone, PartialEq, Eq)]
@@ -66,12 +90,12 @@
     }
 }
 
-pub fn excerpt(text: &str, terms: &[String], width: usize) -> Option<Excerpt> {
+pub fn excerpt(text: &str, terms: &[Term], width: usize) -> Option<Excerpt> {
     let folded = fold_mapped(text);
     let tokens = tokens_of(&folded.chars);
     let mut hits: Vec<(usize, usize)> = terms
         .iter()
-        .flat_map(|term| matches_of(&folded.chars, &tokens, term))
+        .flat_map(|term| spots_of(&folded.chars, &tokens, term))
         .map(|(from, to)| {
             let last = folded.origin[to - 1];
             (
@@ -170,6 +194,47 @@
     tokens
 }
 
+fn spots_of(chars: &[char], tokens: &[(usize, usize)], term: &Term) -> Vec<(usize, usize)> {
+    std::iter::once(&term.text)
+        .chain(&term.alike)
+        .flat_map(|word| {
+            if term.whole {
+                alone_of(chars, word)
+            } else {
+                matches_of(chars, tokens, word)
+            }
+        })
+        .collect()
+}
+
+fn alone_of(chars: &[char], word: &str) -> Vec<(usize, usize)> {
+    let wanted: Vec<char> = word
+        .trim_matches(|c: char| !c.is_alphanumeric())
+        .chars()
+        .collect();
+    if wanted.is_empty() {
+        return Vec::new();
+    }
+    let mut found = Vec::new();
+    let mut at = 0;
+    while at < chars.len() {
+        let start = at;
+        while at < chars.len() && !chars[at].is_whitespace() {
+            at += 1;
+        }
+        let chunk = &chars[start..at];
+        if let (Some(from), Some(to)) = (
+            chunk.iter().position(|c| c.is_alphanumeric()),
+            chunk.iter().rposition(|c| c.is_alphanumeric()),
+        ) && chunk[from..=to] == wanted[..]
+        {
+            found.push((start + from, start + to + 1));
+        }
+        at += 1;
+    }
+    found
+}
+
 fn matches_of(chars: &[char], tokens: &[(usize, usize)], term: &str) -> Vec<(usize, usize)> {
     let wanted: Vec<Vec<char>> = term
         .split(|c: char| !c.is_alphanumeric())

crates/cp-core/src/search_test.rs

@@ -1,4 +1,4 @@
-use super::{EXCERPT_CHARS, Segment, excerpt, fold, fold_mapped, terms_of};
+use super::{EXCERPT_CHARS, Segment, Term, excerpt, fold, fold_mapped, holds, terms_of};
 
 fn plain(text: &str) -> Segment {
     Segment {
@@ -14,8 +14,82 @@ fn hit(text: &str) -> Segment {
     }
 }
 
-fn terms(words: &[&str]) -> Vec<String> {
-    words.iter().map(|word| (*word).to_owned()).collect()
+fn terms(words: &[&str]) -> Vec<Term> {
+    words
+        .iter()
+        .map(|word| Term {
+            text: (*word).to_owned(),
+            ..Term::default()
+        })
+        .collect()
+}
+
+fn texts(query: &str) -> Vec<String> {
+    terms_of(query).into_iter().map(|term| term.text).collect()
+}
+
+fn whole(word: &str) -> Term {
+    Term {
+        text: word.into(),
+        whole: true,
+        ..Term::default()
+    }
+}
+
+#[test]
+fn a_word_followed_by_a_space_is_whole_and_the_one_being_typed_is_not() {
+    let wholes = |query| {
+        terms_of(query)
+            .into_iter()
+            .map(|term| term.whole)
+            .collect::<Vec<_>>()
+    };
+    assert_eq!(wholes("jira"), vec![false]);
+    assert_eq!(wholes("jira "), vec![true]);
+    assert_eq!(wholes("jira abr"), vec![true, false]);
+    assert_eq!(wholes("jira !!"), vec![true]);
+}
+
+#[test]
+fn a_whole_word_skips_links_paths_mail_and_longer_words() {
+    let jira = [whole("jira")];
+    assert!(!holds("https://jira.empresa.com/browse/AB-1", &jira));
+    assert!(!holds("C:\\tools\\jira\\notes.txt", &jira));
+    assert!(!holds("jira@empresa.com", &jira));
+    assert!(!holds("las jiras del equipo", &jira));
+    assert!(holds("abre el ticket en Jira mañana", &jira));
+    assert!(holds("lo vimos en jira.", &jira));
+    assert!(holds("(Jira) y ¿jira?", &jira));
+    assert!(holds("jira", &jira));
+}
+
+#[test]
+fn a_word_being_typed_still_finds_links_and_prefixes() {
+    assert!(holds("https://jira.empresa.com", &terms(&["jir"])));
+    assert!(holds("las jiras", &terms(&["jira"])));
+}
+
+#[test]
+fn every_term_must_hold() {
+    let both = [
+        whole("jira"),
+        Term {
+            text: "abr".into(),
+            ..Term::default()
+        },
+    ];
+    assert!(holds("abrir jira hoy", &both));
+    assert!(!holds("abrir jira.com", &both));
+    assert!(!holds("jira hoy", &both));
+}
+
+#[test]
+fn a_whole_word_marks_only_the_standalone_one() {
+    let found = excerpt("ve jira.com o jira, da igual", &[whole("jira")], 100).expect("a hit");
+    assert_eq!(
+        found.segments,
+        vec![plain("ve jira.com o "), hit("jira"), plain(", da igual")]
+    );
 }
 
 #[test]
@@ -192,7 +266,7 @@ fn nothing_to_look_for_is_nothing_found() {
 
 #[test]
 fn the_terms_are_the_words_the_index_would_look_for() {
-    assert_eq!(terms_of("Café  AB-4417 ... !!"), vec!["cafe", "ab-4417"]);
+    assert_eq!(texts("Café  AB-4417 ... !!"), vec!["cafe", "ab-4417"]);
     assert!(terms_of("   ").is_empty());
     assert!(terms_of("!!! ...").is_empty());
 }
@@ -291,28 +365,43 @@ fn nfc_and_nfd_spellings_of_the_same_letter_fold_the_same() {
 
 #[test]
 fn terms_of_an_empty_or_symbol_only_query_is_empty() {
-    assert_eq!(terms_of(""), Vec::<String>::new());
-    assert_eq!(terms_of("   "), Vec::<String>::new());
-    assert_eq!(terms_of("!?#"), Vec::<String>::new());
+    assert!(terms_of("").is_empty());
+    assert!(terms_of("   ").is_empty());
+    assert!(terms_of("!?#").is_empty());
 }
 
 #[test]
 fn a_leading_dash_stays_glued_to_its_word() {
-    assert_eq!(terms_of("gato -perro"), vec!["gato", "-perro"]);
+    assert_eq!(texts("gato -perro"), vec!["gato", "-perro"]);
 }
 
 #[test]
 fn quotes_are_not_a_token_boundary() {
-    assert_eq!(terms_of("\"hola mundo\""), vec!["\"hola", "mundo\""]);
+    assert_eq!(texts("\"hola mundo\""), vec!["\"hola", "mundo\""]);
 }
 
 #[test]
 fn a_term_longer_than_the_text_finds_nothing_and_does_not_panic() {
-    assert_eq!(excerpt("ab", &["abcdef".to_owned()], 100), None);
+    assert_eq!(excerpt("ab", &terms(&["abcdef"]), 100), None);
 }
 
 #[test]
 fn a_width_of_zero_still_shows_something() {
-    let found = excerpt("a match in here", &["match".to_owned()], 0).expect("a hit");
+    let found = excerpt("a match in here", &terms(&["match"]), 0).expect("a hit");
     assert!(found.segments.iter().any(|one| one.matched));
 }
+
+#[test]
+fn a_guessed_spelling_is_marked_and_found_like_the_typed_one() {
+    let guessed = [Term {
+        text: "cormanillo".into(),
+        whole: true,
+        alike: vec!["cormanilo".into()],
+    }];
+    assert!(holds("si es desde Porter sera cormanilo", &guessed));
+    let found = excerpt("sera cormanilo.", &guessed, 100).expect("a hit");
+    assert_eq!(
+        found.segments,
+        vec![plain("sera "), hit("cormanilo"), plain(".")]
+    );
+}

crates/cp-core/src/typo.rs

@@ -0,0 +1,54 @@
+pub fn tolerance(typed: &str) -> usize {
+    match typed.chars().count() {
+        0..=3 => 0,
+        4..=7 => 1,
+        _ => 2,
+    }
+}
+
+pub fn distance(typed: &str, word: &str, prefix: bool) -> Option<usize> {
+    let limit = tolerance(typed);
+    if limit == 0 {
+        return None;
+    }
+    let a: Vec<char> = typed.chars().collect();
+    let b: Vec<char> = word.chars().collect();
+    let reach = if prefix {
+        b.len().min(a.len() + limit)
+    } else {
+        b.len()
+    };
+    if reach + limit < a.len() || reach > a.len() + limit {
+        return None;
+    }
+    let b = &b[..reach];
+    let mut rows = vec![vec![0usize; b.len() + 1]; a.len() + 1];
+    for (j, cell) in rows[0].iter_mut().enumerate() {
+        *cell = j;
+    }
+    for i in 1..=a.len() {
+        rows[i][0] = i;
+        for j in 1..=b.len() {
+            let cost = usize::from(a[i - 1] != b[j - 1]);
+            let mut best = (rows[i - 1][j] + 1)
+                .min(rows[i][j - 1] + 1)
+                .min(rows[i - 1][j - 1] + cost);
+            if i > 1 && j > 1 && a[i - 1] == b[j - 2] && a[i - 2] == b[j - 1] {
+                best = best.min(rows[i - 2][j - 2] + 1);
+            }
+            rows[i][j] = best;
+        }
+    }
+    let last = &rows[a.len()];
+    let found = if prefix {
+        let from = a.len().saturating_sub(limit).min(b.len());
+        last[from..].iter().copied().min()?
+    } else {
+        last[b.len()]
+    };
+    (found <= limit).then_some(found)
+}
+
+#[cfg(test)]
+#[path = "typo_test.rs"]
+mod tests;

crates/cp-core/src/typo_test.rs

@@ -0,0 +1,40 @@
+use super::{distance, tolerance};
+
+#[test]
+fn short_words_are_never_guessed() {
+    assert_eq!(tolerance("sop"), 0);
+    assert_eq!(distance("sop", "sopa", false), None);
+    assert_eq!(distance("sop", "sopa", true), None);
+}
+
+#[test]
+fn the_longer_the_word_the_more_mistakes_it_forgives() {
+    assert_eq!(tolerance("sopa"), 1);
+    assert_eq!(tolerance("cormanil"), 2);
+    assert_eq!(distance("sopas", "sopa", false), Some(1));
+    assert_eq!(distance("sopxyz", "sopa", false), None);
+    assert_eq!(distance("cormanillo", "cormanilo", false), Some(1));
+    assert_eq!(distance("cromanilo", "cormanilo", false), Some(1));
+    assert_eq!(distance("kormanyllo", "cormanilo", false), None);
+}
+
+#[test]
+fn a_swap_of_two_letters_is_one_mistake() {
+    assert_eq!(distance("piza", "pzia", false), Some(1));
+    assert_eq!(distance("reunoin", "reunion", false), Some(1));
+}
+
+#[test]
+fn a_word_being_typed_is_compared_with_the_start_of_the_other() {
+    assert_eq!(distance("cormanill", "cormanilo", true), Some(1));
+    assert_eq!(distance("cormanil", "cormanilo", true), Some(0));
+    assert_eq!(distance("cromani", "cormanilo", true), Some(1));
+    assert_eq!(distance("cromani", "cormanilo", false), None);
+    assert_eq!(distance("zzzzz", "cormanilo", true), None);
+}
+
+#[test]
+fn letters_beyond_the_basic_alphabet_count_as_one() {
+    assert_eq!(distance("ñandú", "ñandu", false), Some(1));
+    assert_eq!(distance("日本語です", "日本語でず", false), Some(1));
+}

crates/cp-panel/src/view.rs

@@ -606,7 +606,8 @@ pub fn harvest(query: &str) -> (Vec<String>, String) {
 }
 
 pub fn sweeten(query: &str) -> String {
-    query
+    let spaced = query.ends_with(char::is_whitespace);
+    let sweet = query
         .split_whitespace()
         .map(|token| {
             let (sign, rest) = match token.strip_prefix('-') {
@@ -619,7 +620,12 @@ pub fn sweeten(query: &str) -> String {
             }
         })
         .collect::<Vec<_>>()
-        .join(" ")
+        .join(" ");
+    if spaced && !sweet.is_empty() {
+        sweet + " "
+    } else {
+        sweet
+    }
 }
 
 pub fn label_of_form(form: Form) -> &'static str {