diff --git a/boldi-vigna/src/query.rs b/boldi-vigna/src/query.rs index 0046a04..fd924fe 100644 --- a/boldi-vigna/src/query.rs +++ b/boldi-vigna/src/query.rs @@ -174,6 +174,40 @@ impl SpanQuery { SpanQueryWithTerms { query: self, terms } } + /// Calls `f` with the index of every `Term` leaf outside an excluded + /// operand, once per occurrence, left to right. An excluded operand + /// (`NotContaining`'s `little`, `NotContainedBy`'s `big`, + /// `NonOverlapping`'s `b`) only filters its sibling's spans, so no leaf + /// beneath it is visited. + pub fn for_each_positive_term(&self, f: &mut impl FnMut(usize)) { + match self { + SpanQuery::Empty => {} + SpanQuery::Term(i) => f(*i), + SpanQuery::Ordered(children) + | SpanQuery::Unordered(children) + | SpanQuery::Or(children) => { + for child in children { + child.for_each_positive_term(f); + } + } + SpanQuery::MaxGaps { inner, .. } + | SpanQuery::GapsInRange { inner, .. } + | SpanQuery::MaxWidth { inner, .. } + | SpanQuery::WithinPositions { inner, .. } + | SpanQuery::NotContaining { big: inner, .. } + | SpanQuery::NotContainedBy { little: inner, .. } + | SpanQuery::NonOverlapping { a: inner, .. } => inner.for_each_positive_term(f), + SpanQuery::Containing { big: a, little: b } + | SpanQuery::ContainedBy { little: a, big: b } + | SpanQuery::Overlapping { a, b } + | SpanQuery::Before { a, b } + | SpanQuery::After { a, b } => { + a.for_each_positive_term(f); + b.for_each_positive_term(f); + } + } + } + /// Maximum term index referenced anywhere in this query tree. pub fn max_term_index(&self) -> Option { match self { @@ -199,6 +233,30 @@ impl SpanQuery { } } + /// Whether an `Unordered` sits anywhere in this query tree: the operator + /// whose gap-read form walks every child order (`NodeState::compile_gaps`). + pub(crate) fn holds_unordered(&self) -> bool { + match self { + SpanQuery::Unordered(_) => true, + SpanQuery::Empty | SpanQuery::Term(_) => false, + SpanQuery::Ordered(children) | SpanQuery::Or(children) => { + children.iter().any(SpanQuery::holds_unordered) + } + SpanQuery::MaxGaps { inner, .. } + | SpanQuery::GapsInRange { inner, .. } + | SpanQuery::MaxWidth { inner, .. } + | SpanQuery::WithinPositions { inner, .. } => inner.holds_unordered(), + SpanQuery::Containing { big: a, little: b } + | SpanQuery::NotContaining { big: a, little: b } + | SpanQuery::ContainedBy { little: a, big: b } + | SpanQuery::NotContainedBy { little: a, big: b } + | SpanQuery::Overlapping { a, b } + | SpanQuery::NonOverlapping { a, b } + | SpanQuery::Before { a, b } + | SpanQuery::After { a, b } => a.holds_unordered() || b.holds_unordered(), + } + } + /// Number of distinct term indices needed (max_term_index + 1, or 0 if no terms). pub fn num_terms(&self) -> usize { self.max_term_index().map_or(0, |i| i + 1) @@ -292,6 +350,31 @@ mod tests { assert!(q.validate(1).is_err()); } + #[test] + fn positive_terms_repeat_and_skip_excluded_operands() { + let mut leaves = Vec::new(); + let q = SpanQuery::NotContaining { + big: Box::new(SpanQuery::phrase([0, 1, 0])), + little: Box::new(SpanQuery::Term(2)), + }; + q.for_each_positive_term(&mut |idx| leaves.push(idx)); + assert_eq!(leaves, [0, 1, 0]); + + leaves.clear(); + let q = SpanQuery::Containing { + big: Box::new(SpanQuery::NonOverlapping { + a: Box::new(SpanQuery::Term(0)), + b: Box::new(SpanQuery::Term(1)), + }), + little: Box::new(SpanQuery::NotContainedBy { + little: Box::new(SpanQuery::Or(vec![SpanQuery::Term(0), SpanQuery::Term(2)])), + big: Box::new(SpanQuery::Term(3)), + }), + }; + q.for_each_positive_term(&mut |idx| leaves.push(idx)); + assert_eq!(leaves, [0, 0, 2]); + } + #[test] fn num_terms_nested() { let q = SpanQuery::Ordered(vec![ diff --git a/boldi-vigna/src/state/mod.rs b/boldi-vigna/src/state/mod.rs index 28f4a2e..7d2b7fe 100644 --- a/boldi-vigna/src/state/mod.rs +++ b/boldi-vigna/src/state/mod.rs @@ -20,9 +20,13 @@ mod filter; mod ordered; mod phrase; mod relation; +mod shared; mod term; mod unordered; +use std::cell::RefCell; +use std::rc::Rc; + use crate::interval::Interval; use crate::positions::TermPositions; use crate::query::SpanQuery; @@ -35,9 +39,24 @@ pub(crate) use self::filter::{ pub(crate) use self::ordered::OrderedState; pub(crate) use self::phrase::PhraseState; pub(crate) use self::relation::RelationState; +pub(crate) use self::shared::{SharedSource, SharedState}; pub(crate) use self::term::TermState; pub(crate) use self::unordered::UnorderedState; +// States this thread compiled and reset: the unit tests' size pins. +#[cfg(test)] +thread_local! { + static STATES_BUILT: std::cell::Cell = const { std::cell::Cell::new(0) }; + static STATES_RESET: std::cell::Cell = const { std::cell::Cell::new(0) }; +} + +/// Counts one compiled state (unit tests only). +#[inline(always)] +fn built() { + #[cfg(test)] + STATES_BUILT.with(|built| built.set(built.get() + 1)); +} + /// Internal state machine for a single node in the operator tree. /// Mirrors the structure of `SpanQuery` but carries mutable iteration state. pub(crate) enum NodeState { @@ -52,81 +71,91 @@ pub(crate) enum NodeState { MaxWidth(MaxWidthState), WithinPositions(WithinPositionsState), Relation(RelationState), + Shared(SharedState), } impl NodeState { /// Compile a `SpanQuery` tree into a `NodeState` tree. pub(crate) fn compile(query: &SpanQuery) -> Self { + Self::compile_gaps(query, false) + } + + /// Compiles `query`; `gaps_read` says an enclosing gap filter reads this + /// node's gap count. A gap filter passes no overlapping sub-intervals, so a + /// gap-read `Unordered` compiles to its disjoint form: one `Ordered` per + /// distinct child order, unioned. Building the overlapping form there would + /// let an overlapping minimal interval (two children on one position) + /// shadow the disjoint spans that contain it, and the filter would then + /// reject them all. + fn compile_gaps(query: &SpanQuery, gaps_read: bool) -> Self { + built(); if let Some(term_indices) = exact_phrase_term_indices(query) { return NodeState::Phrase(PhraseState::new(term_indices)); } + let compile = |q: &SpanQuery| NodeState::compile_gaps(q, false); + let forward = |q: &SpanQuery| NodeState::compile_gaps(q, gaps_read); + let filtered = |q: &SpanQuery| NodeState::compile_gaps(q, true); match query { SpanQuery::Empty => NodeState::Empty(EmptyState::new()), SpanQuery::Term(i) => NodeState::Term(TermState::new(*i)), SpanQuery::Ordered(children) => { - let child_states: Vec<_> = children.iter().map(NodeState::compile).collect(); - NodeState::Ordered(OrderedState::new(child_states)) + NodeState::Ordered(OrderedState::new(children.iter().map(compile).collect())) } + SpanQuery::Unordered(children) if gaps_read => disjoint_unordered(children), SpanQuery::Unordered(children) => { NodeState::Unordered(UnorderedState::compile(children)) } SpanQuery::Or(children) => { - let child_states: Vec<_> = children.iter().map(NodeState::compile).collect(); - NodeState::Or(OrState::new(child_states)) + NodeState::Or(OrState::new(children.iter().map(forward).collect())) } SpanQuery::MaxGaps { max_gaps, inner } => { - NodeState::MaxGaps(MaxGapsState::new(NodeState::compile(inner), *max_gaps)) + NodeState::MaxGaps(MaxGapsState::new(filtered(inner), *max_gaps)) } SpanQuery::GapsInRange { min_gaps, max_gaps, inner, - } => NodeState::GapsInRange(GapsInRangeState::new( - NodeState::compile(inner), - *min_gaps, - *max_gaps, - )), + } => { + NodeState::GapsInRange(GapsInRangeState::new(filtered(inner), *min_gaps, *max_gaps)) + } SpanQuery::MaxWidth { max_width, inner } => { - NodeState::MaxWidth(MaxWidthState::new(NodeState::compile(inner), *max_width)) + NodeState::MaxWidth(MaxWidthState::new(forward(inner), *max_width)) } - SpanQuery::WithinPositions { inner, lo, hi } => NodeState::WithinPositions( - WithinPositionsState::new(NodeState::compile(inner), *lo, *hi), - ), - SpanQuery::Containing { big, little } => NodeState::Relation( - RelationState::containing(NodeState::compile(big), NodeState::compile(little)), - ), - SpanQuery::ContainedBy { little, big } => NodeState::Relation( - RelationState::contained_by(NodeState::compile(little), NodeState::compile(big)), - ), - SpanQuery::NotContaining { big, little } => NodeState::Relation( - RelationState::not_containing(NodeState::compile(big), NodeState::compile(little)), - ), - SpanQuery::NotContainedBy { little, big } => { - NodeState::Relation(RelationState::not_contained_by( - NodeState::compile(little), - NodeState::compile(big), - )) + SpanQuery::WithinPositions { inner, lo, hi } => { + NodeState::WithinPositions(WithinPositionsState::new(forward(inner), *lo, *hi)) + } + // A relation reports its first operand's gaps. + SpanQuery::Containing { big, little } => { + NodeState::Relation(RelationState::containing(forward(big), compile(little))) + } + SpanQuery::ContainedBy { little, big } => { + NodeState::Relation(RelationState::contained_by(forward(little), compile(big))) + } + SpanQuery::NotContaining { big, little } => { + NodeState::Relation(RelationState::not_containing(forward(big), compile(little))) } - SpanQuery::Overlapping { a, b } => NodeState::Relation(RelationState::overlapping( - NodeState::compile(a), - NodeState::compile(b), - )), - SpanQuery::NonOverlapping { a, b } => NodeState::Relation( - RelationState::non_overlapping(NodeState::compile(a), NodeState::compile(b)), + SpanQuery::NotContainedBy { little, big } => NodeState::Relation( + RelationState::not_contained_by(forward(little), compile(big)), ), - SpanQuery::Before { a, b } => NodeState::Relation(RelationState::before( - NodeState::compile(a), - NodeState::compile(b), - )), - SpanQuery::After { a, b } => NodeState::Relation(RelationState::after( - NodeState::compile(a), - NodeState::compile(b), - )), + SpanQuery::Overlapping { a, b } => { + NodeState::Relation(RelationState::overlapping(forward(a), compile(b))) + } + SpanQuery::NonOverlapping { a, b } => { + NodeState::Relation(RelationState::non_overlapping(forward(a), compile(b))) + } + SpanQuery::Before { a, b } => { + NodeState::Relation(RelationState::before(forward(a), compile(b))) + } + SpanQuery::After { a, b } => { + NodeState::Relation(RelationState::after(forward(a), compile(b))) + } } } pub(crate) fn reset(&mut self) { + #[cfg(test)] + STATES_RESET.with(|reset| reset.set(reset.get() + 1)); match self { NodeState::Empty(s) => s.reset(), NodeState::Term(s) => s.reset(), @@ -139,6 +168,7 @@ impl NodeState { NodeState::MaxWidth(s) => s.reset(), NodeState::WithinPositions(s) => s.reset(), NodeState::Relation(s) => s.reset(), + NodeState::Shared(s) => s.reset(), } } @@ -155,6 +185,7 @@ impl NodeState { NodeState::MaxWidth(s) => s.next_interval(positions), NodeState::WithinPositions(s) => s.next_interval(positions), NodeState::Relation(s) => s.next_interval(positions), + NodeState::Shared(s) => s.next_interval(positions), } } @@ -172,7 +203,78 @@ impl NodeState { NodeState::MaxWidth(s) => s.gaps(), NodeState::WithinPositions(s) => s.gaps(), NodeState::Relation(s) => s.gaps(), + NodeState::Shared(s) => s.gaps(), + } + } +} + +/// Disjoint intervals are strictly ordered, so the minimal spans holding one +/// disjoint interval per child are the union over child orders of `Ordered`, +/// which consumes disjoint intervals by construction. Identical children give +/// identical orders; each distinct order compiles once. tinql's NEAR has two +/// children: at most two ordered walks. A child holding an Unordered compiles +/// once, and each walk reads it through a replay reader of its own +/// ([`SharedState`]): compiled once per walk, a chain of NEARs doubled at every +/// level, 2^65 states for 66 terms (wp-m4.9-gate.md §U6.13). Any other child, +/// a term, a phrase or an OR of them, compiles once per walk, as before. +fn disjoint_unordered(children: &[SpanQuery]) -> NodeState { + let mut orders = Vec::new(); + let mut remaining: Vec<&SpanQuery> = children.iter().collect(); + child_orders(&mut Vec::new(), &mut remaining, &mut orders); + let shared: Vec>>> = (children.iter()) + .map(|child| { + (orders.len() > 1 && child.holds_unordered()) + .then(|| SharedSource::new(NodeState::compile(child))) + }) + .collect(); + let walk = |child: &SpanQuery| { + let index = (children.iter()) + .position(|each| std::ptr::eq(each, child)) + .expect("an order walks the children themselves"); + match &shared[index] { + Some(source) => { + built(); + NodeState::Shared(SharedState::new(source)) + } + None => NodeState::compile(child), + } + }; + let mut ordered: Vec = orders + .iter() + .map(|order| { + built(); + NodeState::Ordered(OrderedState::new( + order.iter().copied().map(&walk).collect(), + )) + }) + .collect(); + if ordered.len() == 1 { + return ordered.pop().expect("one order"); + } + built(); + NodeState::Or(OrState::new(ordered)) +} + +/// Pushes every distinct order of `remaining` after `prefix` onto `orders`. +fn child_orders<'q>( + prefix: &mut Vec<&'q SpanQuery>, + remaining: &mut Vec<&'q SpanQuery>, + orders: &mut Vec>, +) { + if remaining.is_empty() { + orders.push(prefix.clone()); + return; + } + for i in 0..remaining.len() { + // Taking an equal child again at this depth repeats an order. + if remaining[..i].contains(&remaining[i]) { + continue; } + let child = remaining.remove(i); + prefix.push(child); + child_orders(prefix, remaining, orders); + prefix.pop(); + remaining.insert(i, child); } } @@ -200,3 +302,45 @@ fn exact_phrase_term_indices(query: &SpanQuery) -> Option> { } Some(term_indices) } + +#[cfg(test)] +mod tests { + use super::*; + + /// tinql's left-deep `t0 NEAR/n t1 NEAR/n t2 …` over `terms` terms. + fn near_chain(terms: usize, max_gaps: u32) -> SpanQuery { + (1..terms).fold(SpanQuery::Term(0), |chain, term| SpanQuery::MaxGaps { + max_gaps, + inner: Box::new(SpanQuery::Unordered(vec![chain, SpanQuery::Term(term)])), + }) + } + + /// The states `query` compiles to, and those one document's reset + /// reaches. + fn states_built_and_reset(query: &SpanQuery) -> (usize, usize) { + let count = |counter: &'static std::thread::LocalKey>| { + counter.with(std::cell::Cell::get) + }; + let built = count(&STATES_BUILT); + let mut state = NodeState::compile(query); + let reset = count(&STATES_RESET); + state.reset(); + (count(&STATES_BUILT) - built, count(&STATES_RESET) - reset) + } + + /// A NEAR chain compiles to states linear in its depth, and a document's + /// reset reaches each once: each level's two disjoint orders share the + /// chain below them, which only the first reader resets + /// (wp-m4.9-gate.md §U6.13). Compiled once per order, every level doubled + /// the chain below it, and 66 terms took 2^65 states. + #[test] + fn a_near_chain_compiles_and_resets_linear_in_its_depth() { + for terms in [2, 3, 8, 20, 66] { + let (built, reset) = states_built_and_reset(&near_chain(terms, 99)); + assert!( + built <= 10 * terms && reset <= 10 * terms, + "a {terms}-term NEAR chain compiled {built} states and reset {reset}" + ); + } + } +} diff --git a/boldi-vigna/src/state/shared.rs b/boldi-vigna/src/state/shared.rs new file mode 100644 index 0000000..0b0bb45 --- /dev/null +++ b/boldi-vigna/src/state/shared.rs @@ -0,0 +1,106 @@ +// Copyright (C) 2026 PlanetScale +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . +use std::cell::RefCell; +use std::rc::Rc; + +use super::NodeState; +use crate::interval::Interval; +use crate::positions::TermPositions; + +/// One compiled child that several walks read: the order walks of a gap-read +/// `Unordered` (`NodeState::compile_gaps`). The child runs once per document +/// and keeps what it produced, each interval with its gap count, so every +/// reader replays the stream from its own place exactly as a private copy +/// would produce it. A state's output is a function of the document's +/// positions and its call count alone, and `reset` reaches every state from +/// the root before a document's first read. A child returns no interval +/// after its first `None`, so the source asks it no further. +pub(crate) struct SharedSource { + child: NodeState, + produced: Vec<(Interval, Option)>, + exhausted: bool, + /// Reset, and nothing read since: every reader resets the source at a + /// document's start, and only the first reaches the child. Resetting the + /// child once per reader would double the resets at every level of a + /// NEAR chain. + reset: bool, +} + +impl SharedSource { + pub(crate) fn new(child: NodeState) -> Rc> { + Rc::new(RefCell::new(Self { + child, + produced: Vec::new(), + exhausted: false, + reset: false, + })) + } +} + +/// A reader of a [`SharedSource`], at its own place in the child's stream. +pub(crate) struct SharedState { + source: Rc>, + next: usize, + last_gaps: Option, +} + +impl SharedState { + pub(crate) fn new(source: &Rc>) -> Self { + Self { + source: Rc::clone(source), + next: 0, + last_gaps: None, + } + } + + /// Every reader resets the source at a document's start, before any + /// reader reads; the first reset reaches the child, the rest find the + /// source reset already. + pub(crate) fn reset(&mut self) { + let mut source = self.source.borrow_mut(); + if !source.reset { + source.child.reset(); + source.produced.clear(); + source.exhausted = false; + source.reset = true; + } + self.next = 0; + self.last_gaps = None; + } + + pub(crate) fn next_interval(&mut self, positions: &impl TermPositions) -> Option { + let mut source = self.source.borrow_mut(); + source.reset = false; + if self.next == source.produced.len() { + if source.exhausted { + return None; + } + let Some(interval) = source.child.next_interval(positions) else { + source.exhausted = true; + return None; + }; + let gaps = source.child.gaps(); + source.produced.push((interval, gaps)); + } + let (interval, gaps) = source.produced[self.next]; + self.next += 1; + self.last_gaps = gaps; + Some(interval) + } + + pub(crate) fn gaps(&self) -> Option { + self.last_gaps + } +} diff --git a/boldi-vigna/src/state/unordered.rs b/boldi-vigna/src/state/unordered.rs index c1ac66f..065aa25 100644 --- a/boldi-vigna/src/state/unordered.rs +++ b/boldi-vigna/src/state/unordered.rs @@ -16,7 +16,6 @@ // The full license text is available in LICENSE. use std::cmp::Reverse; use std::collections::{BTreeSet, VecDeque}; -use std::ops::Bound::{Excluded, Unbounded}; use rustc_hash::FxHashMap; @@ -28,13 +27,6 @@ use crate::query::SpanQuery; type StartKey = (u32, Reverse, usize); type EndKey = (u32, u32, usize); -#[derive(Clone, Copy)] -struct OperandInterval { - interval: Interval, - coverage: u64, - contains_overlap: bool, -} - enum UnorderedChild { Single(NodeState), Repeated(RepeatingState), @@ -56,15 +48,9 @@ impl UnorderedChild { } } - fn next_interval(&mut self, positions: &impl TermPositions) -> Option { + fn next_interval(&mut self, positions: &impl TermPositions) -> Option { match self { - Self::Single(state) => state - .next_interval(positions) - .map(|interval| OperandInterval { - interval, - coverage: u64::from(interval.width()), - contains_overlap: false, - }), + Self::Single(state) => state.next_interval(positions), Self::Repeated(state) => state.next_interval(positions), } } @@ -75,8 +61,6 @@ struct RepeatingState { child: NodeState, copies: usize, window: VecDeque, - coverage: u64, - overlap_pairs: usize, exhausted: bool, } @@ -87,8 +71,6 @@ impl RepeatingState { child, copies, window: VecDeque::with_capacity(copies), - coverage: 0, - overlap_pairs: 0, exhausted: false, } } @@ -96,17 +78,15 @@ impl RepeatingState { fn reset(&mut self) { self.child.reset(); self.window.clear(); - self.coverage = 0; - self.overlap_pairs = 0; self.exhausted = false; } - fn next_interval(&mut self, positions: &impl TermPositions) -> Option { + fn next_interval(&mut self, positions: &impl TermPositions) -> Option { if self.exhausted { return None; } if self.window.len() == self.copies { - self.pop_front(); + self.window.pop_front(); } while self.window.len() < self.copies { @@ -114,53 +94,20 @@ impl RepeatingState { self.exhausted = true; return None; }; - self.push_back(interval); + self.window.push_back(interval); } let start = self.window.front()?.start; let end = self.window.back()?.end; - Some(OperandInterval { - interval: Interval::new(start, end), - coverage: self.coverage, - contains_overlap: self.overlap_pairs != 0, - }) - } - - fn pop_front(&mut self) { - let Some(front) = self.window.pop_front() else { - return; - }; - self.coverage -= u64::from(front.width()); - if self - .window - .front() - .is_some_and(|next| front.overlaps(*next)) - { - self.overlap_pairs -= 1; - } - } - - fn push_back(&mut self, interval: Interval) { - if self - .window - .back() - .is_some_and(|previous| previous.overlaps(interval)) - { - self.overlap_pairs += 1; - } - self.coverage += u64::from(interval.width()); - self.window.push_back(interval); + Some(Interval::new(start, end)) } } /// Indexed live intervals for an unordered conjunction. struct CurrentIntervals { - slots: Vec>, + slots: Vec>, by_start: BTreeSet, by_end: BTreeSet, - coverage: u64, - overlap_pairs: usize, - overlapping_children: usize, } impl CurrentIntervals { @@ -169,9 +116,6 @@ impl CurrentIntervals { slots: vec![None; children], by_start: BTreeSet::new(), by_end: BTreeSet::new(), - coverage: 0, - overlap_pairs: 0, - overlapping_children: 0, } } @@ -179,9 +123,6 @@ impl CurrentIntervals { self.slots.fill(None); self.by_start.clear(); self.by_end.clear(); - self.coverage = 0; - self.overlap_pairs = 0; - self.overlapping_children = 0; } fn is_present(&self, child: usize) -> bool { @@ -203,59 +144,20 @@ impl CurrentIntervals { Some(Interval::new(start, end)) } - fn gaps(&self, span: Interval) -> Option { - if self.overlap_pairs != 0 || self.overlapping_children != 0 { - return None; - } - u64::from(span.width()).checked_sub(self.coverage) - } - - fn replace(&mut self, child: usize, next: Option) { + fn replace(&mut self, child: usize, next: Option) { if let Some(previous) = self.slots[child] { - self.unlink(child, previous); - self.coverage -= previous.coverage; - self.overlapping_children -= usize::from(previous.contains_overlap); + let removed_start = self.by_start.remove(&start_key(child, previous)); + let removed_end = self.by_end.remove(&end_key(child, previous)); + debug_assert!(removed_start && removed_end); } self.slots[child] = next; if let Some(next) = next { - self.link(child, next); - self.coverage += next.coverage; - self.overlapping_children += usize::from(next.contains_overlap); + let inserted_start = self.by_start.insert(start_key(child, next)); + let inserted_end = self.by_end.insert(end_key(child, next)); + debug_assert!(inserted_start && inserted_end); } } - - fn link(&mut self, child: usize, operand: OperandInterval) { - let key = start_key(child, operand.interval); - let (previous, next) = self.neighbors(key); - self.overlap_pairs -= overlapping_pair(previous, next); - self.overlap_pairs += - overlapping_pair(previous, Some(key)) + overlapping_pair(Some(key), next); - let inserted_start = self.by_start.insert(key); - let inserted_end = self.by_end.insert(end_key(child, operand.interval)); - debug_assert!(inserted_start && inserted_end); - } - - fn unlink(&mut self, child: usize, operand: OperandInterval) { - let key = start_key(child, operand.interval); - let (previous, next) = self.neighbors(key); - self.overlap_pairs -= - overlapping_pair(previous, Some(key)) + overlapping_pair(Some(key), next); - self.overlap_pairs += overlapping_pair(previous, next); - let removed_start = self.by_start.remove(&key); - let removed_end = self.by_end.remove(&end_key(child, operand.interval)); - debug_assert!(removed_start && removed_end); - } - - fn neighbors(&self, key: StartKey) -> (Option, Option) { - let previous = self.by_start.range(..key).next_back().copied(); - let next = self - .by_start - .range((Excluded(key), Unbounded)) - .next() - .copied(); - (previous, next) - } } fn start_key(child: usize, interval: Interval) -> StartKey { @@ -266,24 +168,16 @@ fn end_key(child: usize, interval: Interval) -> EndKey { (interval.end, interval.start, child) } -fn overlapping_pair(left: Option, right: Option) -> usize { - match (left, right) { - (Some((_, Reverse(left_end), _)), Some((right_start, _, _))) => { - usize::from(left_end >= right_start) - } - _ => 0, - } -} - /// BV AND algorithm: unordered conjunction. /// /// Finds minimal intervals containing one interval from each child, -/// in any order. Uses ⪯ priority ordering (left-to-right). +/// in any order. Uses ⪯ priority ordering (left-to-right). Children may +/// overlap, so it reports no gap count: under a gap filter the compiler builds +/// the disjoint form instead (see `NodeState::compile_gaps`). pub(crate) struct UnorderedState { children: Vec, current: CurrentIntervals, prev: Option, - last_gaps: Option, } impl UnorderedState { @@ -313,7 +207,6 @@ impl UnorderedState { children, current, prev: None, - last_gaps: None, } } @@ -323,7 +216,6 @@ impl UnorderedState { } self.current.reset(); self.prev = None; - self.last_gaps = None; } fn advance_child(&mut self, child: usize, positions: &impl TermPositions) { @@ -359,7 +251,6 @@ impl UnorderedState { } let mut candidate = self.current.span()?; - let mut candidate_gaps = self.current.gaps(candidate); loop { if self @@ -380,15 +271,13 @@ impl UnorderedState { break; } candidate = new_span; - candidate_gaps = self.current.gaps(candidate); } self.prev = Some(candidate); - self.last_gaps = candidate_gaps; Some(candidate) } pub(crate) fn gaps(&self) -> Option { - self.last_gaps + None } } diff --git a/boldi-vigna/tests/near_chain_tests.rs b/boldi-vigna/tests/near_chain_tests.rs new file mode 100644 index 0000000..6a4f0d0 --- /dev/null +++ b/boldi-vigna/tests/near_chain_tests.rs @@ -0,0 +1,125 @@ +// Copyright (C) 2026 PlanetScale +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . +//! Chains of NEAR, tinql's left-deep `t0 NEAR/n t1 NEAR/n t2 …`: +//! `MAXGAPS(n, UNORDERED(MAXGAPS(n, UNORDERED(…)), t))`. Under a gap filter an +//! Unordered compiles to its disjoint form, one ordered walk per child order, +//! and those walks share each child that holds a further Unordered rather +//! than compiling it once per order (wp-m4.9-gate.md §U6.13). The shared +//! compile must answer exactly as private copies would: the explicit +//! `MAXGAPS(n, OR(ORDERED(a, b), ORDERED(b, a)))` expansion of every level. + +use boldi_vigna::*; +use proptest::prelude::*; + +fn collect(query: &SpanQuery, positions: &impl TermPositions) -> Vec { + let mut solver = SpanSolver::new(query).unwrap(); + solver.intervals(positions).collect() +} + +fn near(max_gaps: u32, left: SpanQuery, right: SpanQuery) -> SpanQuery { + SpanQuery::MaxGaps { + max_gaps, + inner: Box::new(SpanQuery::Unordered(vec![left, right])), + } +} + +/// A chain over terms `0..gaps.len() + 1`, level `i` joining term `i + 1` +/// under `gaps[i]`, the chain on the left of a level where `left[i]` holds. +fn chain(gaps: &[u32], left: &[bool]) -> SpanQuery { + (gaps.iter().zip(left).enumerate()).fold( + SpanQuery::Term(0), + |chain, (level, (&max_gaps, &left))| { + let term = SpanQuery::Term(level + 1); + if left { + near(max_gaps, chain, term) + } else { + near(max_gaps, term, chain) + } + }, + ) +} + +/// `query` with every NEAR written as the disjoint form private copies +/// compile to: both orders of its two children, each copy its own. +fn private_copies(query: &SpanQuery) -> SpanQuery { + match query { + SpanQuery::MaxGaps { max_gaps, inner } => match inner.as_ref() { + SpanQuery::Unordered(children) => { + let [a, b] = [&children[0], &children[1]].map(private_copies); + SpanQuery::MaxGaps { + max_gaps: *max_gaps, + inner: Box::new(SpanQuery::Or(vec![ + SpanQuery::Ordered(vec![a.clone(), b.clone()]), + SpanQuery::Ordered(vec![b, a]), + ])), + } + } + other => SpanQuery::MaxGaps { + max_gaps: *max_gaps, + inner: Box::new(private_copies(other)), + }, + }, + other => other.clone(), + } +} + +/// The 66-term chain U6.3's audit runs (`dd1 NEAR/99 w000 … NEAR/99 w064`): +/// it compiles (2^65 states when each order compiled its own copy of the +/// chain below) and answers a document holding every term in order, and +/// none missing one. +#[test] +fn a_66_term_near_chain_compiles_and_answers() { + let query = chain(&[99; 65], &[true; 65]); + let in_order: Vec> = (0..66).map(|position| vec![position]).collect(); + assert_eq!(collect(&query, &in_order), [Interval::new(0, 65)]); + let mut missing = in_order.clone(); + missing[40].clear(); + assert!(collect(&query, &missing).is_empty()); + let reversed: Vec> = (0..66).rev().map(|position| vec![position]).collect(); + assert_eq!(collect(&query, &reversed), [Interval::new(0, 65)]); +} + +fn positions_strategy(terms: usize) -> impl Strategy>> { + proptest::collection::vec( + proptest::collection::btree_set(0u32..40, 0..6) + .prop_map(|positions| positions.into_iter().collect::>()), + terms, + ) +} + +fn chain_case() -> impl Strategy, Vec, Vec>)> { + (1usize..=5).prop_flat_map(|depth| { + ( + proptest::collection::vec(0u32..4, depth), + proptest::collection::vec(any::(), depth), + positions_strategy(depth + 1), + ) + }) +} + +proptest! { + #![proptest_config(ProptestConfig::with_cases(2000))] + + /// A chain's shared compile answers exactly as its private copies. + #[test] + fn a_near_chain_answers_as_its_private_copies((gaps, left, positions) in chain_case()) { + let query = chain(&gaps, &left); + prop_assert_eq!( + collect(&query, &positions), + collect(&private_copies(&query), &positions), + "{}", query + ); + } +} diff --git a/boldi-vigna/tests/near_slot_tests.rs b/boldi-vigna/tests/near_slot_tests.rs new file mode 100644 index 0000000..4476de3 --- /dev/null +++ b/boldi-vigna/tests/near_slot_tests.rs @@ -0,0 +1,200 @@ +// Copyright (C) 2026 PlanetScale +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . +//! `MAXGAPS(n, UNORDERED(a, b))` (tinql's `a NEAR/n b`) when the two slots can +//! match the same position: an OR group sharing a term with the other slot, or +//! a wide operand repeated in both slots. +//! +//! Semantics checked here, from the solver's documented contract (`SpanQuery`): +//! `MaxGaps` passes only assignments whose sub-intervals do not overlap, and +//! `Unordered` takes one interval from each child. So a NEAR match is a pair of +//! child intervals that share no position, in either order, whose gap count +//! (`width - covered positions`) is at most `n`; the solver reports the minimal +//! such spans. Two slots never reuse one position. + +use boldi_vigna::*; +use proptest::prelude::*; + +fn collect(query: &SpanQuery, positions: &impl TermPositions) -> Vec { + let mut solver = SpanSolver::new(query).unwrap(); + solver.intervals(positions).collect() +} + +fn near(max_gaps: u32, left: SpanQuery, right: SpanQuery) -> SpanQuery { + SpanQuery::MaxGaps { + max_gaps, + inner: Box::new(SpanQuery::Unordered(vec![left, right])), + } +} + +fn slot(terms: &[usize]) -> SpanQuery { + match terms { + [term] => SpanQuery::Term(*term), + terms => SpanQuery::Or(terms.iter().copied().map(SpanQuery::Term).collect()), + } +} + +/// Brute force over point slots: every pair of distinct positions, one from +/// each slot, within `max_gaps`, reduced to minimal spans. +fn brute_force_near( + max_gaps: u32, + left: &[usize], + right: &[usize], + positions: &[Vec], +) -> Vec { + let slot_positions = |terms: &[usize]| -> Vec { + let mut out: Vec = terms + .iter() + .flat_map(|&t| positions[t].iter().copied()) + .collect(); + out.sort_unstable(); + out.dedup(); + out + }; + let (left, right) = (slot_positions(left), slot_positions(right)); + let mut spans = Vec::new(); + for &p in &left { + for &q in &right { + if p != q && p.abs_diff(q) - 1 <= max_gaps { + spans.push(Interval::new(p.min(q), p.max(q))); + } + } + } + minimize(spans) +} + +fn minimize(mut intervals: Vec) -> Vec { + intervals.sort(); + intervals.dedup(); + intervals + .iter() + .copied() + .filter(|&iv| !intervals.iter().any(|&o| o != iv && iv.contains(o))) + .collect() +} + +#[test] +fn group_slot_pairs_with_the_shared_term_in_the_other_slot() { + // `(alpha OR bravo) NEAR/3 alpha` over "alpha the alpha": alpha=0, bravo=1. + let positions = vec![vec![0, 2], vec![]]; + let q = near(3, slot(&[0, 1]), slot(&[0])); + assert_eq!(collect(&q, &positions), vec![Interval::new(0, 2)]); + // The single-term slot already matches; widening must not lose it. + let narrow = near(3, slot(&[0]), slot(&[0])); + assert_eq!(collect(&narrow, &positions), vec![Interval::new(0, 2)]); +} + +#[test] +fn two_group_slots_sharing_a_term_pair_distinct_occurrences() { + // `(alpha OR bravo) NEAR/2 (alpha OR charlie)` over "alpha of alpha". + let positions = vec![vec![0, 2], vec![], vec![]]; + let q = near(2, slot(&[0, 1]), slot(&[0, 2])); + assert_eq!(collect(&q, &positions), vec![Interval::new(0, 2)]); +} + +#[test] +fn shared_term_does_not_shadow_a_pair_with_the_other_term() { + // "bravo alpha": the alpha alone sits in both groups, but only bravo-alpha + // is a pair of distinct positions. + let positions = vec![vec![1], vec![0], vec![]]; + let q = near(0, slot(&[0, 1]), slot(&[0, 2])); + assert_eq!(collect(&q, &positions), vec![Interval::new(0, 1)]); +} + +#[test] +fn single_shared_occurrence_is_not_a_pair() { + let positions = vec![vec![4], vec![]]; + let q = near(3, slot(&[0, 1]), slot(&[0])); + assert!(collect(&q, &positions).is_empty()); +} + +#[test] +fn repeated_wide_operand_takes_disjoint_occurrences() { + // `"a a" NEAR/0 "a a"` over "a a a a": the phrase occurs at [0,1], [1,2], + // [2,3]; the disjoint pair [0,1] + [2,3] spans [0,3] with no gap. + let positions = vec![vec![0, 1, 2, 3]]; + let phrase = SpanQuery::phrase([0, 0]); + let q = near(0, phrase.clone(), phrase); + assert_eq!(collect(&q, &positions), vec![Interval::new(0, 3)]); +} + +#[test] +fn gaps_in_range_takes_disjoint_occurrences_too() { + let positions = vec![vec![0, 2], vec![]]; + let q = SpanQuery::GapsInRange { + min_gaps: 1, + max_gaps: 1, + inner: Box::new(SpanQuery::Unordered(vec![slot(&[0, 1]), slot(&[0])])), + }; + assert_eq!(collect(&q, &positions), vec![Interval::new(0, 2)]); +} + +fn positions_strategy(terms: usize) -> impl Strategy>> { + // One term per position, as the tokenizer produces. + prop::collection::vec(0..terms + 2, 0..16).prop_map(move |doc| { + let mut positions = vec![Vec::new(); terms]; + for (pos, term) in doc.into_iter().enumerate() { + if term < terms { + positions[term].push(pos as u32); + } + } + positions + }) +} + +fn slot_strategy(terms: usize) -> impl Strategy> { + prop::collection::btree_set(0..terms, 1..=terms).prop_map(|s| s.into_iter().collect()) +} + +proptest! { + #![proptest_config(ProptestConfig::with_cases(2000))] + + #[test] + fn near_over_or_slots_matches_brute_force( + positions in positions_strategy(3), + left in slot_strategy(3), + right in slot_strategy(3), + max_gaps in 0u32..5, + ) { + let q = near(max_gaps, slot(&left), slot(&right)); + prop_assert_eq!( + collect(&q, &positions), + brute_force_near(max_gaps, &left, &right, &positions), + "left={:?} right={:?} positions={:?}", left, right, positions + ); + } + + #[test] + fn widening_a_slot_never_loses_a_match( + positions in positions_strategy(3), + left in slot_strategy(3), + right in slot_strategy(3), + extra in 0usize..3, + max_gaps in 0u32..5, + ) { + let narrow = near(max_gaps, slot(&left), slot(&right)); + let mut wider_left = left.clone(); + if !wider_left.contains(&extra) { + wider_left.push(extra); + wider_left.sort_unstable(); + } + let wide = near(max_gaps, slot(&wider_left), slot(&right)); + if !collect(&narrow, &positions).is_empty() { + prop_assert!( + !collect(&wide, &positions).is_empty(), + "{:?} matched but {:?} did not on {:?}", left, wider_left, positions + ); + } + } +} diff --git a/postgres/src/operator.rs b/postgres/src/operator.rs index 4178069..6f1fa42 100644 --- a/postgres/src/operator.rs +++ b/postgres/src/operator.rs @@ -18,16 +18,24 @@ use crate::am::amhandler; use pgrx::{extension_sql, pg_extern, pg_sys}; use tinql::runtime::{ - Query, SimplificationProfile, evaluate, lower::lower, simplify, subtokenize::sub_tokenize, - tokenize_doc, + Query, SimplificationProfile, evaluate, lower::lower_with_profile, simplify, + subtokenize::sub_tokenize, tokenize_doc, }; use tokenizer::Tokenizer; use tokenizer::presets::default_pipeline; fn parse_search(query_text: &str, tokenizer: &T) -> Result { + parse_search_with(query_text, tokenizer, SimplificationProfile::Structural) +} + +fn parse_search_with( + query_text: &str, + tokenizer: &T, + profile: SimplificationProfile, +) -> Result { let parsed = crate::tinql::parse(query_text).map_err(|e| e.to_string())?; let analyzed = sub_tokenize(parsed, tokenizer).map_err(|e| e.to_string())?; - lower(&analyzed).map_err(|e| e.to_string()) + lower_with_profile(&analyzed, profile).map_err(|e| e.to_string()) } /// Returns the OID of Lead's `==>(text, text)` operator, or `InvalidOid` if @@ -79,9 +87,30 @@ fn invalid_search(error: String) -> ! { /// error `==>` raises for it. The parsed texts are ORed and simplified the way /// lowering simplifies `a OR b`. pub(crate) fn parse_searches(texts: &[String], tokenizer: &T) -> Query { + parse_searches_with(texts, tokenizer, SimplificationProfile::Structural) +} + +/// Like [`parse_searches`], but keeps every written occurrence of a repeated +/// term, which is what scoring weighs: `be AND be` weighs `be` twice. +pub(crate) fn parse_scoring_searches(texts: &[String], tokenizer: &T) -> Query { + parse_searches_with( + texts, + tokenizer, + SimplificationProfile::StructuralPreserveTermMultiplicity, + ) +} + +fn parse_searches_with( + texts: &[String], + tokenizer: &T, + profile: SimplificationProfile, +) -> Query { let mut queries = texts .iter() - .map(|text| parse_search(text, tokenizer).unwrap_or_else(|error| invalid_search(error))) + .map(|text| { + parse_search_with(text, tokenizer, profile) + .unwrap_or_else(|error| invalid_search(error)) + }) .collect::>(); if queries.len() == 1 { return queries.remove(0); @@ -91,7 +120,7 @@ pub(crate) fn parse_searches(texts: &[String], tokenizer: &T) -> Q min: 1, children: queries, }, - SimplificationProfile::Structural, + profile, ) } diff --git a/postgres/src/score.rs b/postgres/src/score.rs index 940dc90..e5908e6 100644 --- a/postgres/src/score.rs +++ b/postgres/src/score.rs @@ -18,7 +18,7 @@ use crate::bm25::{ Bm25Overrides, DenseRatio, ScoreStopWords, ScoringTermInput, TermScorer, TermSetEdit, compile_scoring_terms, sum_scores_in_order, }; -use crate::tinql::parse_tinql_to_query; +use crate::tinql::parse_scoring_tinql_to_query; use pgrx::iter::TableIterator; use pgrx::{ FromDatum, Internal, IntoDatum, PgBox, PgList, PgMemoryContexts, PgRelation, Spi, default, @@ -363,8 +363,9 @@ fn build_group<'a>( pgrx::error!("dense_ratio must be finite and non-negative"); } let query = crate::operator::parse_searches(queries, &tokenizer); + let scoring_query = crate::operator::parse_scoring_searches(queries, &tokenizer); let mut inputs = Vec::new(); - collect_score_terms(&query, 1.0, false, &mut inputs); + collect_score_terms(&scoring_query, 1.0, false, &mut inputs); let edit = TermSetEdit::from_bound_arrays(term_add, term_replace) .unwrap_or_else(|error| pgrx::error!("tin.score(): {error}")) .analyzed_with(|text| { @@ -504,6 +505,9 @@ fn select_read_only(sql: &str, failure: &str) -> Vec>> { }) } +/// Tokenizes the corpus for scoring statistics. As in tin, a document with no +/// tokens is not searchable, so it counts toward neither N nor the average +/// document length. fn tokenize_documents<'a>( documents: impl Iterator, tokenizer: &CompiledTokenizerPipeline, @@ -516,6 +520,7 @@ fn tokenize_documents<'a>( } tokenize(tokenizer, document) }) + .filter(|tokens| !tokens.is_empty()) .collect() } @@ -541,12 +546,16 @@ fn collect_score_terms<'a>( }; match query { Query::Term(text) | Query::Fuzzy { term: text, .. } => push(text), - Query::Span { term_slots, .. } | Query::SpanExpr { term_slots, .. } => { - for slot in term_slots { + Query::Span { .. } | Query::SpanExpr { .. } => { + query.for_each_positive_span_slot(&mut |_, slot, leaf_boost| { if let SpanTermSlot::Term(text) | SpanTermSlot::Fuzzy { term: text, .. } = slot { - push(text); + out.push(ScoringTermInput { + text, + boost: boost * leaf_boost.unwrap_or(1.0), + explicitly_boosted: explicitly_boosted || leaf_boost.is_some(), + }); } - } + }); } Query::And(left, right) | Query::Or(left, right) => { collect_score_terms(left, boost, explicitly_boosted, out); @@ -609,7 +618,7 @@ fn score_inspect( }; } let tokenizer = unsafe { crate::options::tokenizer(index.as_ptr()) }; - let parsed = parse_tinql_to_query(query, &tokenizer) + let parsed = parse_scoring_tinql_to_query(query, &tokenizer) .unwrap_or_else(|error| pgrx::error!("tin.score_inspect() query error: {error}")); let mut inputs = Vec::new(); collect_score_terms(&parsed, 1.0, false, &mut inputs); diff --git a/postgres/src/tinql.rs b/postgres/src/tinql.rs index ccc9f23..12ba7bb 100644 --- a/postgres/src/tinql.rs +++ b/postgres/src/tinql.rs @@ -21,7 +21,11 @@ //! [`MAX_NESTING_DEPTH`] with SQLSTATE 54000 (`program_limit_exceeded`) before //! the parser runs. -use ::tinql::runtime::{Query, QueryError, lower::lower, subtokenize::sub_tokenize}; +use ::tinql::runtime::{ + Query, QueryError, SimplificationProfile, + lower::{lower, lower_with_profile}, + subtokenize::sub_tokenize, +}; use tokenizer::Tokenizer; /// Longest accepted TINQL query text, in bytes. @@ -74,3 +78,15 @@ pub(crate) fn parse_tinql_to_query( ) -> Result { Ok(lower(&sub_tokenize(parse(query)?, tokenizer)?)?) } + +/// Like [`parse_tinql_to_query`], but keeps every written occurrence of a +/// repeated term, which is what scoring weighs. +pub(crate) fn parse_scoring_tinql_to_query( + query: &str, + tokenizer: &T, +) -> Result { + Ok(lower_with_profile( + &sub_tokenize(parse(query)?, tokenizer)?, + SimplificationProfile::StructuralPreserveTermMultiplicity, + )?) +} diff --git a/private-regress.manifest b/private-regress.manifest index 44df0f4..96fc73a 100644 --- a/private-regress.manifest +++ b/private-regress.manifest @@ -35,3 +35,6 @@ highlight_expression_analysis highlight_having_analysis highlight_join_analysis highlight_null_analysis_defaults +phrase_repeat_weights +tinql_query_length_limit +or_dnf_factored_scores diff --git a/tinql/docs/src/alternatives.md b/tinql/docs/src/alternatives.md index 9bebcdf..d85236c 100644 --- a/tinql/docs/src/alternatives.md +++ b/tinql/docs/src/alternatives.md @@ -77,6 +77,24 @@ When the percentage does not divide evenly, the required count rounds up: Place `%` immediately after the number. +### Repeated alternatives + +Each listed alternative counts on its own, even when two are the same +expression: + +``` +AT LEAST 2 OF [beer beer wine] +``` + +A document that contains only "beer" matches, because it satisfies two of +the three listed alternatives. A percentage counts repeats in the list +length too, so `AT LEAST 50% OF [beer beer wine ale]` requires 2. + +### Thresholds outside the list + +`AT LEAST 0 OF [...]` matches every document. A threshold larger than the +list, such as `AT LEAST 4 OF [a b c]`, matches no document. + ### Syntax ``` diff --git a/tinql/docs/src/boost.md b/tinql/docs/src/boost.md index 4cce598..cc04d04 100644 --- a/tinql/docs/src/boost.md +++ b/tinql/docs/src/boost.md @@ -25,6 +25,7 @@ Boost can be applied to: - **Sloppy phrases:** `"big bad"~2^1.5` (slop first, then boost) - **Alternatives:** `[IPA ale]^2` - **Parenthesized expressions:** `(A NEAR/0 B)^3` +- **Operands inside a phrase or span:** `ipa^3 NEAR/5 hoppy`, `"craft [beer^2 ale]"` ## Boost factors @@ -51,6 +52,20 @@ beer)` gives both terms weight 4 in every matching document, including one that matches only the proximity branch. Boosting the phrase provides no separate bonus for matching its positions. +A word written more than once inside one phrase or span adds its weight once +per occurrence too: `"to be or not to be"^2` gives `to` and `be` weight 4 and +`or` and `not` weight 2. The right side of `NOT ENCLOSES`, `NOT ENCLOSED BY` +and `NOT OVERLAPPING` only filters, so its terms add no weight. + +A boost on an operand inside a phrase or span weighs that operand alone, as it +would in a Boolean query: `ipa^3 NEAR/5 hoppy` gives `ipa` weight 3 and `hoppy` +weight 1, like `ipa^3 AND hoppy`. A boost on the whole phrase or span, +`(ipa NEAR/5 hoppy)^3`, multiplies every operand. A boost on the right side of +`NOT ENCLOSES`, `NOT ENCLOSED BY` or `NOT OVERLAPPING` adds nothing, because +that side only filters. As anywhere else, an explicit boost keeps a dense term +in `tin.score()`, and only that operand's term: in `the THEN/0 ipa^2`, a dense +`the` is still elided. + ## Boost on hyphenated terms A boost can follow a hyphenated term: diff --git a/tinql/docs/src/proximity.md b/tinql/docs/src/proximity.md index 547949e..f333381 100644 --- a/tinql/docs/src/proximity.md +++ b/tinql/docs/src/proximity.md @@ -108,6 +108,10 @@ of "IPA", "stout", or "lager". Either "beer" or "ale" within 10 extra positions of either "food" or "dinner", in any order. +Each operand matches at its own position: one occurrence of a word can't +satisfy two operands. `alpha NEAR/3 alpha` needs two occurrences of "alpha", +and `alpha NEAR/0 [alpha beta]` matches "alpha beta" through "beta". + ## Understanding the gap parameter The gap parameter in `THEN/N` and `NEAR/N` controls how many *extra* positions diff --git a/tinql/docs/src/span-relations.md b/tinql/docs/src/span-relations.md index c272be2..b4d47c2 100644 --- a/tinql/docs/src/span-relations.md +++ b/tinql/docs/src/span-relations.md @@ -122,6 +122,10 @@ Every relation operator (except BEFORE and AFTER) has a `NOT` variant: | `A ENCLOSED BY B` | `A NOT ENCLOSED BY B` | | `A OVERLAPPING B` | `A NOT OVERLAPPING B` | +In a negated relation, B only filters: its terms add nothing to the score, as +with `AND NOT`. A term that also appears in A, or elsewhere in the query, +still scores through those occurrences. + ## Combining relations with other operators When combining relation operators with proximity and positional filters, you diff --git a/tinql/src/runtime/display.rs b/tinql/src/runtime/display.rs index ab250a3..fe7ef69 100644 --- a/tinql/src/runtime/display.rs +++ b/tinql/src/runtime/display.rs @@ -37,10 +37,12 @@ impl fmt::Display for Query { term_slots, span_query, position_filter, + .. } => fmt_span_display(f, term_slots, span_query, position_filter.as_ref()), Query::SpanExpr { term_slots, span_expr, + .. } => fmt_span_expr_display(f, term_slots, span_expr), Query::MatchAll => write!(f, "*"), Query::Regex(pat) => write!(f, "REGEX({pat})"), @@ -406,7 +408,7 @@ fn fmt_phrase_term(f: &mut fmt::Formatter<'_>, term: &str) -> fmt::Result { #[cfg(test)] mod tests { use super::*; - use crate::runtime::{PositionFilterBound, SpanPositionFilter}; + use crate::runtime::{PositionFilterBound, SpanLeafBoosts, SpanPositionFilter}; use boldi_vigna::SpanQuery; #[test] @@ -421,6 +423,7 @@ mod tests { inner: Box::new(SpanQuery::Or(vec![SpanQuery::Term(0), SpanQuery::Term(1)])), }, position_filter: None, + leaf_boosts: SpanLeafBoosts::default(), }; assert_eq!(query.to_string(), "SPAN(MAXWIDTH(7, OR(l.a, nood1e)))"); @@ -443,6 +446,7 @@ mod tests { ])), }, position_filter: None, + leaf_boosts: SpanLeafBoosts::default(), }; assert_eq!(query.to_string(), r#"PHRASE("big bad wolf")"#); @@ -463,6 +467,7 @@ mod tests { ])), }, position_filter: None, + leaf_boosts: SpanLeafBoosts::default(), }; assert_eq!(query.to_string(), r#"PHRASE/2("craft beer")"#); @@ -483,6 +488,7 @@ mod tests { ])), }, position_filter: None, + leaf_boosts: SpanLeafBoosts::default(), }; assert_eq!(query.to_string(), r#"PHRASE("say\"hi under\_score")"#); @@ -516,6 +522,7 @@ mod tests { ])), }, position_filter: None, + leaf_boosts: SpanLeafBoosts::default(), }; assert_eq!( @@ -578,6 +585,7 @@ mod tests { term_slots: vec![SpanTermSlot::Term("beer".into())], span_query: SpanQuery::Term(0), position_filter: Some(SpanPositionFilter::Last(PositionFilterBound::Absolute(5))), + leaf_boosts: SpanLeafBoosts::default(), }; assert_eq!(query.to_string(), "SPAN(beer) IN LAST 5 WORDS"); diff --git a/tinql/src/runtime/eval.rs b/tinql/src/runtime/eval.rs index 861124e..959910f 100644 --- a/tinql/src/runtime/eval.rs +++ b/tinql/src/runtime/eval.rs @@ -211,6 +211,7 @@ fn evaluate_searchable(query: &Query, doc: &TokenizedDoc) -> Result { let positions = term_slots .iter() @@ -232,6 +233,7 @@ fn evaluate_searchable(query: &Query, doc: &TokenizedDoc) -> Result { let positions = term_slots .iter() @@ -369,6 +371,7 @@ fn collect_highlight_matches(query: &Query, doc: &TokenizedDoc, out: &mut Vec { let positions = term_slots .iter() @@ -395,6 +398,7 @@ fn collect_highlight_matches(query: &Query, doc: &TokenizedDoc, out: &mut Vec { let positions = term_slots .iter() diff --git a/tinql/src/runtime/lower.rs b/tinql/src/runtime/lower.rs index 2d58d5b..e0b4308 100644 --- a/tinql/src/runtime/lower.rs +++ b/tinql/src/runtime/lower.rs @@ -29,7 +29,7 @@ use rustc_hash::FxHashMap; use super::{ CompiledRegex, PositionFilterBound, Query, RangeBound, SimplificationProfile, SpanExpr, - SpanPositionFilter, SpanTermSlot, simplify, + SpanLeafBoosts, SpanPositionFilter, SpanTermSlot, simplify, }; #[derive(Debug, thiserror::Error)] @@ -137,84 +137,95 @@ fn lower_boolean(expr: &crate::Expr) -> Result { } fn lower_as_span(expr: &crate::Expr) -> Result { - let mut builder = SpanBuilder::new(); + let mut builder = SpanBuilder::default(); let span_expr = builder.lower_span_expr(expr)?; + let leaf_boosts = builder.leaf_boosts(&span_expr); if let Some((span_query, position_filter)) = span_expr.to_fast_path_root() { Ok(Query::Span { term_slots: builder.term_slots, span_query, position_filter, + leaf_boosts, }) } else { Ok(Query::SpanExpr { term_slots: builder.term_slots, span_expr, + leaf_boosts, }) } } +#[derive(Default)] struct SpanBuilder { term_slots: Vec, intern_map: FxHashMap, + /// The product of the `^` factors around the operand being lowered. + boost: Option, + /// `boost` at each leaf, in the order the leaves are lowered. + written_boosts: Vec>, } impl SpanBuilder { - fn new() -> Self { - Self { - term_slots: Vec::new(), - intern_map: FxHashMap::default(), - } + /// One written occurrence of `slot`, under the current boost. Every + /// occurrence of a slot shares its interned index. + fn leaf(&mut self, slot: SpanTermSlot) -> SpanExpr { + self.written_boosts.push(self.boost); + let term_slots = &mut self.term_slots; + let idx = *self.intern_map.entry(slot).or_insert_with_key(|slot| { + term_slots.push(slot.clone()); + term_slots.len() - 1 + }); + SpanExpr::Term(idx) } - fn intern(&mut self, slot: SpanTermSlot) -> usize { - if let Some(&idx) = self.intern_map.get(&slot) { - return idx; + /// The written boosts of the leaves `span_expr` scores. Lowering writes + /// leaves in the tree's left-to-right order, the order `for_each_term` + /// walks, so the two line up one for one. + fn leaf_boosts(&self, span_expr: &SpanExpr) -> SpanLeafBoosts { + let mut written = self.written_boosts.iter(); + let mut included = Vec::new(); + span_expr.for_each_term(true, &mut |_, scored| { + let boost = written.next().expect("lowering writes a boost per leaf"); + if scored { + included.push(*boost); + } + }); + debug_assert!( + written.next().is_none(), + "every written leaf is in the tree" + ); + if included.iter().all(Option::is_none) { + SpanLeafBoosts::default() + } else { + SpanLeafBoosts(included.into()) } - let idx = self.term_slots.len(); - self.intern_map.insert(slot.clone(), idx); - self.term_slots.push(slot); - idx } fn lower_span_expr(&mut self, expr: &crate::Expr) -> Result { use crate::Expr; match expr { - Expr::Term(s) => { - let idx = self.intern(SpanTermSlot::Term(s.clone())); - Ok(SpanExpr::Term(idx)) - } + Expr::Term(s) => Ok(self.leaf(SpanTermSlot::Term(s.clone()))), Expr::MatchAll => Err(LowerError::MatchAllInSpanContext), Expr::MatchNone => Ok(SpanExpr::Empty), Expr::Fuzzy { term, prefix, distance, - } => { - let idx = self.intern(SpanTermSlot::Fuzzy { - term: term.clone(), - prefix: *prefix, - distance: *distance, - }); - Ok(SpanExpr::Term(idx)) - } - Expr::Wildcard(parts) => { - let idx = self.intern(SpanTermSlot::Regex(CompiledRegex::new( - &wildcard_parts_regex(parts), - )?)); - Ok(SpanExpr::Term(idx)) - } - Expr::Regex(pat) => { - let idx = self.intern(SpanTermSlot::Regex(CompiledRegex::new(pat)?)); - Ok(SpanExpr::Term(idx)) - } - Expr::Range { lower, upper } => { - let idx = self.intern(SpanTermSlot::Range { - lower: convert_range_bound(lower), - upper: convert_range_bound(upper), - }); - Ok(SpanExpr::Term(idx)) - } + } => Ok(self.leaf(SpanTermSlot::Fuzzy { + term: term.clone(), + prefix: *prefix, + distance: *distance, + })), + Expr::Wildcard(parts) => Ok(self.leaf(SpanTermSlot::Regex(CompiledRegex::new( + &wildcard_parts_regex(parts), + )?))), + Expr::Regex(pat) => Ok(self.leaf(SpanTermSlot::Regex(CompiledRegex::new(pat)?))), + Expr::Range { lower, upper } => Ok(self.leaf(SpanTermSlot::Range { + lower: convert_range_bound(lower), + upper: convert_range_bound(upper), + })), Expr::Phrase { elements, slop } => self.lower_phrase(elements, *slop), Expr::And(l, r) => { let left = self.lower_span_expr(l)?; @@ -352,7 +363,15 @@ impl SpanBuilder { inner: Box::new(sq), }) } - Expr::Boost { inner, .. } => self.lower_span_expr(inner), + // The factor stays out of the tree: it weighs the leaves beneath + // it (`leaf_boosts`), and matching never sees it. + Expr::Boost { factor, inner } => { + let outer = self.boost; + self.boost = Some(outer.unwrap_or(1.0) * factor.0); + let lowered = self.lower_span_expr(inner); + self.boost = outer; + lowered + } } } @@ -370,8 +389,7 @@ impl SpanBuilder { for elem in elements { match elem { crate::PhraseElement::Term(s) => { - let idx = self.intern(SpanTermSlot::Term(s.clone())); - children.push((pending_gap, SpanExpr::Term(idx))); + children.push((pending_gap, self.leaf(SpanTermSlot::Term(s.clone())))); pending_gap = 0; } crate::PhraseElement::Gap(n) => { @@ -488,6 +506,129 @@ mod tests { lower(&expr).expect("query should lower") } + /// The slots of a lowered span's included leaves, one per written + /// occurrence, left to right. + fn positive_leaves(input: &str) -> Vec { + let query = parse_and_lower(input); + let mut leaves = Vec::new(); + let term_slots = match &query { + Query::Span { + term_slots, + span_query, + .. + } => { + span_query.for_each_positive_term(&mut |idx| leaves.push(idx)); + term_slots + } + Query::SpanExpr { + term_slots, + span_expr, + .. + } => { + span_expr.for_each_positive_term(&mut |idx| leaves.push(idx)); + term_slots + } + other => panic!("{input} lowers to {other:?}, not a span"), + }; + leaves + .into_iter() + .map(|idx| match &term_slots[idx] { + SpanTermSlot::Term(term) => term.clone(), + other => format!("{other:?}"), + }) + .collect() + } + + #[test] + fn span_positive_leaves_are_written_occurrences_outside_excluded_sides() { + let to_be = ["to", "be", "or", "not", "to", "be"]; + assert_eq!(positive_leaves("\"to be or not to be\""), to_be); + assert_eq!(positive_leaves("\"to be or not to be\"~2"), to_be); + assert_eq!(positive_leaves("be NEAR/1 be"), ["be", "be"]); + assert_eq!(positive_leaves("[be or] NEAR/1 be"), ["be", "or", "be"]); + assert_eq!( + positive_leaves("(AT LEAST 2 OF [be be or]) NEAR/3 to"), + ["be", "be", "or", "to"] + ); + assert_eq!( + positive_leaves("(to NEAR/3 not) NOT ENCLOSES be"), + ["to", "not"] + ); + assert_eq!( + positive_leaves("or NOT ENCLOSED BY \"to be or not\""), + ["or"] + ); + assert_eq!(positive_leaves("be NOT OVERLAPPING \"to be\""), ["be"]); + assert_eq!(positive_leaves("(be AND NOT or) NEAR/3 to"), ["be", "to"]); + assert_eq!( + positive_leaves("(be NEAR/3 be) NOT ENCLOSES (or NEAR/1 be)"), + ["be", "be"] + ); + } + + /// A lowered span's included leaves as scoring reads them, left to right: + /// `term^factor` for a leaf with a written boost. + fn boosted_leaves(query: &Query) -> Vec { + let mut leaves = Vec::new(); + query.for_each_positive_span_slot(&mut |_, slot, boost| { + let SpanTermSlot::Term(term) = slot else { + panic!("{slot:?} is not a term slot"); + }; + leaves.push(match boost { + Some(factor) => format!("{term}^{factor}"), + None => term.clone(), + }); + }); + leaves + } + + #[test] + fn span_leaf_boosts_weigh_the_operand_they_are_written_on() { + let leaves = |input| boosted_leaves(&parse_and_lower(input)); + assert_eq!(leaves("ipa^3 NEAR/5 hoppy"), ["ipa^3", "hoppy"]); + assert_eq!(leaves("\"craft [beer^2 ale]\""), ["craft", "beer^2", "ale"]); + assert_eq!( + leaves("(stout^2 OR porter) NEAR/3 chocolate"), + ["stout^2", "porter", "chocolate"] + ); + assert_eq!(leaves("beer^2 IN FIRST 100 WORDS"), ["beer^2"]); + assert_eq!(leaves("beer^0 NEAR/3 tasting"), ["beer^0", "tasting"]); + // Factors multiply down to the leaf; a repeated word weighs per + // occurrence, each with its own boost. + assert_eq!( + leaves("(ipa^2 NEAR/1 pale)^3 THEN/4 ale"), + ["ipa^6", "pale^3", "ale"] + ); + assert_eq!(leaves("beer^2 NEAR/3 beer"), ["beer^2", "beer"]); + // An excluded side only filters: its boosts weigh nothing. + assert_eq!( + leaves("(ipa^2 NEAR/3 hoppy) NOT ENCLOSES stout^5"), + ["ipa^2", "hoppy"] + ); + // The dynamic form (AT LEAST has no fast path) reads them the same way. + let at_least = parse_and_lower("(AT LEAST 2 OF [be^2 be or]) NEAR/3 to"); + assert!(matches!(at_least, Query::SpanExpr { .. })); + assert_eq!(boosted_leaves(&at_least), ["be^2", "be", "or", "to"]); + } + + #[test] + fn unboosted_span_leaves_lower_without_boosts() { + let unboosted = |input| match parse_and_lower(input) { + Query::Span { leaf_boosts, .. } | Query::SpanExpr { leaf_boosts, .. } => { + leaf_boosts == SpanLeafBoosts::default() + } + Query::Boost { inner, .. } => matches!( + *inner, + Query::Span { leaf_boosts, .. } if leaf_boosts == SpanLeafBoosts::default() + ), + other => panic!("{input} lowers to {other:?}"), + }; + assert!(unboosted("ipa NEAR/5 hoppy")); + assert!(unboosted("(ipa NEAR/5 hoppy)^3")); + assert!(unboosted("\"craft beer\"^2")); + assert!(unboosted("ipa NOT ENCLOSES stout^5")); + } + #[test] fn top_level_runtime_position_filter_stays_on_fast_path() { let query = parse_and_lower("beer IN LAST 25%"); @@ -665,6 +806,7 @@ mod tests { let Query::SpanExpr { term_slots, span_expr, + .. } = query else { panic!("expected advanced span expression lowering"); diff --git a/tinql/src/runtime/mod.rs b/tinql/src/runtime/mod.rs index 6e967a4..d0c6d2d 100644 --- a/tinql/src/runtime/mod.rs +++ b/tinql/src/runtime/mod.rs @@ -67,10 +67,12 @@ pub enum Query { term_slots: Vec, span_query: boldi_vigna::SpanQuery, position_filter: Option, + leaf_boosts: SpanLeafBoosts, }, SpanExpr { term_slots: Vec, span_expr: SpanExpr, + leaf_boosts: SpanLeafBoosts, }, MatchAll, Regex(CompiledRegex), @@ -111,6 +113,21 @@ pub enum SpanTermSlot { }, } +/// The `^` factor written on each included leaf of a span, in +/// [`SpanExpr::for_each_positive_term`] order: the product of the boosts +/// between the leaf and the span's root, `None` where there are none. Empty +/// when no included leaf is boosted, so an unboosted span lowers as it always +/// has. Only scoring reads it; matching never does. +#[derive(Debug, Clone, Default, PartialEq)] +pub struct SpanLeafBoosts(Box<[Option]>); + +impl SpanLeafBoosts { + /// The boost written on the `nth` included leaf. + fn get(&self, nth: usize) -> Option { + if self.0.is_empty() { None } else { self.0[nth] } + } +} + /// A bound in a term range expression. #[derive(Debug, Clone, PartialEq, Eq, Hash)] pub enum RangeBound { @@ -140,6 +157,36 @@ impl Query { } } + /// For a span node (`Span` or `SpanExpr`), calls `f` with each included + /// leaf's slot index, slot and written boost ([`SpanLeafBoosts`]), once + /// per written occurrence, left to right + /// ([`SpanExpr::for_each_positive_term`]); nothing for any other node. + /// These are the leaves a span scores: its excluded sides only filter. + pub fn for_each_positive_span_slot<'a>( + &'a self, + f: &mut impl FnMut(usize, &'a SpanTermSlot, Option), + ) { + let mut nth = 0; + let mut visit = |term_slots: &'a [SpanTermSlot], leaf_boosts: &SpanLeafBoosts, idx| { + f(idx, &term_slots[idx], leaf_boosts.get(nth)); + nth += 1; + }; + match self { + Query::Span { + term_slots, + span_query, + leaf_boosts, + .. + } => span_query.for_each_positive_term(&mut |idx| visit(term_slots, leaf_boosts, idx)), + Query::SpanExpr { + term_slots, + span_expr, + leaf_boosts, + } => span_expr.for_each_positive_term(&mut |idx| visit(term_slots, leaf_boosts, idx)), + _ => {} + } + } + pub fn estimate_tuples(&self, total_tuples: u64, lookup: &dyn Fn(&str) -> u64) -> u64 { let n = total_tuples as f64; self.estimate_selectivity(n, lookup, total_tuples) as u64 diff --git a/tinql/src/runtime/simplify.rs b/tinql/src/runtime/simplify.rs index 1e66306..e0412c6 100644 --- a/tinql/src/runtime/simplify.rs +++ b/tinql/src/runtime/simplify.rs @@ -21,6 +21,8 @@ //! - [`SimplificationProfile::Structural`] keeps the old lowering-time work: //! flattening boolean chains, deduplicating flat term siblings, folding //! `MatchAll`, and collapsing one-child boolean nodes +//! - [`SimplificationProfile::Membership`] adds the rewrites across a +//! conjunction's `NOT` children, for match planning only //! - [`SimplificationProfile::LogicalUnscored`] adds implication-based //! absorption for positive unscored trees so planner-time display and //! unscored execution can both reuse the same backend-agnostic rewrites @@ -49,10 +51,14 @@ //! space at this boundary, it is just deleting a small set of provably //! redundant unscored subtrees. //! -//! Future work: -//! -//! - `NOT` remains a negation boundary. We simplify inside negated subtrees, -//! but we don't do implication reasoning across the negative edge yet. +//! Across the negative edge, a match plan's conjunction (the membership +//! profiles) is empty when a positive child implies one of its `NOT` +//! children, and drops the disjuncts they exclude (`exclude_negated`). A +//! negated subtree itself is simplified inside only. Scoring and highlight +//! trees keep those children: a row scores and highlights every positive +//! term it holds, whichever clause matched, so a disjunct no survivor of a +//! nested conjunction matches still counts on a row that matches through a +//! sibling of that conjunction. use super::{Query, SpanExpr, SpanTermSlot}; use rustc_hash::FxHashSet; @@ -60,6 +66,9 @@ use rustc_hash::FxHashSet; #[derive(Clone, Copy)] pub enum SimplificationProfile { Structural, + /// [`Self::Structural`] for match planning, where the set of rows is + /// all that counts: it also rewrites across `NOT` edges. + Membership, StructuralPreserveTermMultiplicity, LogicalUnscored, } @@ -68,6 +77,7 @@ pub fn simplify(query: Query, profile: SimplificationProfile) -> Query { let query = normalize_boolean_query(query, profile); match profile { SimplificationProfile::Structural + | SimplificationProfile::Membership | SimplificationProfile::StructuralPreserveTermMultiplicity => query, SimplificationProfile::LogicalUnscored => reduce_unscored_redundancy(query), } @@ -76,10 +86,19 @@ pub fn simplify(query: Query, profile: SimplificationProfile) -> Query { impl SimplificationProfile { const fn dedup_flat_terms(self) -> bool { match self { - Self::Structural | Self::LogicalUnscored => true, + Self::Structural | Self::Membership | Self::LogicalUnscored => true, Self::StructuralPreserveTermMultiplicity => false, } } + + /// Whether the tree only decides membership, so a conjunction's `NOT` + /// children may rewrite its positive ones (`exclude_negated`). + const fn crosses_negations(self) -> bool { + match self { + Self::Membership | Self::LogicalUnscored => true, + Self::Structural | Self::StructuralPreserveTermMultiplicity => false, + } + } } pub fn all_terms_required_span_query(span_query: &boldi_vigna::SpanQuery) -> bool { @@ -299,10 +318,12 @@ fn implies(lhs: &Query, rhs: &Query) -> bool { term_slots, span_query, position_filter, + .. } => return query_implies_span(lhs, term_slots, span_query, position_filter.as_ref()), Query::SpanExpr { term_slots, span_expr, + .. } => return query_implies_span_expr(lhs, term_slots, span_expr), Query::Disjunction { .. } | Query::Not(_) @@ -352,6 +373,7 @@ fn query_implies_term(query: &Query, rhs_term: &str) -> bool { Query::SpanExpr { term_slots, span_expr, + .. } => { all_terms_required_span_expr(span_expr) && span_expr_references_term(span_expr, term_slots, rhs_term) @@ -377,6 +399,7 @@ fn query_implies_span( term_slots, span_query, position_filter, + .. } => { term_slots == rhs_slots && span_query == rhs_span_query @@ -413,6 +436,7 @@ fn query_implies_span_expr( Query::SpanExpr { term_slots, span_expr, + .. } => term_slots == rhs_slots && span_expr == rhs_span_expr, Query::And(left, right) => { query_implies_span_expr(left, rhs_slots, rhs_span_expr) @@ -591,6 +615,11 @@ fn normalize_conjunction(children: Vec, profile: SimplificationProfile) - if flat.iter().any(is_empty_query) { return empty_query(); } + if profile.crosses_negations() + && let Some(rewritten) = exclude_negated(&mut flat, profile) + { + return rewritten; + } flat.retain(|child| !matches!(child, Query::MatchAll)); if profile.dedup_flat_terms() { @@ -599,6 +628,47 @@ fn normalize_conjunction(children: Vec, profile: SimplificationProfile) - fold_conjunction(flat) } +/// The rewrites a conjunction's `NOT` children allow in a match plan: a +/// positive child that implies a negated one leaves no match, so the +/// conjunction is empty; and a positive disjunction drops each disjunct +/// that implies a negated child, which matches no survivor. `None` when +/// nothing changes. +/// +/// The disjunct's terms go too, which is what keeps this out of scoring +/// and highlight trees: a row scores and highlights every positive term it +/// holds, whichever clause matched (`((a AND c) OR b) AND NOT a` scores `c` +/// on a survivor holding `b c`), and nested under an OR the conjunction's +/// rows are not every row the tree scores (`((a OR b) AND NOT a) OR c` +/// scores `a` on a row holding `a c`). +fn exclude_negated(flat: &mut Vec, profile: SimplificationProfile) -> Option { + let negated: Vec = flat + .iter() + .filter_map(|child| match child { + Query::Not(inner) => Some((**inner).clone()), + _ => None, + }) + .collect(); + if negated.is_empty() { + return None; + } + let excluded = |query: &Query| negated.iter().any(|negated| implies(query, negated)); + if flat.iter().filter_map(positive_root).any(excluded) { + return Some(empty_query()); + } + let mut changed = false; + for child in flat.iter_mut() { + if let Query::Disjunction { min, children } = child + && children.iter().any(excluded) + { + let (min, mut kept) = (*min, std::mem::take(children)); + kept.retain(|disjunct| !excluded(disjunct)); + *child = normalize_disjunction(min, kept, profile); + changed = true; + } + } + changed.then(|| normalize_conjunction(std::mem::take(flat), profile)) +} + fn push_conjunction_child(child: Query, out: &mut Vec) { match child { Query::And(left, right) => { @@ -698,6 +768,7 @@ fn fold_conjunction(mut children: Vec) -> Query { #[cfg(test)] mod tests { use super::*; + use crate::runtime::SpanLeafBoosts; #[test] fn structural_simplify_flattens_and_dedups_terms() { @@ -789,6 +860,7 @@ mod tests { ])), }, position_filter: None, + leaf_boosts: SpanLeafBoosts::default(), }, ], }; @@ -800,9 +872,9 @@ mod tests { } #[test] - fn logical_unscored_does_not_cross_negative_edge() { + fn logical_unscored_keeps_negated_subtrees_whole() { let query = Query::Conjunction(vec![ - Query::Term("beer".into()), + Query::Term("ale".into()), Query::Not(Box::new(Query::Disjunction { min: 1, children: vec![Query::Term("beer".into()), Query::Term("wine".into())], @@ -810,17 +882,216 @@ mod tests { ]); assert_eq!( - simplify(query, SimplificationProfile::LogicalUnscored), + simplify(query.clone(), SimplificationProfile::LogicalUnscored), + query, + ); + } + + /// The profiles that plan membership, which rewrite across `NOT` edges. + const MATCH_PROFILES: [SimplificationProfile; 2] = [ + SimplificationProfile::Membership, + SimplificationProfile::LogicalUnscored, + ]; + + /// The profiles whose trees score or highlight, which keep every + /// positive child. + const TREE_PROFILES: [SimplificationProfile; 2] = [ + SimplificationProfile::Structural, + SimplificationProfile::StructuralPreserveTermMultiplicity, + ]; + + fn term(text: &str) -> Query { + Query::Term(text.into()) + } + + fn not(inner: Query) -> Query { + Query::Not(Box::new(inner)) + } + + fn any(children: Vec) -> Query { + Query::Disjunction { min: 1, children } + } + + fn phrase(left: &str, right: &str) -> Query { + Query::Span { + term_slots: vec![ + SpanTermSlot::Term(left.into()), + SpanTermSlot::Term(right.into()), + ], + span_query: boldi_vigna::SpanQuery::MaxGaps { + max_gaps: 0, + inner: Box::new(boldi_vigna::SpanQuery::Ordered(vec![ + boldi_vigna::SpanQuery::Term(0), + boldi_vigna::SpanQuery::Term(1), + ])), + }, + position_filter: None, + leaf_boosts: SpanLeafBoosts::default(), + } + } + + /// A positive child that implies a negated one leaves no match. + #[test] + fn a_positive_implying_a_negation_is_empty() { + for query in [ + Query::Conjunction(vec![term("a"), not(term("a"))]), + Query::Conjunction(vec![term("a"), term("b"), not(term("a"))]), + Query::And( + Box::new(Query::Conjunction(vec![term("a"), term("b")])), + Box::new(not(term("a"))), + ), + Query::Conjunction(vec![phrase("a", "b"), not(term("a"))]), + Query::Conjunction(vec![term("a"), not(any(vec![term("a"), term("c")]))]), Query::Conjunction(vec![ - Query::Term("beer".into()), - Query::Not(Box::new(Query::Disjunction { - min: 1, - children: vec![Query::Term("beer".into()), Query::Term("wine".into())], - })), + any(vec![term("a"), term("b")]), + not(any(vec![term("a"), term("b")])), ]), + Query::Conjunction(vec![ + Query::Boost { + factor: 2.0, + inner: Box::new(term("a")), + }, + not(term("a")), + ]), + ] { + for profile in MATCH_PROFILES { + assert_eq!(simplify(query.clone(), profile), empty_query(), "{query:?}"); + } + for profile in TREE_PROFILES { + assert_ne!(simplify(query.clone(), profile), empty_query(), "{query:?}"); + } + } + } + + /// A disjunct that implies a negated child goes from the match plan, + /// whatever else it holds; the threshold's tail then applies. + #[test] + fn a_disjunct_the_negation_excludes_goes() { + let cases = [ + ( + Query::Conjunction(vec![any(vec![term("a"), term("b")]), not(term("a"))]), + Query::Conjunction(vec![term("b"), not(term("a"))]), + ), + ( + Query::Conjunction(vec![ + any(vec![term("a"), term("b")]), + not(any(vec![term("a"), term("c")])), + ]), + Query::Conjunction(vec![term("b"), not(any(vec![term("a"), term("c")]))]), + ), + ( + Query::Conjunction(vec![ + any(vec![term("a"), term("b"), term("c")]), + not(term("a")), + ]), + Query::Conjunction(vec![any(vec![term("b"), term("c")]), not(term("a"))]), + ), + ( + Query::Conjunction(vec![ + Query::Disjunction { + min: 2, + children: vec![term("a"), term("b"), term("c")], + }, + not(term("a")), + ]), + Query::Conjunction(vec![term("b"), term("c"), not(term("a"))]), + ), + ( + Query::Conjunction(vec![ + Query::Disjunction { + min: 3, + children: vec![term("a"), term("b"), term("c")], + }, + not(term("a")), + ]), + empty_query(), + ), + ( + Query::Conjunction(vec![any(vec![phrase("a", "b"), term("c")]), not(term("a"))]), + Query::Conjunction(vec![term("c"), not(term("a"))]), + ), + ( + Query::Conjunction(vec![ + any(vec![ + Query::Conjunction(vec![term("a"), term("c")]), + term("b"), + ]), + not(term("a")), + ]), + Query::Conjunction(vec![term("b"), not(term("a"))]), + ), + ]; + for (query, expected) in cases { + for profile in MATCH_PROFILES { + assert_eq!(simplify(query.clone(), profile), expected, "{query:?}"); + } + for profile in TREE_PROFILES { + assert_ne!(simplify(query.clone(), profile), expected, "{query:?}"); + } + } + // The profile that keeps repeats keeps a repeated disjunct too. + let repeated = Query::Conjunction(vec![ + any(vec![term("a"), term("a"), term("b")]), + not(term("a")), + ]); + assert_eq!( + simplify( + repeated.clone(), + SimplificationProfile::StructuralPreserveTermMultiplicity + ), + repeated, ); } + /// Nested under an OR, the rewritten conjunction's rows are not every + /// row the tree matches: a row holding `a c` matches through `c` and + /// scores and highlights `a`. The match plan still rewrites it (its rows + /// are the same); the trees keep it whole. + #[test] + fn a_nested_negation_rewrites_the_match_plan_only() { + let cases = [ + ( + any(vec![ + Query::Conjunction(vec![any(vec![term("a"), term("b")]), not(term("a"))]), + term("c"), + ]), + any(vec![ + Query::Conjunction(vec![term("b"), not(term("a"))]), + term("c"), + ]), + ), + ( + any(vec![ + Query::Conjunction(vec![term("a"), not(term("a"))]), + term("c"), + ]), + term("c"), + ), + ]; + for (query, expected) in cases { + for profile in MATCH_PROFILES { + assert_eq!(simplify(query.clone(), profile), expected, "{query:?}"); + } + for profile in TREE_PROFILES { + assert_eq!(simplify(query.clone(), profile), query, "{query:?}"); + } + } + } + + /// No rewrite where nothing implies the negation. + #[test] + fn a_negation_rewrites_nothing_it_does_not_exclude() { + for query in [ + Query::Conjunction(vec![term("a"), not(phrase("a", "b"))]), + Query::Conjunction(vec![any(vec![term("a"), term("b")]), not(term("c"))]), + Query::Conjunction(vec![any(vec![term("b"), not(term("a"))]), not(term("a"))]), + ] { + for profile in MATCH_PROFILES.into_iter().chain(TREE_PROFILES) { + assert_eq!(simplify(query.clone(), profile), query, "{query:?}"); + } + } + } + #[test] fn logical_unscored_ignores_unused_span_slots() { let query = Query::Disjunction { @@ -833,6 +1104,7 @@ mod tests { ], span_query: boldi_vigna::SpanQuery::Term(0), position_filter: None, + leaf_boosts: SpanLeafBoosts::default(), }, Query::Term("c".into()), ], @@ -850,6 +1122,7 @@ mod tests { ], span_query: boldi_vigna::SpanQuery::Term(0), position_filter: None, + leaf_boosts: SpanLeafBoosts::default(), }, Query::Term("c".into()), ], diff --git a/tinql/src/runtime/span_expr.rs b/tinql/src/runtime/span_expr.rs index b036d53..dc8560c 100644 --- a/tinql/src/runtime/span_expr.rs +++ b/tinql/src/runtime/span_expr.rs @@ -150,6 +150,66 @@ impl SpanExpr { } } + /// Calls `f` with the slot of every `Term` leaf outside an excluded + /// operand, once per occurrence, left to right: the leaves a match's + /// spans are built from ([`Self::for_each_term`]). + pub fn for_each_positive_term(&self, f: &mut impl FnMut(usize)) { + self.for_each_term(true, &mut |idx, included| { + if included { + f(idx); + } + }); + } + + /// Calls `f` with the slot of every `Term` leaf, once per occurrence, left + /// to right, and whether it is included (`included` seeds the root). An + /// excluded operand (`NotContaining`'s `little`, `NotContainedBy`'s + /// `big`, `NonOverlapping`'s `b`) only filters its sibling's spans, so + /// every leaf beneath it is excluded. Left to right is the order lowering + /// writes the leaves in. + pub(super) fn for_each_term(&self, included: bool, f: &mut impl FnMut(usize, bool)) { + match self { + Self::Empty => {} + Self::Term(idx) => f(*idx, included), + Self::Ordered(children) + | Self::Unordered(children) + | Self::Or(children) + | Self::AtLeast { children, .. } => { + for child in children { + child.for_each_term(included, f); + } + } + Self::MaxGaps { inner, .. } + | Self::GapsInRange { inner, .. } + | Self::MaxWidth { inner, .. } + | Self::WithinPositions { inner, .. } + | Self::PositionFilter { inner, .. } => inner.for_each_term(included, f), + Self::NotContaining { + big: kept, + little: excluded, + } + | Self::NotContainedBy { + little: kept, + big: excluded, + } + | Self::NonOverlapping { + a: kept, + b: excluded, + } => { + kept.for_each_term(included, f); + excluded.for_each_term(false, f); + } + Self::Containing { big: a, little: b } + | Self::ContainedBy { little: a, big: b } + | Self::Overlapping { a, b } + | Self::Before { a, b } + | Self::After { a, b } => { + a.for_each_term(included, f); + b.for_each_term(included, f); + } + } + } + pub fn to_fast_path_root( &self, ) -> Option<(boldi_vigna::SpanQuery, Option)> { diff --git a/tinql/tests/near_slot_oracle.rs b/tinql/tests/near_slot_oracle.rs new file mode 100644 index 0000000..284393e --- /dev/null +++ b/tinql/tests/near_slot_oracle.rs @@ -0,0 +1,255 @@ +// Copyright (C) 2026 PlanetScale +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . +//! tinql's evaluator against an independent brute-force NEAR oracle, for slots +//! that are OR groups and may share a word with the other slot. +//! +//! The oracle tokenizes the text with the tokenizer crate's default pipeline, +//! takes each slot's positions directly from the tokens, and enumerates every +//! pair of positions, one per slot. +//! +//! Semantics implemented (book `tinql/proximity.md`, and the span solver's +//! `SpanQuery` contract for what the book leaves open): +//! - `NEAR/n` is unordered: either slot may come first. +//! - Distance: single-word slots at positions `p` and `q` match when at most +//! `n` positions lie strictly between them, `|p - q| - 1 <= n`. +//! - Same-position reuse: the book does not say whether both slots may use one +//! occurrence. The solver's contract does (`MaxGaps` passes no overlapping +//! sub-intervals), and the shipped `alpha NEAR/3 alpha` already requires two +//! distinct occurrences. The oracle follows that: `p != q`. + +use tinql::runtime::{evaluate, lower::lower, subtokenize::sub_tokenize, tokenize_doc}; +use tinql::{ImplicitOp, parse}; +use tokenizer::Tokenizer; +use tokenizer::presets::default_pipeline; + +fn engine_matches(query: &str, text: &str) -> bool { + let expr = parse(query, ImplicitOp::And).expect("query parses"); + let expr = sub_tokenize(expr, default_pipeline()).expect("query sub-tokenizes"); + let query = lower(&expr).expect("query lowers"); + evaluate(&query, &tokenize_doc(text, default_pipeline())) + .expect("query evaluates") + .matched +} + +/// The single token the default pipeline makes of a slot word. +fn normalize(word: &str) -> String { + let mut tokens = default_pipeline().tokenize(word); + let token = tokens + .next() + .expect("slot word tokenizes") + .text + .into_owned(); + assert!(tokens.next().is_none(), "slot word {word:?} is one token"); + token +} + +fn oracle_matches(left: &[&str], right: &[&str], max_gaps: u32, text: &str) -> bool { + let tokens: Vec<(String, u32)> = default_pipeline() + .tokenize(text) + .map(|token| (token.text.into_owned(), token.pos)) + .collect(); + let slot_positions = |slot: &[&str]| -> Vec { + let words: Vec = slot.iter().map(|w| normalize(w)).collect(); + tokens + .iter() + .filter(|(token, _)| words.contains(token)) + .map(|&(_, pos)| pos) + .collect() + }; + let (left, right) = (slot_positions(left), slot_positions(right)); + left.iter().any(|&p| { + right + .iter() + .any(|&q| p != q && p.abs_diff(q) - 1 <= max_gaps) + }) +} + +fn slot_text(slot: &[&str]) -> String { + match slot { + [word] => (*word).to_owned(), + words => format!("({})", words.join(" OR ")), + } +} + +fn near_query(left: &[&str], right: &[&str], max_gaps: u32) -> String { + format!("{} NEAR/{max_gaps} {}", slot_text(left), slot_text(right)) +} + +/// Query, left slot, right slot, gap, text. +type Row = ( + &'static str, + &'static [&'static str], + &'static [&'static str], + u32, + &'static str, +); + +/// The M5c span-fixture queries, on texts that contain a match. +#[test] +fn group_slots_sharing_a_word_match_the_oracle() { + let rows: &[Row] = &[ + ( + "alpha NEAR/3 alpha", + &["alpha"], + &["alpha"], + 3, + "alpha of the alpha", + ), + ( + "(alpha OR bravo) NEAR/3 alpha", + &["alpha", "bravo"], + &["alpha"], + 3, + "alpha of the alpha", + ), + ( + "[alpha bravo] NEAR/2 alpha", + &["alpha", "bravo"], + &["alpha"], + 2, + "the alpha of alpha", + ), + ( + "alpha NEAR/2 alpha", + &["alpha"], + &["alpha"], + 2, + "alpha of the alpha", + ), + ( + "(alpha OR bravo) NEAR/2 (alpha OR charlie)", + &["alpha", "bravo"], + &["alpha", "charlie"], + 2, + "alpha the alpha", + ), + ( + "(alpha OR bravo) NEAR/2 (alpha OR charlie)", + &["alpha", "bravo"], + &["alpha", "charlie"], + 2, + "charlie of the bravo alpha", + ), + ]; + let mut failures = Vec::new(); + for &(query, left, right, max_gaps, text) in rows { + let expected = oracle_matches(left, right, max_gaps, text); + let actual = engine_matches(query, text); + if actual != expected { + failures.push(format!( + "{query:?} on {text:?}: tinql {actual}, oracle {expected}" + )); + } + } + assert!(failures.is_empty(), "{}", failures.join("\n")); +} + +#[test] +fn one_shared_occurrence_is_not_a_pair() { + assert!(!engine_matches( + "(alpha OR bravo) NEAR/3 alpha", + "the alpha of" + )); + assert!(!oracle_matches( + &["alpha", "bravo"], + &["alpha"], + 3, + "the alpha of" + )); +} + +/// xorshift64*: deterministic generated cases without a test dependency. +struct Rng(u64); + +impl Rng { + fn next(&mut self) -> u64 { + self.0 ^= self.0 >> 12; + self.0 ^= self.0 << 25; + self.0 ^= self.0 >> 27; + self.0.wrapping_mul(0x2545_f491_4f6c_dd1d) + } + + fn below(&mut self, n: usize) -> usize { + (self.next() % n as u64) as usize + } +} + +const SLOT_WORDS: [&str; 4] = ["alpha", "bravo", "charlie", "delta"]; +const FILLER: [&str; 3] = ["the", "of", "echo"]; + +fn generated_text(rng: &mut Rng) -> String { + let len = 1 + rng.below(12); + (0..len) + .map(|_| { + if rng.below(2) == 0 { + SLOT_WORDS[rng.below(SLOT_WORDS.len())] + } else { + FILLER[rng.below(FILLER.len())] + } + }) + .collect::>() + .join(" ") +} + +fn generated_slot(rng: &mut Rng) -> Vec<&'static str> { + let mut slot: Vec<&str> = SLOT_WORDS + .iter() + .copied() + .filter(|_| rng.below(3) == 0) + .collect(); + if slot.is_empty() { + slot.push(SLOT_WORDS[rng.below(SLOT_WORDS.len())]); + } + slot +} + +#[test] +fn generated_slot_queries_match_the_oracle_and_widening_never_drops_a_doc() { + let mut rng = Rng(0x9e37_79b9_7f4a_7c15); + let mut failures = Vec::new(); + for _ in 0..3000 { + let text = generated_text(&mut rng); + let left = generated_slot(&mut rng); + let right = generated_slot(&mut rng); + let max_gaps = rng.below(4) as u32; + + let query = near_query(&left, &right, max_gaps); + let actual = engine_matches(&query, &text); + let expected = oracle_matches(&left, &right, max_gaps, &text); + if actual != expected { + failures.push(format!( + "{query:?} on {text:?}: tinql {actual}, oracle {expected}" + )); + } + + let extra = SLOT_WORDS[rng.below(SLOT_WORDS.len())]; + if actual && !left.contains(&extra) { + let mut wider = left.clone(); + wider.push(extra); + let wide_query = near_query(&wider, &right, max_gaps); + if !engine_matches(&wide_query, &text) { + failures.push(format!( + "{query:?} matched {text:?} but {wide_query:?} did not" + )); + } + } + } + assert!( + failures.is_empty(), + "{} failures, first:\n{}", + failures.len(), + failures[..failures.len().min(10)].join("\n") + ); +}