From b8018ac35a1b5328e7bdb472b7fe9316e7a5bad9 Mon Sep 17 00:00:00 2001 From: Patrick Reynolds Date: Wed, 7 Oct 2026 00:03:55 -0400 Subject: [PATCH 1/2] Copy tinql and boldi-vigna bug fixes from TIN --- boldi-vigna/src/query.rs | 83 +++++++ boldi-vigna/src/state/mod.rs | 228 ++++++++++++++++---- boldi-vigna/src/state/shared.rs | 106 +++++++++ boldi-vigna/src/state/unordered.rs | 147 ++----------- boldi-vigna/tests/near_chain_tests.rs | 125 +++++++++++ boldi-vigna/tests/near_slot_tests.rs | 200 +++++++++++++++++ postgres/src/operator.rs | 39 +++- postgres/src/score.rs | 19 +- postgres/src/tinql.rs | 18 +- private-regress.manifest | 2 + tinql/docs/src/alternatives.md | 18 ++ tinql/docs/src/boost.md | 15 ++ tinql/docs/src/proximity.md | 4 + tinql/docs/src/span-relations.md | 4 + tinql/src/runtime/display.rs | 10 +- tinql/src/runtime/eval.rs | 4 + tinql/src/runtime/lower.rs | 234 ++++++++++++++++---- tinql/src/runtime/mod.rs | 47 ++++ tinql/src/runtime/simplify.rs | 299 ++++++++++++++++++++++++-- tinql/src/runtime/span_expr.rs | 60 ++++++ tinql/tests/near_slot_oracle.rs | 255 ++++++++++++++++++++++ 21 files changed, 1673 insertions(+), 244 deletions(-) create mode 100644 boldi-vigna/src/state/shared.rs create mode 100644 boldi-vigna/tests/near_chain_tests.rs create mode 100644 boldi-vigna/tests/near_slot_tests.rs create mode 100644 tinql/tests/near_slot_oracle.rs diff --git a/boldi-vigna/src/query.rs b/boldi-vigna/src/query.rs index 0046a04..fd924fe 100644 --- a/boldi-vigna/src/query.rs +++ b/boldi-vigna/src/query.rs @@ -174,6 +174,40 @@ impl SpanQuery { SpanQueryWithTerms { query: self, terms } } + /// Calls `f` with the index of every `Term` leaf outside an excluded + /// operand, once per occurrence, left to right. An excluded operand + /// (`NotContaining`'s `little`, `NotContainedBy`'s `big`, + /// `NonOverlapping`'s `b`) only filters its sibling's spans, so no leaf + /// beneath it is visited. + pub fn for_each_positive_term(&self, f: &mut impl FnMut(usize)) { + match self { + SpanQuery::Empty => {} + SpanQuery::Term(i) => f(*i), + SpanQuery::Ordered(children) + | SpanQuery::Unordered(children) + | SpanQuery::Or(children) => { + for child in children { + child.for_each_positive_term(f); + } + } + SpanQuery::MaxGaps { inner, .. } + | SpanQuery::GapsInRange { inner, .. } + | SpanQuery::MaxWidth { inner, .. } + | SpanQuery::WithinPositions { inner, .. } + | SpanQuery::NotContaining { big: inner, .. } + | SpanQuery::NotContainedBy { little: inner, .. } + | SpanQuery::NonOverlapping { a: inner, .. } => inner.for_each_positive_term(f), + SpanQuery::Containing { big: a, little: b } + | SpanQuery::ContainedBy { little: a, big: b } + | SpanQuery::Overlapping { a, b } + | SpanQuery::Before { a, b } + | SpanQuery::After { a, b } => { + a.for_each_positive_term(f); + b.for_each_positive_term(f); + } + } + } + /// Maximum term index referenced anywhere in this query tree. pub fn max_term_index(&self) -> Option { match self { @@ -199,6 +233,30 @@ impl SpanQuery { } } + /// Whether an `Unordered` sits anywhere in this query tree: the operator + /// whose gap-read form walks every child order (`NodeState::compile_gaps`). + pub(crate) fn holds_unordered(&self) -> bool { + match self { + SpanQuery::Unordered(_) => true, + SpanQuery::Empty | SpanQuery::Term(_) => false, + SpanQuery::Ordered(children) | SpanQuery::Or(children) => { + children.iter().any(SpanQuery::holds_unordered) + } + SpanQuery::MaxGaps { inner, .. } + | SpanQuery::GapsInRange { inner, .. } + | SpanQuery::MaxWidth { inner, .. } + | SpanQuery::WithinPositions { inner, .. } => inner.holds_unordered(), + SpanQuery::Containing { big: a, little: b } + | SpanQuery::NotContaining { big: a, little: b } + | SpanQuery::ContainedBy { little: a, big: b } + | SpanQuery::NotContainedBy { little: a, big: b } + | SpanQuery::Overlapping { a, b } + | SpanQuery::NonOverlapping { a, b } + | SpanQuery::Before { a, b } + | SpanQuery::After { a, b } => a.holds_unordered() || b.holds_unordered(), + } + } + /// Number of distinct term indices needed (max_term_index + 1, or 0 if no terms). pub fn num_terms(&self) -> usize { self.max_term_index().map_or(0, |i| i + 1) @@ -292,6 +350,31 @@ mod tests { assert!(q.validate(1).is_err()); } + #[test] + fn positive_terms_repeat_and_skip_excluded_operands() { + let mut leaves = Vec::new(); + let q = SpanQuery::NotContaining { + big: Box::new(SpanQuery::phrase([0, 1, 0])), + little: Box::new(SpanQuery::Term(2)), + }; + q.for_each_positive_term(&mut |idx| leaves.push(idx)); + assert_eq!(leaves, [0, 1, 0]); + + leaves.clear(); + let q = SpanQuery::Containing { + big: Box::new(SpanQuery::NonOverlapping { + a: Box::new(SpanQuery::Term(0)), + b: Box::new(SpanQuery::Term(1)), + }), + little: Box::new(SpanQuery::NotContainedBy { + little: Box::new(SpanQuery::Or(vec![SpanQuery::Term(0), SpanQuery::Term(2)])), + big: Box::new(SpanQuery::Term(3)), + }), + }; + q.for_each_positive_term(&mut |idx| leaves.push(idx)); + assert_eq!(leaves, [0, 0, 2]); + } + #[test] fn num_terms_nested() { let q = SpanQuery::Ordered(vec![ diff --git a/boldi-vigna/src/state/mod.rs b/boldi-vigna/src/state/mod.rs index 28f4a2e..7d2b7fe 100644 --- a/boldi-vigna/src/state/mod.rs +++ b/boldi-vigna/src/state/mod.rs @@ -20,9 +20,13 @@ mod filter; mod ordered; mod phrase; mod relation; +mod shared; mod term; mod unordered; +use std::cell::RefCell; +use std::rc::Rc; + use crate::interval::Interval; use crate::positions::TermPositions; use crate::query::SpanQuery; @@ -35,9 +39,24 @@ pub(crate) use self::filter::{ pub(crate) use self::ordered::OrderedState; pub(crate) use self::phrase::PhraseState; pub(crate) use self::relation::RelationState; +pub(crate) use self::shared::{SharedSource, SharedState}; pub(crate) use self::term::TermState; pub(crate) use self::unordered::UnorderedState; +// States this thread compiled and reset: the unit tests' size pins. +#[cfg(test)] +thread_local! { + static STATES_BUILT: std::cell::Cell = const { std::cell::Cell::new(0) }; + static STATES_RESET: std::cell::Cell = const { std::cell::Cell::new(0) }; +} + +/// Counts one compiled state (unit tests only). +#[inline(always)] +fn built() { + #[cfg(test)] + STATES_BUILT.with(|built| built.set(built.get() + 1)); +} + /// Internal state machine for a single node in the operator tree. /// Mirrors the structure of `SpanQuery` but carries mutable iteration state. pub(crate) enum NodeState { @@ -52,81 +71,91 @@ pub(crate) enum NodeState { MaxWidth(MaxWidthState), WithinPositions(WithinPositionsState), Relation(RelationState), + Shared(SharedState), } impl NodeState { /// Compile a `SpanQuery` tree into a `NodeState` tree. pub(crate) fn compile(query: &SpanQuery) -> Self { + Self::compile_gaps(query, false) + } + + /// Compiles `query`; `gaps_read` says an enclosing gap filter reads this + /// node's gap count. A gap filter passes no overlapping sub-intervals, so a + /// gap-read `Unordered` compiles to its disjoint form: one `Ordered` per + /// distinct child order, unioned. Building the overlapping form there would + /// let an overlapping minimal interval (two children on one position) + /// shadow the disjoint spans that contain it, and the filter would then + /// reject them all. + fn compile_gaps(query: &SpanQuery, gaps_read: bool) -> Self { + built(); if let Some(term_indices) = exact_phrase_term_indices(query) { return NodeState::Phrase(PhraseState::new(term_indices)); } + let compile = |q: &SpanQuery| NodeState::compile_gaps(q, false); + let forward = |q: &SpanQuery| NodeState::compile_gaps(q, gaps_read); + let filtered = |q: &SpanQuery| NodeState::compile_gaps(q, true); match query { SpanQuery::Empty => NodeState::Empty(EmptyState::new()), SpanQuery::Term(i) => NodeState::Term(TermState::new(*i)), SpanQuery::Ordered(children) => { - let child_states: Vec<_> = children.iter().map(NodeState::compile).collect(); - NodeState::Ordered(OrderedState::new(child_states)) + NodeState::Ordered(OrderedState::new(children.iter().map(compile).collect())) } + SpanQuery::Unordered(children) if gaps_read => disjoint_unordered(children), SpanQuery::Unordered(children) => { NodeState::Unordered(UnorderedState::compile(children)) } SpanQuery::Or(children) => { - let child_states: Vec<_> = children.iter().map(NodeState::compile).collect(); - NodeState::Or(OrState::new(child_states)) + NodeState::Or(OrState::new(children.iter().map(forward).collect())) } SpanQuery::MaxGaps { max_gaps, inner } => { - NodeState::MaxGaps(MaxGapsState::new(NodeState::compile(inner), *max_gaps)) + NodeState::MaxGaps(MaxGapsState::new(filtered(inner), *max_gaps)) } SpanQuery::GapsInRange { min_gaps, max_gaps, inner, - } => NodeState::GapsInRange(GapsInRangeState::new( - NodeState::compile(inner), - *min_gaps, - *max_gaps, - )), + } => { + NodeState::GapsInRange(GapsInRangeState::new(filtered(inner), *min_gaps, *max_gaps)) + } SpanQuery::MaxWidth { max_width, inner } => { - NodeState::MaxWidth(MaxWidthState::new(NodeState::compile(inner), *max_width)) + NodeState::MaxWidth(MaxWidthState::new(forward(inner), *max_width)) } - SpanQuery::WithinPositions { inner, lo, hi } => NodeState::WithinPositions( - WithinPositionsState::new(NodeState::compile(inner), *lo, *hi), - ), - SpanQuery::Containing { big, little } => NodeState::Relation( - RelationState::containing(NodeState::compile(big), NodeState::compile(little)), - ), - SpanQuery::ContainedBy { little, big } => NodeState::Relation( - RelationState::contained_by(NodeState::compile(little), NodeState::compile(big)), - ), - SpanQuery::NotContaining { big, little } => NodeState::Relation( - RelationState::not_containing(NodeState::compile(big), NodeState::compile(little)), - ), - SpanQuery::NotContainedBy { little, big } => { - NodeState::Relation(RelationState::not_contained_by( - NodeState::compile(little), - NodeState::compile(big), - )) + SpanQuery::WithinPositions { inner, lo, hi } => { + NodeState::WithinPositions(WithinPositionsState::new(forward(inner), *lo, *hi)) + } + // A relation reports its first operand's gaps. + SpanQuery::Containing { big, little } => { + NodeState::Relation(RelationState::containing(forward(big), compile(little))) + } + SpanQuery::ContainedBy { little, big } => { + NodeState::Relation(RelationState::contained_by(forward(little), compile(big))) + } + SpanQuery::NotContaining { big, little } => { + NodeState::Relation(RelationState::not_containing(forward(big), compile(little))) } - SpanQuery::Overlapping { a, b } => NodeState::Relation(RelationState::overlapping( - NodeState::compile(a), - NodeState::compile(b), - )), - SpanQuery::NonOverlapping { a, b } => NodeState::Relation( - RelationState::non_overlapping(NodeState::compile(a), NodeState::compile(b)), + SpanQuery::NotContainedBy { little, big } => NodeState::Relation( + RelationState::not_contained_by(forward(little), compile(big)), ), - SpanQuery::Before { a, b } => NodeState::Relation(RelationState::before( - NodeState::compile(a), - NodeState::compile(b), - )), - SpanQuery::After { a, b } => NodeState::Relation(RelationState::after( - NodeState::compile(a), - NodeState::compile(b), - )), + SpanQuery::Overlapping { a, b } => { + NodeState::Relation(RelationState::overlapping(forward(a), compile(b))) + } + SpanQuery::NonOverlapping { a, b } => { + NodeState::Relation(RelationState::non_overlapping(forward(a), compile(b))) + } + SpanQuery::Before { a, b } => { + NodeState::Relation(RelationState::before(forward(a), compile(b))) + } + SpanQuery::After { a, b } => { + NodeState::Relation(RelationState::after(forward(a), compile(b))) + } } } pub(crate) fn reset(&mut self) { + #[cfg(test)] + STATES_RESET.with(|reset| reset.set(reset.get() + 1)); match self { NodeState::Empty(s) => s.reset(), NodeState::Term(s) => s.reset(), @@ -139,6 +168,7 @@ impl NodeState { NodeState::MaxWidth(s) => s.reset(), NodeState::WithinPositions(s) => s.reset(), NodeState::Relation(s) => s.reset(), + NodeState::Shared(s) => s.reset(), } } @@ -155,6 +185,7 @@ impl NodeState { NodeState::MaxWidth(s) => s.next_interval(positions), NodeState::WithinPositions(s) => s.next_interval(positions), NodeState::Relation(s) => s.next_interval(positions), + NodeState::Shared(s) => s.next_interval(positions), } } @@ -172,7 +203,78 @@ impl NodeState { NodeState::MaxWidth(s) => s.gaps(), NodeState::WithinPositions(s) => s.gaps(), NodeState::Relation(s) => s.gaps(), + NodeState::Shared(s) => s.gaps(), + } + } +} + +/// Disjoint intervals are strictly ordered, so the minimal spans holding one +/// disjoint interval per child are the union over child orders of `Ordered`, +/// which consumes disjoint intervals by construction. Identical children give +/// identical orders; each distinct order compiles once. tinql's NEAR has two +/// children: at most two ordered walks. A child holding an Unordered compiles +/// once, and each walk reads it through a replay reader of its own +/// ([`SharedState`]): compiled once per walk, a chain of NEARs doubled at every +/// level, 2^65 states for 66 terms (wp-m4.9-gate.md §U6.13). Any other child, +/// a term, a phrase or an OR of them, compiles once per walk, as before. +fn disjoint_unordered(children: &[SpanQuery]) -> NodeState { + let mut orders = Vec::new(); + let mut remaining: Vec<&SpanQuery> = children.iter().collect(); + child_orders(&mut Vec::new(), &mut remaining, &mut orders); + let shared: Vec>>> = (children.iter()) + .map(|child| { + (orders.len() > 1 && child.holds_unordered()) + .then(|| SharedSource::new(NodeState::compile(child))) + }) + .collect(); + let walk = |child: &SpanQuery| { + let index = (children.iter()) + .position(|each| std::ptr::eq(each, child)) + .expect("an order walks the children themselves"); + match &shared[index] { + Some(source) => { + built(); + NodeState::Shared(SharedState::new(source)) + } + None => NodeState::compile(child), + } + }; + let mut ordered: Vec = orders + .iter() + .map(|order| { + built(); + NodeState::Ordered(OrderedState::new( + order.iter().copied().map(&walk).collect(), + )) + }) + .collect(); + if ordered.len() == 1 { + return ordered.pop().expect("one order"); + } + built(); + NodeState::Or(OrState::new(ordered)) +} + +/// Pushes every distinct order of `remaining` after `prefix` onto `orders`. +fn child_orders<'q>( + prefix: &mut Vec<&'q SpanQuery>, + remaining: &mut Vec<&'q SpanQuery>, + orders: &mut Vec>, +) { + if remaining.is_empty() { + orders.push(prefix.clone()); + return; + } + for i in 0..remaining.len() { + // Taking an equal child again at this depth repeats an order. + if remaining[..i].contains(&remaining[i]) { + continue; } + let child = remaining.remove(i); + prefix.push(child); + child_orders(prefix, remaining, orders); + prefix.pop(); + remaining.insert(i, child); } } @@ -200,3 +302,45 @@ fn exact_phrase_term_indices(query: &SpanQuery) -> Option> { } Some(term_indices) } + +#[cfg(test)] +mod tests { + use super::*; + + /// tinql's left-deep `t0 NEAR/n t1 NEAR/n t2 …` over `terms` terms. + fn near_chain(terms: usize, max_gaps: u32) -> SpanQuery { + (1..terms).fold(SpanQuery::Term(0), |chain, term| SpanQuery::MaxGaps { + max_gaps, + inner: Box::new(SpanQuery::Unordered(vec![chain, SpanQuery::Term(term)])), + }) + } + + /// The states `query` compiles to, and those one document's reset + /// reaches. + fn states_built_and_reset(query: &SpanQuery) -> (usize, usize) { + let count = |counter: &'static std::thread::LocalKey>| { + counter.with(std::cell::Cell::get) + }; + let built = count(&STATES_BUILT); + let mut state = NodeState::compile(query); + let reset = count(&STATES_RESET); + state.reset(); + (count(&STATES_BUILT) - built, count(&STATES_RESET) - reset) + } + + /// A NEAR chain compiles to states linear in its depth, and a document's + /// reset reaches each once: each level's two disjoint orders share the + /// chain below them, which only the first reader resets + /// (wp-m4.9-gate.md §U6.13). Compiled once per order, every level doubled + /// the chain below it, and 66 terms took 2^65 states. + #[test] + fn a_near_chain_compiles_and_resets_linear_in_its_depth() { + for terms in [2, 3, 8, 20, 66] { + let (built, reset) = states_built_and_reset(&near_chain(terms, 99)); + assert!( + built <= 10 * terms && reset <= 10 * terms, + "a {terms}-term NEAR chain compiled {built} states and reset {reset}" + ); + } + } +} diff --git a/boldi-vigna/src/state/shared.rs b/boldi-vigna/src/state/shared.rs new file mode 100644 index 0000000..0b0bb45 --- /dev/null +++ b/boldi-vigna/src/state/shared.rs @@ -0,0 +1,106 @@ +// Copyright (C) 2026 PlanetScale +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . +use std::cell::RefCell; +use std::rc::Rc; + +use super::NodeState; +use crate::interval::Interval; +use crate::positions::TermPositions; + +/// One compiled child that several walks read: the order walks of a gap-read +/// `Unordered` (`NodeState::compile_gaps`). The child runs once per document +/// and keeps what it produced, each interval with its gap count, so every +/// reader replays the stream from its own place exactly as a private copy +/// would produce it. A state's output is a function of the document's +/// positions and its call count alone, and `reset` reaches every state from +/// the root before a document's first read. A child returns no interval +/// after its first `None`, so the source asks it no further. +pub(crate) struct SharedSource { + child: NodeState, + produced: Vec<(Interval, Option)>, + exhausted: bool, + /// Reset, and nothing read since: every reader resets the source at a + /// document's start, and only the first reaches the child. Resetting the + /// child once per reader would double the resets at every level of a + /// NEAR chain. + reset: bool, +} + +impl SharedSource { + pub(crate) fn new(child: NodeState) -> Rc> { + Rc::new(RefCell::new(Self { + child, + produced: Vec::new(), + exhausted: false, + reset: false, + })) + } +} + +/// A reader of a [`SharedSource`], at its own place in the child's stream. +pub(crate) struct SharedState { + source: Rc>, + next: usize, + last_gaps: Option, +} + +impl SharedState { + pub(crate) fn new(source: &Rc>) -> Self { + Self { + source: Rc::clone(source), + next: 0, + last_gaps: None, + } + } + + /// Every reader resets the source at a document's start, before any + /// reader reads; the first reset reaches the child, the rest find the + /// source reset already. + pub(crate) fn reset(&mut self) { + let mut source = self.source.borrow_mut(); + if !source.reset { + source.child.reset(); + source.produced.clear(); + source.exhausted = false; + source.reset = true; + } + self.next = 0; + self.last_gaps = None; + } + + pub(crate) fn next_interval(&mut self, positions: &impl TermPositions) -> Option { + let mut source = self.source.borrow_mut(); + source.reset = false; + if self.next == source.produced.len() { + if source.exhausted { + return None; + } + let Some(interval) = source.child.next_interval(positions) else { + source.exhausted = true; + return None; + }; + let gaps = source.child.gaps(); + source.produced.push((interval, gaps)); + } + let (interval, gaps) = source.produced[self.next]; + self.next += 1; + self.last_gaps = gaps; + Some(interval) + } + + pub(crate) fn gaps(&self) -> Option { + self.last_gaps + } +} diff --git a/boldi-vigna/src/state/unordered.rs b/boldi-vigna/src/state/unordered.rs index c1ac66f..065aa25 100644 --- a/boldi-vigna/src/state/unordered.rs +++ b/boldi-vigna/src/state/unordered.rs @@ -16,7 +16,6 @@ // The full license text is available in LICENSE. use std::cmp::Reverse; use std::collections::{BTreeSet, VecDeque}; -use std::ops::Bound::{Excluded, Unbounded}; use rustc_hash::FxHashMap; @@ -28,13 +27,6 @@ use crate::query::SpanQuery; type StartKey = (u32, Reverse, usize); type EndKey = (u32, u32, usize); -#[derive(Clone, Copy)] -struct OperandInterval { - interval: Interval, - coverage: u64, - contains_overlap: bool, -} - enum UnorderedChild { Single(NodeState), Repeated(RepeatingState), @@ -56,15 +48,9 @@ impl UnorderedChild { } } - fn next_interval(&mut self, positions: &impl TermPositions) -> Option { + fn next_interval(&mut self, positions: &impl TermPositions) -> Option { match self { - Self::Single(state) => state - .next_interval(positions) - .map(|interval| OperandInterval { - interval, - coverage: u64::from(interval.width()), - contains_overlap: false, - }), + Self::Single(state) => state.next_interval(positions), Self::Repeated(state) => state.next_interval(positions), } } @@ -75,8 +61,6 @@ struct RepeatingState { child: NodeState, copies: usize, window: VecDeque, - coverage: u64, - overlap_pairs: usize, exhausted: bool, } @@ -87,8 +71,6 @@ impl RepeatingState { child, copies, window: VecDeque::with_capacity(copies), - coverage: 0, - overlap_pairs: 0, exhausted: false, } } @@ -96,17 +78,15 @@ impl RepeatingState { fn reset(&mut self) { self.child.reset(); self.window.clear(); - self.coverage = 0; - self.overlap_pairs = 0; self.exhausted = false; } - fn next_interval(&mut self, positions: &impl TermPositions) -> Option { + fn next_interval(&mut self, positions: &impl TermPositions) -> Option { if self.exhausted { return None; } if self.window.len() == self.copies { - self.pop_front(); + self.window.pop_front(); } while self.window.len() < self.copies { @@ -114,53 +94,20 @@ impl RepeatingState { self.exhausted = true; return None; }; - self.push_back(interval); + self.window.push_back(interval); } let start = self.window.front()?.start; let end = self.window.back()?.end; - Some(OperandInterval { - interval: Interval::new(start, end), - coverage: self.coverage, - contains_overlap: self.overlap_pairs != 0, - }) - } - - fn pop_front(&mut self) { - let Some(front) = self.window.pop_front() else { - return; - }; - self.coverage -= u64::from(front.width()); - if self - .window - .front() - .is_some_and(|next| front.overlaps(*next)) - { - self.overlap_pairs -= 1; - } - } - - fn push_back(&mut self, interval: Interval) { - if self - .window - .back() - .is_some_and(|previous| previous.overlaps(interval)) - { - self.overlap_pairs += 1; - } - self.coverage += u64::from(interval.width()); - self.window.push_back(interval); + Some(Interval::new(start, end)) } } /// Indexed live intervals for an unordered conjunction. struct CurrentIntervals { - slots: Vec>, + slots: Vec>, by_start: BTreeSet, by_end: BTreeSet, - coverage: u64, - overlap_pairs: usize, - overlapping_children: usize, } impl CurrentIntervals { @@ -169,9 +116,6 @@ impl CurrentIntervals { slots: vec![None; children], by_start: BTreeSet::new(), by_end: BTreeSet::new(), - coverage: 0, - overlap_pairs: 0, - overlapping_children: 0, } } @@ -179,9 +123,6 @@ impl CurrentIntervals { self.slots.fill(None); self.by_start.clear(); self.by_end.clear(); - self.coverage = 0; - self.overlap_pairs = 0; - self.overlapping_children = 0; } fn is_present(&self, child: usize) -> bool { @@ -203,59 +144,20 @@ impl CurrentIntervals { Some(Interval::new(start, end)) } - fn gaps(&self, span: Interval) -> Option { - if self.overlap_pairs != 0 || self.overlapping_children != 0 { - return None; - } - u64::from(span.width()).checked_sub(self.coverage) - } - - fn replace(&mut self, child: usize, next: Option) { + fn replace(&mut self, child: usize, next: Option) { if let Some(previous) = self.slots[child] { - self.unlink(child, previous); - self.coverage -= previous.coverage; - self.overlapping_children -= usize::from(previous.contains_overlap); + let removed_start = self.by_start.remove(&start_key(child, previous)); + let removed_end = self.by_end.remove(&end_key(child, previous)); + debug_assert!(removed_start && removed_end); } self.slots[child] = next; if let Some(next) = next { - self.link(child, next); - self.coverage += next.coverage; - self.overlapping_children += usize::from(next.contains_overlap); + let inserted_start = self.by_start.insert(start_key(child, next)); + let inserted_end = self.by_end.insert(end_key(child, next)); + debug_assert!(inserted_start && inserted_end); } } - - fn link(&mut self, child: usize, operand: OperandInterval) { - let key = start_key(child, operand.interval); - let (previous, next) = self.neighbors(key); - self.overlap_pairs -= overlapping_pair(previous, next); - self.overlap_pairs += - overlapping_pair(previous, Some(key)) + overlapping_pair(Some(key), next); - let inserted_start = self.by_start.insert(key); - let inserted_end = self.by_end.insert(end_key(child, operand.interval)); - debug_assert!(inserted_start && inserted_end); - } - - fn unlink(&mut self, child: usize, operand: OperandInterval) { - let key = start_key(child, operand.interval); - let (previous, next) = self.neighbors(key); - self.overlap_pairs -= - overlapping_pair(previous, Some(key)) + overlapping_pair(Some(key), next); - self.overlap_pairs += overlapping_pair(previous, next); - let removed_start = self.by_start.remove(&key); - let removed_end = self.by_end.remove(&end_key(child, operand.interval)); - debug_assert!(removed_start && removed_end); - } - - fn neighbors(&self, key: StartKey) -> (Option, Option) { - let previous = self.by_start.range(..key).next_back().copied(); - let next = self - .by_start - .range((Excluded(key), Unbounded)) - .next() - .copied(); - (previous, next) - } } fn start_key(child: usize, interval: Interval) -> StartKey { @@ -266,24 +168,16 @@ fn end_key(child: usize, interval: Interval) -> EndKey { (interval.end, interval.start, child) } -fn overlapping_pair(left: Option, right: Option) -> usize { - match (left, right) { - (Some((_, Reverse(left_end), _)), Some((right_start, _, _))) => { - usize::from(left_end >= right_start) - } - _ => 0, - } -} - /// BV AND algorithm: unordered conjunction. /// /// Finds minimal intervals containing one interval from each child, -/// in any order. Uses ⪯ priority ordering (left-to-right). +/// in any order. Uses ⪯ priority ordering (left-to-right). Children may +/// overlap, so it reports no gap count: under a gap filter the compiler builds +/// the disjoint form instead (see `NodeState::compile_gaps`). pub(crate) struct UnorderedState { children: Vec, current: CurrentIntervals, prev: Option, - last_gaps: Option, } impl UnorderedState { @@ -313,7 +207,6 @@ impl UnorderedState { children, current, prev: None, - last_gaps: None, } } @@ -323,7 +216,6 @@ impl UnorderedState { } self.current.reset(); self.prev = None; - self.last_gaps = None; } fn advance_child(&mut self, child: usize, positions: &impl TermPositions) { @@ -359,7 +251,6 @@ impl UnorderedState { } let mut candidate = self.current.span()?; - let mut candidate_gaps = self.current.gaps(candidate); loop { if self @@ -380,15 +271,13 @@ impl UnorderedState { break; } candidate = new_span; - candidate_gaps = self.current.gaps(candidate); } self.prev = Some(candidate); - self.last_gaps = candidate_gaps; Some(candidate) } pub(crate) fn gaps(&self) -> Option { - self.last_gaps + None } } diff --git a/boldi-vigna/tests/near_chain_tests.rs b/boldi-vigna/tests/near_chain_tests.rs new file mode 100644 index 0000000..6a4f0d0 --- /dev/null +++ b/boldi-vigna/tests/near_chain_tests.rs @@ -0,0 +1,125 @@ +// Copyright (C) 2026 PlanetScale +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . +//! Chains of NEAR, tinql's left-deep `t0 NEAR/n t1 NEAR/n t2 …`: +//! `MAXGAPS(n, UNORDERED(MAXGAPS(n, UNORDERED(…)), t))`. Under a gap filter an +//! Unordered compiles to its disjoint form, one ordered walk per child order, +//! and those walks share each child that holds a further Unordered rather +//! than compiling it once per order (wp-m4.9-gate.md §U6.13). The shared +//! compile must answer exactly as private copies would: the explicit +//! `MAXGAPS(n, OR(ORDERED(a, b), ORDERED(b, a)))` expansion of every level. + +use boldi_vigna::*; +use proptest::prelude::*; + +fn collect(query: &SpanQuery, positions: &impl TermPositions) -> Vec { + let mut solver = SpanSolver::new(query).unwrap(); + solver.intervals(positions).collect() +} + +fn near(max_gaps: u32, left: SpanQuery, right: SpanQuery) -> SpanQuery { + SpanQuery::MaxGaps { + max_gaps, + inner: Box::new(SpanQuery::Unordered(vec![left, right])), + } +} + +/// A chain over terms `0..gaps.len() + 1`, level `i` joining term `i + 1` +/// under `gaps[i]`, the chain on the left of a level where `left[i]` holds. +fn chain(gaps: &[u32], left: &[bool]) -> SpanQuery { + (gaps.iter().zip(left).enumerate()).fold( + SpanQuery::Term(0), + |chain, (level, (&max_gaps, &left))| { + let term = SpanQuery::Term(level + 1); + if left { + near(max_gaps, chain, term) + } else { + near(max_gaps, term, chain) + } + }, + ) +} + +/// `query` with every NEAR written as the disjoint form private copies +/// compile to: both orders of its two children, each copy its own. +fn private_copies(query: &SpanQuery) -> SpanQuery { + match query { + SpanQuery::MaxGaps { max_gaps, inner } => match inner.as_ref() { + SpanQuery::Unordered(children) => { + let [a, b] = [&children[0], &children[1]].map(private_copies); + SpanQuery::MaxGaps { + max_gaps: *max_gaps, + inner: Box::new(SpanQuery::Or(vec![ + SpanQuery::Ordered(vec![a.clone(), b.clone()]), + SpanQuery::Ordered(vec![b, a]), + ])), + } + } + other => SpanQuery::MaxGaps { + max_gaps: *max_gaps, + inner: Box::new(private_copies(other)), + }, + }, + other => other.clone(), + } +} + +/// The 66-term chain U6.3's audit runs (`dd1 NEAR/99 w000 … NEAR/99 w064`): +/// it compiles (2^65 states when each order compiled its own copy of the +/// chain below) and answers a document holding every term in order, and +/// none missing one. +#[test] +fn a_66_term_near_chain_compiles_and_answers() { + let query = chain(&[99; 65], &[true; 65]); + let in_order: Vec> = (0..66).map(|position| vec![position]).collect(); + assert_eq!(collect(&query, &in_order), [Interval::new(0, 65)]); + let mut missing = in_order.clone(); + missing[40].clear(); + assert!(collect(&query, &missing).is_empty()); + let reversed: Vec> = (0..66).rev().map(|position| vec![position]).collect(); + assert_eq!(collect(&query, &reversed), [Interval::new(0, 65)]); +} + +fn positions_strategy(terms: usize) -> impl Strategy>> { + proptest::collection::vec( + proptest::collection::btree_set(0u32..40, 0..6) + .prop_map(|positions| positions.into_iter().collect::>()), + terms, + ) +} + +fn chain_case() -> impl Strategy, Vec, Vec>)> { + (1usize..=5).prop_flat_map(|depth| { + ( + proptest::collection::vec(0u32..4, depth), + proptest::collection::vec(any::(), depth), + positions_strategy(depth + 1), + ) + }) +} + +proptest! { + #![proptest_config(ProptestConfig::with_cases(2000))] + + /// A chain's shared compile answers exactly as its private copies. + #[test] + fn a_near_chain_answers_as_its_private_copies((gaps, left, positions) in chain_case()) { + let query = chain(&gaps, &left); + prop_assert_eq!( + collect(&query, &positions), + collect(&private_copies(&query), &positions), + "{}", query + ); + } +} diff --git a/boldi-vigna/tests/near_slot_tests.rs b/boldi-vigna/tests/near_slot_tests.rs new file mode 100644 index 0000000..4476de3 --- /dev/null +++ b/boldi-vigna/tests/near_slot_tests.rs @@ -0,0 +1,200 @@ +// Copyright (C) 2026 PlanetScale +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . +//! `MAXGAPS(n, UNORDERED(a, b))` (tinql's `a NEAR/n b`) when the two slots can +//! match the same position: an OR group sharing a term with the other slot, or +//! a wide operand repeated in both slots. +//! +//! Semantics checked here, from the solver's documented contract (`SpanQuery`): +//! `MaxGaps` passes only assignments whose sub-intervals do not overlap, and +//! `Unordered` takes one interval from each child. So a NEAR match is a pair of +//! child intervals that share no position, in either order, whose gap count +//! (`width - covered positions`) is at most `n`; the solver reports the minimal +//! such spans. Two slots never reuse one position. + +use boldi_vigna::*; +use proptest::prelude::*; + +fn collect(query: &SpanQuery, positions: &impl TermPositions) -> Vec { + let mut solver = SpanSolver::new(query).unwrap(); + solver.intervals(positions).collect() +} + +fn near(max_gaps: u32, left: SpanQuery, right: SpanQuery) -> SpanQuery { + SpanQuery::MaxGaps { + max_gaps, + inner: Box::new(SpanQuery::Unordered(vec![left, right])), + } +} + +fn slot(terms: &[usize]) -> SpanQuery { + match terms { + [term] => SpanQuery::Term(*term), + terms => SpanQuery::Or(terms.iter().copied().map(SpanQuery::Term).collect()), + } +} + +/// Brute force over point slots: every pair of distinct positions, one from +/// each slot, within `max_gaps`, reduced to minimal spans. +fn brute_force_near( + max_gaps: u32, + left: &[usize], + right: &[usize], + positions: &[Vec], +) -> Vec { + let slot_positions = |terms: &[usize]| -> Vec { + let mut out: Vec = terms + .iter() + .flat_map(|&t| positions[t].iter().copied()) + .collect(); + out.sort_unstable(); + out.dedup(); + out + }; + let (left, right) = (slot_positions(left), slot_positions(right)); + let mut spans = Vec::new(); + for &p in &left { + for &q in &right { + if p != q && p.abs_diff(q) - 1 <= max_gaps { + spans.push(Interval::new(p.min(q), p.max(q))); + } + } + } + minimize(spans) +} + +fn minimize(mut intervals: Vec) -> Vec { + intervals.sort(); + intervals.dedup(); + intervals + .iter() + .copied() + .filter(|&iv| !intervals.iter().any(|&o| o != iv && iv.contains(o))) + .collect() +} + +#[test] +fn group_slot_pairs_with_the_shared_term_in_the_other_slot() { + // `(alpha OR bravo) NEAR/3 alpha` over "alpha the alpha": alpha=0, bravo=1. + let positions = vec![vec![0, 2], vec![]]; + let q = near(3, slot(&[0, 1]), slot(&[0])); + assert_eq!(collect(&q, &positions), vec![Interval::new(0, 2)]); + // The single-term slot already matches; widening must not lose it. + let narrow = near(3, slot(&[0]), slot(&[0])); + assert_eq!(collect(&narrow, &positions), vec![Interval::new(0, 2)]); +} + +#[test] +fn two_group_slots_sharing_a_term_pair_distinct_occurrences() { + // `(alpha OR bravo) NEAR/2 (alpha OR charlie)` over "alpha of alpha". + let positions = vec![vec![0, 2], vec![], vec![]]; + let q = near(2, slot(&[0, 1]), slot(&[0, 2])); + assert_eq!(collect(&q, &positions), vec![Interval::new(0, 2)]); +} + +#[test] +fn shared_term_does_not_shadow_a_pair_with_the_other_term() { + // "bravo alpha": the alpha alone sits in both groups, but only bravo-alpha + // is a pair of distinct positions. + let positions = vec![vec![1], vec![0], vec![]]; + let q = near(0, slot(&[0, 1]), slot(&[0, 2])); + assert_eq!(collect(&q, &positions), vec![Interval::new(0, 1)]); +} + +#[test] +fn single_shared_occurrence_is_not_a_pair() { + let positions = vec![vec![4], vec![]]; + let q = near(3, slot(&[0, 1]), slot(&[0])); + assert!(collect(&q, &positions).is_empty()); +} + +#[test] +fn repeated_wide_operand_takes_disjoint_occurrences() { + // `"a a" NEAR/0 "a a"` over "a a a a": the phrase occurs at [0,1], [1,2], + // [2,3]; the disjoint pair [0,1] + [2,3] spans [0,3] with no gap. + let positions = vec![vec![0, 1, 2, 3]]; + let phrase = SpanQuery::phrase([0, 0]); + let q = near(0, phrase.clone(), phrase); + assert_eq!(collect(&q, &positions), vec![Interval::new(0, 3)]); +} + +#[test] +fn gaps_in_range_takes_disjoint_occurrences_too() { + let positions = vec![vec![0, 2], vec![]]; + let q = SpanQuery::GapsInRange { + min_gaps: 1, + max_gaps: 1, + inner: Box::new(SpanQuery::Unordered(vec![slot(&[0, 1]), slot(&[0])])), + }; + assert_eq!(collect(&q, &positions), vec![Interval::new(0, 2)]); +} + +fn positions_strategy(terms: usize) -> impl Strategy>> { + // One term per position, as the tokenizer produces. + prop::collection::vec(0..terms + 2, 0..16).prop_map(move |doc| { + let mut positions = vec![Vec::new(); terms]; + for (pos, term) in doc.into_iter().enumerate() { + if term < terms { + positions[term].push(pos as u32); + } + } + positions + }) +} + +fn slot_strategy(terms: usize) -> impl Strategy> { + prop::collection::btree_set(0..terms, 1..=terms).prop_map(|s| s.into_iter().collect()) +} + +proptest! { + #![proptest_config(ProptestConfig::with_cases(2000))] + + #[test] + fn near_over_or_slots_matches_brute_force( + positions in positions_strategy(3), + left in slot_strategy(3), + right in slot_strategy(3), + max_gaps in 0u32..5, + ) { + let q = near(max_gaps, slot(&left), slot(&right)); + prop_assert_eq!( + collect(&q, &positions), + brute_force_near(max_gaps, &left, &right, &positions), + "left={:?} right={:?} positions={:?}", left, right, positions + ); + } + + #[test] + fn widening_a_slot_never_loses_a_match( + positions in positions_strategy(3), + left in slot_strategy(3), + right in slot_strategy(3), + extra in 0usize..3, + max_gaps in 0u32..5, + ) { + let narrow = near(max_gaps, slot(&left), slot(&right)); + let mut wider_left = left.clone(); + if !wider_left.contains(&extra) { + wider_left.push(extra); + wider_left.sort_unstable(); + } + let wide = near(max_gaps, slot(&wider_left), slot(&right)); + if !collect(&narrow, &positions).is_empty() { + prop_assert!( + !collect(&wide, &positions).is_empty(), + "{:?} matched but {:?} did not on {:?}", left, wider_left, positions + ); + } + } +} diff --git a/postgres/src/operator.rs b/postgres/src/operator.rs index 4178069..6f1fa42 100644 --- a/postgres/src/operator.rs +++ b/postgres/src/operator.rs @@ -18,16 +18,24 @@ use crate::am::amhandler; use pgrx::{extension_sql, pg_extern, pg_sys}; use tinql::runtime::{ - Query, SimplificationProfile, evaluate, lower::lower, simplify, subtokenize::sub_tokenize, - tokenize_doc, + Query, SimplificationProfile, evaluate, lower::lower_with_profile, simplify, + subtokenize::sub_tokenize, tokenize_doc, }; use tokenizer::Tokenizer; use tokenizer::presets::default_pipeline; fn parse_search(query_text: &str, tokenizer: &T) -> Result { + parse_search_with(query_text, tokenizer, SimplificationProfile::Structural) +} + +fn parse_search_with( + query_text: &str, + tokenizer: &T, + profile: SimplificationProfile, +) -> Result { let parsed = crate::tinql::parse(query_text).map_err(|e| e.to_string())?; let analyzed = sub_tokenize(parsed, tokenizer).map_err(|e| e.to_string())?; - lower(&analyzed).map_err(|e| e.to_string()) + lower_with_profile(&analyzed, profile).map_err(|e| e.to_string()) } /// Returns the OID of Lead's `==>(text, text)` operator, or `InvalidOid` if @@ -79,9 +87,30 @@ fn invalid_search(error: String) -> ! { /// error `==>` raises for it. The parsed texts are ORed and simplified the way /// lowering simplifies `a OR b`. pub(crate) fn parse_searches(texts: &[String], tokenizer: &T) -> Query { + parse_searches_with(texts, tokenizer, SimplificationProfile::Structural) +} + +/// Like [`parse_searches`], but keeps every written occurrence of a repeated +/// term, which is what scoring weighs: `be AND be` weighs `be` twice. +pub(crate) fn parse_scoring_searches(texts: &[String], tokenizer: &T) -> Query { + parse_searches_with( + texts, + tokenizer, + SimplificationProfile::StructuralPreserveTermMultiplicity, + ) +} + +fn parse_searches_with( + texts: &[String], + tokenizer: &T, + profile: SimplificationProfile, +) -> Query { let mut queries = texts .iter() - .map(|text| parse_search(text, tokenizer).unwrap_or_else(|error| invalid_search(error))) + .map(|text| { + parse_search_with(text, tokenizer, profile) + .unwrap_or_else(|error| invalid_search(error)) + }) .collect::>(); if queries.len() == 1 { return queries.remove(0); @@ -91,7 +120,7 @@ pub(crate) fn parse_searches(texts: &[String], tokenizer: &T) -> Q min: 1, children: queries, }, - SimplificationProfile::Structural, + profile, ) } diff --git a/postgres/src/score.rs b/postgres/src/score.rs index 940dc90..2984421 100644 --- a/postgres/src/score.rs +++ b/postgres/src/score.rs @@ -18,7 +18,7 @@ use crate::bm25::{ Bm25Overrides, DenseRatio, ScoreStopWords, ScoringTermInput, TermScorer, TermSetEdit, compile_scoring_terms, sum_scores_in_order, }; -use crate::tinql::parse_tinql_to_query; +use crate::tinql::parse_scoring_tinql_to_query; use pgrx::iter::TableIterator; use pgrx::{ FromDatum, Internal, IntoDatum, PgBox, PgList, PgMemoryContexts, PgRelation, Spi, default, @@ -363,8 +363,9 @@ fn build_group<'a>( pgrx::error!("dense_ratio must be finite and non-negative"); } let query = crate::operator::parse_searches(queries, &tokenizer); + let scoring_query = crate::operator::parse_scoring_searches(queries, &tokenizer); let mut inputs = Vec::new(); - collect_score_terms(&query, 1.0, false, &mut inputs); + collect_score_terms(&scoring_query, 1.0, false, &mut inputs); let edit = TermSetEdit::from_bound_arrays(term_add, term_replace) .unwrap_or_else(|error| pgrx::error!("tin.score(): {error}")) .analyzed_with(|text| { @@ -541,12 +542,16 @@ fn collect_score_terms<'a>( }; match query { Query::Term(text) | Query::Fuzzy { term: text, .. } => push(text), - Query::Span { term_slots, .. } | Query::SpanExpr { term_slots, .. } => { - for slot in term_slots { + Query::Span { .. } | Query::SpanExpr { .. } => { + query.for_each_positive_span_slot(&mut |_, slot, leaf_boost| { if let SpanTermSlot::Term(text) | SpanTermSlot::Fuzzy { term: text, .. } = slot { - push(text); + out.push(ScoringTermInput { + text, + boost: boost * leaf_boost.unwrap_or(1.0), + explicitly_boosted: explicitly_boosted || leaf_boost.is_some(), + }); } - } + }); } Query::And(left, right) | Query::Or(left, right) => { collect_score_terms(left, boost, explicitly_boosted, out); @@ -609,7 +614,7 @@ fn score_inspect( }; } let tokenizer = unsafe { crate::options::tokenizer(index.as_ptr()) }; - let parsed = parse_tinql_to_query(query, &tokenizer) + let parsed = parse_scoring_tinql_to_query(query, &tokenizer) .unwrap_or_else(|error| pgrx::error!("tin.score_inspect() query error: {error}")); let mut inputs = Vec::new(); collect_score_terms(&parsed, 1.0, false, &mut inputs); diff --git a/postgres/src/tinql.rs b/postgres/src/tinql.rs index ccc9f23..12ba7bb 100644 --- a/postgres/src/tinql.rs +++ b/postgres/src/tinql.rs @@ -21,7 +21,11 @@ //! [`MAX_NESTING_DEPTH`] with SQLSTATE 54000 (`program_limit_exceeded`) before //! the parser runs. -use ::tinql::runtime::{Query, QueryError, lower::lower, subtokenize::sub_tokenize}; +use ::tinql::runtime::{ + Query, QueryError, SimplificationProfile, + lower::{lower, lower_with_profile}, + subtokenize::sub_tokenize, +}; use tokenizer::Tokenizer; /// Longest accepted TINQL query text, in bytes. @@ -74,3 +78,15 @@ pub(crate) fn parse_tinql_to_query( ) -> Result { Ok(lower(&sub_tokenize(parse(query)?, tokenizer)?)?) } + +/// Like [`parse_tinql_to_query`], but keeps every written occurrence of a +/// repeated term, which is what scoring weighs. +pub(crate) fn parse_scoring_tinql_to_query( + query: &str, + tokenizer: &T, +) -> Result { + Ok(lower_with_profile( + &sub_tokenize(parse(query)?, tokenizer)?, + SimplificationProfile::StructuralPreserveTermMultiplicity, + )?) +} diff --git a/private-regress.manifest b/private-regress.manifest index 44df0f4..613b030 100644 --- a/private-regress.manifest +++ b/private-regress.manifest @@ -35,3 +35,5 @@ highlight_expression_analysis highlight_having_analysis highlight_join_analysis highlight_null_analysis_defaults +phrase_repeat_weights +tinql_query_length_limit diff --git a/tinql/docs/src/alternatives.md b/tinql/docs/src/alternatives.md index 9bebcdf..d85236c 100644 --- a/tinql/docs/src/alternatives.md +++ b/tinql/docs/src/alternatives.md @@ -77,6 +77,24 @@ When the percentage does not divide evenly, the required count rounds up: Place `%` immediately after the number. +### Repeated alternatives + +Each listed alternative counts on its own, even when two are the same +expression: + +``` +AT LEAST 2 OF [beer beer wine] +``` + +A document that contains only "beer" matches, because it satisfies two of +the three listed alternatives. A percentage counts repeats in the list +length too, so `AT LEAST 50% OF [beer beer wine ale]` requires 2. + +### Thresholds outside the list + +`AT LEAST 0 OF [...]` matches every document. A threshold larger than the +list, such as `AT LEAST 4 OF [a b c]`, matches no document. + ### Syntax ``` diff --git a/tinql/docs/src/boost.md b/tinql/docs/src/boost.md index 4cce598..cc04d04 100644 --- a/tinql/docs/src/boost.md +++ b/tinql/docs/src/boost.md @@ -25,6 +25,7 @@ Boost can be applied to: - **Sloppy phrases:** `"big bad"~2^1.5` (slop first, then boost) - **Alternatives:** `[IPA ale]^2` - **Parenthesized expressions:** `(A NEAR/0 B)^3` +- **Operands inside a phrase or span:** `ipa^3 NEAR/5 hoppy`, `"craft [beer^2 ale]"` ## Boost factors @@ -51,6 +52,20 @@ beer)` gives both terms weight 4 in every matching document, including one that matches only the proximity branch. Boosting the phrase provides no separate bonus for matching its positions. +A word written more than once inside one phrase or span adds its weight once +per occurrence too: `"to be or not to be"^2` gives `to` and `be` weight 4 and +`or` and `not` weight 2. The right side of `NOT ENCLOSES`, `NOT ENCLOSED BY` +and `NOT OVERLAPPING` only filters, so its terms add no weight. + +A boost on an operand inside a phrase or span weighs that operand alone, as it +would in a Boolean query: `ipa^3 NEAR/5 hoppy` gives `ipa` weight 3 and `hoppy` +weight 1, like `ipa^3 AND hoppy`. A boost on the whole phrase or span, +`(ipa NEAR/5 hoppy)^3`, multiplies every operand. A boost on the right side of +`NOT ENCLOSES`, `NOT ENCLOSED BY` or `NOT OVERLAPPING` adds nothing, because +that side only filters. As anywhere else, an explicit boost keeps a dense term +in `tin.score()`, and only that operand's term: in `the THEN/0 ipa^2`, a dense +`the` is still elided. + ## Boost on hyphenated terms A boost can follow a hyphenated term: diff --git a/tinql/docs/src/proximity.md b/tinql/docs/src/proximity.md index 547949e..f333381 100644 --- a/tinql/docs/src/proximity.md +++ b/tinql/docs/src/proximity.md @@ -108,6 +108,10 @@ of "IPA", "stout", or "lager". Either "beer" or "ale" within 10 extra positions of either "food" or "dinner", in any order. +Each operand matches at its own position: one occurrence of a word can't +satisfy two operands. `alpha NEAR/3 alpha` needs two occurrences of "alpha", +and `alpha NEAR/0 [alpha beta]` matches "alpha beta" through "beta". + ## Understanding the gap parameter The gap parameter in `THEN/N` and `NEAR/N` controls how many *extra* positions diff --git a/tinql/docs/src/span-relations.md b/tinql/docs/src/span-relations.md index c272be2..b4d47c2 100644 --- a/tinql/docs/src/span-relations.md +++ b/tinql/docs/src/span-relations.md @@ -122,6 +122,10 @@ Every relation operator (except BEFORE and AFTER) has a `NOT` variant: | `A ENCLOSED BY B` | `A NOT ENCLOSED BY B` | | `A OVERLAPPING B` | `A NOT OVERLAPPING B` | +In a negated relation, B only filters: its terms add nothing to the score, as +with `AND NOT`. A term that also appears in A, or elsewhere in the query, +still scores through those occurrences. + ## Combining relations with other operators When combining relation operators with proximity and positional filters, you diff --git a/tinql/src/runtime/display.rs b/tinql/src/runtime/display.rs index ab250a3..fe7ef69 100644 --- a/tinql/src/runtime/display.rs +++ b/tinql/src/runtime/display.rs @@ -37,10 +37,12 @@ impl fmt::Display for Query { term_slots, span_query, position_filter, + .. } => fmt_span_display(f, term_slots, span_query, position_filter.as_ref()), Query::SpanExpr { term_slots, span_expr, + .. } => fmt_span_expr_display(f, term_slots, span_expr), Query::MatchAll => write!(f, "*"), Query::Regex(pat) => write!(f, "REGEX({pat})"), @@ -406,7 +408,7 @@ fn fmt_phrase_term(f: &mut fmt::Formatter<'_>, term: &str) -> fmt::Result { #[cfg(test)] mod tests { use super::*; - use crate::runtime::{PositionFilterBound, SpanPositionFilter}; + use crate::runtime::{PositionFilterBound, SpanLeafBoosts, SpanPositionFilter}; use boldi_vigna::SpanQuery; #[test] @@ -421,6 +423,7 @@ mod tests { inner: Box::new(SpanQuery::Or(vec![SpanQuery::Term(0), SpanQuery::Term(1)])), }, position_filter: None, + leaf_boosts: SpanLeafBoosts::default(), }; assert_eq!(query.to_string(), "SPAN(MAXWIDTH(7, OR(l.a, nood1e)))"); @@ -443,6 +446,7 @@ mod tests { ])), }, position_filter: None, + leaf_boosts: SpanLeafBoosts::default(), }; assert_eq!(query.to_string(), r#"PHRASE("big bad wolf")"#); @@ -463,6 +467,7 @@ mod tests { ])), }, position_filter: None, + leaf_boosts: SpanLeafBoosts::default(), }; assert_eq!(query.to_string(), r#"PHRASE/2("craft beer")"#); @@ -483,6 +488,7 @@ mod tests { ])), }, position_filter: None, + leaf_boosts: SpanLeafBoosts::default(), }; assert_eq!(query.to_string(), r#"PHRASE("say\"hi under\_score")"#); @@ -516,6 +522,7 @@ mod tests { ])), }, position_filter: None, + leaf_boosts: SpanLeafBoosts::default(), }; assert_eq!( @@ -578,6 +585,7 @@ mod tests { term_slots: vec![SpanTermSlot::Term("beer".into())], span_query: SpanQuery::Term(0), position_filter: Some(SpanPositionFilter::Last(PositionFilterBound::Absolute(5))), + leaf_boosts: SpanLeafBoosts::default(), }; assert_eq!(query.to_string(), "SPAN(beer) IN LAST 5 WORDS"); diff --git a/tinql/src/runtime/eval.rs b/tinql/src/runtime/eval.rs index 861124e..959910f 100644 --- a/tinql/src/runtime/eval.rs +++ b/tinql/src/runtime/eval.rs @@ -211,6 +211,7 @@ fn evaluate_searchable(query: &Query, doc: &TokenizedDoc) -> Result { let positions = term_slots .iter() @@ -232,6 +233,7 @@ fn evaluate_searchable(query: &Query, doc: &TokenizedDoc) -> Result { let positions = term_slots .iter() @@ -369,6 +371,7 @@ fn collect_highlight_matches(query: &Query, doc: &TokenizedDoc, out: &mut Vec { let positions = term_slots .iter() @@ -395,6 +398,7 @@ fn collect_highlight_matches(query: &Query, doc: &TokenizedDoc, out: &mut Vec { let positions = term_slots .iter() diff --git a/tinql/src/runtime/lower.rs b/tinql/src/runtime/lower.rs index 2d58d5b..e0b4308 100644 --- a/tinql/src/runtime/lower.rs +++ b/tinql/src/runtime/lower.rs @@ -29,7 +29,7 @@ use rustc_hash::FxHashMap; use super::{ CompiledRegex, PositionFilterBound, Query, RangeBound, SimplificationProfile, SpanExpr, - SpanPositionFilter, SpanTermSlot, simplify, + SpanLeafBoosts, SpanPositionFilter, SpanTermSlot, simplify, }; #[derive(Debug, thiserror::Error)] @@ -137,84 +137,95 @@ fn lower_boolean(expr: &crate::Expr) -> Result { } fn lower_as_span(expr: &crate::Expr) -> Result { - let mut builder = SpanBuilder::new(); + let mut builder = SpanBuilder::default(); let span_expr = builder.lower_span_expr(expr)?; + let leaf_boosts = builder.leaf_boosts(&span_expr); if let Some((span_query, position_filter)) = span_expr.to_fast_path_root() { Ok(Query::Span { term_slots: builder.term_slots, span_query, position_filter, + leaf_boosts, }) } else { Ok(Query::SpanExpr { term_slots: builder.term_slots, span_expr, + leaf_boosts, }) } } +#[derive(Default)] struct SpanBuilder { term_slots: Vec, intern_map: FxHashMap, + /// The product of the `^` factors around the operand being lowered. + boost: Option, + /// `boost` at each leaf, in the order the leaves are lowered. + written_boosts: Vec>, } impl SpanBuilder { - fn new() -> Self { - Self { - term_slots: Vec::new(), - intern_map: FxHashMap::default(), - } + /// One written occurrence of `slot`, under the current boost. Every + /// occurrence of a slot shares its interned index. + fn leaf(&mut self, slot: SpanTermSlot) -> SpanExpr { + self.written_boosts.push(self.boost); + let term_slots = &mut self.term_slots; + let idx = *self.intern_map.entry(slot).or_insert_with_key(|slot| { + term_slots.push(slot.clone()); + term_slots.len() - 1 + }); + SpanExpr::Term(idx) } - fn intern(&mut self, slot: SpanTermSlot) -> usize { - if let Some(&idx) = self.intern_map.get(&slot) { - return idx; + /// The written boosts of the leaves `span_expr` scores. Lowering writes + /// leaves in the tree's left-to-right order, the order `for_each_term` + /// walks, so the two line up one for one. + fn leaf_boosts(&self, span_expr: &SpanExpr) -> SpanLeafBoosts { + let mut written = self.written_boosts.iter(); + let mut included = Vec::new(); + span_expr.for_each_term(true, &mut |_, scored| { + let boost = written.next().expect("lowering writes a boost per leaf"); + if scored { + included.push(*boost); + } + }); + debug_assert!( + written.next().is_none(), + "every written leaf is in the tree" + ); + if included.iter().all(Option::is_none) { + SpanLeafBoosts::default() + } else { + SpanLeafBoosts(included.into()) } - let idx = self.term_slots.len(); - self.intern_map.insert(slot.clone(), idx); - self.term_slots.push(slot); - idx } fn lower_span_expr(&mut self, expr: &crate::Expr) -> Result { use crate::Expr; match expr { - Expr::Term(s) => { - let idx = self.intern(SpanTermSlot::Term(s.clone())); - Ok(SpanExpr::Term(idx)) - } + Expr::Term(s) => Ok(self.leaf(SpanTermSlot::Term(s.clone()))), Expr::MatchAll => Err(LowerError::MatchAllInSpanContext), Expr::MatchNone => Ok(SpanExpr::Empty), Expr::Fuzzy { term, prefix, distance, - } => { - let idx = self.intern(SpanTermSlot::Fuzzy { - term: term.clone(), - prefix: *prefix, - distance: *distance, - }); - Ok(SpanExpr::Term(idx)) - } - Expr::Wildcard(parts) => { - let idx = self.intern(SpanTermSlot::Regex(CompiledRegex::new( - &wildcard_parts_regex(parts), - )?)); - Ok(SpanExpr::Term(idx)) - } - Expr::Regex(pat) => { - let idx = self.intern(SpanTermSlot::Regex(CompiledRegex::new(pat)?)); - Ok(SpanExpr::Term(idx)) - } - Expr::Range { lower, upper } => { - let idx = self.intern(SpanTermSlot::Range { - lower: convert_range_bound(lower), - upper: convert_range_bound(upper), - }); - Ok(SpanExpr::Term(idx)) - } + } => Ok(self.leaf(SpanTermSlot::Fuzzy { + term: term.clone(), + prefix: *prefix, + distance: *distance, + })), + Expr::Wildcard(parts) => Ok(self.leaf(SpanTermSlot::Regex(CompiledRegex::new( + &wildcard_parts_regex(parts), + )?))), + Expr::Regex(pat) => Ok(self.leaf(SpanTermSlot::Regex(CompiledRegex::new(pat)?))), + Expr::Range { lower, upper } => Ok(self.leaf(SpanTermSlot::Range { + lower: convert_range_bound(lower), + upper: convert_range_bound(upper), + })), Expr::Phrase { elements, slop } => self.lower_phrase(elements, *slop), Expr::And(l, r) => { let left = self.lower_span_expr(l)?; @@ -352,7 +363,15 @@ impl SpanBuilder { inner: Box::new(sq), }) } - Expr::Boost { inner, .. } => self.lower_span_expr(inner), + // The factor stays out of the tree: it weighs the leaves beneath + // it (`leaf_boosts`), and matching never sees it. + Expr::Boost { factor, inner } => { + let outer = self.boost; + self.boost = Some(outer.unwrap_or(1.0) * factor.0); + let lowered = self.lower_span_expr(inner); + self.boost = outer; + lowered + } } } @@ -370,8 +389,7 @@ impl SpanBuilder { for elem in elements { match elem { crate::PhraseElement::Term(s) => { - let idx = self.intern(SpanTermSlot::Term(s.clone())); - children.push((pending_gap, SpanExpr::Term(idx))); + children.push((pending_gap, self.leaf(SpanTermSlot::Term(s.clone())))); pending_gap = 0; } crate::PhraseElement::Gap(n) => { @@ -488,6 +506,129 @@ mod tests { lower(&expr).expect("query should lower") } + /// The slots of a lowered span's included leaves, one per written + /// occurrence, left to right. + fn positive_leaves(input: &str) -> Vec { + let query = parse_and_lower(input); + let mut leaves = Vec::new(); + let term_slots = match &query { + Query::Span { + term_slots, + span_query, + .. + } => { + span_query.for_each_positive_term(&mut |idx| leaves.push(idx)); + term_slots + } + Query::SpanExpr { + term_slots, + span_expr, + .. + } => { + span_expr.for_each_positive_term(&mut |idx| leaves.push(idx)); + term_slots + } + other => panic!("{input} lowers to {other:?}, not a span"), + }; + leaves + .into_iter() + .map(|idx| match &term_slots[idx] { + SpanTermSlot::Term(term) => term.clone(), + other => format!("{other:?}"), + }) + .collect() + } + + #[test] + fn span_positive_leaves_are_written_occurrences_outside_excluded_sides() { + let to_be = ["to", "be", "or", "not", "to", "be"]; + assert_eq!(positive_leaves("\"to be or not to be\""), to_be); + assert_eq!(positive_leaves("\"to be or not to be\"~2"), to_be); + assert_eq!(positive_leaves("be NEAR/1 be"), ["be", "be"]); + assert_eq!(positive_leaves("[be or] NEAR/1 be"), ["be", "or", "be"]); + assert_eq!( + positive_leaves("(AT LEAST 2 OF [be be or]) NEAR/3 to"), + ["be", "be", "or", "to"] + ); + assert_eq!( + positive_leaves("(to NEAR/3 not) NOT ENCLOSES be"), + ["to", "not"] + ); + assert_eq!( + positive_leaves("or NOT ENCLOSED BY \"to be or not\""), + ["or"] + ); + assert_eq!(positive_leaves("be NOT OVERLAPPING \"to be\""), ["be"]); + assert_eq!(positive_leaves("(be AND NOT or) NEAR/3 to"), ["be", "to"]); + assert_eq!( + positive_leaves("(be NEAR/3 be) NOT ENCLOSES (or NEAR/1 be)"), + ["be", "be"] + ); + } + + /// A lowered span's included leaves as scoring reads them, left to right: + /// `term^factor` for a leaf with a written boost. + fn boosted_leaves(query: &Query) -> Vec { + let mut leaves = Vec::new(); + query.for_each_positive_span_slot(&mut |_, slot, boost| { + let SpanTermSlot::Term(term) = slot else { + panic!("{slot:?} is not a term slot"); + }; + leaves.push(match boost { + Some(factor) => format!("{term}^{factor}"), + None => term.clone(), + }); + }); + leaves + } + + #[test] + fn span_leaf_boosts_weigh_the_operand_they_are_written_on() { + let leaves = |input| boosted_leaves(&parse_and_lower(input)); + assert_eq!(leaves("ipa^3 NEAR/5 hoppy"), ["ipa^3", "hoppy"]); + assert_eq!(leaves("\"craft [beer^2 ale]\""), ["craft", "beer^2", "ale"]); + assert_eq!( + leaves("(stout^2 OR porter) NEAR/3 chocolate"), + ["stout^2", "porter", "chocolate"] + ); + assert_eq!(leaves("beer^2 IN FIRST 100 WORDS"), ["beer^2"]); + assert_eq!(leaves("beer^0 NEAR/3 tasting"), ["beer^0", "tasting"]); + // Factors multiply down to the leaf; a repeated word weighs per + // occurrence, each with its own boost. + assert_eq!( + leaves("(ipa^2 NEAR/1 pale)^3 THEN/4 ale"), + ["ipa^6", "pale^3", "ale"] + ); + assert_eq!(leaves("beer^2 NEAR/3 beer"), ["beer^2", "beer"]); + // An excluded side only filters: its boosts weigh nothing. + assert_eq!( + leaves("(ipa^2 NEAR/3 hoppy) NOT ENCLOSES stout^5"), + ["ipa^2", "hoppy"] + ); + // The dynamic form (AT LEAST has no fast path) reads them the same way. + let at_least = parse_and_lower("(AT LEAST 2 OF [be^2 be or]) NEAR/3 to"); + assert!(matches!(at_least, Query::SpanExpr { .. })); + assert_eq!(boosted_leaves(&at_least), ["be^2", "be", "or", "to"]); + } + + #[test] + fn unboosted_span_leaves_lower_without_boosts() { + let unboosted = |input| match parse_and_lower(input) { + Query::Span { leaf_boosts, .. } | Query::SpanExpr { leaf_boosts, .. } => { + leaf_boosts == SpanLeafBoosts::default() + } + Query::Boost { inner, .. } => matches!( + *inner, + Query::Span { leaf_boosts, .. } if leaf_boosts == SpanLeafBoosts::default() + ), + other => panic!("{input} lowers to {other:?}"), + }; + assert!(unboosted("ipa NEAR/5 hoppy")); + assert!(unboosted("(ipa NEAR/5 hoppy)^3")); + assert!(unboosted("\"craft beer\"^2")); + assert!(unboosted("ipa NOT ENCLOSES stout^5")); + } + #[test] fn top_level_runtime_position_filter_stays_on_fast_path() { let query = parse_and_lower("beer IN LAST 25%"); @@ -665,6 +806,7 @@ mod tests { let Query::SpanExpr { term_slots, span_expr, + .. } = query else { panic!("expected advanced span expression lowering"); diff --git a/tinql/src/runtime/mod.rs b/tinql/src/runtime/mod.rs index 6e967a4..d0c6d2d 100644 --- a/tinql/src/runtime/mod.rs +++ b/tinql/src/runtime/mod.rs @@ -67,10 +67,12 @@ pub enum Query { term_slots: Vec, span_query: boldi_vigna::SpanQuery, position_filter: Option, + leaf_boosts: SpanLeafBoosts, }, SpanExpr { term_slots: Vec, span_expr: SpanExpr, + leaf_boosts: SpanLeafBoosts, }, MatchAll, Regex(CompiledRegex), @@ -111,6 +113,21 @@ pub enum SpanTermSlot { }, } +/// The `^` factor written on each included leaf of a span, in +/// [`SpanExpr::for_each_positive_term`] order: the product of the boosts +/// between the leaf and the span's root, `None` where there are none. Empty +/// when no included leaf is boosted, so an unboosted span lowers as it always +/// has. Only scoring reads it; matching never does. +#[derive(Debug, Clone, Default, PartialEq)] +pub struct SpanLeafBoosts(Box<[Option]>); + +impl SpanLeafBoosts { + /// The boost written on the `nth` included leaf. + fn get(&self, nth: usize) -> Option { + if self.0.is_empty() { None } else { self.0[nth] } + } +} + /// A bound in a term range expression. #[derive(Debug, Clone, PartialEq, Eq, Hash)] pub enum RangeBound { @@ -140,6 +157,36 @@ impl Query { } } + /// For a span node (`Span` or `SpanExpr`), calls `f` with each included + /// leaf's slot index, slot and written boost ([`SpanLeafBoosts`]), once + /// per written occurrence, left to right + /// ([`SpanExpr::for_each_positive_term`]); nothing for any other node. + /// These are the leaves a span scores: its excluded sides only filter. + pub fn for_each_positive_span_slot<'a>( + &'a self, + f: &mut impl FnMut(usize, &'a SpanTermSlot, Option), + ) { + let mut nth = 0; + let mut visit = |term_slots: &'a [SpanTermSlot], leaf_boosts: &SpanLeafBoosts, idx| { + f(idx, &term_slots[idx], leaf_boosts.get(nth)); + nth += 1; + }; + match self { + Query::Span { + term_slots, + span_query, + leaf_boosts, + .. + } => span_query.for_each_positive_term(&mut |idx| visit(term_slots, leaf_boosts, idx)), + Query::SpanExpr { + term_slots, + span_expr, + leaf_boosts, + } => span_expr.for_each_positive_term(&mut |idx| visit(term_slots, leaf_boosts, idx)), + _ => {} + } + } + pub fn estimate_tuples(&self, total_tuples: u64, lookup: &dyn Fn(&str) -> u64) -> u64 { let n = total_tuples as f64; self.estimate_selectivity(n, lookup, total_tuples) as u64 diff --git a/tinql/src/runtime/simplify.rs b/tinql/src/runtime/simplify.rs index 1e66306..e0412c6 100644 --- a/tinql/src/runtime/simplify.rs +++ b/tinql/src/runtime/simplify.rs @@ -21,6 +21,8 @@ //! - [`SimplificationProfile::Structural`] keeps the old lowering-time work: //! flattening boolean chains, deduplicating flat term siblings, folding //! `MatchAll`, and collapsing one-child boolean nodes +//! - [`SimplificationProfile::Membership`] adds the rewrites across a +//! conjunction's `NOT` children, for match planning only //! - [`SimplificationProfile::LogicalUnscored`] adds implication-based //! absorption for positive unscored trees so planner-time display and //! unscored execution can both reuse the same backend-agnostic rewrites @@ -49,10 +51,14 @@ //! space at this boundary, it is just deleting a small set of provably //! redundant unscored subtrees. //! -//! Future work: -//! -//! - `NOT` remains a negation boundary. We simplify inside negated subtrees, -//! but we don't do implication reasoning across the negative edge yet. +//! Across the negative edge, a match plan's conjunction (the membership +//! profiles) is empty when a positive child implies one of its `NOT` +//! children, and drops the disjuncts they exclude (`exclude_negated`). A +//! negated subtree itself is simplified inside only. Scoring and highlight +//! trees keep those children: a row scores and highlights every positive +//! term it holds, whichever clause matched, so a disjunct no survivor of a +//! nested conjunction matches still counts on a row that matches through a +//! sibling of that conjunction. use super::{Query, SpanExpr, SpanTermSlot}; use rustc_hash::FxHashSet; @@ -60,6 +66,9 @@ use rustc_hash::FxHashSet; #[derive(Clone, Copy)] pub enum SimplificationProfile { Structural, + /// [`Self::Structural`] for match planning, where the set of rows is + /// all that counts: it also rewrites across `NOT` edges. + Membership, StructuralPreserveTermMultiplicity, LogicalUnscored, } @@ -68,6 +77,7 @@ pub fn simplify(query: Query, profile: SimplificationProfile) -> Query { let query = normalize_boolean_query(query, profile); match profile { SimplificationProfile::Structural + | SimplificationProfile::Membership | SimplificationProfile::StructuralPreserveTermMultiplicity => query, SimplificationProfile::LogicalUnscored => reduce_unscored_redundancy(query), } @@ -76,10 +86,19 @@ pub fn simplify(query: Query, profile: SimplificationProfile) -> Query { impl SimplificationProfile { const fn dedup_flat_terms(self) -> bool { match self { - Self::Structural | Self::LogicalUnscored => true, + Self::Structural | Self::Membership | Self::LogicalUnscored => true, Self::StructuralPreserveTermMultiplicity => false, } } + + /// Whether the tree only decides membership, so a conjunction's `NOT` + /// children may rewrite its positive ones (`exclude_negated`). + const fn crosses_negations(self) -> bool { + match self { + Self::Membership | Self::LogicalUnscored => true, + Self::Structural | Self::StructuralPreserveTermMultiplicity => false, + } + } } pub fn all_terms_required_span_query(span_query: &boldi_vigna::SpanQuery) -> bool { @@ -299,10 +318,12 @@ fn implies(lhs: &Query, rhs: &Query) -> bool { term_slots, span_query, position_filter, + .. } => return query_implies_span(lhs, term_slots, span_query, position_filter.as_ref()), Query::SpanExpr { term_slots, span_expr, + .. } => return query_implies_span_expr(lhs, term_slots, span_expr), Query::Disjunction { .. } | Query::Not(_) @@ -352,6 +373,7 @@ fn query_implies_term(query: &Query, rhs_term: &str) -> bool { Query::SpanExpr { term_slots, span_expr, + .. } => { all_terms_required_span_expr(span_expr) && span_expr_references_term(span_expr, term_slots, rhs_term) @@ -377,6 +399,7 @@ fn query_implies_span( term_slots, span_query, position_filter, + .. } => { term_slots == rhs_slots && span_query == rhs_span_query @@ -413,6 +436,7 @@ fn query_implies_span_expr( Query::SpanExpr { term_slots, span_expr, + .. } => term_slots == rhs_slots && span_expr == rhs_span_expr, Query::And(left, right) => { query_implies_span_expr(left, rhs_slots, rhs_span_expr) @@ -591,6 +615,11 @@ fn normalize_conjunction(children: Vec, profile: SimplificationProfile) - if flat.iter().any(is_empty_query) { return empty_query(); } + if profile.crosses_negations() + && let Some(rewritten) = exclude_negated(&mut flat, profile) + { + return rewritten; + } flat.retain(|child| !matches!(child, Query::MatchAll)); if profile.dedup_flat_terms() { @@ -599,6 +628,47 @@ fn normalize_conjunction(children: Vec, profile: SimplificationProfile) - fold_conjunction(flat) } +/// The rewrites a conjunction's `NOT` children allow in a match plan: a +/// positive child that implies a negated one leaves no match, so the +/// conjunction is empty; and a positive disjunction drops each disjunct +/// that implies a negated child, which matches no survivor. `None` when +/// nothing changes. +/// +/// The disjunct's terms go too, which is what keeps this out of scoring +/// and highlight trees: a row scores and highlights every positive term it +/// holds, whichever clause matched (`((a AND c) OR b) AND NOT a` scores `c` +/// on a survivor holding `b c`), and nested under an OR the conjunction's +/// rows are not every row the tree scores (`((a OR b) AND NOT a) OR c` +/// scores `a` on a row holding `a c`). +fn exclude_negated(flat: &mut Vec, profile: SimplificationProfile) -> Option { + let negated: Vec = flat + .iter() + .filter_map(|child| match child { + Query::Not(inner) => Some((**inner).clone()), + _ => None, + }) + .collect(); + if negated.is_empty() { + return None; + } + let excluded = |query: &Query| negated.iter().any(|negated| implies(query, negated)); + if flat.iter().filter_map(positive_root).any(excluded) { + return Some(empty_query()); + } + let mut changed = false; + for child in flat.iter_mut() { + if let Query::Disjunction { min, children } = child + && children.iter().any(excluded) + { + let (min, mut kept) = (*min, std::mem::take(children)); + kept.retain(|disjunct| !excluded(disjunct)); + *child = normalize_disjunction(min, kept, profile); + changed = true; + } + } + changed.then(|| normalize_conjunction(std::mem::take(flat), profile)) +} + fn push_conjunction_child(child: Query, out: &mut Vec) { match child { Query::And(left, right) => { @@ -698,6 +768,7 @@ fn fold_conjunction(mut children: Vec) -> Query { #[cfg(test)] mod tests { use super::*; + use crate::runtime::SpanLeafBoosts; #[test] fn structural_simplify_flattens_and_dedups_terms() { @@ -789,6 +860,7 @@ mod tests { ])), }, position_filter: None, + leaf_boosts: SpanLeafBoosts::default(), }, ], }; @@ -800,9 +872,9 @@ mod tests { } #[test] - fn logical_unscored_does_not_cross_negative_edge() { + fn logical_unscored_keeps_negated_subtrees_whole() { let query = Query::Conjunction(vec![ - Query::Term("beer".into()), + Query::Term("ale".into()), Query::Not(Box::new(Query::Disjunction { min: 1, children: vec![Query::Term("beer".into()), Query::Term("wine".into())], @@ -810,17 +882,216 @@ mod tests { ]); assert_eq!( - simplify(query, SimplificationProfile::LogicalUnscored), + simplify(query.clone(), SimplificationProfile::LogicalUnscored), + query, + ); + } + + /// The profiles that plan membership, which rewrite across `NOT` edges. + const MATCH_PROFILES: [SimplificationProfile; 2] = [ + SimplificationProfile::Membership, + SimplificationProfile::LogicalUnscored, + ]; + + /// The profiles whose trees score or highlight, which keep every + /// positive child. + const TREE_PROFILES: [SimplificationProfile; 2] = [ + SimplificationProfile::Structural, + SimplificationProfile::StructuralPreserveTermMultiplicity, + ]; + + fn term(text: &str) -> Query { + Query::Term(text.into()) + } + + fn not(inner: Query) -> Query { + Query::Not(Box::new(inner)) + } + + fn any(children: Vec) -> Query { + Query::Disjunction { min: 1, children } + } + + fn phrase(left: &str, right: &str) -> Query { + Query::Span { + term_slots: vec![ + SpanTermSlot::Term(left.into()), + SpanTermSlot::Term(right.into()), + ], + span_query: boldi_vigna::SpanQuery::MaxGaps { + max_gaps: 0, + inner: Box::new(boldi_vigna::SpanQuery::Ordered(vec![ + boldi_vigna::SpanQuery::Term(0), + boldi_vigna::SpanQuery::Term(1), + ])), + }, + position_filter: None, + leaf_boosts: SpanLeafBoosts::default(), + } + } + + /// A positive child that implies a negated one leaves no match. + #[test] + fn a_positive_implying_a_negation_is_empty() { + for query in [ + Query::Conjunction(vec![term("a"), not(term("a"))]), + Query::Conjunction(vec![term("a"), term("b"), not(term("a"))]), + Query::And( + Box::new(Query::Conjunction(vec![term("a"), term("b")])), + Box::new(not(term("a"))), + ), + Query::Conjunction(vec![phrase("a", "b"), not(term("a"))]), + Query::Conjunction(vec![term("a"), not(any(vec![term("a"), term("c")]))]), Query::Conjunction(vec![ - Query::Term("beer".into()), - Query::Not(Box::new(Query::Disjunction { - min: 1, - children: vec![Query::Term("beer".into()), Query::Term("wine".into())], - })), + any(vec![term("a"), term("b")]), + not(any(vec![term("a"), term("b")])), ]), + Query::Conjunction(vec![ + Query::Boost { + factor: 2.0, + inner: Box::new(term("a")), + }, + not(term("a")), + ]), + ] { + for profile in MATCH_PROFILES { + assert_eq!(simplify(query.clone(), profile), empty_query(), "{query:?}"); + } + for profile in TREE_PROFILES { + assert_ne!(simplify(query.clone(), profile), empty_query(), "{query:?}"); + } + } + } + + /// A disjunct that implies a negated child goes from the match plan, + /// whatever else it holds; the threshold's tail then applies. + #[test] + fn a_disjunct_the_negation_excludes_goes() { + let cases = [ + ( + Query::Conjunction(vec![any(vec![term("a"), term("b")]), not(term("a"))]), + Query::Conjunction(vec![term("b"), not(term("a"))]), + ), + ( + Query::Conjunction(vec![ + any(vec![term("a"), term("b")]), + not(any(vec![term("a"), term("c")])), + ]), + Query::Conjunction(vec![term("b"), not(any(vec![term("a"), term("c")]))]), + ), + ( + Query::Conjunction(vec![ + any(vec![term("a"), term("b"), term("c")]), + not(term("a")), + ]), + Query::Conjunction(vec![any(vec![term("b"), term("c")]), not(term("a"))]), + ), + ( + Query::Conjunction(vec![ + Query::Disjunction { + min: 2, + children: vec![term("a"), term("b"), term("c")], + }, + not(term("a")), + ]), + Query::Conjunction(vec![term("b"), term("c"), not(term("a"))]), + ), + ( + Query::Conjunction(vec![ + Query::Disjunction { + min: 3, + children: vec![term("a"), term("b"), term("c")], + }, + not(term("a")), + ]), + empty_query(), + ), + ( + Query::Conjunction(vec![any(vec![phrase("a", "b"), term("c")]), not(term("a"))]), + Query::Conjunction(vec![term("c"), not(term("a"))]), + ), + ( + Query::Conjunction(vec![ + any(vec![ + Query::Conjunction(vec![term("a"), term("c")]), + term("b"), + ]), + not(term("a")), + ]), + Query::Conjunction(vec![term("b"), not(term("a"))]), + ), + ]; + for (query, expected) in cases { + for profile in MATCH_PROFILES { + assert_eq!(simplify(query.clone(), profile), expected, "{query:?}"); + } + for profile in TREE_PROFILES { + assert_ne!(simplify(query.clone(), profile), expected, "{query:?}"); + } + } + // The profile that keeps repeats keeps a repeated disjunct too. + let repeated = Query::Conjunction(vec![ + any(vec![term("a"), term("a"), term("b")]), + not(term("a")), + ]); + assert_eq!( + simplify( + repeated.clone(), + SimplificationProfile::StructuralPreserveTermMultiplicity + ), + repeated, ); } + /// Nested under an OR, the rewritten conjunction's rows are not every + /// row the tree matches: a row holding `a c` matches through `c` and + /// scores and highlights `a`. The match plan still rewrites it (its rows + /// are the same); the trees keep it whole. + #[test] + fn a_nested_negation_rewrites_the_match_plan_only() { + let cases = [ + ( + any(vec![ + Query::Conjunction(vec![any(vec![term("a"), term("b")]), not(term("a"))]), + term("c"), + ]), + any(vec![ + Query::Conjunction(vec![term("b"), not(term("a"))]), + term("c"), + ]), + ), + ( + any(vec![ + Query::Conjunction(vec![term("a"), not(term("a"))]), + term("c"), + ]), + term("c"), + ), + ]; + for (query, expected) in cases { + for profile in MATCH_PROFILES { + assert_eq!(simplify(query.clone(), profile), expected, "{query:?}"); + } + for profile in TREE_PROFILES { + assert_eq!(simplify(query.clone(), profile), query, "{query:?}"); + } + } + } + + /// No rewrite where nothing implies the negation. + #[test] + fn a_negation_rewrites_nothing_it_does_not_exclude() { + for query in [ + Query::Conjunction(vec![term("a"), not(phrase("a", "b"))]), + Query::Conjunction(vec![any(vec![term("a"), term("b")]), not(term("c"))]), + Query::Conjunction(vec![any(vec![term("b"), not(term("a"))]), not(term("a"))]), + ] { + for profile in MATCH_PROFILES.into_iter().chain(TREE_PROFILES) { + assert_eq!(simplify(query.clone(), profile), query, "{query:?}"); + } + } + } + #[test] fn logical_unscored_ignores_unused_span_slots() { let query = Query::Disjunction { @@ -833,6 +1104,7 @@ mod tests { ], span_query: boldi_vigna::SpanQuery::Term(0), position_filter: None, + leaf_boosts: SpanLeafBoosts::default(), }, Query::Term("c".into()), ], @@ -850,6 +1122,7 @@ mod tests { ], span_query: boldi_vigna::SpanQuery::Term(0), position_filter: None, + leaf_boosts: SpanLeafBoosts::default(), }, Query::Term("c".into()), ], diff --git a/tinql/src/runtime/span_expr.rs b/tinql/src/runtime/span_expr.rs index b036d53..dc8560c 100644 --- a/tinql/src/runtime/span_expr.rs +++ b/tinql/src/runtime/span_expr.rs @@ -150,6 +150,66 @@ impl SpanExpr { } } + /// Calls `f` with the slot of every `Term` leaf outside an excluded + /// operand, once per occurrence, left to right: the leaves a match's + /// spans are built from ([`Self::for_each_term`]). + pub fn for_each_positive_term(&self, f: &mut impl FnMut(usize)) { + self.for_each_term(true, &mut |idx, included| { + if included { + f(idx); + } + }); + } + + /// Calls `f` with the slot of every `Term` leaf, once per occurrence, left + /// to right, and whether it is included (`included` seeds the root). An + /// excluded operand (`NotContaining`'s `little`, `NotContainedBy`'s + /// `big`, `NonOverlapping`'s `b`) only filters its sibling's spans, so + /// every leaf beneath it is excluded. Left to right is the order lowering + /// writes the leaves in. + pub(super) fn for_each_term(&self, included: bool, f: &mut impl FnMut(usize, bool)) { + match self { + Self::Empty => {} + Self::Term(idx) => f(*idx, included), + Self::Ordered(children) + | Self::Unordered(children) + | Self::Or(children) + | Self::AtLeast { children, .. } => { + for child in children { + child.for_each_term(included, f); + } + } + Self::MaxGaps { inner, .. } + | Self::GapsInRange { inner, .. } + | Self::MaxWidth { inner, .. } + | Self::WithinPositions { inner, .. } + | Self::PositionFilter { inner, .. } => inner.for_each_term(included, f), + Self::NotContaining { + big: kept, + little: excluded, + } + | Self::NotContainedBy { + little: kept, + big: excluded, + } + | Self::NonOverlapping { + a: kept, + b: excluded, + } => { + kept.for_each_term(included, f); + excluded.for_each_term(false, f); + } + Self::Containing { big: a, little: b } + | Self::ContainedBy { little: a, big: b } + | Self::Overlapping { a, b } + | Self::Before { a, b } + | Self::After { a, b } => { + a.for_each_term(included, f); + b.for_each_term(included, f); + } + } + } + pub fn to_fast_path_root( &self, ) -> Option<(boldi_vigna::SpanQuery, Option)> { diff --git a/tinql/tests/near_slot_oracle.rs b/tinql/tests/near_slot_oracle.rs new file mode 100644 index 0000000..284393e --- /dev/null +++ b/tinql/tests/near_slot_oracle.rs @@ -0,0 +1,255 @@ +// Copyright (C) 2026 PlanetScale +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . +//! tinql's evaluator against an independent brute-force NEAR oracle, for slots +//! that are OR groups and may share a word with the other slot. +//! +//! The oracle tokenizes the text with the tokenizer crate's default pipeline, +//! takes each slot's positions directly from the tokens, and enumerates every +//! pair of positions, one per slot. +//! +//! Semantics implemented (book `tinql/proximity.md`, and the span solver's +//! `SpanQuery` contract for what the book leaves open): +//! - `NEAR/n` is unordered: either slot may come first. +//! - Distance: single-word slots at positions `p` and `q` match when at most +//! `n` positions lie strictly between them, `|p - q| - 1 <= n`. +//! - Same-position reuse: the book does not say whether both slots may use one +//! occurrence. The solver's contract does (`MaxGaps` passes no overlapping +//! sub-intervals), and the shipped `alpha NEAR/3 alpha` already requires two +//! distinct occurrences. The oracle follows that: `p != q`. + +use tinql::runtime::{evaluate, lower::lower, subtokenize::sub_tokenize, tokenize_doc}; +use tinql::{ImplicitOp, parse}; +use tokenizer::Tokenizer; +use tokenizer::presets::default_pipeline; + +fn engine_matches(query: &str, text: &str) -> bool { + let expr = parse(query, ImplicitOp::And).expect("query parses"); + let expr = sub_tokenize(expr, default_pipeline()).expect("query sub-tokenizes"); + let query = lower(&expr).expect("query lowers"); + evaluate(&query, &tokenize_doc(text, default_pipeline())) + .expect("query evaluates") + .matched +} + +/// The single token the default pipeline makes of a slot word. +fn normalize(word: &str) -> String { + let mut tokens = default_pipeline().tokenize(word); + let token = tokens + .next() + .expect("slot word tokenizes") + .text + .into_owned(); + assert!(tokens.next().is_none(), "slot word {word:?} is one token"); + token +} + +fn oracle_matches(left: &[&str], right: &[&str], max_gaps: u32, text: &str) -> bool { + let tokens: Vec<(String, u32)> = default_pipeline() + .tokenize(text) + .map(|token| (token.text.into_owned(), token.pos)) + .collect(); + let slot_positions = |slot: &[&str]| -> Vec { + let words: Vec = slot.iter().map(|w| normalize(w)).collect(); + tokens + .iter() + .filter(|(token, _)| words.contains(token)) + .map(|&(_, pos)| pos) + .collect() + }; + let (left, right) = (slot_positions(left), slot_positions(right)); + left.iter().any(|&p| { + right + .iter() + .any(|&q| p != q && p.abs_diff(q) - 1 <= max_gaps) + }) +} + +fn slot_text(slot: &[&str]) -> String { + match slot { + [word] => (*word).to_owned(), + words => format!("({})", words.join(" OR ")), + } +} + +fn near_query(left: &[&str], right: &[&str], max_gaps: u32) -> String { + format!("{} NEAR/{max_gaps} {}", slot_text(left), slot_text(right)) +} + +/// Query, left slot, right slot, gap, text. +type Row = ( + &'static str, + &'static [&'static str], + &'static [&'static str], + u32, + &'static str, +); + +/// The M5c span-fixture queries, on texts that contain a match. +#[test] +fn group_slots_sharing_a_word_match_the_oracle() { + let rows: &[Row] = &[ + ( + "alpha NEAR/3 alpha", + &["alpha"], + &["alpha"], + 3, + "alpha of the alpha", + ), + ( + "(alpha OR bravo) NEAR/3 alpha", + &["alpha", "bravo"], + &["alpha"], + 3, + "alpha of the alpha", + ), + ( + "[alpha bravo] NEAR/2 alpha", + &["alpha", "bravo"], + &["alpha"], + 2, + "the alpha of alpha", + ), + ( + "alpha NEAR/2 alpha", + &["alpha"], + &["alpha"], + 2, + "alpha of the alpha", + ), + ( + "(alpha OR bravo) NEAR/2 (alpha OR charlie)", + &["alpha", "bravo"], + &["alpha", "charlie"], + 2, + "alpha the alpha", + ), + ( + "(alpha OR bravo) NEAR/2 (alpha OR charlie)", + &["alpha", "bravo"], + &["alpha", "charlie"], + 2, + "charlie of the bravo alpha", + ), + ]; + let mut failures = Vec::new(); + for &(query, left, right, max_gaps, text) in rows { + let expected = oracle_matches(left, right, max_gaps, text); + let actual = engine_matches(query, text); + if actual != expected { + failures.push(format!( + "{query:?} on {text:?}: tinql {actual}, oracle {expected}" + )); + } + } + assert!(failures.is_empty(), "{}", failures.join("\n")); +} + +#[test] +fn one_shared_occurrence_is_not_a_pair() { + assert!(!engine_matches( + "(alpha OR bravo) NEAR/3 alpha", + "the alpha of" + )); + assert!(!oracle_matches( + &["alpha", "bravo"], + &["alpha"], + 3, + "the alpha of" + )); +} + +/// xorshift64*: deterministic generated cases without a test dependency. +struct Rng(u64); + +impl Rng { + fn next(&mut self) -> u64 { + self.0 ^= self.0 >> 12; + self.0 ^= self.0 << 25; + self.0 ^= self.0 >> 27; + self.0.wrapping_mul(0x2545_f491_4f6c_dd1d) + } + + fn below(&mut self, n: usize) -> usize { + (self.next() % n as u64) as usize + } +} + +const SLOT_WORDS: [&str; 4] = ["alpha", "bravo", "charlie", "delta"]; +const FILLER: [&str; 3] = ["the", "of", "echo"]; + +fn generated_text(rng: &mut Rng) -> String { + let len = 1 + rng.below(12); + (0..len) + .map(|_| { + if rng.below(2) == 0 { + SLOT_WORDS[rng.below(SLOT_WORDS.len())] + } else { + FILLER[rng.below(FILLER.len())] + } + }) + .collect::>() + .join(" ") +} + +fn generated_slot(rng: &mut Rng) -> Vec<&'static str> { + let mut slot: Vec<&str> = SLOT_WORDS + .iter() + .copied() + .filter(|_| rng.below(3) == 0) + .collect(); + if slot.is_empty() { + slot.push(SLOT_WORDS[rng.below(SLOT_WORDS.len())]); + } + slot +} + +#[test] +fn generated_slot_queries_match_the_oracle_and_widening_never_drops_a_doc() { + let mut rng = Rng(0x9e37_79b9_7f4a_7c15); + let mut failures = Vec::new(); + for _ in 0..3000 { + let text = generated_text(&mut rng); + let left = generated_slot(&mut rng); + let right = generated_slot(&mut rng); + let max_gaps = rng.below(4) as u32; + + let query = near_query(&left, &right, max_gaps); + let actual = engine_matches(&query, &text); + let expected = oracle_matches(&left, &right, max_gaps, &text); + if actual != expected { + failures.push(format!( + "{query:?} on {text:?}: tinql {actual}, oracle {expected}" + )); + } + + let extra = SLOT_WORDS[rng.below(SLOT_WORDS.len())]; + if actual && !left.contains(&extra) { + let mut wider = left.clone(); + wider.push(extra); + let wide_query = near_query(&wider, &right, max_gaps); + if !engine_matches(&wide_query, &text) { + failures.push(format!( + "{query:?} matched {text:?} but {wide_query:?} did not" + )); + } + } + } + assert!( + failures.is_empty(), + "{} failures, first:\n{}", + failures.len(), + failures[..failures.len().min(10)].join("\n") + ); +} From 615e9cea4e1885fb32363ea90000e762dc9d478b Mon Sep 17 00:00:00 2001 From: Patrick Reynolds Date: Wed, 7 Oct 2026 00:19:38 -0400 Subject: [PATCH 2/2] Exclude empty documents from BM25 scoring TIN never inserts empty documents into the index, which means they don't count toward the number of documents or average doc length for BM25 calculations. Lead now matches that behavior. --- postgres/src/score.rs | 4 ++++ private-regress.manifest | 1 + 2 files changed, 5 insertions(+) diff --git a/postgres/src/score.rs b/postgres/src/score.rs index 2984421..e5908e6 100644 --- a/postgres/src/score.rs +++ b/postgres/src/score.rs @@ -505,6 +505,9 @@ fn select_read_only(sql: &str, failure: &str) -> Vec>> { }) } +/// Tokenizes the corpus for scoring statistics. As in tin, a document with no +/// tokens is not searchable, so it counts toward neither N nor the average +/// document length. fn tokenize_documents<'a>( documents: impl Iterator, tokenizer: &CompiledTokenizerPipeline, @@ -517,6 +520,7 @@ fn tokenize_documents<'a>( } tokenize(tokenizer, document) }) + .filter(|tokens| !tokens.is_empty()) .collect() } diff --git a/private-regress.manifest b/private-regress.manifest index 613b030..96fc73a 100644 --- a/private-regress.manifest +++ b/private-regress.manifest @@ -37,3 +37,4 @@ highlight_join_analysis highlight_null_analysis_defaults phrase_repeat_weights tinql_query_length_limit +or_dnf_factored_scores