From d69afa43c681b81c4bdcc61ab9abe69675e70e54 Mon Sep 17 00:00:00 2001 From: Adrian Garcia Badaracco <1755071+adriangb@users.noreply.github.com> Date: Sun, 28 Jun 2026 16:55:42 +0200 Subject: [PATCH 01/27] feat(physical-plan): adaptive conjunct reordering in FilterExec (compact-once core) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add runtime, statistics-based conjunct reordering for `FilterExec`, off by default behind `datafusion.execution.adaptive_filter_reordering`. A conjunctive predicate is evaluated through a compact-once loop: conjunct masks are AND-combined and the working batch is physically compacted to the surviving rows once the accumulated mask is selective enough, so a selective conjunct shrinks the batch the conjuncts after it must decode. This compaction — not reordering a fused `BinaryExpr` AND, which does not compact between conjuncts — is the source of the win, and reordering compounds it. Each conjunct is timed and counted on the rows that reach it during a short warm-up; the conjuncts are then ranked by rows discarded per nanosecond (`(1 - pass_rate) / cost_per_row`) and, if the ranked order is materially cheaper than the written one, it is adopted and frozen. Results, plan, and EXPLAIN are unchanged; volatile predicates are never reordered. This is the minimal core. Benchmarks (predicate_eval) confirm it captures the "buried selective conjunct" wins (costsel_q01 ~-14%, width ~-12%) but also that compact-once regresses cheap-predicate conjunctions (cardinality k8 ~+37%) where the compaction overhead is not repaid — a guard that keeps the plain fused evaluation for those is added in the next commit. Cross-stream sharing, drift re-measurement, and confidence-interval statistics are later layers. Tested by unit tests (compact-once result-equivalence in any order, ranking, expected-cost weighting, adopt/keep decisions) and an end-to-end `adaptive_filter.slt` asserting identical results and EXPLAIN with the flag on and off. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01Lh7i9DyeFWuTFWjogrVNkb --- datafusion/common/src/config.rs | 11 + .../physical-plan/src/adaptive_filter.rs | 538 ++++++++++++++++++ datafusion/physical-plan/src/filter.rs | 26 +- datafusion/physical-plan/src/lib.rs | 1 + .../test_files/adaptive_filter.slt | 80 +++ .../test_files/information_schema.slt | 2 + docs/source/user-guide/configs.md | 1 + 7 files changed, 656 insertions(+), 3 deletions(-) create mode 100644 datafusion/physical-plan/src/adaptive_filter.rs create mode 100644 datafusion/sqllogictest/test_files/adaptive_filter.slt diff --git a/datafusion/common/src/config.rs b/datafusion/common/src/config.rs index 360586b0e9bae..e39047f9ce097 100644 --- a/datafusion/common/src/config.rs +++ b/datafusion/common/src/config.rs @@ -1138,6 +1138,17 @@ config_namespace! { /// Note: this option currently only applies to the symmetric hash join. pub enforce_batch_size_in_joins: bool, default = false + /// (experimental) When enabled, `FilterExec` adaptively reorders the + /// conjuncts of a conjunctive predicate at runtime. It measures each + /// conjunct's selectivity and evaluation cost on the rows that reach it + /// and runs the conjuncts that discard the most rows per unit of CPU + /// time first, so cheap-and-selective predicates gate expensive ones. + /// Reordering never changes query results (only the evaluation order of + /// a conjunction) but can change observable side effects of fallible + /// predicates, so it is off by default. Predicates containing volatile + /// expressions are never reordered. + pub adaptive_filter_reordering: bool, default = false + /// Size (bytes) of data buffer DataFusion uses when writing output files. /// This affects the size of the data chunks that are uploaded to remote /// object stores (e.g. AWS S3). If very large (>= 100 GiB) output files are being diff --git a/datafusion/physical-plan/src/adaptive_filter.rs b/datafusion/physical-plan/src/adaptive_filter.rs new file mode 100644 index 0000000000000..46e76d6ddc591 --- /dev/null +++ b/datafusion/physical-plan/src/adaptive_filter.rs @@ -0,0 +1,538 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Runtime-adaptive evaluation of a conjunctive (`AND`) predicate in +//! [`FilterExec`](crate::filter::FilterExec). +//! +//! Predicate evaluation order matters: a selective predicate run first gates +//! the work of the predicates after it. DataFusion's `BinaryExpr` `AND` +//! short-circuit only gates on the *leftmost* conjunct, so a conjunction whose +//! selective member is written last (e.g. +//! `regexp_like(s,'a') AND … AND regexp_like(s,'rare')`) evaluates every +//! predicate against ~every row. +//! +//! ## How it evaluates: the compact-once loop +//! +//! The conjuncts are evaluated sequentially, combining their boolean results +//! with `AND`. The working batch is physically compacted to the surviving rows +//! once the accumulated mask becomes selective enough — so a run of +//! non-selective conjuncts costs only cheap bitwise `AND`s, while a selective +//! conjunct shrinks the batch the conjuncts after it must decode. This +//! compaction is what makes ordering pay off (and is itself a win even without +//! reordering): a left-deep fused `BinaryExpr` `AND` does *not* compact between +//! conjuncts, so it evaluates ~every conjunct on ~every row regardless of order. +//! +//! ## How it orders +//! +//! Each conjunct is timed and counted on exactly the rows it evaluated, giving +//! its marginal selectivity and per-row cost. After a short warm-up the +//! conjuncts are ranked by rows discarded per nanosecond +//! (`(1 - pass_rate) / cost_per_row`, the classic optimal ordering key for +//! independent conjuncts), and if the ranked order is *materially* cheaper than +//! the written one it is adopted. The order then stays fixed. +//! +//! It is **off by default** +//! (`datafusion.execution.adaptive_filter_reordering`) and never changes query +//! results: a conjunction's value is independent of evaluation order. Predicates +//! containing volatile expressions are never reordered (their observable side +//! effects depend on order). +//! +//! This is the minimal core of the adaptive evaluator. Richer policies +//! (A/B-validated adoption, cross-stream sharing, drift re-measurement) build on +//! top of it. + +use std::sync::Arc; + +use arrow::array::{Array, ArrayRef, BooleanArray, BooleanBufferBuilder, UInt32Array}; +use arrow::buffer::BooleanBuffer; +use arrow::compute::kernels::boolean::and; +use arrow::compute::{filter, filter_record_batch, prep_null_mask_filter}; +use arrow::record_batch::RecordBatch; +use datafusion_common::Result; +use datafusion_common::cast::as_boolean_array; +use datafusion_common::instant::Instant; +use datafusion_physical_expr::PhysicalExpr; +use datafusion_physical_expr::utils::split_conjunction; +use datafusion_physical_expr_common::physical_expr::is_volatile; + +/// Batches measured before the order is settled. +const WARMUP_BATCHES: u64 = 8; + +/// Fraction of the conjunction's expected per-row cost below which a reorder is +/// immaterial. A candidate order is adopted only if it is expected to cost less +/// than `(1 - TIE_COST_FRACTION)` of the written order, so interchangeable +/// conjuncts never trigger a reorder. +const TIE_COST_FRACTION: f64 = 0.05; + +/// Physically compact the working batch to the surviving rows only when the +/// accumulated mask keeps at most this fraction of them. Above this, the cost +/// of materializing a barely-smaller batch is not repaid, so we keep evaluating +/// against the full working batch and just `AND` the boolean masks. +const COMPACTION_SELECTIVITY_THRESHOLD: f64 = 0.2; + +/// Per-conjunct measurement: marginal pass rate and per-row evaluation cost, +/// accumulated over the warm-up window on exactly the rows that reached the +/// conjunct. +#[derive(Debug, Default, Clone)] +struct ConjunctStats { + /// Total rows the conjunct was evaluated on. + rows: u64, + /// Rows that passed (non-null `true`, matching SQL filter semantics). + matched: u64, + /// Total evaluation time, nanoseconds. + nanos: u64, +} + +impl ConjunctStats { + fn record(&mut self, matched: u64, rows: u64, nanos: u64) { + self.rows += rows; + self.matched += matched; + self.nanos += nanos; + } + + /// Fraction of rows that pass, or `None` if never evaluated on any row. + fn pass_rate(&self) -> Option { + (self.rows > 0).then(|| self.matched as f64 / self.rows as f64) + } + + /// Per-row evaluation cost in nanoseconds, or `None` if unmeasured. + fn cost_per_row(&self) -> Option { + (self.rows > 0 && self.nanos > 0).then(|| self.nanos as f64 / self.rows as f64) + } + + /// Ranking key: rows discarded per nanosecond of evaluation + /// (`(1 - pass_rate) / cost_per_row`). Maximising this is exactly + /// minimising `cost_per_row / (1 - pass_rate)`, the classic optimal + /// ordering key for independent conjuncts — so a selective-but-expensive + /// predicate correctly sorts ahead of a cheap-but-unselective one. + /// `None` when unmeasured, so such conjuncts sort last. + fn effectiveness(&self) -> Option { + let cost = self.cost_per_row()?; + let pass = self.pass_rate()?; + Some((1.0 - pass) / cost) + } +} + +/// Adaptive evaluator for a single conjunctive predicate, owned per partition +/// stream (single-threaded, no locking). +#[derive(Debug)] +pub(crate) struct AdaptiveConjunction { + /// The split conjuncts. `stats`/`order` indices refer to positions here. + conjuncts: Vec>, + /// Per-conjunct measurements, indexed by conjunct position. + stats: Vec, + /// Evaluation order: indices into `conjuncts`. Starts as the written order + /// and may be permuted once after the warm-up. + order: Vec, + /// Warm-up batches still to measure; `0` means the order has settled. + warmup_left: u64, +} + +impl AdaptiveConjunction { + /// Build an adaptive evaluator for `predicate`, or `None` if adaptive + /// reordering does not apply: + /// + /// - `enabled` is false (the config flag is off); + /// - the predicate has fewer than two `AND` conjuncts (nothing to reorder); + /// - any conjunct is volatile (reordering could change side effects). + pub(crate) fn try_new( + predicate: &Arc, + enabled: bool, + ) -> Option { + if !enabled { + return None; + } + let conjuncts: Vec> = split_conjunction(predicate) + .into_iter() + .map(Arc::clone) + .collect(); + if conjuncts.len() < 2 || conjuncts.iter().any(is_volatile) { + return None; + } + let stats = vec![ConjunctStats::default(); conjuncts.len()]; + let order = (0..conjuncts.len()).collect(); + Some(Self { + conjuncts, + stats, + order, + warmup_left: WARMUP_BATCHES, + }) + } + + /// Evaluate the conjunction against `batch`, returning the boolean mask + /// (over the batch's rows) of rows that passed every conjunct. + /// + /// During the warm-up the conjuncts are measured (on the rows that reach + /// each one); once the window ends the order is settled and subsequent + /// batches evaluate with no instrumentation. + pub(crate) fn evaluate(&mut self, batch: &RecordBatch) -> Result { + if self.warmup_left == 0 { + return eval_conjuncts(&self.conjuncts, &self.order, batch, None); + } + let result = + eval_conjuncts(&self.conjuncts, &self.order, batch, Some(&mut self.stats))?; + self.warmup_left -= 1; + if self.warmup_left == 0 { + self.settle(); + } + Ok(result) + } + + /// Rank the conjuncts by measured effectiveness and, if the resulting order + /// is materially cheaper than the written order, adopt it. + fn settle(&mut self) { + let candidate = rank_by_effectiveness(&self.stats); + if candidate == self.order { + return; + } + let candidate_cost = expected_cost_per_row(&self.stats, &candidate); + let current_cost = expected_cost_per_row(&self.stats, &self.order); + if candidate_cost < (1.0 - TIE_COST_FRACTION) * current_cost { + self.order = candidate; + } + } +} + +/// Evaluate `conjuncts` in `order` against `batch` via the compact-once loop, +/// returning the boolean mask (over the batch's original rows) of rows that +/// passed every conjunct. With `stats`, each conjunct is additionally timed and +/// counted on exactly the rows it evaluated (its marginal selectivity and cost +/// on the current working population). +/// +/// The working batch is physically compacted to the surviving rows only once +/// the accumulated mask becomes selective enough (see +/// [`COMPACTION_SELECTIVITY_THRESHOLD`]); until then masks are combined with a +/// cheap bitwise `AND`, so a run of non-selective conjuncts pays no +/// materialization cost. Unlike a fused `BinaryExpr` chain, survivors stay +/// compacted across the remaining conjuncts instead of being re-evaluated on +/// every row. +fn eval_conjuncts( + conjuncts: &[Arc], + order: &[usize], + batch: &RecordBatch, + mut stats: Option<&mut [ConjunctStats]>, +) -> Result { + let num_rows = batch.num_rows(); + if num_rows == 0 { + return Ok(Arc::new(BooleanArray::from(Vec::::new()))); + } + + // `working` is the batch conjuncts are evaluated against. `acc` is the + // accumulated (`AND`-combined, null-free) result over `working`'s rows since + // the last compaction; `None` means all of them are still live. `live` maps + // `working`'s rows back to original row indices; `None` until a compaction + // first drops rows. + let mut working = batch.clone(); + let mut acc: Option = None; + let mut live: Option = None; + + for &id in order { + let rows_in = working.num_rows(); + + let timer = stats.is_some().then(Instant::now); + let array = conjuncts[id].evaluate(&working)?.into_array(rows_in)?; + let mask = as_boolean_array(&array)?; + // `matched` counts non-null trues (SQL filter semantics). + let matched = mask.true_count() as u64; + + if let (Some(stats), Some(timer)) = (stats.as_deref_mut(), timer) { + let eval_nanos = timer.elapsed().as_nanos() as u64; + stats[id].record(matched, rows_in as u64, eval_nanos); + } + + // An all-true mask leaves the accumulated result untouched. + if matched == rows_in as u64 && mask.null_count() == 0 { + continue; + } + + // Fold this conjunct into the accumulated mask (null -> false). + let mask = if mask.null_count() > 0 { + prep_null_mask_filter(mask) + } else { + mask.clone() + }; + let folded = match &acc { + None => mask, + Some(prev) => and(prev, &mask)?, + }; + + let alive = folded.true_count(); + if alive == 0 { + // Nothing survives; the result is all-false over the original rows. + return Ok(Arc::new(BooleanArray::new( + BooleanBuffer::new_unset(num_rows), + None, + ))); + } + // Compact only when the survivors are a small fraction of the working + // batch — otherwise the copy is not worth it. + if (alive as f64) <= COMPACTION_SELECTIVITY_THRESHOLD * rows_in as f64 { + working = filter_record_batch(&working, &folded)?; + let indices = live.take().unwrap_or_else(|| { + Arc::new(UInt32Array::from_iter_values(0..num_rows as u32)) + }); + live = Some(filter(&indices, &folded)?); + acc = None; + } else { + acc = Some(folded); + } + } + + match live { + // Never compacted: `acc` (or all-true) already covers the original rows. + None => Ok(match acc { + Some(acc) => Arc::new(acc), + None => Arc::new(BooleanArray::new(BooleanBuffer::new_set(num_rows), None)), + }), + // Compacted at least once: scatter the surviving original indices + // (`live`, narrowed by any residual `acc`) into a full-length mask. + Some(indices) => { + let indices = match acc { + Some(acc) => filter(&indices, &acc)?, + None => indices, + }; + let indices = indices + .as_any() + .downcast_ref::() + .expect("u32 live"); + let mut builder = BooleanBufferBuilder::new(num_rows); + builder.append_n(num_rows, false); + for &idx in indices.values() { + builder.set_bit(idx as usize, true); + } + Ok(Arc::new(BooleanArray::new(builder.finish(), None))) + } + } +} + +/// Rank conjunct ids by effectiveness (discards per nanosecond) descending; +/// ids without measurements sort last. Stable, so equal ids keep their order. +fn rank_by_effectiveness(stats: &[ConjunctStats]) -> Vec { + let mut ids: Vec = (0..stats.len()).collect(); + ids.sort_by( + |&a, &b| match (stats[a].effectiveness(), stats[b].effectiveness()) { + (Some(x), Some(y)) => y.partial_cmp(&x).unwrap_or(std::cmp::Ordering::Equal), + (Some(_), None) => std::cmp::Ordering::Less, + (None, Some(_)) => std::cmp::Ordering::Greater, + (None, None) => std::cmp::Ordering::Equal, + }, + ); + ids +} + +/// Expected cost of evaluating the conjuncts in `order`, in nanoseconds per +/// input row: each conjunct's measured per-row cost weighted by the fraction of +/// rows expected to reach it (the product of the pass rates of the conjuncts +/// before it, treated as independent). Unmeasured conjuncts contribute nothing. +fn expected_cost_per_row(stats: &[ConjunctStats], order: &[usize]) -> f64 { + let mut weight = 1.0_f64; + let mut total = 0.0_f64; + for &id in order { + let (Some(cost), Some(pass)) = (stats[id].cost_per_row(), stats[id].pass_rate()) + else { + continue; + }; + total += weight * cost; + weight *= pass; + } + total +} + +#[cfg(test)] +mod tests { + use super::*; + + use arrow::array::Int32Array; + use arrow::datatypes::{DataType, Field, Schema}; + use datafusion_expr::Operator; + use datafusion_physical_expr::expressions::{binary, col, lit}; + + fn schema() -> Arc { + Arc::new(Schema::new(vec![ + Field::new("a", DataType::Int32, false), + Field::new("b", DataType::Int32, false), + ])) + } + + fn batch(schema: &Arc, a: Vec, b: Vec) -> RecordBatch { + RecordBatch::try_new( + Arc::clone(schema), + vec![Arc::new(Int32Array::from(a)), Arc::new(Int32Array::from(b))], + ) + .unwrap() + } + + /// `a > 2 AND b < 5` + fn predicate(schema: &Arc) -> Arc { + let left = + binary(col("a", schema).unwrap(), Operator::Gt, lit(2i32), schema).unwrap(); + let right = + binary(col("b", schema).unwrap(), Operator::Lt, lit(5i32), schema).unwrap(); + binary(left, Operator::And, right, schema).unwrap() + } + + fn conjuncts(schema: &Arc) -> Vec> { + split_conjunction(&predicate(schema)) + .into_iter() + .map(Arc::clone) + .collect() + } + + fn passing_rows(mask: &ArrayRef) -> Vec { + let mask = as_boolean_array(mask).unwrap(); + (0..mask.len()) + .filter(|&i| !mask.is_null(i) && mask.value(i)) + .collect() + } + + fn stats(rows: u64, matched: u64, nanos: u64) -> ConjunctStats { + ConjunctStats { + rows, + matched, + nanos, + } + } + + #[test] + fn disabled_is_not_adaptive() { + let schema = schema(); + assert!(AdaptiveConjunction::try_new(&predicate(&schema), false).is_none()); + } + + #[test] + fn single_conjunct_is_not_adaptive() { + let schema = schema(); + let p = + binary(col("a", &schema).unwrap(), Operator::Gt, lit(2i32), &schema).unwrap(); + assert!(AdaptiveConjunction::try_new(&p, true).is_none()); + } + + #[test] + fn two_conjuncts_are_adaptive() { + let schema = schema(); + let adaptive = AdaptiveConjunction::try_new(&predicate(&schema), true).unwrap(); + assert_eq!(adaptive.conjuncts.len(), 2); + assert_eq!(adaptive.order, vec![0, 1]); + assert_eq!(adaptive.warmup_left, WARMUP_BATCHES); + } + + #[test] + fn ranks_by_discards_per_nanosecond() { + // id 0: cheap (1ns/row), unselective (pass 0.9): eff = 0.1 / 1 = 0.1 + // id 1: expensive (5ns/row), selective (pass 0.01): eff = 0.99 / 5 = 0.198 (first) + // id 2: cheap (1ns/row), very unselective (pass 0.95): eff = 0.05 / 1 = 0.05 (last) + let s = vec![ + stats(1000, 900, 1000), + stats(1000, 10, 5000), + stats(1000, 950, 1000), + ]; + assert_eq!(rank_by_effectiveness(&s), vec![1, 0, 2]); + } + + #[test] + fn unmeasured_conjuncts_sort_last() { + let s = vec![ + stats(0, 0, 0), // unmeasured -> last + stats(1000, 10, 1000), // selective + stats(1000, 900, 1000), // unselective + ]; + assert_eq!(rank_by_effectiveness(&s), vec![1, 2, 0]); + } + + #[test] + fn expected_cost_weights_by_upstream_pass_rate() { + // a: cost 1, pass 0.5 ; b: cost 10, pass 0.5 + let s = vec![stats(1000, 500, 1000), stats(1000, 500, 10_000)]; + // order [0,1]: 1 + 0.5*10 = 6 + assert!((expected_cost_per_row(&s, &[0, 1]) - 6.0).abs() < 1e-9); + // order [1,0]: 10 + 0.5*1 = 10.5 + assert!((expected_cost_per_row(&s, &[1, 0]) - 10.5).abs() < 1e-9); + } + + /// The compact-once loop returns exactly the rows the plain predicate + /// keeps, in any order and whether or not compaction triggers. + #[test] + fn eval_conjuncts_matches_predicate_in_any_order() { + let schema = schema(); + let cs = conjuncts(&schema); + let p = predicate(&schema); + + // A batch where `b < 5` is rare (forces a compaction) and one where it + // is common (no compaction). + for b in [ + (0..100).map(|x| x % 50).collect::>(), // b<5 rare + (0..100).map(|x| x % 3).collect::>(), // b<5 common + ] { + let a: Vec = (0..100).collect(); + let rb = batch(&schema, a, b); + let want = p.evaluate(&rb).unwrap().into_array(rb.num_rows()).unwrap(); + for order in [vec![0, 1], vec![1, 0]] { + let got = eval_conjuncts(&cs, &order, &rb, None).unwrap(); + assert_eq!(passing_rows(&got), passing_rows(&want), "order {order:?}"); + } + } + } + + /// Across the warm-up boundary the mask must always equal the plain + /// predicate's, before and after the order settles. + #[test] + fn evaluate_matches_predicate_across_warmup() { + let schema = schema(); + let p = predicate(&schema); + let mut adaptive = AdaptiveConjunction::try_new(&p, true).unwrap(); + + for round in 0..(WARMUP_BATCHES as i32 + 4) { + let base = round * 10; + let a: Vec = (base..base + 10).collect(); + let b: Vec = (base..base + 10).map(|x| x.rem_euclid(9)).collect(); + let rb = batch(&schema, a, b); + + let got = adaptive.evaluate(&rb).unwrap(); + let want = p.evaluate(&rb).unwrap().into_array(rb.num_rows()).unwrap(); + assert_eq!( + passing_rows(&got), + passing_rows(&want), + "mismatch on round {round}" + ); + } + assert_eq!(adaptive.warmup_left, 0); + } + + /// A reorder is adopted only when materially cheaper; an already-good order + /// is left untouched. + #[test] + fn settle_keeps_order_when_not_materially_better() { + let schema = schema(); + let mut adaptive = + AdaptiveConjunction::try_new(&predicate(&schema), true).unwrap(); + // Two equally cheap, equally selective conjuncts: swapping cannot help. + adaptive.stats = vec![stats(1000, 500, 1000), stats(1000, 500, 1000)]; + adaptive.settle(); + assert_eq!(adaptive.order, vec![0, 1]); + } + + #[test] + fn settle_adopts_materially_cheaper_order() { + let schema = schema(); + let mut adaptive = + AdaptiveConjunction::try_new(&predicate(&schema), true).unwrap(); + // id 1 is far more selective and equally cheap: it should move first. + adaptive.stats = vec![stats(1000, 900, 1000), stats(1000, 10, 1000)]; + adaptive.settle(); + assert_eq!(adaptive.order, vec![1, 0]); + } +} diff --git a/datafusion/physical-plan/src/filter.rs b/datafusion/physical-plan/src/filter.rs index ae87168ec7598..ae01a03ea7cbe 100644 --- a/datafusion/physical-plan/src/filter.rs +++ b/datafusion/physical-plan/src/filter.rs @@ -28,6 +28,7 @@ use super::{ ColumnStatistics, DisplayAs, ExecutionPlanProperties, PlanProperties, RecordBatchStream, SendableRecordBatchStream, Statistics, }; +use crate::adaptive_filter::AdaptiveConjunction; use crate::coalesce::{LimitedBatchCoalescer, PushBatchStatus}; use crate::common::can_project; use crate::execution_plan::{CardinalityEffect, replace_children_if_necessary}; @@ -646,9 +647,18 @@ impl ExecutionPlan for FilterExec { context.task_id() ); let metrics = FilterExecMetrics::new(&self.metrics, partition); + let adaptive = AdaptiveConjunction::try_new( + &self.predicate, + context + .session_config() + .options() + .execution + .adaptive_filter_reordering, + ); Ok(Box::pin(FilterExecStream { schema: self.schema(), predicate: Arc::clone(&self.predicate), + adaptive, input: self.input.execute(partition, context)?, metrics, projection: self.projection.clone(), @@ -1356,6 +1366,10 @@ struct FilterExecStream { schema: SchemaRef, /// The expression to filter on. This expression must evaluate to a boolean value. predicate: Arc, + /// When set, the predicate is a reorderable conjunction evaluated + /// adaptively (conjuncts measured, then reordered) instead of via + /// `predicate`. + adaptive: Option, /// The input partition to filter. input: SendableRecordBatchStream, /// Runtime metrics recording @@ -1451,9 +1465,15 @@ impl Stream for FilterExecStream { } Some(Ok(batch)) => { let timer = elapsed_compute.timer(); - let status = self.predicate.as_ref() - .evaluate(&batch) - .and_then(|v| v.into_array(batch.num_rows())) + let array = match self.adaptive.as_mut() { + Some(adaptive) => adaptive.evaluate(&batch), + None => self + .predicate + .as_ref() + .evaluate(&batch) + .and_then(|v| v.into_array(batch.num_rows())), + }; + let status = array .and_then(|array| { Ok(match self.projection.as_ref() { Some(projection) => { diff --git a/datafusion/physical-plan/src/lib.rs b/datafusion/physical-plan/src/lib.rs index 5ff6cec374ee1..fb87e9c118c2f 100644 --- a/datafusion/physical-plan/src/lib.rs +++ b/datafusion/physical-plan/src/lib.rs @@ -67,6 +67,7 @@ mod render_tree; mod topk; mod visitor; +mod adaptive_filter; pub mod aggregates; pub mod analyze; pub mod async_func; diff --git a/datafusion/sqllogictest/test_files/adaptive_filter.slt b/datafusion/sqllogictest/test_files/adaptive_filter.slt new file mode 100644 index 0000000000000..b0e2d958a3be9 --- /dev/null +++ b/datafusion/sqllogictest/test_files/adaptive_filter.slt @@ -0,0 +1,80 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Tests for execution.adaptive_filter_reordering. Runtime reordering of a +# conjunction must never change query results, only evaluation order. + +statement ok +CREATE TABLE t AS +SELECT + i AS a, + i % 7 AS b, + arrow_cast(i, 'Utf8') AS s +FROM generate_series(1, 1000) AS tbl(i); + +# Baseline (flag off): multi-conjunct filter mixing a cheap comparison with an +# expensive LIKE. +query I +SELECT count(*) FROM t WHERE b = 3 AND s LIKE '1%'; +---- +17 + +statement ok +SET datafusion.execution.adaptive_filter_reordering = true; + +# Same query with adaptive reordering on must return the same result. +query I +SELECT count(*) FROM t WHERE b = 3 AND s LIKE '1%'; +---- +17 + +# A three-conjunct predicate, including an expensive-but-selective LIKE. +query I +SELECT count(*) FROM t WHERE a > 100 AND s LIKE '5%' AND b <> 0; +---- +86 + +# Full row materialization (not just count) is unchanged by reordering. +query IIT +SELECT a, b, s FROM t WHERE b = 1 AND a > 990 AND s LIKE '99%' ORDER BY a; +---- +995 1 995 + +# EXPLAIN: runtime reordering is invisible to the plan — the FilterExec +# predicate is unchanged whether the flag is on or off. +query TT +EXPLAIN SELECT count(*) FROM t WHERE b = 3 AND s LIKE '1%'; +---- +logical_plan +01)Projection: count(Int64(1)) AS count(*) +02)--Aggregate: groupBy=[[]], aggr=[[count(Int64(1))]] +03)----Projection: +04)------Filter: t.b = Int64(3) AND t.s LIKE Utf8("1%") +05)--------TableScan: t projection=[b, s] +physical_plan +01)ProjectionExec: expr=[count(Int64(1))@0 as count(*)] +02)--AggregateExec: mode=Final, gby=[], aggr=[count(Int64(1))] +03)----CoalescePartitionsExec +04)------AggregateExec: mode=Partial, gby=[], aggr=[count(Int64(1))] +05)--------FilterExec: b@0 = 3 AND s@1 LIKE 1%, projection=[] +06)----------DataSourceExec: partitions=4, partition_sizes=[1, 0, 0, 0] + +statement ok +SET datafusion.execution.adaptive_filter_reordering = false; + +statement ok +DROP TABLE t; diff --git a/datafusion/sqllogictest/test_files/information_schema.slt b/datafusion/sqllogictest/test_files/information_schema.slt index b270eba99d7b0..69bfcbcea4114 100644 --- a/datafusion/sqllogictest/test_files/information_schema.slt +++ b/datafusion/sqllogictest/test_files/information_schema.slt @@ -213,6 +213,7 @@ datafusion.catalog.has_header true datafusion.catalog.information_schema true datafusion.catalog.location NULL datafusion.catalog.newlines_in_values false +datafusion.execution.adaptive_filter_reordering false datafusion.execution.batch_size 8192 datafusion.execution.coalesce_batches true datafusion.execution.collect_statistics true @@ -374,6 +375,7 @@ datafusion.catalog.has_header true Default value for `format.has_header` for `CR datafusion.catalog.information_schema true Should DataFusion provide access to `information_schema` virtual tables for displaying schema information datafusion.catalog.location NULL Location scanned to load tables for `default` schema datafusion.catalog.newlines_in_values false Specifies whether newlines in (quoted) CSV values are supported. This is the default value for `format.newlines_in_values` for `CREATE EXTERNAL TABLE` if not specified explicitly in the statement. Parsing newlines in quoted values may be affected by execution behaviour such as parallel file scanning. Setting this to `true` ensures that newlines in values are parsed successfully, which may reduce performance. +datafusion.execution.adaptive_filter_reordering false (experimental) When enabled, `FilterExec` adaptively reorders the conjuncts of a conjunctive predicate at runtime. It measures each conjunct's selectivity and evaluation cost on the rows that reach it and runs the conjuncts that discard the most rows per unit of CPU time first, so cheap-and-selective predicates gate expensive ones. Reordering never changes query results (only the evaluation order of a conjunction) but can change observable side effects of fallible predicates, so it is off by default. Predicates containing volatile expressions are never reordered. datafusion.execution.batch_size 8192 Default batch size while creating new batches, it's especially useful for buffer-in-memory batches since creating tiny batches would result in too much metadata memory consumption datafusion.execution.coalesce_batches true When set to true, record batches will be examined between each operator and small batches will be coalesced into larger batches. This is helpful when there are highly selective filters or joins that could produce tiny output batches. The target batch size is determined by the configuration setting datafusion.execution.collect_statistics true Should DataFusion collect statistics when first creating a table. Has no effect after the table is created. Defaults to true. diff --git a/docs/source/user-guide/configs.md b/docs/source/user-guide/configs.md index 0085d4ac7c1fa..4fe049662a050 100644 --- a/docs/source/user-guide/configs.md +++ b/docs/source/user-guide/configs.md @@ -142,6 +142,7 @@ The following configuration settings are available: | datafusion.execution.skip_partial_aggregation_probe_rows_threshold | 100000 | Number of input rows partial aggregation partition should process, before aggregation ratio check and trying to switch to skipping aggregation mode | | datafusion.execution.use_row_number_estimates_to_optimize_partitioning | false | Should DataFusion use row number estimates at the input to decide whether increasing parallelism is beneficial or not. By default, only exact row numbers (not estimates) are used for this decision. Setting this flag to `true` will likely produce better plans. if the source of statistics is accurate. We plan to make this the default in the future. | | datafusion.execution.enforce_batch_size_in_joins | false | Should DataFusion enforce batch size in joins or not. By default, DataFusion will not enforce batch size in joins. Enforcing batch size in joins can reduce memory usage when joining large tables with a highly-selective join filter, but is also slightly slower. Note: this option currently only applies to the symmetric hash join. | +| datafusion.execution.adaptive_filter_reordering | false | (experimental) When enabled, `FilterExec` adaptively reorders the conjuncts of a conjunctive predicate at runtime. It measures each conjunct's selectivity and evaluation cost on the rows that reach it and runs the conjuncts that discard the most rows per unit of CPU time first, so cheap-and-selective predicates gate expensive ones. Reordering never changes query results (only the evaluation order of a conjunction) but can change observable side effects of fallible predicates, so it is off by default. Predicates containing volatile expressions are never reordered. | | datafusion.execution.objectstore_writer_buffer_size | 10485760 | Size (bytes) of data buffer DataFusion uses when writing output files. This affects the size of the data chunks that are uploaded to remote object stores (e.g. AWS S3). If very large (>= 100 GiB) output files are being written, it may be necessary to increase this size to avoid errors from the remote end point. | | datafusion.execution.enable_ansi_mode | false | Whether to enable ANSI SQL mode. The flag is experimental and relevant only for DataFusion Spark built-in functions When `enable_ansi_mode` is set to `true`, the query engine follows ANSI SQL semantics for expressions, casting, and error handling. This means: - **Strict type coercion rules:** implicit casts between incompatible types are disallowed. - **Standard SQL arithmetic behavior:** operations such as division by zero, numeric overflow, or invalid casts raise runtime errors rather than returning `NULL` or adjusted values. - **Consistent ANSI behavior** for string concatenation, comparisons, and `NULL` handling. When `enable_ansi_mode` is `false` (the default), the engine uses a more permissive, non-ANSI mode designed for user convenience and backward compatibility. In this mode: - Implicit casts between types are allowed (e.g., string to integer when possible). - Arithmetic operations are more lenient — for example, `abs()` on the minimum representable integer value returns the input value instead of raising overflow. - Division by zero or invalid casts may return `NULL` instead of failing. # Default `false` — ANSI SQL mode is disabled by default. | | datafusion.execution.hash_join_buffering_capacity | 0 | How many bytes to buffer in the probe side of hash joins while the build side is concurrently being built. Without this, hash joins will wait until the full materialization of the build side before polling the probe side. This is useful in scenarios where the query is not completely CPU bounded, allowing to do some early work concurrently and reducing the latency of the query. Note that when hash join buffering is enabled, the probe side will start eagerly polling data, not giving time for the producer side of dynamic filters to produce any meaningful predicate. Queries with dynamic filters might see performance degradation. Disabled by default, set to a number greater than 0 for enabling it. | From ffe07f460fd8abb12bc16c4d36fbee4e67fea045 Mon Sep 17 00:00:00 2001 From: Adrian Garcia Badaracco <1755071+adriangb@users.noreply.github.com> Date: Mon, 29 Jun 2026 00:32:00 +0200 Subject: [PATCH 02/27] feat(physical-plan): pool adaptive measurements across partition streams MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A `FilterExec` is split across many partition streams, each seeing a slice of the data. With per-stream warm-up, every stream pays its own measurement cost, and when each stream is only a handful of batches long that warm-up is most of its work — so the reordering win never materialises (and the warm-up overhead shows up as a regression). Benchmarked: at 12 partitions the costsel_q01 win collapsed from -67% (single stream) to -14%. Share the measurements. `AdaptiveFilterShared` holds a per-conjunct stats pool plus a settled-order epoch, common to every stream of one `FilterExec`. Each stream measures a batch into a local accumulator and folds it into the pool; the first stream to reach `WARMUP_BATCHES` pooled batches decides the order and publishes it by bumping the epoch. Other streams poll the epoch with one relaxed atomic load per batch and adopt the published order without paying warm-up. The warm-up is thus paid ~once per query, not once per stream. Restores the recovered win at default partitioning: width -56..-66%, costsel_q01 -58%, cardinality k16 -26% (was -12%, -14%, -6% without sharing), matching or beating the full design. Steady-state regressions on conjunctions where compaction does not pay (neutral_q61 ~+11%, cardinality k4 ~+5%, costsel_q02/q03 ~+3-5%) remain — a Fused-vs-CompactOnce guard addresses those in the next commit. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01Lh7i9DyeFWuTFWjogrVNkb --- .../physical-plan/src/adaptive_filter.rs | 233 ++++++++++++++---- datafusion/physical-plan/src/filter.rs | 15 +- 2 files changed, 199 insertions(+), 49 deletions(-) diff --git a/datafusion/physical-plan/src/adaptive_filter.rs b/datafusion/physical-plan/src/adaptive_filter.rs index 46e76d6ddc591..b444b1a7c1704 100644 --- a/datafusion/physical-plan/src/adaptive_filter.rs +++ b/datafusion/physical-plan/src/adaptive_filter.rs @@ -56,6 +56,8 @@ //! top of it. use std::sync::Arc; +use std::sync::Mutex; +use std::sync::atomic::{AtomicU64, Ordering}; use arrow::array::{Array, ArrayRef, BooleanArray, BooleanBufferBuilder, UInt32Array}; use arrow::buffer::BooleanBuffer; @@ -104,6 +106,14 @@ impl ConjunctStats { self.nanos += nanos; } + /// Fold another accumulator's counts into this one (they are plain sums, so + /// merging is addition). Used to pool measurements across partition streams. + fn merge(&mut self, other: &Self) { + self.rows += other.rows; + self.matched += other.matched; + self.nanos += other.nanos; + } + /// Fraction of rows that pass, or `None` if never evaluated on any row. fn pass_rate(&self) -> Option { (self.rows > 0).then(|| self.matched as f64 / self.rows as f64) @@ -127,19 +137,61 @@ impl ConjunctStats { } } +/// State shared by every partition stream of one `FilterExec`, so the streams +/// learn as one: per-conjunct measurements are pooled across streams and the +/// first stream to accumulate enough samples settles the order for all of them. +/// +/// This matters because a `FilterExec` is split across many partition streams, +/// each seeing only a slice of the data. Without sharing, every stream pays its +/// own warm-up — and when each stream is only a handful of batches long, that +/// warm-up is most of its work, so the reordering win never materialises. With +/// sharing the warm-up is paid roughly once per query, not once per stream. +#[derive(Debug, Default)] +pub(crate) struct AdaptiveFilterShared { + /// `0` until an order is published; bumped once when the first stream + /// settles. Streams poll it with one relaxed atomic load per batch. + epoch: AtomicU64, + inner: Mutex, +} + +#[derive(Debug, Default)] +struct SharedInner { + /// Per-conjunct counts pooled across all streams (indexed by conjunct + /// position). Empty until the first measured batch sizes it. + stats: Vec, + /// Measured batches contributed by all streams so far. + measured_batches: u64, + /// The settled evaluation order, once decided; `None` while learning. + settled: Option>, +} + +impl AdaptiveFilterShared { + pub(crate) fn new() -> Self { + Self::default() + } + + /// The published settled order, or `None` if the streams are still learning. + fn settled_order(&self) -> Option> { + self.inner.lock().expect("poisoned").settled.clone() + } +} + /// Adaptive evaluator for a single conjunctive predicate, owned per partition -/// stream (single-threaded, no locking). +/// stream. Measurements are pooled into the shared [`AdaptiveFilterShared`]; +/// the per-stream state is just the current order and how far it has caught up. #[derive(Debug)] pub(crate) struct AdaptiveConjunction { - /// The split conjuncts. `stats`/`order` indices refer to positions here. + /// The split conjuncts. `order` indices refer to positions here. conjuncts: Vec>, - /// Per-conjunct measurements, indexed by conjunct position. - stats: Vec, - /// Evaluation order: indices into `conjuncts`. Starts as the written order - /// and may be permuted once after the warm-up. + /// Measurements and the settled order, shared by every partition stream. + shared: Arc, + /// Evaluation order: indices into `conjuncts`. The written order until a + /// settled order is adopted. order: Vec, - /// Warm-up batches still to measure; `0` means the order has settled. - warmup_left: u64, + /// Shared epoch this stream has caught up to. + epoch_seen: u64, + /// Whether the order is settled (frozen): this stream no longer measures. + settled: bool, } impl AdaptiveConjunction { @@ -149,9 +201,13 @@ impl AdaptiveConjunction { /// - `enabled` is false (the config flag is off); /// - the predicate has fewer than two `AND` conjuncts (nothing to reorder); /// - any conjunct is volatile (reordering could change side effects). + /// + /// `shared` is the state common to all partition streams of the owning + /// `FilterExec`. pub(crate) fn try_new( predicate: &Arc, enabled: bool, + shared: Arc, ) -> Option { if !enabled { return None; @@ -163,47 +219,83 @@ impl AdaptiveConjunction { if conjuncts.len() < 2 || conjuncts.iter().any(is_volatile) { return None; } - let stats = vec![ConjunctStats::default(); conjuncts.len()]; let order = (0..conjuncts.len()).collect(); Some(Self { conjuncts, - stats, + shared, order, - warmup_left: WARMUP_BATCHES, + epoch_seen: 0, + settled: false, }) } /// Evaluate the conjunction against `batch`, returning the boolean mask /// (over the batch's rows) of rows that passed every conjunct. /// - /// During the warm-up the conjuncts are measured (on the rows that reach - /// each one); once the window ends the order is settled and subsequent - /// batches evaluate with no instrumentation. + /// Until the order settles, each batch is measured and its counts pooled + /// into the shared registry; a stream adopts the settled order another + /// stream published as soon as it sees the epoch advance. pub(crate) fn evaluate(&mut self, batch: &RecordBatch) -> Result { - if self.warmup_left == 0 { + // Adopt a settled order another stream published since we last looked: + // one relaxed atomic load per batch, a lock only on the transition. + if !self.settled { + let epoch = self.shared.epoch.load(Ordering::Acquire); + if epoch != self.epoch_seen { + self.epoch_seen = epoch; + if let Some(order) = self.shared.settled_order() { + self.order = order; + self.settled = true; + } + } + } + if self.settled { return eval_conjuncts(&self.conjuncts, &self.order, batch, None); } + + // Measure this batch into a local accumulator, then pool it. + let mut local = vec![ConjunctStats::default(); self.conjuncts.len()]; let result = - eval_conjuncts(&self.conjuncts, &self.order, batch, Some(&mut self.stats))?; - self.warmup_left -= 1; - if self.warmup_left == 0 { - self.settle(); - } + eval_conjuncts(&self.conjuncts, &self.order, batch, Some(&mut local))?; + self.pool_and_maybe_settle(&local); Ok(result) } - /// Rank the conjuncts by measured effectiveness and, if the resulting order - /// is materially cheaper than the written order, adopt it. - fn settle(&mut self) { - let candidate = rank_by_effectiveness(&self.stats); - if candidate == self.order { - return; + /// Merge this batch's measurements into the shared pool and, once enough + /// batches have accrued across all streams, decide and publish the order. + fn pool_and_maybe_settle(&mut self, local: &[ConjunctStats]) { + let mut inner = self.shared.inner.lock().expect("poisoned"); + if inner.stats.len() != local.len() { + inner.stats = vec![ConjunctStats::default(); local.len()]; } - let candidate_cost = expected_cost_per_row(&self.stats, &candidate); - let current_cost = expected_cost_per_row(&self.stats, &self.order); - if candidate_cost < (1.0 - TIE_COST_FRACTION) * current_cost { - self.order = candidate; + for (s, l) in inner.stats.iter_mut().zip(local) { + s.merge(l); + } + inner.measured_batches += 1; + if inner.settled.is_some() || inner.measured_batches < WARMUP_BATCHES { + return; } + let order = settle_order(&inner.stats); + inner.settled = Some(order.clone()); + drop(inner); + self.order = order; + self.settled = true; + self.shared.epoch.fetch_add(1, Ordering::Release); + } +} + +/// Rank the conjuncts by effectiveness and adopt the ranking only if it is +/// materially cheaper than the written order; otherwise keep the written order +/// (so interchangeable conjuncts are never reshuffled). +fn settle_order(stats: &[ConjunctStats]) -> Vec { + let identity: Vec = (0..stats.len()).collect(); + let candidate = rank_by_effectiveness(stats); + if candidate != identity + && expected_cost_per_row(stats, &candidate) + < (1.0 - TIE_COST_FRACTION) * expected_cost_per_row(stats, &identity) + { + candidate + } else { + identity } } @@ -407,10 +499,22 @@ mod tests { } } + /// `try_new` with a fresh, unshared registry. + fn try_new( + predicate: &Arc, + enabled: bool, + ) -> Option { + AdaptiveConjunction::try_new( + predicate, + enabled, + Arc::new(AdaptiveFilterShared::new()), + ) + } + #[test] fn disabled_is_not_adaptive() { let schema = schema(); - assert!(AdaptiveConjunction::try_new(&predicate(&schema), false).is_none()); + assert!(try_new(&predicate(&schema), false).is_none()); } #[test] @@ -418,16 +522,16 @@ mod tests { let schema = schema(); let p = binary(col("a", &schema).unwrap(), Operator::Gt, lit(2i32), &schema).unwrap(); - assert!(AdaptiveConjunction::try_new(&p, true).is_none()); + assert!(try_new(&p, true).is_none()); } #[test] fn two_conjuncts_are_adaptive() { let schema = schema(); - let adaptive = AdaptiveConjunction::try_new(&predicate(&schema), true).unwrap(); + let adaptive = try_new(&predicate(&schema), true).unwrap(); assert_eq!(adaptive.conjuncts.len(), 2); assert_eq!(adaptive.order, vec![0, 1]); - assert_eq!(adaptive.warmup_left, WARMUP_BATCHES); + assert!(!adaptive.settled); } #[test] @@ -493,7 +597,7 @@ mod tests { fn evaluate_matches_predicate_across_warmup() { let schema = schema(); let p = predicate(&schema); - let mut adaptive = AdaptiveConjunction::try_new(&p, true).unwrap(); + let mut adaptive = try_new(&p, true).unwrap(); for round in 0..(WARMUP_BATCHES as i32 + 4) { let base = round * 10; @@ -509,30 +613,63 @@ mod tests { "mismatch on round {round}" ); } - assert_eq!(adaptive.warmup_left, 0); + assert!(adaptive.settled); } /// A reorder is adopted only when materially cheaper; an already-good order /// is left untouched. #[test] fn settle_keeps_order_when_not_materially_better() { - let schema = schema(); - let mut adaptive = - AdaptiveConjunction::try_new(&predicate(&schema), true).unwrap(); // Two equally cheap, equally selective conjuncts: swapping cannot help. - adaptive.stats = vec![stats(1000, 500, 1000), stats(1000, 500, 1000)]; - adaptive.settle(); - assert_eq!(adaptive.order, vec![0, 1]); + let s = vec![stats(1000, 500, 1000), stats(1000, 500, 1000)]; + assert_eq!(settle_order(&s), vec![0, 1]); } #[test] fn settle_adopts_materially_cheaper_order() { - let schema = schema(); - let mut adaptive = - AdaptiveConjunction::try_new(&predicate(&schema), true).unwrap(); // id 1 is far more selective and equally cheap: it should move first. - adaptive.stats = vec![stats(1000, 900, 1000), stats(1000, 10, 1000)]; - adaptive.settle(); - assert_eq!(adaptive.order, vec![1, 0]); + let s = vec![stats(1000, 900, 1000), stats(1000, 10, 1000)]; + assert_eq!(settle_order(&s), vec![1, 0]); + } + + /// Two streams sharing one registry settle the order together: the pooled + /// warm-up is `WARMUP_BATCHES` total across both streams, and once one + /// stream publishes the order the other adopts it on its next batch. + #[test] + fn streams_pool_measurements_and_share_settled_order() { + let schema = schema(); + let p = predicate(&schema); + let shared = Arc::new(AdaptiveFilterShared::new()); + let mut s1 = AdaptiveConjunction::try_new(&p, true, Arc::clone(&shared)).unwrap(); + let mut s2 = AdaptiveConjunction::try_new(&p, true, Arc::clone(&shared)).unwrap(); + + // `b < 5` (conjunct 1) is the selective one; drive both streams with + // batches where it keeps ~1 row in 25. + let mk = |round: i32| { + let base = round * 100; + let a: Vec = (base..base + 100).collect(); + let b: Vec = (base..base + 100).map(|x| x.rem_euclid(25)).collect(); + batch(&schema, a, b) + }; + + // Alternate the two streams for `WARMUP_BATCHES` pooled batches; the + // order settles partway through and both streams must end settled. + for round in 0..(WARMUP_BATCHES as i32) { + let rb = mk(round); + for s in [&mut s1, &mut s2] { + let got = s.evaluate(&rb).unwrap(); + let want = p.evaluate(&rb).unwrap().into_array(rb.num_rows()).unwrap(); + assert_eq!(passing_rows(&got), passing_rows(&want)); + } + } + + assert!(shared.settled_order().is_some()); + // One more batch each lets a not-yet-settled stream adopt the epoch. + s1.evaluate(&mk(99)).unwrap(); + s2.evaluate(&mk(99)).unwrap(); + assert!(s1.settled && s2.settled); + // The selective conjunct was promoted to the front for both. + assert_eq!(s1.order, vec![1, 0]); + assert_eq!(s2.order, vec![1, 0]); } } diff --git a/datafusion/physical-plan/src/filter.rs b/datafusion/physical-plan/src/filter.rs index ae01a03ea7cbe..ab3951f50d4bd 100644 --- a/datafusion/physical-plan/src/filter.rs +++ b/datafusion/physical-plan/src/filter.rs @@ -28,7 +28,7 @@ use super::{ ColumnStatistics, DisplayAs, ExecutionPlanProperties, PlanProperties, RecordBatchStream, SendableRecordBatchStream, Statistics, }; -use crate::adaptive_filter::AdaptiveConjunction; +use crate::adaptive_filter::{AdaptiveConjunction, AdaptiveFilterShared}; use crate::coalesce::{LimitedBatchCoalescer, PushBatchStatus}; use crate::common::can_project; use crate::execution_plan::{CardinalityEffect, replace_children_if_necessary}; @@ -101,6 +101,10 @@ pub struct FilterExec { batch_size: usize, /// Number of rows to fetch fetch: Option, + /// Measurements shared by all partition streams, used by adaptive conjunct + /// reordering (see [`AdaptiveConjunction`]) so the streams learn as one. + /// Fresh per plan node; never affects the plan. + adaptive_stats: Arc, } /// Builder for [`FilterExec`] to set optional parameters @@ -224,6 +228,7 @@ impl FilterExecBuilder { projection: self.projection, batch_size: self.batch_size, fetch: self.fetch, + adaptive_stats: Arc::new(AdaptiveFilterShared::new()), }) } } @@ -300,6 +305,7 @@ impl FilterExec { projection: self.projection.clone(), batch_size, fetch: self.fetch, + adaptive_stats: Arc::clone(&self.adaptive_stats), }) } @@ -654,6 +660,7 @@ impl ExecutionPlan for FilterExec { .options() .execution .adaptive_filter_reordering, + Arc::clone(&self.adaptive_stats), ); Ok(Box::pin(FilterExecStream { schema: self.schema(), @@ -865,6 +872,9 @@ impl ExecutionPlan for FilterExec { projection: self.projection.clone(), batch_size: self.batch_size, fetch: self.fetch, + // The predicate changed; pooled per-conjunct stats no longer + // describe it. + adaptive_stats: Arc::new(AdaptiveFilterShared::new()), }; Some(Arc::new(new) as _) }; @@ -889,6 +899,7 @@ impl ExecutionPlan for FilterExec { projection: self.projection.clone(), batch_size: self.batch_size, fetch, + adaptive_stats: Arc::clone(&self.adaptive_stats), })) } @@ -925,6 +936,8 @@ impl ExecutionPlan for FilterExec { projection, batch_size, fetch, + // Per-execution adaptive measurements, not part of the plan shape. + adaptive_stats: _, } = self; let input_node = ctx.encode_child(input)?; let expr = ctx.encode_expr(predicate)?; From e2055c460616ef8aa4fe647e8fdcaba6910ad1ad Mon Sep 17 00:00:00 2001 From: Adrian Garcia Badaracco <1755071+adriangb@users.noreply.github.com> Date: Mon, 29 Jun 2026 01:41:00 +0200 Subject: [PATCH 03/27] feat(physical-plan): use compact-once only in service of a reorder MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The compact-once loop wins by gating expensive conjuncts behind a selective one, but its per-conjunct bookkeeping (mask AND, true_count, the compaction copy) is pure overhead when there is nothing to gate. On a conjunction of interchangeable predicates — several equally expensive, equally unselective regexps, say — the warm-up settles on the written order (nothing to reorder) yet still paid compact-once on every batch, regressing ~11% vs the plain predicate (predicate_eval neutral_q61). Guard it: compact-once is adopted only when the warm-up actually reorders the conjuncts. When the settled order equals the written order, evaluate the predicate as-is — byte-for-byte the flag-off path, zero overhead. Since every real win reorders (a selective conjunct moves toward the front), this keeps the full win while removing the no-reorder regression. predicate_eval (vs flag off): neutral_q61 +11% -> ~0; wins preserved (costsel_q01 -60%, width -58..-66%, cardinality k16 -31%). A small residual remains on low-cardinality cheap conjunctions that do reorder (k4/k8 ~+3-4%), where compaction's cost is not repaid by gating so few/cheap predicates; the full champion/challenger arbiter regresses these more (~+10%), so a heavier guard is not worth it here. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01Lh7i9DyeFWuTFWjogrVNkb --- .../physical-plan/src/adaptive_filter.rs | 160 ++++++++++++++---- 1 file changed, 131 insertions(+), 29 deletions(-) diff --git a/datafusion/physical-plan/src/adaptive_filter.rs b/datafusion/physical-plan/src/adaptive_filter.rs index b444b1a7c1704..ca4a683d4e9fa 100644 --- a/datafusion/physical-plan/src/adaptive_filter.rs +++ b/datafusion/physical-plan/src/adaptive_filter.rs @@ -45,15 +45,31 @@ //! independent conjuncts), and if the ranked order is *materially* cheaper than //! the written one it is adopted. The order then stays fixed. //! +//! Compact-once is used **only in service of a reorder**: if the warm-up does +//! not reorder the conjuncts (e.g. they are interchangeable), the written +//! predicate is evaluated as-is, so a conjunction that does not benefit from +//! reordering pays no compact-once overhead and behaves exactly as it would +//! with the feature off. +//! +//! ## How it shares +//! +//! A `FilterExec` is split across many partition streams, each seeing only a +//! slice of the data. Measurements are pooled into a shared +//! [`AdaptiveFilterShared`] so the streams learn as one: the first stream to +//! accumulate enough samples settles the order and publishes it, and the others +//! adopt it (one relaxed atomic load per batch) without each re-paying the +//! warm-up — which is what makes the win materialise when each stream is only a +//! handful of batches long. +//! //! It is **off by default** //! (`datafusion.execution.adaptive_filter_reordering`) and never changes query //! results: a conjunction's value is independent of evaluation order. Predicates //! containing volatile expressions are never reordered (their observable side //! effects depend on order). //! -//! This is the minimal core of the adaptive evaluator. Richer policies -//! (A/B-validated adoption, cross-stream sharing, drift re-measurement) build on -//! top of it. +//! This is the core of the adaptive evaluator. Further policies (drift +//! re-measurement, confidence-interval statistics, A/B-validated adoption for +//! the cases a cost model cannot separate) can build on top of it. use std::sync::Arc; use std::sync::Mutex; @@ -161,8 +177,19 @@ struct SharedInner { stats: Vec, /// Measured batches contributed by all streams so far. measured_batches: u64, - /// The settled evaluation order, once decided; `None` while learning. - settled: Option>, + /// The settled decision, once made; `None` while learning. + settled: Option, +} + +/// The settled outcome of the warm-up: the evaluation order, and whether to run +/// it through the compact-once loop or the plain predicate. +#[derive(Debug, Clone)] +struct Settled { + /// Evaluation order: indices into the conjunct list. + order: Vec, + /// `true` to run `order` through the compact-once loop; `false` to evaluate + /// the written predicate as-is (see [`settle`]). + compact: bool, } impl AdaptiveFilterShared { @@ -170,8 +197,8 @@ impl AdaptiveFilterShared { Self::default() } - /// The published settled order, or `None` if the streams are still learning. - fn settled_order(&self) -> Option> { + /// The published settled decision, or `None` if streams are still learning. + fn settled(&self) -> Option { self.inner.lock().expect("poisoned").settled.clone() } } @@ -183,11 +210,18 @@ impl AdaptiveFilterShared { pub(crate) struct AdaptiveConjunction { /// The split conjuncts. `order` indices refer to positions here. conjuncts: Vec>, - /// Measurements and the settled order, shared by every partition stream. + /// The written predicate, evaluated as-is when the settled order does not + /// reorder it (so a settled non-reorder costs exactly what the flag-off path + /// costs — no compact-once overhead). + predicate: Arc, + /// Measurements and the settled decision, shared by every partition stream. shared: Arc, /// Evaluation order: indices into `conjuncts`. The written order until a /// settled order is adopted. order: Vec, + /// Whether the settled order runs through the compact-once loop; `false` + /// means evaluate [`predicate`](Self::predicate) directly. + compact: bool, /// Shared epoch this stream has caught up to. epoch_seen: u64, /// Whether the order is settled (frozen): this stream no longer measures. @@ -222,8 +256,10 @@ impl AdaptiveConjunction { let order = (0..conjuncts.len()).collect(); Some(Self { conjuncts, + predicate: Arc::clone(predicate), shared, order, + compact: false, epoch_seen: 0, settled: false, }) @@ -242,14 +278,13 @@ impl AdaptiveConjunction { let epoch = self.shared.epoch.load(Ordering::Acquire); if epoch != self.epoch_seen { self.epoch_seen = epoch; - if let Some(order) = self.shared.settled_order() { - self.order = order; - self.settled = true; + if let Some(decision) = self.shared.settled() { + self.adopt(decision); } } } if self.settled { - return eval_conjuncts(&self.conjuncts, &self.order, batch, None); + return self.evaluate_settled(batch); } // Measure this batch into a local accumulator, then pool it. @@ -260,6 +295,23 @@ impl AdaptiveConjunction { Ok(result) } + /// Evaluate the settled arrangement with no instrumentation: the + /// compact-once loop when the order was reordered, or the written predicate + /// directly otherwise (identical to the feature being off). + fn evaluate_settled(&self, batch: &RecordBatch) -> Result { + if self.compact { + eval_conjuncts(&self.conjuncts, &self.order, batch, None) + } else { + self.predicate.evaluate(batch)?.into_array(batch.num_rows()) + } + } + + fn adopt(&mut self, decision: Settled) { + self.order = decision.order; + self.compact = decision.compact; + self.settled = true; + } + /// Merge this batch's measurements into the shared pool and, once enough /// batches have accrued across all streams, decide and publish the order. fn pool_and_maybe_settle(&mut self, local: &[ConjunctStats]) { @@ -274,28 +326,40 @@ impl AdaptiveConjunction { if inner.settled.is_some() || inner.measured_batches < WARMUP_BATCHES { return; } - let order = settle_order(&inner.stats); - inner.settled = Some(order.clone()); + let decision = settle(&inner.stats); + inner.settled = Some(decision.clone()); drop(inner); - self.order = order; - self.settled = true; + self.adopt(decision); self.shared.epoch.fetch_add(1, Ordering::Release); } } +/// Decide the settled arrangement from the pooled measurements. +/// /// Rank the conjuncts by effectiveness and adopt the ranking only if it is -/// materially cheaper than the written order; otherwise keep the written order -/// (so interchangeable conjuncts are never reshuffled). -fn settle_order(stats: &[ConjunctStats]) -> Vec { +/// materially cheaper than the written order. A genuine reorder runs through the +/// compact-once loop (the source of the win); otherwise the written predicate is +/// kept and evaluated as-is. This is the guard: compact-once is only ever used +/// in service of a reorder, so a conjunction that does not benefit from +/// reordering (interchangeable conjuncts, e.g. several equally expensive +/// unselective predicates) pays no compact-once overhead and behaves exactly as +/// it would with the feature off. +fn settle(stats: &[ConjunctStats]) -> Settled { let identity: Vec = (0..stats.len()).collect(); let candidate = rank_by_effectiveness(stats); if candidate != identity && expected_cost_per_row(stats, &candidate) < (1.0 - TIE_COST_FRACTION) * expected_cost_per_row(stats, &identity) { - candidate + Settled { + order: candidate, + compact: true, + } } else { - identity + Settled { + order: identity, + compact: false, + } } } @@ -617,19 +681,55 @@ mod tests { } /// A reorder is adopted only when materially cheaper; an already-good order - /// is left untouched. + /// is left untouched and runs the plain predicate (no compact-once). #[test] fn settle_keeps_order_when_not_materially_better() { - // Two equally cheap, equally selective conjuncts: swapping cannot help. + // Two equally cheap, equally selective conjuncts: swapping cannot help, + // so the written order stands and compact-once is not used. let s = vec![stats(1000, 500, 1000), stats(1000, 500, 1000)]; - assert_eq!(settle_order(&s), vec![0, 1]); + let d = settle(&s); + assert_eq!(d.order, vec![0, 1]); + assert!(!d.compact); } #[test] - fn settle_adopts_materially_cheaper_order() { - // id 1 is far more selective and equally cheap: it should move first. + fn settle_adopts_materially_cheaper_order_with_compaction() { + // id 1 is far more selective and equally cheap: it should move first and + // run through the compact-once loop. let s = vec![stats(1000, 900, 1000), stats(1000, 10, 1000)]; - assert_eq!(settle_order(&s), vec![1, 0]); + let d = settle(&s); + assert_eq!(d.order, vec![1, 0]); + assert!(d.compact); + } + + /// When the order does not change, the settled evaluator runs the plain + /// predicate (compact-once is only used in service of a reorder), so an + /// interchangeable conjunction costs exactly what the flag-off path costs. + #[test] + fn no_reorder_evaluates_plain_predicate() { + let schema = schema(); + // Both conjuncts equally cheap and selective: nothing to reorder. + let left = + binary(col("a", &schema).unwrap(), Operator::Gt, lit(2i32), &schema).unwrap(); + let right = + binary(col("b", &schema).unwrap(), Operator::Gt, lit(2i32), &schema).unwrap(); + let p = binary(left, Operator::And, right, &schema).unwrap(); + let mut adaptive = try_new(&p, true).unwrap(); + + for round in 0..(WARMUP_BATCHES as i32 + 2) { + let base = round * 10; + let a: Vec = (base..base + 10).collect(); + let b: Vec = (base..base + 10).collect(); + let rb = batch(&schema, a, b); + let got = adaptive.evaluate(&rb).unwrap(); + let want = p.evaluate(&rb).unwrap().into_array(rb.num_rows()).unwrap(); + assert_eq!(passing_rows(&got), passing_rows(&want)); + } + assert!(adaptive.settled); + assert!( + !adaptive.compact, + "interchangeable conjuncts stay on the plain predicate" + ); } /// Two streams sharing one registry settle the order together: the pooled @@ -663,13 +763,15 @@ mod tests { } } - assert!(shared.settled_order().is_some()); + assert!(shared.settled().is_some()); // One more batch each lets a not-yet-settled stream adopt the epoch. s1.evaluate(&mk(99)).unwrap(); s2.evaluate(&mk(99)).unwrap(); assert!(s1.settled && s2.settled); - // The selective conjunct was promoted to the front for both. + // The selective conjunct was promoted to the front for both, and the + // reorder runs through the compact-once loop. assert_eq!(s1.order, vec![1, 0]); assert_eq!(s2.order, vec![1, 0]); + assert!(s1.compact && s2.compact); } } From 8beea6cac3972444922e9b5c55e80402f0df8bc5 Mon Sep 17 00:00:00 2001 From: Adrian Garcia Badaracco <1755071+adriangb@users.noreply.github.com> Date: Fri, 3 Jul 2026 00:28:19 +0200 Subject: [PATCH 04/27] fix(physical-plan): address review feedback on adaptive filter Two points raised by @xudong963 that carried over into the compact-once rewrite of the adaptive `FilterExec` conjunct evaluator: - Replace `.expect("u32 live")` on the live-row index downcast with a let-else returning `internal_err!`, so a broken invariant surfaces as a clean error rather than a panic. - Add a `debug_assert!` documenting that live-row indices are tracked in arrow's `u32` `filter`/`take` index space, making the `num_rows as u32` cast's precondition explicit. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01R1mvYrjFyTy2kbBoGrWzT6 --- datafusion/physical-plan/src/adaptive_filter.rs | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/datafusion/physical-plan/src/adaptive_filter.rs b/datafusion/physical-plan/src/adaptive_filter.rs index ca4a683d4e9fa..9100956ddbe4c 100644 --- a/datafusion/physical-plan/src/adaptive_filter.rs +++ b/datafusion/physical-plan/src/adaptive_filter.rs @@ -80,9 +80,9 @@ use arrow::buffer::BooleanBuffer; use arrow::compute::kernels::boolean::and; use arrow::compute::{filter, filter_record_batch, prep_null_mask_filter}; use arrow::record_batch::RecordBatch; -use datafusion_common::Result; use datafusion_common::cast::as_boolean_array; use datafusion_common::instant::Instant; +use datafusion_common::{Result, internal_err}; use datafusion_physical_expr::PhysicalExpr; use datafusion_physical_expr::utils::split_conjunction; use datafusion_physical_expr_common::physical_expr::is_volatile; @@ -386,6 +386,12 @@ fn eval_conjuncts( if num_rows == 0 { return Ok(Arc::new(BooleanArray::from(Vec::::new()))); } + // Live-row indices are tracked as `u32` (arrow's `filter`/`take` index + // space), so a batch cannot exceed `u32::MAX` rows here. + debug_assert!( + num_rows <= u32::MAX as usize, + "adaptive filter: batch exceeds u32::MAX rows" + ); // `working` is the batch conjuncts are evaluated against. `acc` is the // accumulated (`AND`-combined, null-free) result over `working`'s rows since @@ -461,10 +467,11 @@ fn eval_conjuncts( Some(acc) => filter(&indices, &acc)?, None => indices, }; - let indices = indices - .as_any() - .downcast_ref::() - .expect("u32 live"); + let Some(indices) = indices.as_any().downcast_ref::() else { + return internal_err!( + "adaptive filter: live row indices are not a UInt32Array" + ); + }; let mut builder = BooleanBufferBuilder::new(num_rows); builder.append_n(num_rows, false); for &idx in indices.values() { From fdbbf14913483b57ba9ab727319abbf132f75b75 Mon Sep 17 00:00:00 2001 From: Adrian Garcia Badaracco <1755071+adriangb@users.noreply.github.com> Date: Fri, 3 Jul 2026 08:39:26 +0200 Subject: [PATCH 05/27] fix(physical-plan): reset adaptive filter state on re-execution MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit @xudong963 noted the pooled adaptive-conjunct measurements live on the `FilterExec` plan node and are reused by every `execute()` call, leaking the learned conjunct order across independent executions. Implement `ExecutionPlan::reset_state` for `FilterExec` — the sanctioned mechanism for exactly this (its trait docs cite `DynamicFilterPhysicalExpr`; `CrossJoinExec`, `HashJoinExec`, and `SortExec` use it for their build-side / dynamic-filter state). It returns a fresh node with a new `AdaptiveFilterShared` (and fresh metrics), so a re-executed plan re-learns from scratch, while preserving the still-valid predicate, input, and cached plan properties. Reordering only ever affects performance, never results, so this closes a perf-staleness gap, not a correctness bug. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01R1mvYrjFyTy2kbBoGrWzT6 --- datafusion/physical-plan/src/filter.rs | 46 ++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/datafusion/physical-plan/src/filter.rs b/datafusion/physical-plan/src/filter.rs index ab3951f50d4bd..1fa4fc397917f 100644 --- a/datafusion/physical-plan/src/filter.rs +++ b/datafusion/physical-plan/src/filter.rs @@ -641,6 +641,24 @@ impl ExecutionPlan for FilterExec { ) } + /// Reset per-execution state so an independent re-execution (e.g. a + /// recursive query) does not inherit runtime state from a prior run. + /// + /// The pooled adaptive-conjunct measurements (`AdaptiveFilterShared`) and + /// the execution metrics are the per-execution state that must be reset — + /// otherwise the adaptive reordering learned in one execution would leak + /// into the next. The predicate, input, and cached plan properties are + /// unchanged and remain valid, so they are preserved (unlike + /// [`with_new_children`](Self::with_new_children), this does not recompute + /// them). Any dynamic filters *inside* the predicate are owned and reset by + /// the operator that created them, not by `FilterExec`. + fn reset_state(self: Arc) -> Result> { + let mut new = (*self).clone(); + new.adaptive_stats = Arc::new(AdaptiveFilterShared::new()); + new.metrics = ExecutionPlanMetricsSet::new(); + Ok(Arc::new(new)) + } + fn execute( &self, partition: usize, @@ -2475,6 +2493,34 @@ mod tests { Ok(()) } + #[tokio::test] + async fn test_reset_state_gives_fresh_adaptive_stats() -> Result<()> { + let schema = Schema::new(vec![Field::new("a", DataType::Int32, false)]); + let input = Arc::new(StatisticsExec::new( + Statistics::new_unknown(&schema), + schema, + )); + let predicate = Arc::new(BinaryExpr::new( + Arc::new(Column::new("a", 0)), + Operator::Eq, + Arc::new(Literal::new(ScalarValue::Int32(Some(10)))), + )); + let filter = Arc::new(FilterExec::try_new(predicate, input)?); + + let reset = Arc::clone(&filter).reset_state()?; + let reset = reset + .as_ref() + .downcast_ref::() + .expect("reset_state returns a FilterExec"); + + // The per-execution adaptive state is a fresh instance, so learning + // cannot leak across independent executions... + assert!(!Arc::ptr_eq(&filter.adaptive_stats, &reset.adaptive_stats)); + // ...while the predicate (which the reset does not touch) is preserved. + assert!(Arc::ptr_eq(&filter.predicate, &reset.predicate)); + Ok(()) + } + #[test] fn test_equivalence_properties_union_type() -> Result<()> { let union_type = DataType::Union( From b99a92a86013aa9df7cb19f911ce68c7b01c5105 Mon Sep 17 00:00:00 2001 From: Adrian Garcia Badaracco <1755071+adriangb@users.noreply.github.com> Date: Fri, 10 Jul 2026 13:15:04 -0500 Subject: [PATCH 06/27] fix(physical-plan): harden adaptive filter edge cases MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - An empty batch no longer consumes the warm-up: a run of empty batches would settle the written order on no evidence, permanently disabling adaptation for the stream. - A conjunct evaluated faster than the timer's resolution now clamps its cost to 1ns instead of dropping out of the ranking as unmeasured (which sorted the cheapest conjunct last — backwards). - The u32::MAX row-count guard is now a real internal error instead of a debug_assert; in release the indices would have silently wrapped. Also documents the known limitations of the one-shot, conditional-stats settle (correlated conjuncts, no drift re-measurement) in the module doc. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01DgXQmKrKgre4epbcSxUHNo --- .../physical-plan/src/adaptive_filter.rs | 83 ++++++++++++++++--- 1 file changed, 70 insertions(+), 13 deletions(-) diff --git a/datafusion/physical-plan/src/adaptive_filter.rs b/datafusion/physical-plan/src/adaptive_filter.rs index 9100956ddbe4c..feccf2d7fb62c 100644 --- a/datafusion/physical-plan/src/adaptive_filter.rs +++ b/datafusion/physical-plan/src/adaptive_filter.rs @@ -47,9 +47,11 @@ //! //! Compact-once is used **only in service of a reorder**: if the warm-up does //! not reorder the conjuncts (e.g. they are interchangeable), the written -//! predicate is evaluated as-is, so a conjunction that does not benefit from -//! reordering pays no compact-once overhead and behaves exactly as it would -//! with the feature off. +//! predicate is evaluated as-is, so once settled a conjunction that does not +//! benefit from reordering pays no compact-once overhead and evaluates exactly +//! as it would with the feature off. During the warm-up itself the conjuncts +//! are necessarily evaluated individually (with compaction) so they can be +//! measured — see the side-effects caveat below. //! //! ## How it shares //! @@ -67,9 +69,26 @@ //! containing volatile expressions are never reordered (their observable side //! effects depend on order). //! -//! This is the core of the adaptive evaluator. Further policies (drift -//! re-measurement, confidence-interval statistics, A/B-validated adoption for -//! the cases a cost model cannot separate) can build on top of it. +//! Observable *side effects* of fallible predicates can change even when no +//! reorder is adopted: whenever conjuncts are evaluated individually (during +//! warm-up, or settled with a reorder), a conjunct after a compaction sees only +//! the surviving rows, so an error a fused evaluation would have raised on an +//! already-filtered row (e.g. `b <> 0 AND 1/b > 2` with the fused `AND` +//! evaluating `1/b` on every row) may not occur. +//! +//! ## Known limitations +//! +//! The statistics are *conditional*: each conjunct is measured on the rows that +//! survived the conjuncts before it in written order, and (after a compaction) +//! on small survivor batches whose per-row cost is inflated by fixed overheads. +//! Correlated conjuncts can therefore look more selective in a late position +//! than they would be up front, and the settle decision is one-shot: once +//! adopted, the order is never re-measured, so a misjudged reorder (or drifting +//! data) is kept for the stream's lifetime. The material-win guard +//! ([`TIE_COST_FRACTION`]) makes adoption conservative but cannot detect +//! correlation. Further policies (drift re-measurement, confidence-interval +//! statistics, A/B-validated adoption for the cases a cost model cannot +//! separate) can build on top of this core. use std::sync::Arc; use std::sync::Mutex; @@ -135,9 +154,12 @@ impl ConjunctStats { (self.rows > 0).then(|| self.matched as f64 / self.rows as f64) } - /// Per-row evaluation cost in nanoseconds, or `None` if unmeasured. + /// Per-row evaluation cost in nanoseconds, or `None` if the conjunct was + /// never evaluated on any row. An evaluation faster than the timer's + /// resolution is clamped to one nanosecond total: "too cheap to measure" + /// must rank as very cheap, not drop out of the ranking as unmeasured. fn cost_per_row(&self) -> Option { - (self.rows > 0 && self.nanos > 0).then(|| self.nanos as f64 / self.rows as f64) + (self.rows > 0).then(|| self.nanos.max(1) as f64 / self.rows as f64) } /// Ranking key: rows discarded per nanosecond of evaluation @@ -287,6 +309,13 @@ impl AdaptiveConjunction { return self.evaluate_settled(batch); } + // An empty batch measures nothing; evaluating it must not consume the + // warm-up (a run of empty batches would otherwise settle the written + // order on no evidence, permanently). + if batch.num_rows() == 0 { + return eval_conjuncts(&self.conjuncts, &self.order, batch, None); + } + // Measure this batch into a local accumulator, then pool it. let mut local = vec![ConjunctStats::default(); self.conjuncts.len()]; let result = @@ -387,11 +416,10 @@ fn eval_conjuncts( return Ok(Arc::new(BooleanArray::from(Vec::::new()))); } // Live-row indices are tracked as `u32` (arrow's `filter`/`take` index - // space), so a batch cannot exceed `u32::MAX` rows here. - debug_assert!( - num_rows <= u32::MAX as usize, - "adaptive filter: batch exceeds u32::MAX rows" - ); + // space); a larger batch would silently wrap the indices, so refuse it. + if num_rows > u32::MAX as usize { + return internal_err!("adaptive filter: batch exceeds u32::MAX rows"); + } // `working` is the batch conjuncts are evaluated against. `acc` is the // accumulated (`AND`-combined, null-free) result over `working`'s rows since @@ -628,6 +656,35 @@ mod tests { assert_eq!(rank_by_effectiveness(&s), vec![1, 2, 0]); } + #[test] + fn zero_nanos_ranks_as_very_cheap() { + // A conjunct evaluated faster than the timer's resolution must rank as + // very cheap (its cost clamps to 1ns total), not drop out of the + // ranking as unmeasured (which would sort it last — backwards). + let s = vec![ + stats(1000, 500, 0), // immeasurably cheap, somewhat selective + stats(1000, 500, 1000), // same selectivity, 1ns/row + ]; + assert_eq!(rank_by_effectiveness(&s), vec![0, 1]); + } + + /// Empty batches measure nothing, so they must not consume the warm-up: + /// a stream fed only empty batches keeps learning instead of settling the + /// written order on no evidence. + #[test] + fn empty_batches_do_not_consume_warmup() { + let schema = schema(); + let mut adaptive = try_new(&predicate(&schema), true).unwrap(); + + for _ in 0..(2 * WARMUP_BATCHES) { + let rb = batch(&schema, vec![], vec![]); + let got = adaptive.evaluate(&rb).unwrap(); + assert_eq!(got.len(), 0); + } + assert!(!adaptive.settled); + assert_eq!(adaptive.shared.inner.lock().unwrap().measured_batches, 0); + } + #[test] fn expected_cost_weights_by_upstream_pass_rate() { // a: cost 1, pass 0.5 ; b: cost 10, pass 0.5 From 66e890044ad1989e52071923e79db1a21ab1afd6 Mon Sep 17 00:00:00 2001 From: Adrian Garcia Badaracco <1755071+adriangb@users.noreply.github.com> Date: Fri, 10 Jul 2026 13:15:04 -0500 Subject: [PATCH 07/27] test(sqllogictest): drive adaptive filter across the settle boundary The previous test stored the table as a single batch, so with WARMUP_BATCHES = 8 the flag-on queries only ever exercised the measuring path. Store 4000 rows as 64-row batches so the warm-up completes and the settled (possibly reordered) path runs end-to-end, and add a query whose conjunct produces NULLs to exercise the null-mask path through real SQL. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01DgXQmKrKgre4epbcSxUHNo --- .../test_files/adaptive_filter.slt | 33 +++++++++++++++---- 1 file changed, 26 insertions(+), 7 deletions(-) diff --git a/datafusion/sqllogictest/test_files/adaptive_filter.slt b/datafusion/sqllogictest/test_files/adaptive_filter.slt index b0e2d958a3be9..3db9adc23b517 100644 --- a/datafusion/sqllogictest/test_files/adaptive_filter.slt +++ b/datafusion/sqllogictest/test_files/adaptive_filter.slt @@ -18,20 +18,28 @@ # Tests for execution.adaptive_filter_reordering. Runtime reordering of a # conjunction must never change query results, only evaluation order. +# Store the table as many small batches so that, with the flag on, the +# adaptive evaluator's warm-up (8 measured batches) completes and the settled +# (possibly reordered) path is exercised end-to-end — not just the measuring +# path a single-batch table would reach. +statement ok +SET datafusion.execution.batch_size = 64; + statement ok CREATE TABLE t AS SELECT i AS a, i % 7 AS b, - arrow_cast(i, 'Utf8') AS s -FROM generate_series(1, 1000) AS tbl(i); + arrow_cast(i, 'Utf8') AS s, + CASE WHEN i % 5 = 0 THEN NULL ELSE i END AS n +FROM generate_series(1, 4000) AS tbl(i); # Baseline (flag off): multi-conjunct filter mixing a cheap comparison with an # expensive LIKE. query I SELECT count(*) FROM t WHERE b = 3 AND s LIKE '1%'; ---- -17 +160 statement ok SET datafusion.execution.adaptive_filter_reordering = true; @@ -40,7 +48,7 @@ SET datafusion.execution.adaptive_filter_reordering = true; query I SELECT count(*) FROM t WHERE b = 3 AND s LIKE '1%'; ---- -17 +160 # A three-conjunct predicate, including an expensive-but-selective LIKE. query I @@ -48,11 +56,19 @@ SELECT count(*) FROM t WHERE a > 100 AND s LIKE '5%' AND b <> 0; ---- 86 +# A conjunct that produces NULLs: SQL filter semantics treat NULL as false, +# with or without reordering. +query I +SELECT count(*) FROM t WHERE n % 2 = 0 AND s LIKE '2%'; +---- +445 + # Full row materialization (not just count) is unchanged by reordering. query IIT -SELECT a, b, s FROM t WHERE b = 1 AND a > 990 AND s LIKE '99%' ORDER BY a; +SELECT a, b, s FROM t WHERE b = 1 AND a > 3990 AND s LIKE '39%' ORDER BY a; ---- -995 1 995 +3991 1 3991 +3998 1 3998 # EXPLAIN: runtime reordering is invisible to the plan — the FilterExec # predicate is unchanged whether the flag is on or off. @@ -71,10 +87,13 @@ physical_plan 03)----CoalescePartitionsExec 04)------AggregateExec: mode=Partial, gby=[], aggr=[count(Int64(1))] 05)--------FilterExec: b@0 = 3 AND s@1 LIKE 1%, projection=[] -06)----------DataSourceExec: partitions=4, partition_sizes=[1, 0, 0, 0] +06)----------DataSourceExec: partitions=4, partition_sizes=[16, 16, 16, 15] statement ok SET datafusion.execution.adaptive_filter_reordering = false; +statement ok +SET datafusion.execution.batch_size = 8192; + statement ok DROP TABLE t; From 1f0a68e47b6c91d334077bf3d352cb2ec3e27d73 Mon Sep 17 00:00:00 2001 From: Adrian Garcia Badaracco <1755071+adriangb@users.noreply.github.com> Date: Fri, 10 Jul 2026 13:15:04 -0500 Subject: [PATCH 08/27] docs: tighten adaptive filter behavior claims The config docs claimed reordering was the only observable difference; in fact side effects of fallible predicates can change even when no reorder is adopted, because while measuring (and after a reorder) conjuncts are evaluated only on rows that survived the conjuncts before them. Say so explicitly, with an example, and note in FilterExec that Clone sharing the pooled measurements is deliberate. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01DgXQmKrKgre4epbcSxUHNo --- datafusion/common/src/config.rs | 12 ++++++++---- datafusion/physical-plan/src/filter.rs | 5 ++++- .../sqllogictest/test_files/information_schema.slt | 2 +- docs/source/user-guide/configs.md | 2 +- 4 files changed, 14 insertions(+), 7 deletions(-) diff --git a/datafusion/common/src/config.rs b/datafusion/common/src/config.rs index e39047f9ce097..ac81570a0fdeb 100644 --- a/datafusion/common/src/config.rs +++ b/datafusion/common/src/config.rs @@ -1143,10 +1143,14 @@ config_namespace! { /// conjunct's selectivity and evaluation cost on the rows that reach it /// and runs the conjuncts that discard the most rows per unit of CPU /// time first, so cheap-and-selective predicates gate expensive ones. - /// Reordering never changes query results (only the evaluation order of - /// a conjunction) but can change observable side effects of fallible - /// predicates, so it is off by default. Predicates containing volatile - /// expressions are never reordered. + /// This never changes query results, but it is off by default because + /// it can change observable side effects of fallible predicates (even + /// when no reorder is adopted): while measuring, and after a reorder, + /// conjuncts are evaluated only on the rows that survived the conjuncts + /// before them, so an error the fused predicate would have raised on an + /// already-filtered row (e.g. `b <> 0 AND 1/b > 2` evaluating `1/b` on + /// every row) may not occur. Predicates containing volatile expressions + /// are never reordered. pub adaptive_filter_reordering: bool, default = false /// Size (bytes) of data buffer DataFusion uses when writing output files. diff --git a/datafusion/physical-plan/src/filter.rs b/datafusion/physical-plan/src/filter.rs index 1fa4fc397917f..20b646c0b4a94 100644 --- a/datafusion/physical-plan/src/filter.rs +++ b/datafusion/physical-plan/src/filter.rs @@ -103,7 +103,10 @@ pub struct FilterExec { fetch: Option, /// Measurements shared by all partition streams, used by adaptive conjunct /// reordering (see [`AdaptiveConjunction`]) so the streams learn as one. - /// Fresh per plan node; never affects the plan. + /// Fresh per plan node; never affects the plan. `Clone` deliberately shares + /// it (the clone filters the same predicate over the same input, so pooled + /// learning still applies); [`reset_state`](ExecutionPlan::reset_state) + /// and predicate rewrites replace it with a fresh instance. adaptive_stats: Arc, } diff --git a/datafusion/sqllogictest/test_files/information_schema.slt b/datafusion/sqllogictest/test_files/information_schema.slt index 69bfcbcea4114..7905d34fc9b56 100644 --- a/datafusion/sqllogictest/test_files/information_schema.slt +++ b/datafusion/sqllogictest/test_files/information_schema.slt @@ -375,7 +375,7 @@ datafusion.catalog.has_header true Default value for `format.has_header` for `CR datafusion.catalog.information_schema true Should DataFusion provide access to `information_schema` virtual tables for displaying schema information datafusion.catalog.location NULL Location scanned to load tables for `default` schema datafusion.catalog.newlines_in_values false Specifies whether newlines in (quoted) CSV values are supported. This is the default value for `format.newlines_in_values` for `CREATE EXTERNAL TABLE` if not specified explicitly in the statement. Parsing newlines in quoted values may be affected by execution behaviour such as parallel file scanning. Setting this to `true` ensures that newlines in values are parsed successfully, which may reduce performance. -datafusion.execution.adaptive_filter_reordering false (experimental) When enabled, `FilterExec` adaptively reorders the conjuncts of a conjunctive predicate at runtime. It measures each conjunct's selectivity and evaluation cost on the rows that reach it and runs the conjuncts that discard the most rows per unit of CPU time first, so cheap-and-selective predicates gate expensive ones. Reordering never changes query results (only the evaluation order of a conjunction) but can change observable side effects of fallible predicates, so it is off by default. Predicates containing volatile expressions are never reordered. +datafusion.execution.adaptive_filter_reordering false (experimental) When enabled, `FilterExec` adaptively reorders the conjuncts of a conjunctive predicate at runtime. It measures each conjunct's selectivity and evaluation cost on the rows that reach it and runs the conjuncts that discard the most rows per unit of CPU time first, so cheap-and-selective predicates gate expensive ones. This never changes query results, but it is off by default because it can change observable side effects of fallible predicates (even when no reorder is adopted): while measuring, and after a reorder, conjuncts are evaluated only on the rows that survived the conjuncts before them, so an error the fused predicate would have raised on an already-filtered row (e.g. `b <> 0 AND 1/b > 2` evaluating `1/b` on every row) may not occur. Predicates containing volatile expressions are never reordered. datafusion.execution.batch_size 8192 Default batch size while creating new batches, it's especially useful for buffer-in-memory batches since creating tiny batches would result in too much metadata memory consumption datafusion.execution.coalesce_batches true When set to true, record batches will be examined between each operator and small batches will be coalesced into larger batches. This is helpful when there are highly selective filters or joins that could produce tiny output batches. The target batch size is determined by the configuration setting datafusion.execution.collect_statistics true Should DataFusion collect statistics when first creating a table. Has no effect after the table is created. Defaults to true. diff --git a/docs/source/user-guide/configs.md b/docs/source/user-guide/configs.md index 4fe049662a050..ccb11af76fe45 100644 --- a/docs/source/user-guide/configs.md +++ b/docs/source/user-guide/configs.md @@ -142,7 +142,7 @@ The following configuration settings are available: | datafusion.execution.skip_partial_aggregation_probe_rows_threshold | 100000 | Number of input rows partial aggregation partition should process, before aggregation ratio check and trying to switch to skipping aggregation mode | | datafusion.execution.use_row_number_estimates_to_optimize_partitioning | false | Should DataFusion use row number estimates at the input to decide whether increasing parallelism is beneficial or not. By default, only exact row numbers (not estimates) are used for this decision. Setting this flag to `true` will likely produce better plans. if the source of statistics is accurate. We plan to make this the default in the future. | | datafusion.execution.enforce_batch_size_in_joins | false | Should DataFusion enforce batch size in joins or not. By default, DataFusion will not enforce batch size in joins. Enforcing batch size in joins can reduce memory usage when joining large tables with a highly-selective join filter, but is also slightly slower. Note: this option currently only applies to the symmetric hash join. | -| datafusion.execution.adaptive_filter_reordering | false | (experimental) When enabled, `FilterExec` adaptively reorders the conjuncts of a conjunctive predicate at runtime. It measures each conjunct's selectivity and evaluation cost on the rows that reach it and runs the conjuncts that discard the most rows per unit of CPU time first, so cheap-and-selective predicates gate expensive ones. Reordering never changes query results (only the evaluation order of a conjunction) but can change observable side effects of fallible predicates, so it is off by default. Predicates containing volatile expressions are never reordered. | +| datafusion.execution.adaptive_filter_reordering | false | (experimental) When enabled, `FilterExec` adaptively reorders the conjuncts of a conjunctive predicate at runtime. It measures each conjunct's selectivity and evaluation cost on the rows that reach it and runs the conjuncts that discard the most rows per unit of CPU time first, so cheap-and-selective predicates gate expensive ones. This never changes query results, but it is off by default because it can change observable side effects of fallible predicates (even when no reorder is adopted): while measuring, and after a reorder, conjuncts are evaluated only on the rows that survived the conjuncts before them, so an error the fused predicate would have raised on an already-filtered row (e.g. `b <> 0 AND 1/b > 2` evaluating `1/b` on every row) may not occur. Predicates containing volatile expressions are never reordered. | | datafusion.execution.objectstore_writer_buffer_size | 10485760 | Size (bytes) of data buffer DataFusion uses when writing output files. This affects the size of the data chunks that are uploaded to remote object stores (e.g. AWS S3). If very large (>= 100 GiB) output files are being written, it may be necessary to increase this size to avoid errors from the remote end point. | | datafusion.execution.enable_ansi_mode | false | Whether to enable ANSI SQL mode. The flag is experimental and relevant only for DataFusion Spark built-in functions When `enable_ansi_mode` is set to `true`, the query engine follows ANSI SQL semantics for expressions, casting, and error handling. This means: - **Strict type coercion rules:** implicit casts between incompatible types are disallowed. - **Standard SQL arithmetic behavior:** operations such as division by zero, numeric overflow, or invalid casts raise runtime errors rather than returning `NULL` or adjusted values. - **Consistent ANSI behavior** for string concatenation, comparisons, and `NULL` handling. When `enable_ansi_mode` is `false` (the default), the engine uses a more permissive, non-ANSI mode designed for user convenience and backward compatibility. In this mode: - Implicit casts between types are allowed (e.g., string to integer when possible). - Arithmetic operations are more lenient — for example, `abs()` on the minimum representable integer value returns the input value instead of raising overflow. - Division by zero or invalid casts may return `NULL` instead of failing. # Default `false` — ANSI SQL mode is disabled by default. | | datafusion.execution.hash_join_buffering_capacity | 0 | How many bytes to buffer in the probe side of hash joins while the build side is concurrently being built. Without this, hash joins will wait until the full materialization of the build side before polling the probe side. This is useful in scenarios where the query is not completely CPU bounded, allowing to do some early work concurrently and reducing the latency of the query. Note that when hash join buffering is enabled, the probe side will start eagerly polling data, not giving time for the producer side of dynamic filters to produce any meaningful predicate. Queries with dynamic filters might see performance degradation. Disabled by default, set to a number greater than 0 for enabling it. | From c9c7c36931ea948c0094a869ac1d032e9b7584bb Mon Sep 17 00:00:00 2001 From: Adrian Garcia Badaracco <1755071+adriangb@users.noreply.github.com> Date: Fri, 10 Jul 2026 17:13:40 -0500 Subject: [PATCH 09/27] refactor(physical-plan): FilterExec owns the enable gate; trace per-batch strategy MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two review responses: - AdaptiveConjunction::try_new no longer takes an `enabled` bool that short-circuits to None; whether the feature is on is FilterExec's policy, so the flag check moves to FilterExec::execute and try_new answers only the structural question (reorderable, non-volatile conjunction). - The evaluator's per-batch behaviour is now observable: evaluate is a thin wrapper over evaluate_traced, which also reports the BatchStrategy used (Measure / Fused / Reordered). Two scenario tests exercise the input/output contract end to end — batches in, masks + strategy trace out — with per-conjunct costs injected by seeding the shared pool with synthetic measurements (the stand-in for a mocked clock), so which strategy gets adopted is deterministic: warm-up settles on a reorder for cheap-unselective + expensive-selective conjuncts, and on the written fused predicate for interchangeable ones. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01DgXQmKrKgre4epbcSxUHNo --- .../physical-plan/src/adaptive_filter.rs | 177 +++++++++++++++--- datafusion/physical-plan/src/filter.rs | 22 ++- 2 files changed, 159 insertions(+), 40 deletions(-) diff --git a/datafusion/physical-plan/src/adaptive_filter.rs b/datafusion/physical-plan/src/adaptive_filter.rs index feccf2d7fb62c..9d5fa1a2d9679 100644 --- a/datafusion/physical-plan/src/adaptive_filter.rs +++ b/datafusion/physical-plan/src/adaptive_filter.rs @@ -203,6 +203,26 @@ struct SharedInner { settled: Option, } +/// How one batch was evaluated. Reported by +/// [`AdaptiveConjunction::evaluate_traced`] so the evaluator's behaviour is +/// observable batch by batch (its input/output contract is exercised by the +/// `scenario_*` tests). Borrows the adopted order rather than cloning it, so +/// reporting costs nothing on the per-batch path. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum BatchStrategy<'a> { + /// Still learning: the written order through the compact-once loop, each + /// conjunct instrumented and its counts pooled. (Empty batches are + /// evaluated but measure nothing and do not consume the warm-up.) + Measure, + /// Settled without a reorder: the written fused predicate, exactly as if + /// the feature were off. + Fused, + /// Settled on an adopted reorder, evaluated through the compact-once + /// loop. The payload is the adopted evaluation order: positions in the + /// written conjunct list, first-evaluated first. + Reordered(&'a [usize]), +} + /// The settled outcome of the warm-up: the evaluation order, and whether to run /// it through the compact-once loop or the plain predicate. #[derive(Debug, Clone)] @@ -252,22 +272,19 @@ pub(crate) struct AdaptiveConjunction { impl AdaptiveConjunction { /// Build an adaptive evaluator for `predicate`, or `None` if adaptive - /// reordering does not apply: + /// reordering does not structurally apply: /// - /// - `enabled` is false (the config flag is off); /// - the predicate has fewer than two `AND` conjuncts (nothing to reorder); /// - any conjunct is volatile (reordering could change side effects). /// - /// `shared` is the state common to all partition streams of the owning - /// `FilterExec`. + /// Whether adaptive reordering is *enabled* is the caller's policy (the + /// config flag lives with `FilterExec`); this constructor only answers + /// whether the predicate is a reorderable conjunction. `shared` is the + /// state common to all partition streams of the owning `FilterExec`. pub(crate) fn try_new( predicate: &Arc, - enabled: bool, shared: Arc, ) -> Option { - if !enabled { - return None; - } let conjuncts: Vec> = split_conjunction(predicate) .into_iter() .map(Arc::clone) @@ -294,6 +311,16 @@ impl AdaptiveConjunction { /// into the shared registry; a stream adopts the settled order another /// stream published as soon as it sees the epoch advance. pub(crate) fn evaluate(&mut self, batch: &RecordBatch) -> Result { + self.evaluate_traced(batch).map(|(mask, _)| mask) + } + + /// [`evaluate`](Self::evaluate), additionally reporting the + /// [`BatchStrategy`] used for this batch, so the evaluator's behaviour is + /// observable batch by batch (see the scenario tests). + fn evaluate_traced( + &mut self, + batch: &RecordBatch, + ) -> Result<(ArrayRef, BatchStrategy<'_>)> { // Adopt a settled order another stream published since we last looked: // one relaxed atomic load per batch, a lock only on the transition. if !self.settled { @@ -306,14 +333,21 @@ impl AdaptiveConjunction { } } if self.settled { - return self.evaluate_settled(batch); + let mask = self.evaluate_settled(batch)?; + let strategy = if self.compact { + BatchStrategy::Reordered(&self.order) + } else { + BatchStrategy::Fused + }; + return Ok((mask, strategy)); } // An empty batch measures nothing; evaluating it must not consume the // warm-up (a run of empty batches would otherwise settle the written // order on no evidence, permanently). if batch.num_rows() == 0 { - return eval_conjuncts(&self.conjuncts, &self.order, batch, None); + let mask = eval_conjuncts(&self.conjuncts, &self.order, batch, None)?; + return Ok((mask, BatchStrategy::Measure)); } // Measure this batch into a local accumulator, then pool it. @@ -321,7 +355,7 @@ impl AdaptiveConjunction { let result = eval_conjuncts(&self.conjuncts, &self.order, batch, Some(&mut local))?; self.pool_and_maybe_settle(&local); - Ok(result) + Ok((result, BatchStrategy::Measure)) } /// Evaluate the settled arrangement with no instrumentation: the @@ -599,21 +633,18 @@ mod tests { } /// `try_new` with a fresh, unshared registry. - fn try_new( - predicate: &Arc, - enabled: bool, - ) -> Option { - AdaptiveConjunction::try_new( - predicate, - enabled, - Arc::new(AdaptiveFilterShared::new()), - ) + fn try_new(predicate: &Arc) -> Option { + AdaptiveConjunction::try_new(predicate, Arc::new(AdaptiveFilterShared::new())) } - #[test] - fn disabled_is_not_adaptive() { - let schema = schema(); - assert!(try_new(&predicate(&schema), false).is_none()); + /// Seed the shared pool as if `batches` instrumented batches had already + /// recorded `stats` — a stand-in for a mocked clock, giving scenario tests + /// deterministic control over each conjunct's measured cost and + /// selectivity. + fn seed(shared: &AdaptiveFilterShared, stats: Vec, batches: u64) { + let mut inner = shared.inner.lock().unwrap(); + inner.stats = stats; + inner.measured_batches = batches; } #[test] @@ -621,13 +652,13 @@ mod tests { let schema = schema(); let p = binary(col("a", &schema).unwrap(), Operator::Gt, lit(2i32), &schema).unwrap(); - assert!(try_new(&p, true).is_none()); + assert!(try_new(&p).is_none()); } #[test] fn two_conjuncts_are_adaptive() { let schema = schema(); - let adaptive = try_new(&predicate(&schema), true).unwrap(); + let adaptive = try_new(&predicate(&schema)).unwrap(); assert_eq!(adaptive.conjuncts.len(), 2); assert_eq!(adaptive.order, vec![0, 1]); assert!(!adaptive.settled); @@ -674,7 +705,7 @@ mod tests { #[test] fn empty_batches_do_not_consume_warmup() { let schema = schema(); - let mut adaptive = try_new(&predicate(&schema), true).unwrap(); + let mut adaptive = try_new(&predicate(&schema)).unwrap(); for _ in 0..(2 * WARMUP_BATCHES) { let rb = batch(&schema, vec![], vec![]); @@ -725,7 +756,7 @@ mod tests { fn evaluate_matches_predicate_across_warmup() { let schema = schema(); let p = predicate(&schema); - let mut adaptive = try_new(&p, true).unwrap(); + let mut adaptive = try_new(&p).unwrap(); for round in 0..(WARMUP_BATCHES as i32 + 4) { let base = round * 10; @@ -778,7 +809,7 @@ mod tests { let right = binary(col("b", &schema).unwrap(), Operator::Gt, lit(2i32), &schema).unwrap(); let p = binary(left, Operator::And, right, &schema).unwrap(); - let mut adaptive = try_new(&p, true).unwrap(); + let mut adaptive = try_new(&p).unwrap(); for round in 0..(WARMUP_BATCHES as i32 + 2) { let base = round * 10; @@ -804,8 +835,8 @@ mod tests { let schema = schema(); let p = predicate(&schema); let shared = Arc::new(AdaptiveFilterShared::new()); - let mut s1 = AdaptiveConjunction::try_new(&p, true, Arc::clone(&shared)).unwrap(); - let mut s2 = AdaptiveConjunction::try_new(&p, true, Arc::clone(&shared)).unwrap(); + let mut s1 = AdaptiveConjunction::try_new(&p, Arc::clone(&shared)).unwrap(); + let mut s2 = AdaptiveConjunction::try_new(&p, Arc::clone(&shared)).unwrap(); // `b < 5` (conjunct 1) is the selective one; drive both streams with // batches where it keeps ~1 row in 25. @@ -838,4 +869,88 @@ mod tests { assert_eq!(s2.order, vec![1, 0]); assert!(s1.compact && s2.compact); } + + /// End-to-end input/output-contract scenario: feed batches, observe the + /// strategy used for each one alongside the masks. + /// + /// Per-conjunct costs are injected by seeding the shared pool with + /// synthetic measurements (the stand-in for a mocked clock): conjunct 0 is + /// cheap but unselective, conjunct 1 is expensive but very selective, so + /// the warm-up must settle on promoting conjunct 1 and run the reorder + /// through the compact-once loop. The seeded magnitudes dominate the one + /// real measured batch, so the decision is deterministic regardless of + /// real timer values. + #[test] + fn scenario_measure_batches_then_settle_on_reorder() { + let schema = schema(); + let p = predicate(&schema); // `a > 2 AND b < 5`, written order [0, 1] + let shared = Arc::new(AdaptiveFilterShared::new()); + // One batch short of the warm-up: the next measured batch settles. + seed( + &shared, + vec![ + stats(70_000_000, 63_000_000, 70_000_000), // pass 0.9, ~1ns/row + stats(70_000_000, 700_000, 350_000_000), // pass 0.01, ~5ns/row + ], + WARMUP_BATCHES - 1, + ); + let mut adaptive = AdaptiveConjunction::try_new(&p, Arc::clone(&shared)).unwrap(); + + let mut trace = vec![]; + for round in 0..3 { + let base = round * 100; + let a: Vec = (base..base + 100).collect(); + let b: Vec = (base..base + 100).map(|x| x.rem_euclid(25)).collect(); + let rb = batch(&schema, a, b); + let (got, strategy) = adaptive.evaluate_traced(&rb).unwrap(); + trace.push(format!("{strategy:?}")); + let want = p.evaluate(&rb).unwrap().into_array(rb.num_rows()).unwrap(); + assert_eq!(passing_rows(&got), passing_rows(&want), "round {round}"); + } + + // Batch 1 completes the warm-up and settles; batches 2+ run the + // adopted reorder (selective conjunct promoted to the front) through + // the compact-once loop. + assert_eq!( + trace, + vec!["Measure", "Reordered([1, 0])", "Reordered([1, 0])"] + ); + } + + /// Contract scenario for the no-win case: interchangeable conjuncts settle + /// on the written fused predicate (as if the feature were off), never the + /// compact-once loop. + #[test] + fn scenario_measure_batches_then_settle_on_fused() { + let schema = schema(); + let p = predicate(&schema); + let shared = Arc::new(AdaptiveFilterShared::new()); + // Identical cost and selectivity: no order can be materially cheaper. + // The seeded magnitudes dominate the one real measured batch, so even + // if real timings nudge the ranking, the 5% material-win guard holds. + seed( + &shared, + vec![ + stats(70_000_000, 35_000_000, 70_000_000), + stats(70_000_000, 35_000_000, 70_000_000), + ], + WARMUP_BATCHES - 1, + ); + let mut adaptive = AdaptiveConjunction::try_new(&p, Arc::clone(&shared)).unwrap(); + + let mut trace = vec![]; + for round in 0..3 { + let base = round * 100; + let a: Vec = (base..base + 100).collect(); + let b: Vec = (base..base + 100).collect(); + let rb = batch(&schema, a, b); + let (got, strategy) = adaptive.evaluate_traced(&rb).unwrap(); + trace.push(format!("{strategy:?}")); + let want = p.evaluate(&rb).unwrap().into_array(rb.num_rows()).unwrap(); + assert_eq!(passing_rows(&got), passing_rows(&want), "round {round}"); + } + + assert_eq!(trace, vec!["Measure", "Fused", "Fused"]); + assert_eq!(adaptive.order, vec![0, 1]); + } } diff --git a/datafusion/physical-plan/src/filter.rs b/datafusion/physical-plan/src/filter.rs index 20b646c0b4a94..a9f1af231978e 100644 --- a/datafusion/physical-plan/src/filter.rs +++ b/datafusion/physical-plan/src/filter.rs @@ -674,15 +674,19 @@ impl ExecutionPlan for FilterExec { context.task_id() ); let metrics = FilterExecMetrics::new(&self.metrics, partition); - let adaptive = AdaptiveConjunction::try_new( - &self.predicate, - context - .session_config() - .options() - .execution - .adaptive_filter_reordering, - Arc::clone(&self.adaptive_stats), - ); + let enabled = context + .session_config() + .options() + .execution + .adaptive_filter_reordering; + let adaptive = enabled + .then(|| { + AdaptiveConjunction::try_new( + &self.predicate, + Arc::clone(&self.adaptive_stats), + ) + }) + .flatten(); Ok(Box::pin(FilterExecStream { schema: self.schema(), predicate: Arc::clone(&self.predicate), From 1bf6b204e68a79b69203164de9943529da1bcef6 Mon Sep 17 00:00:00 2001 From: Adrian Garcia Badaracco <1755071+adriangb@users.noreply.github.com> Date: Sun, 6 Sep 2026 17:48:15 -0700 Subject: [PATCH 10/27] refactor(physical-plan): drop redundant adaptive filter epoch MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `AdaptiveFilterShared` carried an `epoch` atomic that unsettled streams polled once per batch, taking the mutex only when it changed. The atomic bought nothing: every unsettled stream already locks `inner` on each non-empty measured batch to pool its counts, so it was one extra word of state and a second synchronisation point for the same information. Unsettled streams now read the published decision straight from the mutex at the top of `evaluate_traced` — the same adoption point the epoch had, before the current batch is evaluated — and `pool_and_maybe_settle` adopts a decision another stream published between the two lock acquisitions instead of returning early. A settled stream never touches the shared state at all, as before. Also replace the defensive stats-length reset in `pool_and_maybe_settle` with lazy init plus a `debug_assert_eq!`: no site shares one `AdaptiveFilterShared` across different predicates (the builder, predicate rewrites and `reset_state` all allocate fresh; `Clone`, `with_fetch` and `with_batch_size` share it for the same predicate), so a length mismatch is a bug, not a case to paper over. No behavioural change. Co-Authored-By: Claude Fable 5.1 --- .../physical-plan/src/adaptive_filter.rs | 54 ++++++++++--------- datafusion/physical-plan/src/filter.rs | 3 ++ 2 files changed, 32 insertions(+), 25 deletions(-) diff --git a/datafusion/physical-plan/src/adaptive_filter.rs b/datafusion/physical-plan/src/adaptive_filter.rs index 9d5fa1a2d9679..8c451027f0a3d 100644 --- a/datafusion/physical-plan/src/adaptive_filter.rs +++ b/datafusion/physical-plan/src/adaptive_filter.rs @@ -59,9 +59,11 @@ //! slice of the data. Measurements are pooled into a shared //! [`AdaptiveFilterShared`] so the streams learn as one: the first stream to //! accumulate enough samples settles the order and publishes it, and the others -//! adopt it (one relaxed atomic load per batch) without each re-paying the -//! warm-up — which is what makes the win materialise when each stream is only a -//! handful of batches long. +//! adopt it on their next batch without each re-paying the warm-up — which is +//! what makes the win materialise when each stream is only a handful of batches +//! long. Only unsettled streams touch the shared mutex, and only to pool a +//! batch's counts or pick up a published decision; a settled stream never +//! locks it again. //! //! It is **off by default** //! (`datafusion.execution.adaptive_filter_reordering`) and never changes query @@ -92,7 +94,6 @@ use std::sync::Arc; use std::sync::Mutex; -use std::sync::atomic::{AtomicU64, Ordering}; use arrow::array::{Array, ArrayRef, BooleanArray, BooleanBufferBuilder, UInt32Array}; use arrow::buffer::BooleanBuffer; @@ -186,9 +187,6 @@ impl ConjunctStats { /// sharing the warm-up is paid roughly once per query, not once per stream. #[derive(Debug, Default)] pub(crate) struct AdaptiveFilterShared { - /// `0` until an order is published; bumped once when the first stream - /// settles. Streams poll it with one relaxed atomic load per batch. - epoch: AtomicU64, inner: Mutex, } @@ -264,8 +262,6 @@ pub(crate) struct AdaptiveConjunction { /// Whether the settled order runs through the compact-once loop; `false` /// means evaluate [`predicate`](Self::predicate) directly. compact: bool, - /// Shared epoch this stream has caught up to. - epoch_seen: u64, /// Whether the order is settled (frozen): this stream no longer measures. settled: bool, } @@ -299,7 +295,6 @@ impl AdaptiveConjunction { shared, order, compact: false, - epoch_seen: 0, settled: false, }) } @@ -309,7 +304,7 @@ impl AdaptiveConjunction { /// /// Until the order settles, each batch is measured and its counts pooled /// into the shared registry; a stream adopts the settled order another - /// stream published as soon as it sees the epoch advance. + /// stream published on its next batch. pub(crate) fn evaluate(&mut self, batch: &RecordBatch) -> Result { self.evaluate_traced(batch).map(|(mask, _)| mask) } @@ -321,16 +316,14 @@ impl AdaptiveConjunction { &mut self, batch: &RecordBatch, ) -> Result<(ArrayRef, BatchStrategy<'_>)> { - // Adopt a settled order another stream published since we last looked: - // one relaxed atomic load per batch, a lock only on the transition. - if !self.settled { - let epoch = self.shared.epoch.load(Ordering::Acquire); - if epoch != self.epoch_seen { - self.epoch_seen = epoch; - if let Some(decision) = self.shared.settled() { - self.adopt(decision); - } - } + // Adopt a settled order another stream published since our last batch. + // An unsettled stream already locks the shared state once per measured + // batch to pool its counts, so this brief extra lock is in the same + // cost class; a settled stream never touches it again. + if !self.settled + && let Some(decision) = self.shared.settled() + { + self.adopt(decision); } if self.settled { let mask = self.evaluate_settled(batch)?; @@ -379,21 +372,32 @@ impl AdaptiveConjunction { /// batches have accrued across all streams, decide and publish the order. fn pool_and_maybe_settle(&mut self, local: &[ConjunctStats]) { let mut inner = self.shared.inner.lock().expect("poisoned"); - if inner.stats.len() != local.len() { + if inner.stats.is_empty() { inner.stats = vec![ConjunctStats::default(); local.len()]; } + // One `AdaptiveFilterShared` only ever backs one predicate: the builder, + // predicate rewrites and `reset_state` each allocate a fresh instance, + // and the paths that share one (`Clone`, `with_fetch`, + // `with_batch_size`) keep the same predicate. + debug_assert_eq!(inner.stats.len(), local.len()); for (s, l) in inner.stats.iter_mut().zip(local) { s.merge(l); } inner.measured_batches += 1; - if inner.settled.is_some() || inner.measured_batches < WARMUP_BATCHES { + // Another stream settled between our two lock acquisitions: adopt its + // decision rather than measuring on. + if let Some(decision) = inner.settled.clone() { + drop(inner); + self.adopt(decision); + return; + } + if inner.measured_batches < WARMUP_BATCHES { return; } let decision = settle(&inner.stats); inner.settled = Some(decision.clone()); drop(inner); self.adopt(decision); - self.shared.epoch.fetch_add(1, Ordering::Release); } } @@ -859,7 +863,7 @@ mod tests { } assert!(shared.settled().is_some()); - // One more batch each lets a not-yet-settled stream adopt the epoch. + // One more batch each lets a not-yet-settled stream adopt the decision. s1.evaluate(&mk(99)).unwrap(); s2.evaluate(&mk(99)).unwrap(); assert!(s1.settled && s2.settled); diff --git a/datafusion/physical-plan/src/filter.rs b/datafusion/physical-plan/src/filter.rs index a9f1af231978e..f1b7b987bfcf5 100644 --- a/datafusion/physical-plan/src/filter.rs +++ b/datafusion/physical-plan/src/filter.rs @@ -609,6 +609,9 @@ impl ExecutionPlan for FilterExec { ) -> Result> { validate_child_count!(self, children); match options.children_properties { + // `adaptive_stats` is deliberately carried over by the struct + // update: the predicate is unchanged, and no measurements exist + // before execution. `reset_state` is what replaces it. ChildrenPropertiesMode::Keep => Ok(Arc::new(Self { input: children.swap_remove(0), metrics: ExecutionPlanMetricsSet::new(), From f76986b6608c5323b9d48369028022d97f554c2b Mon Sep 17 00:00:00 2001 From: Adrian Garcia Badaracco <1755071+adriangb@users.noreply.github.com> Date: Sun, 6 Sep 2026 23:46:56 -0500 Subject: [PATCH 11/27] feat(physical-plan): report adaptive filter reordering in FilterExec metrics MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adaptive conjunct reordering was invisible from the outside: with the flag on there was no way to tell from a real query whether the runtime actually adopted a reordered evaluation order, or settled on the written one. Add an `adaptive_reorders` counter to `FilterExecMetrics`, incremented once per partition stream at the batch on which that stream adopts a reordered (compacting) decision — including streams that pick up a decision another stream published. `AdaptiveConjunction` stays free of metrics types: it exposes a one-shot `take_adopted_reorder()` transition signal and the `FilterExec` stream does the counting. The counter is registered only when adaptive reordering is enabled for the execution, so the default flag-off path's metrics are unchanged. Also assert in `adaptive_filter.slt` that the reorder happens, via `EXPLAIN ANALYZE` on a predicate written selective-conjunct-last, and add the flag-off `EXPLAIN` that the file's "identical on and off" claim was asserting against nothing. Co-Authored-By: Claude Fable 5.1 --- .../physical-plan/src/adaptive_filter.rs | 94 +++++++++++++++++++ datafusion/physical-plan/src/filter.rs | 55 +++++++++-- .../test_files/adaptive_filter.slt | 41 ++++++++ docs/source/user-guide/metrics.md | 7 +- 4 files changed, 184 insertions(+), 13 deletions(-) diff --git a/datafusion/physical-plan/src/adaptive_filter.rs b/datafusion/physical-plan/src/adaptive_filter.rs index 8c451027f0a3d..a89284a32f42b 100644 --- a/datafusion/physical-plan/src/adaptive_filter.rs +++ b/datafusion/physical-plan/src/adaptive_filter.rs @@ -264,6 +264,11 @@ pub(crate) struct AdaptiveConjunction { compact: bool, /// Whether the order is settled (frozen): this stream no longer measures. settled: bool, + /// Set when this stream adopts a *reordered* decision, and cleared by + /// [`take_adopted_reorder`](Self::take_adopted_reorder) — a one-shot + /// transition signal so the owner (`FilterExec`'s stream) can report that + /// the reorder happened without this type knowing about metrics. + adopted_reorder: bool, } impl AdaptiveConjunction { @@ -296,9 +301,21 @@ impl AdaptiveConjunction { order, compact: false, settled: false, + adopted_reorder: false, }) } + /// Whether this stream has just adopted a reordered evaluation order, + /// clearing the signal. + /// + /// Fires exactly once per stream, on the batch at which the stream settles + /// on a reorder — whether it settled the order itself or picked up one + /// another stream published. A stream that settles on the written order + /// (no reorder) never fires. + pub(crate) fn take_adopted_reorder(&mut self) -> bool { + std::mem::take(&mut self.adopted_reorder) + } + /// Evaluate the conjunction against `batch`, returning the boolean mask /// (over the batch's rows) of rows that passed every conjunct. /// @@ -366,6 +383,9 @@ impl AdaptiveConjunction { self.order = decision.order; self.compact = decision.compact; self.settled = true; + // Only a genuine reorder is worth reporting; settling on the written + // order is indistinguishable from the feature being off. + self.adopted_reorder = self.compact; } /// Merge this batch's measurements into the shared pool and, once enough @@ -874,6 +894,80 @@ mod tests { assert!(s1.compact && s2.compact); } + /// The reorder-adoption signal (`take_adopted_reorder`, what `FilterExec` + /// counts into its `adaptive_reorders` metric) fires exactly once per + /// stream: once for the stream that settles the order, and once for a + /// stream that later picks up the decision that stream published. + /// + /// The per-conjunct costs are seeded (see [`seed`]) so the settle decision + /// is a reorder deterministically, regardless of real timer values. + #[test] + fn adopted_reorder_signals_once_per_stream() { + let schema = schema(); + let p = predicate(&schema); // `a > 2 AND b < 5`, written order [0, 1] + let shared = Arc::new(AdaptiveFilterShared::new()); + // Conjunct 1 is far more selective; promoting it is materially cheaper, + // so the warm-up settles on a reorder. One batch short of the warm-up. + seed( + &shared, + vec![ + stats(70_000_000, 63_000_000, 70_000_000), // pass 0.9, ~1ns/row + stats(70_000_000, 700_000, 350_000_000), // pass 0.01, ~5ns/row + ], + WARMUP_BATCHES - 1, + ); + let mut settler = AdaptiveConjunction::try_new(&p, Arc::clone(&shared)).unwrap(); + let mut adopter = AdaptiveConjunction::try_new(&p, Arc::clone(&shared)).unwrap(); + + let a: Vec = (0..100).collect(); + let b: Vec = a.iter().map(|x| x.rem_euclid(25)).collect(); + let rb = batch(&schema, a, b); + + // Nothing adopted yet. + assert!(!settler.take_adopted_reorder()); + + // This batch completes the warm-up: `settler` settles on the reorder + // and signals it, exactly once. + settler.evaluate(&rb).unwrap(); + assert!(settler.compact); + assert!(settler.take_adopted_reorder()); + settler.evaluate(&rb).unwrap(); + assert!(!settler.take_adopted_reorder()); + + // `adopter` never measured its way to a decision: it picks up the + // published one on its next batch, and signals that once too. + adopter.evaluate(&rb).unwrap(); + assert!(adopter.compact); + assert!(adopter.take_adopted_reorder()); + adopter.evaluate(&rb).unwrap(); + assert!(!adopter.take_adopted_reorder()); + } + + /// Settling on the written order is indistinguishable from the feature + /// being off, so it must not signal a reorder. + #[test] + fn settling_without_reorder_signals_nothing() { + let schema = schema(); + let p = predicate(&schema); + let shared = Arc::new(AdaptiveFilterShared::new()); + // Identical cost and selectivity: no order can be materially cheaper. + seed( + &shared, + vec![ + stats(70_000_000, 35_000_000, 70_000_000), + stats(70_000_000, 35_000_000, 70_000_000), + ], + WARMUP_BATCHES - 1, + ); + let mut adaptive = AdaptiveConjunction::try_new(&p, Arc::clone(&shared)).unwrap(); + + let a: Vec = (0..100).collect(); + let rb = batch(&schema, a.clone(), a); + adaptive.evaluate(&rb).unwrap(); + assert!(adaptive.settled && !adaptive.compact); + assert!(!adaptive.take_adopted_reorder()); + } + /// End-to-end input/output-contract scenario: feed batches, observe the /// strategy used for each one alongside the masks. /// diff --git a/datafusion/physical-plan/src/filter.rs b/datafusion/physical-plan/src/filter.rs index f1b7b987bfcf5..2f2d5fbdf66ac 100644 --- a/datafusion/physical-plan/src/filter.rs +++ b/datafusion/physical-plan/src/filter.rs @@ -37,7 +37,7 @@ use crate::filter_pushdown::{ FilterPushdownPropagation, FilterRemapper, PushedDown, }; use crate::limit::LocalLimitExec; -use crate::metrics::{MetricBuilder, MetricType}; +use crate::metrics::{Count, MetricBuilder, MetricCategory, MetricType}; use crate::projection::{ EmbeddedProjection, ProjectionExec, ProjectionExpr, make_with_child, try_embed_projection, update_expr, @@ -676,7 +676,6 @@ impl ExecutionPlan for FilterExec { context.session_id(), context.task_id() ); - let metrics = FilterExecMetrics::new(&self.metrics, partition); let enabled = context .session_config() .options() @@ -690,6 +689,8 @@ impl ExecutionPlan for FilterExec { ) }) .flatten(); + let metrics = + FilterExecMetrics::new(&self.metrics, partition, adaptive.is_some()); Ok(Box::pin(FilterExecStream { schema: self.schema(), predicate: Arc::clone(&self.predicate), @@ -1427,17 +1428,41 @@ struct FilterExecMetrics { baseline_metrics: BaselineMetrics, /// Selectivity of the filter, calculated as output_rows / input_rows selectivity: RatioMetrics, + /// Number of partition streams that adopted an adaptively reordered + /// evaluation order for the predicate's conjuncts (at most one per + /// stream). Registered only when adaptive conjunct reordering is enabled + /// for this execution, so the metrics of the (default) flag-off path are + /// unchanged. + adaptive_reorders: Option, // Remember to update `docs/source/user-guide/metrics.md` when adding new metrics, // or modifying metrics comments } impl FilterExecMetrics { - pub fn new(metrics: &ExecutionPlanMetricsSet, partition: usize) -> Self { + pub fn new( + metrics: &ExecutionPlanMetricsSet, + partition: usize, + adaptive: bool, + ) -> Self { Self { baseline_metrics: BaselineMetrics::new(metrics, partition), selectivity: MetricBuilder::new(metrics) .with_type(MetricType::Summary) .ratio_metrics("selectivity", partition), + adaptive_reorders: adaptive.then(|| { + MetricBuilder::new(metrics) + // A deterministic, dimensionless counter: it depends on + // the plan and the data, not on wall-clock timings. + .with_category(MetricCategory::Rows) + .counter("adaptive_reorders", partition) + }), + } + } + + /// Record that this stream adopted a reordered evaluation order. + fn record_adaptive_reorder(&self) { + if let Some(count) = &self.adaptive_reorders { + count.add(1); } } } @@ -1506,14 +1531,24 @@ impl Stream for FilterExecStream { } Some(Ok(batch)) => { let timer = elapsed_compute.timer(); - let array = match self.adaptive.as_mut() { - Some(adaptive) => adaptive.evaluate(&batch), - None => self - .predicate - .as_ref() - .evaluate(&batch) - .and_then(|v| v.into_array(batch.num_rows())), + let (array, adopted_reorder) = match self.adaptive.as_mut() { + Some(adaptive) => { + let array = adaptive.evaluate(&batch); + // Report the settle-on-a-reorder transition, which + // fires at most once per stream. + (array, adaptive.take_adopted_reorder()) + } + None => ( + self.predicate + .as_ref() + .evaluate(&batch) + .and_then(|v| v.into_array(batch.num_rows())), + false, + ), }; + if adopted_reorder { + self.metrics.record_adaptive_reorder(); + } let status = array .and_then(|array| { Ok(match self.projection.as_ref() { diff --git a/datafusion/sqllogictest/test_files/adaptive_filter.slt b/datafusion/sqllogictest/test_files/adaptive_filter.slt index 3db9adc23b517..d8462127a8333 100644 --- a/datafusion/sqllogictest/test_files/adaptive_filter.slt +++ b/datafusion/sqllogictest/test_files/adaptive_filter.slt @@ -41,6 +41,25 @@ SELECT count(*) FROM t WHERE b = 3 AND s LIKE '1%'; ---- 160 +# Baseline plan (flag off), so the "EXPLAIN is identical on and off" claim +# below is asserted against something rather than merely stated. +query TT +EXPLAIN SELECT count(*) FROM t WHERE b = 3 AND s LIKE '1%'; +---- +logical_plan +01)Projection: count(Int64(1)) AS count(*) +02)--Aggregate: groupBy=[[]], aggr=[[count(Int64(1))]] +03)----Projection: +04)------Filter: t.b = Int64(3) AND t.s LIKE Utf8("1%") +05)--------TableScan: t projection=[b, s] +physical_plan +01)ProjectionExec: expr=[count(Int64(1))@0 as count(*)] +02)--AggregateExec: mode=Final, gby=[], aggr=[count(Int64(1))] +03)----CoalescePartitionsExec +04)------AggregateExec: mode=Partial, gby=[], aggr=[count(Int64(1))] +05)--------FilterExec: b@0 = 3 AND s@1 LIKE 1%, projection=[] +06)----------DataSourceExec: partitions=4, partition_sizes=[16, 16, 16, 15] + statement ok SET datafusion.execution.adaptive_filter_reordering = true; @@ -89,6 +108,28 @@ physical_plan 05)--------FilterExec: b@0 = 3 AND s@1 LIKE 1%, projection=[] 06)----------DataSourceExec: partitions=4, partition_sizes=[16, 16, 16, 15] +# EXPLAIN ANALYZE: the `adaptive_reorders` metric makes a reorder that actually +# happened at runtime visible. The predicate is written selective-conjunct-last: +# the arithmetic chain is the more expensive conjunct and matches every row, +# while `a > 3990` is a single comparison keeping 10 rows in 4000 — so the +# learned order flips the two. (Both conjuncts are "cheap" to the static +# cheap-first `reorder_predicates` rule, which is cost-only and blind to +# selectivity, so the written order reaches `FilterExec` untouched.) Each of the +# 4 partition streams adopts the reorder exactly once — the first stream to +# finish the pooled warm-up publishes the order and the others pick it up on +# their next batch — so the aggregated counter is 4. +query I +SELECT count(*) FROM t WHERE a % 97 + a % 89 + a % 83 + b % 13 >= 0 AND a > 3990; +---- +10 + +query TT +EXPLAIN ANALYZE +SELECT count(*) FROM t WHERE a % 97 + a % 89 + a % 83 + b % 13 >= 0 AND a > 3990; +---- +Plan with Metrics +adaptive_reorders=4 + statement ok SET datafusion.execution.adaptive_filter_reordering = false; diff --git a/docs/source/user-guide/metrics.md b/docs/source/user-guide/metrics.md index 5bb4895a3da1e..f672045507135 100644 --- a/docs/source/user-guide/metrics.md +++ b/docs/source/user-guide/metrics.md @@ -38,9 +38,10 @@ DataFusion operators expose runtime metrics so you can understand where time is ### FilterExec -| Metric | Description | -| ----------- | ----------------------------------------------------------------- | -| selectivity | Selectivity of the filter, calculated as output_rows / input_rows | +| Metric | Description | +| ----------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| selectivity | Selectivity of the filter, calculated as output_rows / input_rows | +| adaptive_reorders | Number of partition streams that adopted an adaptively reordered evaluation order for the predicate's conjuncts (at most one per stream). Only present when `datafusion.execution.adaptive_filter_reordering` is enabled; `0` means the measured order was kept. | ### HashJoinExec From c94e22deaa8945391a0595a3a33a8f10f257c267 Mon Sep 17 00:00:00 2001 From: Adrian Garcia Badaracco <1755071+adriangb@users.noreply.github.com> Date: Sun, 6 Sep 2026 23:57:52 -0500 Subject: [PATCH 12/27] test(physical-plan): deterministic adaptive filter tests and flag-on FilterExec coverage MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `streams_pool_measurements_and_share_settled_order` derived its `order == [1, 0]` / `compact` assertions from the real `Instant` timings of eight hundred-row batches, so the settle guard made it depend on the measured cost ratio staying inside the material-win window — a scheduling hiccup on a shared runner flips it. Seed the shared pool two batches short of the warm-up, as the scenario tests already do, so the two real batches cannot move the ranking. Also correct its comment: `rem_euclid(25) < 5` keeps 5 rows in 25 (20%), exactly the compact-once threshold. `no_reorder_evaluates_plain_predicate` still measures real timings and is left that way on purpose: both conjuncts pass ~96% of rows, and above `1 - TIE_COST_FRACTION` the material-win guard cannot hold for any positive costs, so no timing can produce a reorder there. Its doc now says so. New tests: - `adaptive_filter_reordering_end_to_end` (filter.rs): a real four- partition, sixteen-batch-per-partition `FilterExec` run with `adaptive_filter_reordering` on. Asserts the rows equal the flag-off output, NULLs are dropped, `adaptive_reorders` is registered only with the flag on and counts an adoption, and that re-executing the node (state kept) and re-executing after `reset_state` (state dropped) both produce identical rows. - `adopted_reorder_can_introduce_a_divide_by_zero` and `adopted_reorder_can_avoid_a_divide_by_zero_the_written_order_raises`: the two directions of the side effect the config option's doc warns about. The fused `BinaryExpr` `AND` pre-selects at its own 20% threshold, so `b <> 0 AND 1 / b > 2` succeeds flag-off when `b <> 0` holds for 15% of rows and errors once the conjuncts are reordered; mirrored, `1 / b > 2 AND a < 10` errors as written and succeeds once the selective conjunct is promoted and compacts the zeros away. - `first_measured_batch_initialises_the_shared_pool`: the lazy `stats.is_empty()` sizing of the pooled registry, including that an empty batch does not size it. The `FilterExec` conjuncts are too cheap for real timings to separate reliably, so the pool is seeded through a new `#[cfg(test)]` `AdaptiveFilterShared::seed_one_batch_short_of_warmup` — the same mocked-clock stand-in the scenario tests use, reachable from filter.rs. Co-Authored-By: Claude Fable 5.1 --- .../physical-plan/src/adaptive_filter.rs | 229 +++++++++++++++++- datafusion/physical-plan/src/filter.rs | 162 +++++++++++++ 2 files changed, 388 insertions(+), 3 deletions(-) diff --git a/datafusion/physical-plan/src/adaptive_filter.rs b/datafusion/physical-plan/src/adaptive_filter.rs index a89284a32f42b..5b90327f52e54 100644 --- a/datafusion/physical-plan/src/adaptive_filter.rs +++ b/datafusion/physical-plan/src/adaptive_filter.rs @@ -241,6 +241,32 @@ impl AdaptiveFilterShared { fn settled(&self) -> Option { self.inner.lock().expect("poisoned").settled.clone() } + + /// Test-only: seed the pooled measurements with `(rows, matched, nanos)` + /// per conjunct and leave the pool exactly one batch short of the warm-up, + /// so the next measured batch settles on the seeded decision. + /// + /// The stand-in for a mocked clock: it lets a test pin the settle decision + /// instead of depending on real timer values, which on a shared CI runner + /// can be perturbed by a scheduling hiccup far larger than the evaluation + /// being measured. Used by `FilterExec`'s end-to-end tests; the tests in + /// this module use the equivalent `tests::seed`. + #[cfg(test)] + pub(crate) fn seed_one_batch_short_of_warmup( + &self, + per_conjunct: &[(u64, u64, u64)], + ) { + let mut inner = self.inner.lock().expect("poisoned"); + inner.stats = per_conjunct + .iter() + .map(|&(rows, matched, nanos)| ConjunctStats { + rows, + matched, + nanos, + }) + .collect(); + inner.measured_batches = WARMUP_BATCHES - 1; + } } /// Adaptive evaluator for a single conjunctive predicate, owned per partition @@ -605,7 +631,7 @@ fn expected_cost_per_row(stats: &[ConjunctStats], order: &[usize]) -> f64 { mod tests { use super::*; - use arrow::array::Int32Array; + use arrow::array::{Int32Array, Int64Array}; use arrow::datatypes::{DataType, Field, Schema}; use datafusion_expr::Operator; use datafusion_physical_expr::expressions::{binary, col, lit}; @@ -824,6 +850,12 @@ mod tests { /// When the order does not change, the settled evaluator runs the plain /// predicate (compact-once is only used in service of a reorder), so an /// interchangeable conjunction costs exactly what the flag-off path costs. + /// + /// This one measures real timings on purpose and is still deterministic: + /// both conjuncts pass the same ~96% of rows, and with a pass rate `p` + /// above `1 - TIE_COST_FRACTION` the material-win guard + /// (`c1 + p*c0 < 0.95 * (c0 + p*c1)`) cannot hold for any positive costs, + /// so no timing can produce a reorder here. #[test] fn no_reorder_evaluates_plain_predicate() { let schema = schema(); @@ -854,16 +886,36 @@ mod tests { /// Two streams sharing one registry settle the order together: the pooled /// warm-up is `WARMUP_BATCHES` total across both streams, and once one /// stream publishes the order the other adopts it on its next batch. + /// + /// The pool is seeded (see [`seed`]) two batches short of the warm-up, so + /// the two real measured batches that complete it cannot move the ranking: + /// their counts are several orders of magnitude smaller than the seeded + /// ones. Deriving the ranking from the real `Instant` timings of two + /// hundred-row batches instead would make the assertions below depend on + /// the measured cost ratio staying inside the material-win guard, which a + /// scheduling hiccup on a shared runner can flip. #[test] fn streams_pool_measurements_and_share_settled_order() { let schema = schema(); - let p = predicate(&schema); + let p = predicate(&schema); // `a > 2 AND b < 5`, written order [0, 1] let shared = Arc::new(AdaptiveFilterShared::new()); + // Conjunct 1 is far more selective, so promoting it is materially + // cheaper. Two batches short of the warm-up: the two streams below + // pool one measured batch each to complete it. + seed( + &shared, + vec![ + stats(70_000_000, 63_000_000, 70_000_000), // pass 0.9, ~1ns/row + stats(70_000_000, 700_000, 350_000_000), // pass 0.01, ~5ns/row + ], + WARMUP_BATCHES - 2, + ); let mut s1 = AdaptiveConjunction::try_new(&p, Arc::clone(&shared)).unwrap(); let mut s2 = AdaptiveConjunction::try_new(&p, Arc::clone(&shared)).unwrap(); // `b < 5` (conjunct 1) is the selective one; drive both streams with - // batches where it keeps ~1 row in 25. + // batches where it keeps 5 rows in 25 (20%, exactly the compact-once + // threshold). let mk = |round: i32| { let base = round * 100; let a: Vec = (base..base + 100).collect(); @@ -1051,4 +1103,175 @@ mod tests { assert_eq!(trace, vec!["Measure", "Fused", "Fused"]); assert_eq!(adaptive.order, vec![0, 1]); } + + /// The pooled registry is sized lazily by the first measured batch (the + /// conjunct count is not known to `AdaptiveFilterShared`, which is built + /// before the predicate is split), and the counts of that first batch land + /// in it. + #[test] + fn first_measured_batch_initialises_the_shared_pool() { + let schema = schema(); + let p = predicate(&schema); // `a > 2 AND b < 5` + let shared = Arc::new(AdaptiveFilterShared::new()); + assert!(shared.inner.lock().unwrap().stats.is_empty()); + let mut adaptive = AdaptiveConjunction::try_new(&p, Arc::clone(&shared)).unwrap(); + // Empty batches measure nothing, so the pool is still unsized after one. + adaptive.evaluate(&batch(&schema, vec![], vec![])).unwrap(); + assert!(shared.inner.lock().unwrap().stats.is_empty()); + + let a: Vec = (0..10).collect(); + adaptive.evaluate(&batch(&schema, a.clone(), a)).unwrap(); + + let inner = shared.inner.lock().unwrap(); + assert_eq!(inner.stats.len(), 2, "sized to the conjunct count"); + assert_eq!(inner.measured_batches, 1); + // `a > 2` keeps 7 of 10 rows: too many to compact, so `b < 5` is + // evaluated on all 10 rows too and keeps 5 of them. + assert_eq!((inner.stats[0].rows, inner.stats[0].matched), (10, 7)); + assert_eq!((inner.stats[1].rows, inner.stats[1].matched), (10, 5)); + } + + /// `Int64` schema for the divide-by-zero side-effect tests below. + fn int64_schema() -> Arc { + Arc::new(Schema::new(vec![ + Field::new("a", DataType::Int64, false), + Field::new("b", DataType::Int64, false), + ])) + } + + fn int64_batch(schema: &Arc, a: Vec, b: Vec) -> RecordBatch { + RecordBatch::try_new( + Arc::clone(schema), + vec![Arc::new(Int64Array::from(a)), Arc::new(Int64Array::from(b))], + ) + .unwrap() + } + + /// The two conjuncts `b <> 0` and `1 / b > 2`. + fn divide_by_zero_conjuncts( + schema: &Arc, + ) -> (Arc, Arc) { + let non_zero = binary( + col("b", schema).unwrap(), + Operator::NotEq, + lit(0i64), + schema, + ) + .unwrap(); + let divide = binary( + binary( + lit(1i64), + Operator::Divide, + col("b", schema).unwrap(), + schema, + ) + .unwrap(), + Operator::Gt, + lit(2i64), + schema, + ) + .unwrap(); + (non_zero, divide) + } + + /// This is the behaviour the config option's doc warns about: adopting a + /// reorder can introduce an error the written order avoided. + /// + /// `b <> 0 AND 1 / b > 2` on data where `b <> 0` holds for 15% of the rows. + /// The written fused `BinaryExpr` `AND` pre-selects (its own threshold is + /// also 20%), so `1 / b` never sees a zero and the flag-off query succeeds. + /// Once the conjuncts are reordered, `1 / b > 2` runs first — on every row, + /// zeros included — and integer division by zero is an error. + #[test] + fn adopted_reorder_can_introduce_a_divide_by_zero() { + let schema = int64_schema(); + let (non_zero, divide) = divide_by_zero_conjuncts(&schema); + // Written order [0, 1] = [`b <> 0`, `1 / b > 2`]. + let p = binary(non_zero, Operator::And, divide, &schema).unwrap(); + let a: Vec = (0..100).collect(); + let b: Vec = (0..100).map(|i| i64::from(i < 15)).collect(); + let rb = int64_batch(&schema, a, b); + + // Flag off: the fused predicate pre-selects on `b <> 0` (15 of 100 + // rows, within its 20% threshold) and succeeds. + assert!(p.evaluate(&rb).is_ok(), "flag-off evaluation must succeed"); + + let shared = Arc::new(AdaptiveFilterShared::new()); + // `1 / b > 2` (conjunct 1) seeded as cheap and very selective, so the + // warm-up settles on promoting it. One batch short of the warm-up. + seed( + &shared, + vec![ + stats(70_000_000, 63_000_000, 70_000_000), // pass 0.9, ~1ns/row + stats(70_000_000, 700_000, 70_000_000), // pass 0.01, ~1ns/row + ], + WARMUP_BATCHES - 1, + ); + let mut adaptive = AdaptiveConjunction::try_new(&p, Arc::clone(&shared)).unwrap(); + + // The settling batch is still measured in the written order, whose + // compaction on `b <> 0` also keeps `1 / b` away from the zeros. + adaptive.evaluate(&rb).unwrap(); + assert_eq!(adaptive.order, vec![1, 0]); + assert!(adaptive.compact); + + // The next batch runs the adopted reorder, and errors. + let err = adaptive.evaluate(&rb).unwrap_err().to_string(); + assert!(err.contains("Divide by zero"), "unexpected error: {err}"); + } + + /// The mirror of the case above: an error the written fused order *does* + /// raise, which the reordered compact-once evaluation avoids. + /// + /// `1 / b > 2 AND a < 10`, with `b = 0` on exactly the rows `a < 10` + /// discards. The fused `AND` evaluates its left side on every row and + /// errors; the adopted order runs `a < 10` first, compacts to its 10 + /// survivors (all with `b = 1`), and never divides by zero. + #[test] + fn adopted_reorder_can_avoid_a_divide_by_zero_the_written_order_raises() { + let schema = int64_schema(); + let (_, divide) = divide_by_zero_conjuncts(&schema); + let selective = binary( + col("a", &schema).unwrap(), + Operator::Lt, + lit(10i64), + &schema, + ) + .unwrap(); + // Written order [0, 1] = [`1 / b > 2`, `a < 10`]. + let p = binary(divide, Operator::And, selective, &schema).unwrap(); + + let shared = Arc::new(AdaptiveFilterShared::new()); + // Conjunct 1 (`a < 10`) seeded as cheap and very selective, conjunct 0 + // as expensive and unselective, so the warm-up promotes conjunct 1. + seed( + &shared, + vec![ + stats(70_000_000, 63_000_000, 350_000_000), // pass 0.9, ~5ns/row + stats(70_000_000, 700_000, 70_000_000), // pass 0.01, ~1ns/row + ], + WARMUP_BATCHES - 1, + ); + let mut adaptive = AdaptiveConjunction::try_new(&p, Arc::clone(&shared)).unwrap(); + + // Settle on a batch with no zeros at all, so the warm-up itself (which + // evaluates in the written order) cannot hit the error. + let a: Vec = (0..100).collect(); + adaptive + .evaluate(&int64_batch(&schema, a.clone(), vec![1; 100])) + .unwrap(); + assert_eq!(adaptive.order, vec![1, 0]); + + // Now a batch whose `b` is zero on every row `a < 10` discards. + let b: Vec = (0..100).map(|i| i64::from(i < 10)).collect(); + let rb = int64_batch(&schema, a, b); + + // The written fused order divides by zero... + let err = p.evaluate(&rb).unwrap_err().to_string(); + assert!(err.contains("Divide by zero"), "unexpected error: {err}"); + // ...while the adopted order compacts `1 / b > 2` down to the rows + // `a < 10` kept, none of which is zero. + let got = adaptive.evaluate(&rb).unwrap(); + assert!(passing_rows(&got).is_empty(), "1 / 1 > 2 is false"); + } } diff --git a/datafusion/physical-plan/src/filter.rs b/datafusion/physical-plan/src/filter.rs index 2f2d5fbdf66ac..87eee8b104424 100644 --- a/datafusion/physical-plan/src/filter.rs +++ b/datafusion/physical-plan/src/filter.rs @@ -1671,8 +1671,11 @@ mod tests { use crate::expressions::*; use crate::statistics::{StatisticsArgs, StatisticsContext}; use crate::test; + use crate::test::TestMemoryExec; use crate::test::exec::StatisticsExec; + use arrow::array::{Array, Int64Array}; use arrow::datatypes::{Field, Schema, UnionFields, UnionMode}; + use datafusion_execution::config::SessionConfig; #[test] fn filter_rejects_zero_batch_size() -> Result<()> { @@ -2566,6 +2569,165 @@ mod tests { Ok(()) } + /// End-to-end `FilterExec` run with `adaptive_filter_reordering` enabled: + /// four partitions of sixteen small batches, a nullable column, and a + /// conjunction written selective-*last*. + /// + /// Both conjuncts are cheap arithmetic, so real timings cannot separate + /// them reliably; the shared pool is therefore seeded one batch short of + /// the warm-up (the stand-in for a mocked clock) so the reorder is adopted + /// deterministically. Everything else — the streams, the metric, the + /// coalescer, the projection-free output path — is the real thing. + #[tokio::test] + async fn adaptive_filter_reordering_end_to_end() -> Result<()> { + const PARTITIONS: usize = 4; + const BATCHES: usize = 16; + const ROWS: i64 = 64; + // `b` is NULL on every 37th row, including inside the range the + // predicate selects, so the run exercises SQL filter semantics. + const NULL_EVERY: i64 = 37; + + let schema = Arc::new(Schema::new(vec![ + Field::new("a", DataType::Int64, false), + Field::new("b", DataType::Int64, true), + ])); + + let mut partitions = Vec::with_capacity(PARTITIONS); + for p in 0..PARTITIONS { + let mut batches = Vec::with_capacity(BATCHES); + for batch in 0..BATCHES { + let base = (p * BATCHES + batch) as i64 * ROWS; + let a: Vec = (base..base + ROWS).collect(); + let b: Vec> = a + .iter() + .map(|&v| (v % NULL_EVERY != 0).then_some(v)) + .collect(); + batches.push(RecordBatch::try_new( + Arc::clone(&schema), + vec![Arc::new(Int64Array::from(a)), Arc::new(Int64Array::from(b))], + )?); + } + partitions.push(batches); + } + let total_rows = (PARTITIONS * BATCHES) as i64 * ROWS; + let input: Arc = + TestMemoryExec::try_new_exec(&partitions, Arc::clone(&schema), None)?; + + // `(a % 97 + a % 89 >= 0) AND (b > 3990)`: the left conjunct is always + // true and the more expensive of the two, the right one keeps a + // hundred-odd rows out of four thousand — the selective conjunct is + // written last, and both are cheap arithmetic, so the logical + // heuristics would not have reordered this. + let threshold = total_rows - 106; // 3990 + let cheap = binary( + binary( + binary(col("a", &schema)?, Operator::Modulo, lit(97i64), &schema)?, + Operator::Plus, + binary(col("a", &schema)?, Operator::Modulo, lit(89i64), &schema)?, + &schema, + )?, + Operator::GtEq, + lit(0i64), + &schema, + )?; + let selective = + binary(col("b", &schema)?, Operator::Gt, lit(threshold), &schema)?; + let predicate = binary(cheap, Operator::And, selective, &schema)?; + let filter = Arc::new(FilterExec::try_new(predicate, input)?); + + // Execute every partition with the flag set as given and return the + // output rows, sorted so partition interleaving cannot matter. + async fn run( + filter: &Arc, + adaptive: bool, + ) -> Result> { + let mut config = SessionConfig::new(); + config.options_mut().execution.adaptive_filter_reordering = adaptive; + let ctx = Arc::new(TaskContext::default().with_session_config(config)); + let mut rows = vec![]; + for partition in 0..PARTITIONS { + let stream = filter.execute(partition, Arc::clone(&ctx))?; + for batch in crate::common::collect(stream).await? { + let a = batch + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + let b = batch + .column(1) + .as_any() + .downcast_ref::() + .unwrap(); + for i in 0..batch.num_rows() { + // SQL filter semantics: `NULL > 3990` is not `true`. + assert!(!b.is_null(i), "a NULL row survived the filter"); + rows.push((a.value(i), b.value(i))); + } + } + } + rows.sort_unstable(); + Ok(rows) + } + + // The rows the predicate selects, computed independently: `b > 3990` + // spans 105 values, three of which have a NULL `b` and must be dropped. + let candidates: Vec = (threshold + 1..total_rows).collect(); + assert_eq!(candidates.len(), 105); + let expected: Vec<(i64, i64)> = candidates + .into_iter() + .filter(|v| v % NULL_EVERY != 0) + .map(|v| (v, v)) + .collect(); + assert_eq!(expected.len(), 102, "three NULL `b` rows are dropped"); + + let flag_off = run(&filter, false).await?; + assert_eq!(flag_off, expected); + assert!( + filter + .metrics() + .unwrap() + .sum_by_name("adaptive_reorders") + .is_none(), + "the flag-off path must not register the adaptive metric" + ); + + // Seed the pooled measurements so the warm-up settles on promoting the + // selective conjunct: conjunct 0 keeps every row and is ~5x the cost of + // conjunct 1, which keeps 1%. + filter.adaptive_stats.seed_one_batch_short_of_warmup(&[ + (70_000_000, 70_000_000, 350_000_000), + (70_000_000, 700_000, 70_000_000), + ]); + + let flag_on = run(&filter, true).await?; + assert_eq!(flag_on, flag_off, "reordering must not change results"); + let reorders = filter + .metrics() + .unwrap() + .sum_by_name("adaptive_reorders") + .map(|m| m.as_usize()) + .unwrap_or(0); + assert!(reorders >= 1, "expected an adopted reorder, got {reorders}"); + + // Re-executing the same node keeps the learned state (the streams adopt + // the already-published order on their first batch) but cannot change + // the rows. + assert_eq!(run(&filter, true).await?, flag_off, "state persists"); + + // `reset_state` drops the pooled measurements, so the fresh node learns + // from scratch — and still produces the same rows. + let reset = Arc::clone(&filter).reset_state()?; + let reset: Arc = Arc::new( + reset + .as_ref() + .downcast_ref::() + .expect("reset_state returns a FilterExec") + .clone(), + ); + assert_eq!(run(&reset, true).await?, flag_off, "after reset_state"); + Ok(()) + } + #[test] fn test_equivalence_properties_union_type() -> Result<()> { let union_type = DataType::Union( From 113b05cef89d50632e57b3a0e223c6b3a69c0e03 Mon Sep 17 00:00:00 2001 From: Adrian Garcia Badaracco <1755071+adriangb@users.noreply.github.com> Date: Mon, 7 Sep 2026 00:09:39 -0500 Subject: [PATCH 13/27] docs(physical-plan): tighten adaptive filter documentation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Documentation-only pass over the adaptive filter, applying alamb's review nits and correcting the stated baseline. - Module doc: describe what conjunct evaluation already does today — `reorder_predicates` orders conjuncts cheap-before-expensive by a static cost class, and `BinaryExpr`'s `AND` pre-selects when the conjuncts so far keep <= 20% of the rows and produce no nulls. Spell out what pre-selection cannot do (gate on a later conjunct, fire through nulls, carry survivors compacted across a nested chain) instead of claiming the `AND` evaluates every conjunct on every row regardless of order. - Add an intra-doc link to `BinaryExpr`, drop the "left-deep fused" jargon, expand the `regexp_like` example into a before/after evaluation order, and leave the flag's default value documented on the flag itself. - Drop the unsupported "compact-once is itself a win even without reordering" claim; point at the PR for the measurements rather than quoting numbers. - Fold the side-effect caveat, the conditional-statistics caveat and the one-shot caveat into a single "Known limitations" list instead of repeating them across the module; settle on one vocabulary (a decision is *settled*, a stream *adopts* it) and remove the leftover "publishes", "frozen" and "A/B-validated" wording. - Config doc: shorter, and the side-effect caveat is now bidirectional (a divide-by-zero can appear *or* disappear). Co-Authored-By: Claude Fable 5.1 --- datafusion/common/src/config.rs | 22 +- .../physical-plan/src/adaptive_filter.rs | 300 +++++++++--------- datafusion/physical-plan/src/filter.rs | 47 ++- .../test_files/adaptive_filter.slt | 6 +- .../test_files/information_schema.slt | 2 +- docs/source/user-guide/configs.md | 2 +- docs/source/user-guide/metrics.md | 8 +- 7 files changed, 197 insertions(+), 190 deletions(-) diff --git a/datafusion/common/src/config.rs b/datafusion/common/src/config.rs index ac81570a0fdeb..97a2d61f8e5d5 100644 --- a/datafusion/common/src/config.rs +++ b/datafusion/common/src/config.rs @@ -1138,19 +1138,15 @@ config_namespace! { /// Note: this option currently only applies to the symmetric hash join. pub enforce_batch_size_in_joins: bool, default = false - /// (experimental) When enabled, `FilterExec` adaptively reorders the - /// conjuncts of a conjunctive predicate at runtime. It measures each - /// conjunct's selectivity and evaluation cost on the rows that reach it - /// and runs the conjuncts that discard the most rows per unit of CPU - /// time first, so cheap-and-selective predicates gate expensive ones. - /// This never changes query results, but it is off by default because - /// it can change observable side effects of fallible predicates (even - /// when no reorder is adopted): while measuring, and after a reorder, - /// conjuncts are evaluated only on the rows that survived the conjuncts - /// before them, so an error the fused predicate would have raised on an - /// already-filtered row (e.g. `b <> 0 AND 1/b > 2` evaluating `1/b` on - /// every row) may not occur. Predicates containing volatile expressions - /// are never reordered. + /// (experimental) When enabled, `FilterExec` measures the selectivity + /// and evaluation cost of each conjunct of an `AND` predicate at + /// runtime and reorders them to run the ones that discard the most + /// rows per unit of CPU time first. Query results never change, but + /// the observable side effects of a fallible predicate can, in either + /// direction: reordering `b <> 0 AND 1/b > 2` can make a + /// divide-by-zero error appear or disappear, since each conjunct is + /// evaluated only on the rows the conjuncts before it kept. Predicates + /// containing volatile expressions are never reordered. pub adaptive_filter_reordering: bool, default = false /// Size (bytes) of data buffer DataFusion uses when writing output files. diff --git a/datafusion/physical-plan/src/adaptive_filter.rs b/datafusion/physical-plan/src/adaptive_filter.rs index 5b90327f52e54..39e74df39d205 100644 --- a/datafusion/physical-plan/src/adaptive_filter.rs +++ b/datafusion/physical-plan/src/adaptive_filter.rs @@ -18,23 +18,68 @@ //! Runtime-adaptive evaluation of a conjunctive (`AND`) predicate in //! [`FilterExec`](crate::filter::FilterExec). //! -//! Predicate evaluation order matters: a selective predicate run first gates -//! the work of the predicates after it. DataFusion's `BinaryExpr` `AND` -//! short-circuit only gates on the *leftmost* conjunct, so a conjunction whose -//! selective member is written last (e.g. -//! `regexp_like(s,'a') AND … AND regexp_like(s,'rare')`) evaluates every -//! predicate against ~every row. +//! Evaluation order matters: a selective conjunct run first gates the work of +//! the conjuncts after it. Two mechanisms already order and gate conjuncts +//! before this module sees them, and both decide statically: +//! +//! - the logical optimizer's `reorder_predicates` pass sorts the conjuncts +//! cheap-before-expensive by a static cost class +//! (). It is blind to +//! selectivity, so a cheap-but-unselective conjunct still sorts ahead of an +//! expensive-but-very-selective one, and conjuncts in the same cost class +//! keep the order they were written in. That is the order this module calls +//! the *written order* and measures against. +//! - [`BinaryExpr`](datafusion_physical_expr::expressions::BinaryExpr)'s `AND` +//! pre-selects: when the conjuncts evaluated so far keep at most 20% of the +//! rows and produce no nulls, it filters the batch down to those rows before +//! evaluating the next conjunct. +//! +//! Pre-selection can only gate a conjunct on the conjuncts written *before* it, +//! never on a more selective one written after it; it does not fire while the +//! accumulated result still has nulls; and in a left-nested `AND` chain each +//! level filters the original batch and scatters its result back to full +//! length, so survivors are not carried forward compacted from one level to +//! the next. +//! +//! This module measures each conjunct's selectivity and cost at runtime, +//! reorders them accordingly, and evaluates them through a loop that compacts +//! the survivors once and keeps them compacted. Whether it runs at all is +//! controlled by `datafusion.execution.adaptive_filter_reordering`. +//! +//! For example: +//! +//! ```sql +//! WHERE regexp_like(s,'a') AND regexp_like(s,'b') AND regexp_like(s,'rare') +//! ``` +//! +//! All three conjuncts are equally expensive to the static cost class, so they +//! reach `FilterExec` as written. The first two each keep most rows, so `AND` +//! pre-selection never fires and every conjunct runs on the whole batch: +//! +//! ```text +//! regexp_like(s,'a') evaluated on every row +//! regexp_like(s,'b') evaluated on every row +//! regexp_like(s,'rare') evaluated on every row +//! ``` +//! +//! Once the warm-up has measured the three, the selective one is promoted and +//! the batch is compacted behind it: +//! +//! ```text +//! regexp_like(s,'rare') evaluated on every row, keeps ~1% -> batch compacted +//! regexp_like(s,'a') evaluated on those survivors only +//! regexp_like(s,'b') evaluated on those survivors only +//! ``` //! //! ## How it evaluates: the compact-once loop //! -//! The conjuncts are evaluated sequentially, combining their boolean results +//! The conjuncts are evaluated one at a time, combining their boolean results //! with `AND`. The working batch is physically compacted to the surviving rows -//! once the accumulated mask becomes selective enough — so a run of -//! non-selective conjuncts costs only cheap bitwise `AND`s, while a selective -//! conjunct shrinks the batch the conjuncts after it must decode. This -//! compaction is what makes ordering pay off (and is itself a win even without -//! reordering): a left-deep fused `BinaryExpr` `AND` does *not* compact between -//! conjuncts, so it evaluates ~every conjunct on ~every row regardless of order. +//! once the accumulated mask becomes selective enough +//! ([`COMPACTION_SELECTIVITY_THRESHOLD`]), and every conjunct after that point +//! is evaluated against the compacted batch. So a run of non-selective +//! conjuncts costs only cheap bitwise `AND`s, and everything after a selective +//! conjunct decodes just its survivors. //! //! ## How it orders //! @@ -42,55 +87,47 @@ //! its marginal selectivity and per-row cost. After a short warm-up the //! conjuncts are ranked by rows discarded per nanosecond //! (`(1 - pass_rate) / cost_per_row`, the classic optimal ordering key for -//! independent conjuncts), and if the ranked order is *materially* cheaper than -//! the written one it is adopted. The order then stays fixed. +//! independent conjuncts), and the ranking is adopted only if it is materially +//! cheaper than the written order ([`TIE_COST_FRACTION`]). The order then stays +//! fixed. //! -//! Compact-once is used **only in service of a reorder**: if the warm-up does -//! not reorder the conjuncts (e.g. they are interchangeable), the written -//! predicate is evaluated as-is, so once settled a conjunction that does not -//! benefit from reordering pays no compact-once overhead and evaluates exactly -//! as it would with the feature off. During the warm-up itself the conjuncts -//! are necessarily evaluated individually (with compaction) so they can be -//! measured — see the side-effects caveat below. +//! The compact-once loop is used only in service of a reorder: if the warm-up +//! does not reorder the conjuncts, the written predicate is from then on +//! evaluated as one expression, exactly as it would be with the flag off. +//! During the warm-up the conjuncts are necessarily evaluated one at a time so +//! they can be measured individually. //! //! ## How it shares //! //! A `FilterExec` is split across many partition streams, each seeing only a //! slice of the data. Measurements are pooled into a shared //! [`AdaptiveFilterShared`] so the streams learn as one: the first stream to -//! accumulate enough samples settles the order and publishes it, and the others -//! adopt it on their next batch without each re-paying the warm-up — which is -//! what makes the win materialise when each stream is only a handful of batches -//! long. Only unsettled streams touch the shared mutex, and only to pool a -//! batch's counts or pick up a published decision; a settled stream never -//! locks it again. -//! -//! It is **off by default** -//! (`datafusion.execution.adaptive_filter_reordering`) and never changes query -//! results: a conjunction's value is independent of evaluation order. Predicates -//! containing volatile expressions are never reordered (their observable side -//! effects depend on order). -//! -//! Observable *side effects* of fallible predicates can change even when no -//! reorder is adopted: whenever conjuncts are evaluated individually (during -//! warm-up, or settled with a reorder), a conjunct after a compaction sees only -//! the surviving rows, so an error a fused evaluation would have raised on an -//! already-filtered row (e.g. `b <> 0 AND 1/b > 2` with the fused `AND` -//! evaluating `1/b` on every row) may not occur. +//! accumulate enough samples settles the order for all of them, and the rest +//! adopt it on their next batch instead of each re-paying the warm-up — which +//! is what makes the win materialise when each stream is only a handful of +//! batches long. Only unsettled streams take the shared lock; a settled stream +//! never touches it again. //! //! ## Known limitations //! -//! The statistics are *conditional*: each conjunct is measured on the rows that -//! survived the conjuncts before it in written order, and (after a compaction) -//! on small survivor batches whose per-row cost is inflated by fixed overheads. -//! Correlated conjuncts can therefore look more selective in a late position -//! than they would be up front, and the settle decision is one-shot: once -//! adopted, the order is never re-measured, so a misjudged reorder (or drifting -//! data) is kept for the stream's lifetime. The material-win guard -//! ([`TIE_COST_FRACTION`]) makes adoption conservative but cannot detect -//! correlation. Further policies (drift re-measurement, confidence-interval -//! statistics, A/B-validated adoption for the cases a cost model cannot -//! separate) can build on top of this core. +//! - Reordering never changes query *results* — the value of a conjunction does +//! not depend on evaluation order — but it can change the observable *side +//! effects* of fallible predicates, in either direction: a conjunct evaluated +//! after a compaction sees only the rows that survived, so an error the +//! written order raises can disappear and one it avoided can appear. +//! Predicates containing volatile expressions are never reordered. +//! - The measurements are *conditional*: each conjunct is measured on the rows +//! that survived the conjuncts before it in written order, and after a +//! compaction on small survivor batches whose per-row cost is inflated by +//! fixed overheads. Correlated conjuncts can therefore look more selective in +//! a late position than they would be up front; the material-win guard makes +//! adoption conservative but cannot detect correlation. +//! - The decision is one-shot: once settled, the order is never re-measured, so +//! a misjudged reorder — or data whose selectivity drifts — is kept for the +//! rest of the query. +//! +//! See for the measurements +//! behind this design. use std::sync::Arc; use std::sync::Mutex; @@ -178,13 +215,8 @@ impl ConjunctStats { /// State shared by every partition stream of one `FilterExec`, so the streams /// learn as one: per-conjunct measurements are pooled across streams and the -/// first stream to accumulate enough samples settles the order for all of them. -/// -/// This matters because a `FilterExec` is split across many partition streams, -/// each seeing only a slice of the data. Without sharing, every stream pays its -/// own warm-up — and when each stream is only a handful of batches long, that -/// warm-up is most of its work, so the reordering win never materialises. With -/// sharing the warm-up is paid roughly once per query, not once per stream. +/// first stream to accumulate enough samples settles the order for all of them +/// (see "How it shares" in the [module docs](self)). #[derive(Debug, Default)] pub(crate) struct AdaptiveFilterShared { inner: Mutex, @@ -201,28 +233,27 @@ struct SharedInner { settled: Option, } -/// How one batch was evaluated. Reported by -/// [`AdaptiveConjunction::evaluate_traced`] so the evaluator's behaviour is -/// observable batch by batch (its input/output contract is exercised by the -/// `scenario_*` tests). Borrows the adopted order rather than cloning it, so -/// reporting costs nothing on the per-batch path. +/// How one batch was evaluated, reported by +/// [`AdaptiveConjunction::evaluate_traced`] so the `scenario_*` tests can +/// assert the strategy batch by batch. Borrows the adopted order rather than +/// cloning it, so reporting costs nothing on the per-batch path. #[derive(Debug, Clone, Copy, PartialEq, Eq)] enum BatchStrategy<'a> { /// Still learning: the written order through the compact-once loop, each /// conjunct instrumented and its counts pooled. (Empty batches are /// evaluated but measure nothing and do not consume the warm-up.) Measure, - /// Settled without a reorder: the written fused predicate, exactly as if - /// the feature were off. + /// Settled without a reorder: the written predicate evaluated as one + /// expression, exactly as if the feature were off. Fused, - /// Settled on an adopted reorder, evaluated through the compact-once - /// loop. The payload is the adopted evaluation order: positions in the - /// written conjunct list, first-evaluated first. + /// Settled on an adopted reorder, evaluated through the compact-once loop. + /// The payload is the adopted order: positions in the written conjunct + /// list, first-evaluated first. Reordered(&'a [usize]), } /// The settled outcome of the warm-up: the evaluation order, and whether to run -/// it through the compact-once loop or the plain predicate. +/// it through the compact-once loop or as the written predicate. #[derive(Debug, Clone)] struct Settled { /// Evaluation order: indices into the conjunct list. @@ -237,7 +268,7 @@ impl AdaptiveFilterShared { Self::default() } - /// The published settled decision, or `None` if streams are still learning. + /// The settled decision, or `None` if the streams are still learning. fn settled(&self) -> Option { self.inner.lock().expect("poisoned").settled.clone() } @@ -246,11 +277,11 @@ impl AdaptiveFilterShared { /// per conjunct and leave the pool exactly one batch short of the warm-up, /// so the next measured batch settles on the seeded decision. /// - /// The stand-in for a mocked clock: it lets a test pin the settle decision - /// instead of depending on real timer values, which on a shared CI runner - /// can be perturbed by a scheduling hiccup far larger than the evaluation - /// being measured. Used by `FilterExec`'s end-to-end tests; the tests in - /// this module use the equivalent `tests::seed`. + /// This stands in for a mocked clock: it pins the settle decision instead + /// of leaving it to real timer values, which a scheduling hiccup on a + /// shared CI runner can perturb by far more than the evaluation being + /// measured. Used by `FilterExec`'s end-to-end tests; the tests in this + /// module use the equivalent `tests::seed`. #[cfg(test)] pub(crate) fn seed_one_batch_short_of_warmup( &self, @@ -276,9 +307,8 @@ impl AdaptiveFilterShared { pub(crate) struct AdaptiveConjunction { /// The split conjuncts. `order` indices refer to positions here. conjuncts: Vec>, - /// The written predicate, evaluated as-is when the settled order does not - /// reorder it (so a settled non-reorder costs exactly what the flag-off path - /// costs — no compact-once overhead). + /// The written predicate as one expression, evaluated as-is when the + /// settled order does not reorder it. predicate: Arc, /// Measurements and the settled decision, shared by every partition stream. shared: Arc, @@ -288,7 +318,7 @@ pub(crate) struct AdaptiveConjunction { /// Whether the settled order runs through the compact-once loop; `false` /// means evaluate [`predicate`](Self::predicate) directly. compact: bool, - /// Whether the order is settled (frozen): this stream no longer measures. + /// Whether the order is settled: this stream no longer measures. settled: bool, /// Set when this stream adopts a *reordered* decision, and cleared by /// [`take_adopted_reorder`](Self::take_adopted_reorder) — a one-shot @@ -334,10 +364,9 @@ impl AdaptiveConjunction { /// Whether this stream has just adopted a reordered evaluation order, /// clearing the signal. /// - /// Fires exactly once per stream, on the batch at which the stream settles - /// on a reorder — whether it settled the order itself or picked up one - /// another stream published. A stream that settles on the written order - /// (no reorder) never fires. + /// Fires exactly once per stream, on the batch at which it adopts a + /// reorder — whether it settled the order itself or took up one another + /// stream settled. A stream that settles on the written order never fires. pub(crate) fn take_adopted_reorder(&mut self) -> bool { std::mem::take(&mut self.adopted_reorder) } @@ -346,21 +375,19 @@ impl AdaptiveConjunction { /// (over the batch's rows) of rows that passed every conjunct. /// /// Until the order settles, each batch is measured and its counts pooled - /// into the shared registry; a stream adopts the settled order another - /// stream published on its next batch. + /// into the shared state. pub(crate) fn evaluate(&mut self, batch: &RecordBatch) -> Result { self.evaluate_traced(batch).map(|(mask, _)| mask) } /// [`evaluate`](Self::evaluate), additionally reporting the - /// [`BatchStrategy`] used for this batch, so the evaluator's behaviour is - /// observable batch by batch (see the scenario tests). + /// [`BatchStrategy`] used for this batch. fn evaluate_traced( &mut self, batch: &RecordBatch, ) -> Result<(ArrayRef, BatchStrategy<'_>)> { - // Adopt a settled order another stream published since our last batch. - // An unsettled stream already locks the shared state once per measured + // Take up an order another stream settled since our last batch. An + // unsettled stream already locks the shared state once per measured // batch to pool its counts, so this brief extra lock is in the same // cost class; a settled stream never touches it again. if !self.settled @@ -395,8 +422,8 @@ impl AdaptiveConjunction { } /// Evaluate the settled arrangement with no instrumentation: the - /// compact-once loop when the order was reordered, or the written predicate - /// directly otherwise (identical to the feature being off). + /// compact-once loop when the order was reordered, the written predicate + /// directly otherwise. fn evaluate_settled(&self, batch: &RecordBatch) -> Result { if self.compact { eval_conjuncts(&self.conjuncts, &self.order, batch, None) @@ -415,7 +442,8 @@ impl AdaptiveConjunction { } /// Merge this batch's measurements into the shared pool and, once enough - /// batches have accrued across all streams, decide and publish the order. + /// batches have accrued across all streams, settle the order for all of + /// them. fn pool_and_maybe_settle(&mut self, local: &[ConjunctStats]) { let mut inner = self.shared.inner.lock().expect("poisoned"); if inner.stats.is_empty() { @@ -449,14 +477,11 @@ impl AdaptiveConjunction { /// Decide the settled arrangement from the pooled measurements. /// -/// Rank the conjuncts by effectiveness and adopt the ranking only if it is -/// materially cheaper than the written order. A genuine reorder runs through the -/// compact-once loop (the source of the win); otherwise the written predicate is -/// kept and evaluated as-is. This is the guard: compact-once is only ever used -/// in service of a reorder, so a conjunction that does not benefit from -/// reordering (interchangeable conjuncts, e.g. several equally expensive -/// unselective predicates) pays no compact-once overhead and behaves exactly as -/// it would with the feature off. +/// Rank the conjuncts by effectiveness and take the ranking only if it is +/// materially cheaper than the written order; otherwise keep the written +/// predicate and evaluate it as one expression. This is the guard that keeps +/// the compact-once loop in service of a reorder, so a conjunction that does +/// not benefit from reordering pays none of its overhead. fn settle(stats: &[ConjunctStats]) -> Settled { let identity: Vec = (0..stats.len()).collect(); let candidate = rank_by_effectiveness(stats); @@ -486,9 +511,8 @@ fn settle(stats: &[ConjunctStats]) -> Settled { /// the accumulated mask becomes selective enough (see /// [`COMPACTION_SELECTIVITY_THRESHOLD`]); until then masks are combined with a /// cheap bitwise `AND`, so a run of non-selective conjuncts pays no -/// materialization cost. Unlike a fused `BinaryExpr` chain, survivors stay -/// compacted across the remaining conjuncts instead of being re-evaluated on -/// every row. +/// materialization cost. Once compacted, the survivors stay compacted for the +/// conjuncts that follow. fn eval_conjuncts( conjuncts: &[Arc], order: &[usize], @@ -776,7 +800,7 @@ mod tests { assert!((expected_cost_per_row(&s, &[1, 0]) - 10.5).abs() < 1e-9); } - /// The compact-once loop returns exactly the rows the plain predicate + /// The compact-once loop returns exactly the rows the written predicate /// keeps, in any order and whether or not compaction triggers. #[test] fn eval_conjuncts_matches_predicate_in_any_order() { @@ -800,7 +824,7 @@ mod tests { } } - /// Across the warm-up boundary the mask must always equal the plain + /// Across the warm-up boundary the mask must always equal the written /// predicate's, before and after the order settles. #[test] fn evaluate_matches_predicate_across_warmup() { @@ -826,7 +850,7 @@ mod tests { } /// A reorder is adopted only when materially cheaper; an already-good order - /// is left untouched and runs the plain predicate (no compact-once). + /// is left untouched and runs the written predicate (no compact-once). #[test] fn settle_keeps_order_when_not_materially_better() { // Two equally cheap, equally selective conjuncts: swapping cannot help, @@ -847,11 +871,10 @@ mod tests { assert!(d.compact); } - /// When the order does not change, the settled evaluator runs the plain - /// predicate (compact-once is only used in service of a reorder), so an - /// interchangeable conjunction costs exactly what the flag-off path costs. + /// When the order does not change, the settled evaluator runs the written + /// predicate as one expression. /// - /// This one measures real timings on purpose and is still deterministic: + /// This test measures real timings on purpose and is still deterministic: /// both conjuncts pass the same ~96% of rows, and with a pass rate `p` /// above `1 - TIE_COST_FRACTION` the material-win guard /// (`c1 + p*c0 < 0.95 * (c0 + p*c1)`) cannot hold for any positive costs, @@ -883,17 +906,13 @@ mod tests { ); } - /// Two streams sharing one registry settle the order together: the pooled - /// warm-up is `WARMUP_BATCHES` total across both streams, and once one - /// stream publishes the order the other adopts it on its next batch. + /// Two streams sharing one pool settle the order together: the warm-up is + /// `WARMUP_BATCHES` batches total across both streams, and once one stream + /// settles the order the other adopts it on its next batch. /// /// The pool is seeded (see [`seed`]) two batches short of the warm-up, so /// the two real measured batches that complete it cannot move the ranking: - /// their counts are several orders of magnitude smaller than the seeded - /// ones. Deriving the ranking from the real `Instant` timings of two - /// hundred-row batches instead would make the assertions below depend on - /// the measured cost ratio staying inside the material-win guard, which a - /// scheduling hiccup on a shared runner can flip. + /// their counts are orders of magnitude smaller than the seeded ones. #[test] fn streams_pool_measurements_and_share_settled_order() { let schema = schema(); @@ -949,10 +968,10 @@ mod tests { /// The reorder-adoption signal (`take_adopted_reorder`, what `FilterExec` /// counts into its `adaptive_reorders` metric) fires exactly once per /// stream: once for the stream that settles the order, and once for a - /// stream that later picks up the decision that stream published. + /// stream that later takes it up. /// - /// The per-conjunct costs are seeded (see [`seed`]) so the settle decision - /// is a reorder deterministically, regardless of real timer values. + /// The per-conjunct costs are seeded (see [`seed`]) so the decision is a + /// reorder regardless of real timer values. #[test] fn adopted_reorder_signals_once_per_stream() { let schema = schema(); @@ -986,8 +1005,8 @@ mod tests { settler.evaluate(&rb).unwrap(); assert!(!settler.take_adopted_reorder()); - // `adopter` never measured its way to a decision: it picks up the - // published one on its next batch, and signals that once too. + // `adopter` never measured its way to a decision: it takes up the + // settled one on its next batch, and signals that once too. adopter.evaluate(&rb).unwrap(); assert!(adopter.compact); assert!(adopter.take_adopted_reorder()); @@ -1020,16 +1039,13 @@ mod tests { assert!(!adaptive.take_adopted_reorder()); } - /// End-to-end input/output-contract scenario: feed batches, observe the - /// strategy used for each one alongside the masks. + /// End-to-end contract scenario: feed batches, observe the strategy used + /// for each one alongside the masks. /// - /// Per-conjunct costs are injected by seeding the shared pool with - /// synthetic measurements (the stand-in for a mocked clock): conjunct 0 is - /// cheap but unselective, conjunct 1 is expensive but very selective, so - /// the warm-up must settle on promoting conjunct 1 and run the reorder - /// through the compact-once loop. The seeded magnitudes dominate the one - /// real measured batch, so the decision is deterministic regardless of - /// real timer values. + /// The seeded costs (see [`seed`]) make conjunct 0 cheap but unselective + /// and conjunct 1 expensive but very selective, so the warm-up must settle + /// on promoting conjunct 1 and run it through the compact-once loop, + /// regardless of real timer values. #[test] fn scenario_measure_batches_then_settle_on_reorder() { let schema = schema(); @@ -1068,8 +1084,8 @@ mod tests { } /// Contract scenario for the no-win case: interchangeable conjuncts settle - /// on the written fused predicate (as if the feature were off), never the - /// compact-once loop. + /// on the written predicate evaluated as one expression (as if the feature + /// were off), never the compact-once loop. #[test] fn scenario_measure_batches_then_settle_on_fused() { let schema = schema(); @@ -1178,8 +1194,8 @@ mod tests { /// reorder can introduce an error the written order avoided. /// /// `b <> 0 AND 1 / b > 2` on data where `b <> 0` holds for 15% of the rows. - /// The written fused `BinaryExpr` `AND` pre-selects (its own threshold is - /// also 20%), so `1 / b` never sees a zero and the flag-off query succeeds. + /// The written `BinaryExpr` `AND` pre-selects (its own threshold is also + /// 20%), so `1 / b` never sees a zero and the flag-off query succeeds. /// Once the conjuncts are reordered, `1 / b > 2` runs first — on every row, /// zeros included — and integer division by zero is an error. #[test] @@ -1192,7 +1208,7 @@ mod tests { let b: Vec = (0..100).map(|i| i64::from(i < 15)).collect(); let rb = int64_batch(&schema, a, b); - // Flag off: the fused predicate pre-selects on `b <> 0` (15 of 100 + // Flag off: the written predicate pre-selects on `b <> 0` (15 of 100 // rows, within its 20% threshold) and succeeds. assert!(p.evaluate(&rb).is_ok(), "flag-off evaluation must succeed"); @@ -1220,11 +1236,11 @@ mod tests { assert!(err.contains("Divide by zero"), "unexpected error: {err}"); } - /// The mirror of the case above: an error the written fused order *does* - /// raise, which the reordered compact-once evaluation avoids. + /// The mirror of the case above: an error the written order *does* raise, + /// which the reordered compact-once evaluation avoids. /// /// `1 / b > 2 AND a < 10`, with `b = 0` on exactly the rows `a < 10` - /// discards. The fused `AND` evaluates its left side on every row and + /// discards. The written `AND` evaluates its left side on every row and /// errors; the adopted order runs `a < 10` first, compacts to its 10 /// survivors (all with `b = 1`), and never divides by zero. #[test] @@ -1266,7 +1282,7 @@ mod tests { let b: Vec = (0..100).map(|i| i64::from(i < 10)).collect(); let rb = int64_batch(&schema, a, b); - // The written fused order divides by zero... + // The written order divides by zero... let err = p.evaluate(&rb).unwrap_err().to_string(); assert!(err.contains("Divide by zero"), "unexpected error: {err}"); // ...while the adopted order compacts `1 / b > 2` down to the rows diff --git a/datafusion/physical-plan/src/filter.rs b/datafusion/physical-plan/src/filter.rs index 87eee8b104424..10e9a9e16e42b 100644 --- a/datafusion/physical-plan/src/filter.rs +++ b/datafusion/physical-plan/src/filter.rs @@ -101,12 +101,11 @@ pub struct FilterExec { batch_size: usize, /// Number of rows to fetch fetch: Option, - /// Measurements shared by all partition streams, used by adaptive conjunct - /// reordering (see [`AdaptiveConjunction`]) so the streams learn as one. - /// Fresh per plan node; never affects the plan. `Clone` deliberately shares - /// it (the clone filters the same predicate over the same input, so pooled - /// learning still applies); [`reset_state`](ExecutionPlan::reset_state) - /// and predicate rewrites replace it with a fresh instance. + /// Per-execution measurements pooled across this node's partition streams + /// by adaptive conjunct reordering (see [`AdaptiveConjunction`]). Not part + /// of the plan shape. `Clone` shares it, since a clone filters the same + /// predicate; [`reset_state`](ExecutionPlan::reset_state) and predicate + /// rewrites replace it with a fresh instance. adaptive_stats: Arc, } @@ -609,9 +608,9 @@ impl ExecutionPlan for FilterExec { ) -> Result> { validate_child_count!(self, children); match options.children_properties { - // `adaptive_stats` is deliberately carried over by the struct - // update: the predicate is unchanged, and no measurements exist - // before execution. `reset_state` is what replaces it. + // `adaptive_stats` is carried over by the struct update: the + // predicate is unchanged, so any pooled measurements still + // describe it. ChildrenPropertiesMode::Keep => Ok(Arc::new(Self { input: children.swap_remove(0), metrics: ExecutionPlanMetricsSet::new(), @@ -650,11 +649,10 @@ impl ExecutionPlan for FilterExec { /// Reset per-execution state so an independent re-execution (e.g. a /// recursive query) does not inherit runtime state from a prior run. /// - /// The pooled adaptive-conjunct measurements (`AdaptiveFilterShared`) and - /// the execution metrics are the per-execution state that must be reset — - /// otherwise the adaptive reordering learned in one execution would leak - /// into the next. The predicate, input, and cached plan properties are - /// unchanged and remain valid, so they are preserved (unlike + /// The per-execution state is the pooled adaptive-conjunct measurements + /// (`AdaptiveFilterShared`) and the execution metrics; both are replaced + /// with fresh instances. The predicate, input, and cached plan properties + /// remain valid, so they are preserved (unlike /// [`with_new_children`](Self::with_new_children), this does not recompute /// them). Any dynamic filters *inside* the predicate are owned and reset by /// the operator that created them, not by `FilterExec`. @@ -1409,8 +1407,7 @@ struct FilterExecStream { /// The expression to filter on. This expression must evaluate to a boolean value. predicate: Arc, /// When set, the predicate is a reorderable conjunction evaluated - /// adaptively (conjuncts measured, then reordered) instead of via - /// `predicate`. + /// adaptively instead of via `predicate`. adaptive: Option, /// The input partition to filter. input: SendableRecordBatchStream, @@ -1431,8 +1428,8 @@ struct FilterExecMetrics { /// Number of partition streams that adopted an adaptively reordered /// evaluation order for the predicate's conjuncts (at most one per /// stream). Registered only when adaptive conjunct reordering is enabled - /// for this execution, so the metrics of the (default) flag-off path are - /// unchanged. + /// and the predicate is a reorderable conjunction, so the flag-off path's + /// metrics are unchanged. adaptive_reorders: Option, // Remember to update `docs/source/user-guide/metrics.md` when adding new metrics, // or modifying metrics comments @@ -1534,8 +1531,6 @@ impl Stream for FilterExecStream { let (array, adopted_reorder) = match self.adaptive.as_mut() { Some(adaptive) => { let array = adaptive.evaluate(&batch); - // Report the settle-on-a-reorder transition, which - // fires at most once per stream. (array, adaptive.take_adopted_reorder()) } None => ( @@ -2575,9 +2570,9 @@ mod tests { /// /// Both conjuncts are cheap arithmetic, so real timings cannot separate /// them reliably; the shared pool is therefore seeded one batch short of - /// the warm-up (the stand-in for a mocked clock) so the reorder is adopted - /// deterministically. Everything else — the streams, the metric, the - /// coalescer, the projection-free output path — is the real thing. + /// the warm-up so the reorder is adopted deterministically. Everything + /// else — the streams, the metric, the coalescer, the projection-free + /// output path — is the real thing. #[tokio::test] async fn adaptive_filter_reordering_end_to_end() -> Result<()> { const PARTITIONS: usize = 4; @@ -2709,9 +2704,9 @@ mod tests { .unwrap_or(0); assert!(reorders >= 1, "expected an adopted reorder, got {reorders}"); - // Re-executing the same node keeps the learned state (the streams adopt - // the already-published order on their first batch) but cannot change - // the rows. + // Re-executing the same node keeps the learned state (the streams + // adopt the settled order on their first batch) but cannot change the + // rows. assert_eq!(run(&filter, true).await?, flag_off, "state persists"); // `reset_state` drops the pooled measurements, so the fresh node learns diff --git a/datafusion/sqllogictest/test_files/adaptive_filter.slt b/datafusion/sqllogictest/test_files/adaptive_filter.slt index d8462127a8333..4678c5ce4e25b 100644 --- a/datafusion/sqllogictest/test_files/adaptive_filter.slt +++ b/datafusion/sqllogictest/test_files/adaptive_filter.slt @@ -115,9 +115,9 @@ physical_plan # learned order flips the two. (Both conjuncts are "cheap" to the static # cheap-first `reorder_predicates` rule, which is cost-only and blind to # selectivity, so the written order reaches `FilterExec` untouched.) Each of the -# 4 partition streams adopts the reorder exactly once — the first stream to -# finish the pooled warm-up publishes the order and the others pick it up on -# their next batch — so the aggregated counter is 4. +# 4 partition streams adopts the reorder exactly once — the first to finish the +# pooled warm-up settles the order and the others take it up on their next +# batch — so the aggregated counter is 4. query I SELECT count(*) FROM t WHERE a % 97 + a % 89 + a % 83 + b % 13 >= 0 AND a > 3990; ---- diff --git a/datafusion/sqllogictest/test_files/information_schema.slt b/datafusion/sqllogictest/test_files/information_schema.slt index 7905d34fc9b56..81004c6176c81 100644 --- a/datafusion/sqllogictest/test_files/information_schema.slt +++ b/datafusion/sqllogictest/test_files/information_schema.slt @@ -375,7 +375,7 @@ datafusion.catalog.has_header true Default value for `format.has_header` for `CR datafusion.catalog.information_schema true Should DataFusion provide access to `information_schema` virtual tables for displaying schema information datafusion.catalog.location NULL Location scanned to load tables for `default` schema datafusion.catalog.newlines_in_values false Specifies whether newlines in (quoted) CSV values are supported. This is the default value for `format.newlines_in_values` for `CREATE EXTERNAL TABLE` if not specified explicitly in the statement. Parsing newlines in quoted values may be affected by execution behaviour such as parallel file scanning. Setting this to `true` ensures that newlines in values are parsed successfully, which may reduce performance. -datafusion.execution.adaptive_filter_reordering false (experimental) When enabled, `FilterExec` adaptively reorders the conjuncts of a conjunctive predicate at runtime. It measures each conjunct's selectivity and evaluation cost on the rows that reach it and runs the conjuncts that discard the most rows per unit of CPU time first, so cheap-and-selective predicates gate expensive ones. This never changes query results, but it is off by default because it can change observable side effects of fallible predicates (even when no reorder is adopted): while measuring, and after a reorder, conjuncts are evaluated only on the rows that survived the conjuncts before them, so an error the fused predicate would have raised on an already-filtered row (e.g. `b <> 0 AND 1/b > 2` evaluating `1/b` on every row) may not occur. Predicates containing volatile expressions are never reordered. +datafusion.execution.adaptive_filter_reordering false (experimental) When enabled, `FilterExec` measures the selectivity and evaluation cost of each conjunct of an `AND` predicate at runtime and reorders them to run the ones that discard the most rows per unit of CPU time first. Query results never change, but the observable side effects of a fallible predicate can, in either direction: reordering `b <> 0 AND 1/b > 2` can make a divide-by-zero error appear or disappear, since each conjunct is evaluated only on the rows the conjuncts before it kept. Predicates containing volatile expressions are never reordered. datafusion.execution.batch_size 8192 Default batch size while creating new batches, it's especially useful for buffer-in-memory batches since creating tiny batches would result in too much metadata memory consumption datafusion.execution.coalesce_batches true When set to true, record batches will be examined between each operator and small batches will be coalesced into larger batches. This is helpful when there are highly selective filters or joins that could produce tiny output batches. The target batch size is determined by the configuration setting datafusion.execution.collect_statistics true Should DataFusion collect statistics when first creating a table. Has no effect after the table is created. Defaults to true. diff --git a/docs/source/user-guide/configs.md b/docs/source/user-guide/configs.md index ccb11af76fe45..e85caacc1ff9f 100644 --- a/docs/source/user-guide/configs.md +++ b/docs/source/user-guide/configs.md @@ -142,7 +142,7 @@ The following configuration settings are available: | datafusion.execution.skip_partial_aggregation_probe_rows_threshold | 100000 | Number of input rows partial aggregation partition should process, before aggregation ratio check and trying to switch to skipping aggregation mode | | datafusion.execution.use_row_number_estimates_to_optimize_partitioning | false | Should DataFusion use row number estimates at the input to decide whether increasing parallelism is beneficial or not. By default, only exact row numbers (not estimates) are used for this decision. Setting this flag to `true` will likely produce better plans. if the source of statistics is accurate. We plan to make this the default in the future. | | datafusion.execution.enforce_batch_size_in_joins | false | Should DataFusion enforce batch size in joins or not. By default, DataFusion will not enforce batch size in joins. Enforcing batch size in joins can reduce memory usage when joining large tables with a highly-selective join filter, but is also slightly slower. Note: this option currently only applies to the symmetric hash join. | -| datafusion.execution.adaptive_filter_reordering | false | (experimental) When enabled, `FilterExec` adaptively reorders the conjuncts of a conjunctive predicate at runtime. It measures each conjunct's selectivity and evaluation cost on the rows that reach it and runs the conjuncts that discard the most rows per unit of CPU time first, so cheap-and-selective predicates gate expensive ones. This never changes query results, but it is off by default because it can change observable side effects of fallible predicates (even when no reorder is adopted): while measuring, and after a reorder, conjuncts are evaluated only on the rows that survived the conjuncts before them, so an error the fused predicate would have raised on an already-filtered row (e.g. `b <> 0 AND 1/b > 2` evaluating `1/b` on every row) may not occur. Predicates containing volatile expressions are never reordered. | +| datafusion.execution.adaptive_filter_reordering | false | (experimental) When enabled, `FilterExec` measures the selectivity and evaluation cost of each conjunct of an `AND` predicate at runtime and reorders them to run the ones that discard the most rows per unit of CPU time first. Query results never change, but the observable side effects of a fallible predicate can, in either direction: reordering `b <> 0 AND 1/b > 2` can make a divide-by-zero error appear or disappear, since each conjunct is evaluated only on the rows the conjuncts before it kept. Predicates containing volatile expressions are never reordered. | | datafusion.execution.objectstore_writer_buffer_size | 10485760 | Size (bytes) of data buffer DataFusion uses when writing output files. This affects the size of the data chunks that are uploaded to remote object stores (e.g. AWS S3). If very large (>= 100 GiB) output files are being written, it may be necessary to increase this size to avoid errors from the remote end point. | | datafusion.execution.enable_ansi_mode | false | Whether to enable ANSI SQL mode. The flag is experimental and relevant only for DataFusion Spark built-in functions When `enable_ansi_mode` is set to `true`, the query engine follows ANSI SQL semantics for expressions, casting, and error handling. This means: - **Strict type coercion rules:** implicit casts between incompatible types are disallowed. - **Standard SQL arithmetic behavior:** operations such as division by zero, numeric overflow, or invalid casts raise runtime errors rather than returning `NULL` or adjusted values. - **Consistent ANSI behavior** for string concatenation, comparisons, and `NULL` handling. When `enable_ansi_mode` is `false` (the default), the engine uses a more permissive, non-ANSI mode designed for user convenience and backward compatibility. In this mode: - Implicit casts between types are allowed (e.g., string to integer when possible). - Arithmetic operations are more lenient — for example, `abs()` on the minimum representable integer value returns the input value instead of raising overflow. - Division by zero or invalid casts may return `NULL` instead of failing. # Default `false` — ANSI SQL mode is disabled by default. | | datafusion.execution.hash_join_buffering_capacity | 0 | How many bytes to buffer in the probe side of hash joins while the build side is concurrently being built. Without this, hash joins will wait until the full materialization of the build side before polling the probe side. This is useful in scenarios where the query is not completely CPU bounded, allowing to do some early work concurrently and reducing the latency of the query. Note that when hash join buffering is enabled, the probe side will start eagerly polling data, not giving time for the producer side of dynamic filters to produce any meaningful predicate. Queries with dynamic filters might see performance degradation. Disabled by default, set to a number greater than 0 for enabling it. | diff --git a/docs/source/user-guide/metrics.md b/docs/source/user-guide/metrics.md index f672045507135..46aad02fa806d 100644 --- a/docs/source/user-guide/metrics.md +++ b/docs/source/user-guide/metrics.md @@ -38,10 +38,10 @@ DataFusion operators expose runtime metrics so you can understand where time is ### FilterExec -| Metric | Description | -| ----------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| selectivity | Selectivity of the filter, calculated as output_rows / input_rows | -| adaptive_reorders | Number of partition streams that adopted an adaptively reordered evaluation order for the predicate's conjuncts (at most one per stream). Only present when `datafusion.execution.adaptive_filter_reordering` is enabled; `0` means the measured order was kept. | +| Metric | Description | +| ----------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| selectivity | Selectivity of the filter, calculated as output_rows / input_rows | +| adaptive_reorders | Number of partition streams that adopted an adaptively reordered evaluation order for the predicate's conjuncts (at most one per stream). Only present when `datafusion.execution.adaptive_filter_reordering` is enabled and the predicate is a reorderable conjunction; `0` means the written order was kept. | ### HashJoinExec From 2b043532ed57bc54473ebf0cc921852cb23e4ec1 Mon Sep 17 00:00:00 2001 From: Adrian Garcia Badaracco <1755071+adriangb@users.noreply.github.com> Date: Mon, 7 Sep 2026 01:03:07 -0500 Subject: [PATCH 14/27] refactor(physical-plan): evaluate the adopted order as a right-nested AND Once the warm-up adopts a reorder, materialise the learned order once as a right-nested `AND` chain, `(c_first AND (c_second AND (... AND c_last)))`, and let `BinaryExpr` evaluate it like any other predicate, instead of running the settled path through the per-conjunct compact-once loop. Right-nesting is what makes this cheap: `BinaryExpr`'s pre-selection filters the batch it is handed before evaluating its right-hand side, so the survivors of the first (most selective) conjunct stay compacted for the entire remainder of the chain. A left-nested chain -- what `conjunction()` builds -- re-filters the original batch and scatters at every level, which measures materially slower than the flag off on cheap 4-8 conjunct predicates; right-nested is within noise of the compact-once loop. The measuring path keeps the per-conjunct loop with compaction: it has to time each conjunct on exactly the rows that reach it. `Settled` now carries the expression to evaluate alongside the order, and `settle` builds it. Co-Authored-By: Claude Fable 5.1 --- .../physical-plan/src/adaptive_filter.rs | 335 ++++++++++++------ 1 file changed, 234 insertions(+), 101 deletions(-) diff --git a/datafusion/physical-plan/src/adaptive_filter.rs b/datafusion/physical-plan/src/adaptive_filter.rs index 39e74df39d205..812a98c479327 100644 --- a/datafusion/physical-plan/src/adaptive_filter.rs +++ b/datafusion/physical-plan/src/adaptive_filter.rs @@ -29,10 +29,9 @@ //! expensive-but-very-selective one, and conjuncts in the same cost class //! keep the order they were written in. That is the order this module calls //! the *written order* and measures against. -//! - [`BinaryExpr`](datafusion_physical_expr::expressions::BinaryExpr)'s `AND` -//! pre-selects: when the conjuncts evaluated so far keep at most 20% of the -//! rows and produce no nulls, it filters the batch down to those rows before -//! evaluating the next conjunct. +//! - [`BinaryExpr`]'s `AND` pre-selects: when the conjuncts evaluated so far +//! keep at most 20% of the rows and produce no nulls, it filters the batch +//! down to those rows before evaluating the next conjunct. //! //! Pre-selection can only gate a conjunct on the conjuncts written *before* it, //! never on a more selective one written after it; it does not fire while the @@ -41,10 +40,12 @@ //! length, so survivors are not carried forward compacted from one level to //! the next. //! -//! This module measures each conjunct's selectivity and cost at runtime, -//! reorders them accordingly, and evaluates them through a loop that compacts -//! the survivors once and keeps them compacted. Whether it runs at all is -//! controlled by `datafusion.execution.adaptive_filter_reordering`. +//! This module measures each conjunct's selectivity and cost at runtime and +//! reorders them accordingly, handing the learned order back to `BinaryExpr` +//! as a right-nested `AND` chain so that pre-selection fires on the conjunct +//! that discards the most rows and its survivors stay compacted for the rest +//! of the chain. Whether it runs at all is controlled by +//! `datafusion.execution.adaptive_filter_reordering`. //! //! For example: //! @@ -71,15 +72,32 @@ //! regexp_like(s,'b') evaluated on those survivors only //! ``` //! -//! ## How it evaluates: the compact-once loop +//! ## How it evaluates //! -//! The conjuncts are evaluated one at a time, combining their boolean results -//! with `AND`. The working batch is physically compacted to the surviving rows -//! once the accumulated mask becomes selective enough -//! ([`COMPACTION_SELECTIVITY_THRESHOLD`]), and every conjunct after that point -//! is evaluated against the compacted batch. So a run of non-selective -//! conjuncts costs only cheap bitwise `AND`s, and everything after a selective -//! conjunct decodes just its survivors. +//! While the order is being learned, the conjuncts are evaluated one at a time +//! so each can be timed and counted on exactly the rows that reached it. Their +//! boolean results are combined with `AND`, and the working batch is +//! physically compacted to the surviving rows once the accumulated mask +//! becomes selective enough ([`COMPACTION_SELECTIVITY_THRESHOLD`]); every +//! conjunct after that point is evaluated against the compacted batch. So a +//! run of non-selective conjuncts costs only cheap bitwise `AND`s, everything +//! after a selective conjunct decodes just its survivors, and each measurement +//! is taken on the population that conjunct would really see. +//! +//! Once the order is settled that loop is gone. If the warm-up kept the +//! written order, the written predicate is evaluated as one expression, +//! exactly as it would be with the flag off. If it adopted a reorder, the +//! learned order is materialised once as a right-nested `AND` chain, +//! `(c_first AND (c_second AND (... AND c_last)))`, and from then on evaluated +//! by [`BinaryExpr`] like any other predicate. +//! +//! Right-nesting is what makes that cheap. Pre-selection filters the batch the +//! `AND` is handed before evaluating its right-hand side, so under right +//! nesting the survivors of the first (most selective) conjunct stay compacted +//! for the entire remainder of the chain. A left-nested chain — what +//! [`conjunction`](datafusion_physical_expr::utils::conjunction) builds — +//! would instead re-filter the original batch at every level and scatter each +//! level's result back to full length. //! //! ## How it orders //! @@ -87,15 +105,11 @@ //! its marginal selectivity and per-row cost. After a short warm-up the //! conjuncts are ranked by rows discarded per nanosecond //! (`(1 - pass_rate) / cost_per_row`, the classic optimal ordering key for -//! independent conjuncts), and the ranking is adopted only if it is materially -//! cheaper than the written order ([`TIE_COST_FRACTION`]). The order then stays -//! fixed. -//! -//! The compact-once loop is used only in service of a reorder: if the warm-up -//! does not reorder the conjuncts, the written predicate is from then on -//! evaluated as one expression, exactly as it would be with the flag off. -//! During the warm-up the conjuncts are necessarily evaluated one at a time so -//! they can be measured individually. +//! independent conjuncts). The ranking is adopted only if it is materially +//! cheaper than the written order ([`TIE_COST_FRACTION`]); otherwise the +//! written predicate is evaluated unchanged, so a conjunction that does not +//! benefit from reordering carries none of this module's machinery past the +//! warm-up. The decision then stays fixed. //! //! ## How it shares //! @@ -140,7 +154,9 @@ use arrow::record_batch::RecordBatch; use datafusion_common::cast::as_boolean_array; use datafusion_common::instant::Instant; use datafusion_common::{Result, internal_err}; +use datafusion_expr::Operator; use datafusion_physical_expr::PhysicalExpr; +use datafusion_physical_expr::expressions::BinaryExpr; use datafusion_physical_expr::utils::split_conjunction; use datafusion_physical_expr_common::physical_expr::is_volatile; @@ -239,28 +255,33 @@ struct SharedInner { /// cloning it, so reporting costs nothing on the per-batch path. #[derive(Debug, Clone, Copy, PartialEq, Eq)] enum BatchStrategy<'a> { - /// Still learning: the written order through the compact-once loop, each + /// Still learning: the written order through the measuring loop, each /// conjunct instrumented and its counts pooled. (Empty batches are /// evaluated but measure nothing and do not consume the warm-up.) Measure, /// Settled without a reorder: the written predicate evaluated as one /// expression, exactly as if the feature were off. Fused, - /// Settled on an adopted reorder, evaluated through the compact-once loop. - /// The payload is the adopted order: positions in the written conjunct - /// list, first-evaluated first. + /// Settled on an adopted reorder, evaluated as the right-nested `AND` + /// chain built from it. The payload is the adopted order: positions in the + /// written conjunct list, first-evaluated first. Reordered(&'a [usize]), } -/// The settled outcome of the warm-up: the evaluation order, and whether to run -/// it through the compact-once loop or as the written predicate. +/// The settled outcome of the warm-up: the expression every subsequent batch +/// is evaluated with, and the order it evaluates the conjuncts in. #[derive(Debug, Clone)] struct Settled { - /// Evaluation order: indices into the conjunct list. + /// The predicate to evaluate from now on: the written predicate when the + /// warm-up kept the written order, otherwise `order` materialised as a + /// right-nested `AND` chain (see [`settle`]). + predicate: Arc, + /// Evaluation order: indices into the conjunct list. Carried for reporting + /// ([`BatchStrategy::Reordered`]) rather than for evaluation. order: Vec, - /// `true` to run `order` through the compact-once loop; `false` to evaluate - /// the written predicate as-is (see [`settle`]). - compact: bool, + /// Whether `order` reorders the written conjuncts — equivalently, whether + /// `predicate` is the rebuilt chain rather than the written predicate. + reordered: bool, } impl AdaptiveFilterShared { @@ -307,17 +328,24 @@ impl AdaptiveFilterShared { pub(crate) struct AdaptiveConjunction { /// The split conjuncts. `order` indices refer to positions here. conjuncts: Vec>, - /// The written predicate as one expression, evaluated as-is when the - /// settled order does not reorder it. + /// The written predicate as one expression: what the measured conjuncts + /// are ranked against, and what is evaluated when the settled order does + /// not reorder them. predicate: Arc, /// Measurements and the settled decision, shared by every partition stream. shared: Arc, + /// The expression [`evaluate_settled`](Self::evaluate_settled) evaluates: + /// the written predicate, or — once a reorder is adopted — the learned + /// order as a right-nested `AND` chain. Unused until `settled`, and equal + /// to the written predicate until then. + settled_predicate: Arc, /// Evaluation order: indices into `conjuncts`. The written order until a /// settled order is adopted. order: Vec, - /// Whether the settled order runs through the compact-once loop; `false` - /// means evaluate [`predicate`](Self::predicate) directly. - compact: bool, + /// Whether the settled decision reordered the conjuncts — equivalently, + /// whether [`settled_predicate`](Self::settled_predicate) is the rebuilt + /// chain rather than [`predicate`](Self::predicate). + reordered: bool, /// Whether the order is settled: this stream no longer measures. settled: bool, /// Set when this stream adopts a *reordered* decision, and cleared by @@ -354,8 +382,9 @@ impl AdaptiveConjunction { conjuncts, predicate: Arc::clone(predicate), shared, + settled_predicate: Arc::clone(predicate), order, - compact: false, + reordered: false, settled: false, adopted_reorder: false, }) @@ -397,7 +426,7 @@ impl AdaptiveConjunction { } if self.settled { let mask = self.evaluate_settled(batch)?; - let strategy = if self.compact { + let strategy = if self.reordered { BatchStrategy::Reordered(&self.order) } else { BatchStrategy::Fused @@ -421,24 +450,23 @@ impl AdaptiveConjunction { Ok((result, BatchStrategy::Measure)) } - /// Evaluate the settled arrangement with no instrumentation: the - /// compact-once loop when the order was reordered, the written predicate - /// directly otherwise. + /// Evaluate the settled arrangement with no instrumentation: one + /// expression, either the written predicate or the right-nested `AND` + /// chain built from the adopted order. fn evaluate_settled(&self, batch: &RecordBatch) -> Result { - if self.compact { - eval_conjuncts(&self.conjuncts, &self.order, batch, None) - } else { - self.predicate.evaluate(batch)?.into_array(batch.num_rows()) - } + self.settled_predicate + .evaluate(batch)? + .into_array(batch.num_rows()) } fn adopt(&mut self, decision: Settled) { + self.settled_predicate = decision.predicate; self.order = decision.order; - self.compact = decision.compact; + self.reordered = decision.reordered; self.settled = true; // Only a genuine reorder is worth reporting; settling on the written // order is indistinguishable from the feature being off. - self.adopted_reorder = self.compact; + self.adopted_reorder = self.reordered; } /// Merge this batch's measurements into the shared pool and, once enough @@ -468,7 +496,7 @@ impl AdaptiveConjunction { if inner.measured_batches < WARMUP_BATCHES { return; } - let decision = settle(&inner.stats); + let decision = settle(&inner.stats, &self.conjuncts, &self.predicate); inner.settled = Some(decision.clone()); drop(inner); self.adopt(decision); @@ -478,34 +506,66 @@ impl AdaptiveConjunction { /// Decide the settled arrangement from the pooled measurements. /// /// Rank the conjuncts by effectiveness and take the ranking only if it is -/// materially cheaper than the written order; otherwise keep the written -/// predicate and evaluate it as one expression. This is the guard that keeps -/// the compact-once loop in service of a reorder, so a conjunction that does -/// not benefit from reordering pays none of its overhead. -fn settle(stats: &[ConjunctStats]) -> Settled { +/// materially cheaper than the written order, materialising it as a +/// right-nested `AND` chain over `conjuncts`; otherwise keep `predicate` and +/// evaluate it as one expression, so a conjunction that does not benefit from +/// reordering pays nothing for the attempt. +fn settle( + stats: &[ConjunctStats], + conjuncts: &[Arc], + predicate: &Arc, +) -> Settled { let identity: Vec = (0..stats.len()).collect(); let candidate = rank_by_effectiveness(stats); if candidate != identity && expected_cost_per_row(stats, &candidate) < (1.0 - TIE_COST_FRACTION) * expected_cost_per_row(stats, &identity) + && let Some(reordered) = right_nested_conjunction(conjuncts, &candidate) { Settled { + predicate: reordered, order: candidate, - compact: true, + reordered: true, } } else { Settled { + predicate: Arc::clone(predicate), order: identity, - compact: false, + reordered: false, } } } -/// Evaluate `conjuncts` in `order` against `batch` via the compact-once loop, -/// returning the boolean mask (over the batch's original rows) of rows that -/// passed every conjunct. With `stats`, each conjunct is additionally timed and -/// counted on exactly the rows it evaluated (its marginal selectivity and cost -/// on the current working population). +/// Build `conjuncts` in `order` into one right-nested `AND` chain, +/// `(c_first AND (c_second AND (... AND c_last)))`, or `None` if `order` is +/// empty. +/// +/// The nesting is the point. [`BinaryExpr`]'s `AND` pre-selects by filtering +/// the batch it is given before evaluating its right-hand side, so nesting to +/// the right keeps the survivors of the first conjunct compacted for every +/// conjunct after it. Nesting to the left — what +/// [`conjunction`](datafusion_physical_expr::utils::conjunction) builds — +/// would re-filter the original batch at each level instead. +fn right_nested_conjunction( + conjuncts: &[Arc], + order: &[usize], +) -> Option> { + order.iter().rev().fold(None, |acc, &id| { + let conjunct = Arc::clone(&conjuncts[id]); + Some(match acc { + None => conjunct, + Some(acc) => Arc::new(BinaryExpr::new(conjunct, Operator::And, acc)) as _, + }) + }) +} + +/// The measuring loop: evaluate `conjuncts` one at a time in `order` against +/// `batch`, returning the boolean mask (over the batch's original rows) of rows +/// that passed every conjunct. With `stats`, each conjunct is additionally +/// timed and counted on exactly the rows it evaluated (its marginal selectivity +/// and cost on the current working population) — which is why the warm-up +/// evaluates conjunct by conjunct rather than handing the predicate to +/// [`BinaryExpr`], as the settled path does. /// /// The working batch is physically compacted to the surviving rows only once /// the accumulated mask becomes selective enough (see @@ -657,7 +717,6 @@ mod tests { use arrow::array::{Int32Array, Int64Array}; use arrow::datatypes::{DataType, Field, Schema}; - use datafusion_expr::Operator; use datafusion_physical_expr::expressions::{binary, col, lit}; fn schema() -> Arc { @@ -684,13 +743,19 @@ mod tests { binary(left, Operator::And, right, schema).unwrap() } - fn conjuncts(schema: &Arc) -> Vec> { - split_conjunction(&predicate(schema)) + /// The conjuncts of `predicate`, exactly as `AdaptiveConjunction` splits + /// them (so the `Arc`s are pointer-equal to the ones inside `predicate`). + fn split(predicate: &Arc) -> Vec> { + split_conjunction(predicate) .into_iter() .map(Arc::clone) .collect() } + fn conjuncts(schema: &Arc) -> Vec> { + split(&predicate(schema)) + } + fn passing_rows(mask: &ArrayRef) -> Vec { let mask = as_boolean_array(mask).unwrap(); (0..mask.len()) @@ -800,7 +865,7 @@ mod tests { assert!((expected_cost_per_row(&s, &[1, 0]) - 10.5).abs() < 1e-9); } - /// The compact-once loop returns exactly the rows the written predicate + /// The measuring loop returns exactly the rows the written predicate /// keeps, in any order and whether or not compaction triggers. #[test] fn eval_conjuncts_matches_predicate_in_any_order() { @@ -850,25 +915,89 @@ mod tests { } /// A reorder is adopted only when materially cheaper; an already-good order - /// is left untouched and runs the written predicate (no compact-once). + /// is left untouched and keeps evaluating the written predicate itself. #[test] fn settle_keeps_order_when_not_materially_better() { + let schema = schema(); + let p = predicate(&schema); // Two equally cheap, equally selective conjuncts: swapping cannot help, - // so the written order stands and compact-once is not used. + // so the written order stands and nothing is rebuilt. let s = vec![stats(1000, 500, 1000), stats(1000, 500, 1000)]; - let d = settle(&s); + let d = settle(&s, &split(&p), &p); assert_eq!(d.order, vec![0, 1]); - assert!(!d.compact); + assert!(!d.reordered); + assert!( + Arc::ptr_eq(&d.predicate, &p), + "the written predicate itself" + ); } #[test] - fn settle_adopts_materially_cheaper_order_with_compaction() { - // id 1 is far more selective and equally cheap: it should move first and - // run through the compact-once loop. + fn settle_adopts_materially_cheaper_order() { + let schema = schema(); + let p = predicate(&schema); + let cs = split(&p); + // id 1 is far more selective and equally cheap: it should move first, + // as the outermost left operand of the rebuilt chain. let s = vec![stats(1000, 900, 1000), stats(1000, 10, 1000)]; - let d = settle(&s); + let d = settle(&s, &cs, &p); assert_eq!(d.order, vec![1, 0]); - assert!(d.compact); + assert!(d.reordered); + let chain = d.predicate.downcast_ref::().expect("an AND"); + assert!(Arc::ptr_eq(chain.left(), &cs[1])); + assert!(Arc::ptr_eq(chain.right(), &cs[0])); + } + + /// The adopted order is materialised as a *right*-nested `AND` chain: + /// `(c_first AND (c_second AND c_last))`. That is what lets `BinaryExpr`'s + /// pre-selection keep the survivors of the first conjunct compacted for the + /// whole remainder of the chain; a left-nested chain would re-filter the + /// original batch at every level. + #[test] + fn adopted_order_is_a_right_nested_and_chain() { + let schema = schema(); + // Three conjuncts: `a > 2 AND b < 5 AND a < 90`. + let p = binary( + predicate(&schema), + Operator::And, + binary( + col("a", &schema).unwrap(), + Operator::Lt, + lit(90i32), + &schema, + ) + .unwrap(), + &schema, + ) + .unwrap(); + let cs = split(&p); + assert_eq!(cs.len(), 3); + + // Equal cost, decreasing pass rate: the written order is exactly + // reversed, and reversing it is materially cheaper. + let s = vec![ + stats(1000, 900, 1000), + stats(1000, 500, 1000), + stats(1000, 10, 1000), + ]; + let d = settle(&s, &cs, &p); + assert_eq!(d.order, vec![2, 1, 0]); + assert!(d.reordered); + + // `(cs[2] AND (cs[1] AND cs[0]))`. + let outer = d.predicate.downcast_ref::().expect("an AND"); + assert_eq!(*outer.op(), Operator::And); + assert!( + Arc::ptr_eq(outer.left(), &cs[2]), + "first conjunct is outermost" + ); + let inner = outer + .right() + .downcast_ref::() + .expect("the tail is itself an AND"); + assert_eq!(*inner.op(), Operator::And); + assert!(Arc::ptr_eq(inner.left(), &cs[1])); + assert!(Arc::ptr_eq(inner.right(), &cs[0])); } /// When the order does not change, the settled evaluator runs the written @@ -901,9 +1030,10 @@ mod tests { } assert!(adaptive.settled); assert!( - !adaptive.compact, + !adaptive.reordered, "interchangeable conjuncts stay on the plain predicate" ); + assert!(Arc::ptr_eq(&adaptive.settled_predicate, &p)); } /// Two streams sharing one pool settle the order together: the warm-up is @@ -933,7 +1063,7 @@ mod tests { let mut s2 = AdaptiveConjunction::try_new(&p, Arc::clone(&shared)).unwrap(); // `b < 5` (conjunct 1) is the selective one; drive both streams with - // batches where it keeps 5 rows in 25 (20%, exactly the compact-once + // batches where it keeps 5 rows in 25 (20%, exactly the compaction // threshold). let mk = |round: i32| { let base = round * 100; @@ -959,10 +1089,10 @@ mod tests { s2.evaluate(&mk(99)).unwrap(); assert!(s1.settled && s2.settled); // The selective conjunct was promoted to the front for both, and the - // reorder runs through the compact-once loop. + // reorder is evaluated as the rebuilt chain. assert_eq!(s1.order, vec![1, 0]); assert_eq!(s2.order, vec![1, 0]); - assert!(s1.compact && s2.compact); + assert!(s1.reordered && s2.reordered); } /// The reorder-adoption signal (`take_adopted_reorder`, what `FilterExec` @@ -1000,7 +1130,7 @@ mod tests { // This batch completes the warm-up: `settler` settles on the reorder // and signals it, exactly once. settler.evaluate(&rb).unwrap(); - assert!(settler.compact); + assert!(settler.reordered); assert!(settler.take_adopted_reorder()); settler.evaluate(&rb).unwrap(); assert!(!settler.take_adopted_reorder()); @@ -1008,7 +1138,7 @@ mod tests { // `adopter` never measured its way to a decision: it takes up the // settled one on its next batch, and signals that once too. adopter.evaluate(&rb).unwrap(); - assert!(adopter.compact); + assert!(adopter.reordered); assert!(adopter.take_adopted_reorder()); adopter.evaluate(&rb).unwrap(); assert!(!adopter.take_adopted_reorder()); @@ -1035,7 +1165,7 @@ mod tests { let a: Vec = (0..100).collect(); let rb = batch(&schema, a.clone(), a); adaptive.evaluate(&rb).unwrap(); - assert!(adaptive.settled && !adaptive.compact); + assert!(adaptive.settled && !adaptive.reordered); assert!(!adaptive.take_adopted_reorder()); } @@ -1044,8 +1174,8 @@ mod tests { /// /// The seeded costs (see [`seed`]) make conjunct 0 cheap but unselective /// and conjunct 1 expensive but very selective, so the warm-up must settle - /// on promoting conjunct 1 and run it through the compact-once loop, - /// regardless of real timer values. + /// on promoting conjunct 1 and evaluate the rebuilt chain, regardless of + /// real timer values. #[test] fn scenario_measure_batches_then_settle_on_reorder() { let schema = schema(); @@ -1074,9 +1204,9 @@ mod tests { assert_eq!(passing_rows(&got), passing_rows(&want), "round {round}"); } - // Batch 1 completes the warm-up and settles; batches 2+ run the - // adopted reorder (selective conjunct promoted to the front) through - // the compact-once loop. + // Batch 1 completes the warm-up and settles; batches 2+ evaluate the + // adopted reorder (selective conjunct promoted to the front) as a + // right-nested `AND` chain. assert_eq!( trace, vec!["Measure", "Reordered([1, 0])", "Reordered([1, 0])"] @@ -1085,7 +1215,7 @@ mod tests { /// Contract scenario for the no-win case: interchangeable conjuncts settle /// on the written predicate evaluated as one expression (as if the feature - /// were off), never the compact-once loop. + /// were off), never a rebuilt chain. #[test] fn scenario_measure_batches_then_settle_on_fused() { let schema = schema(); @@ -1194,10 +1324,11 @@ mod tests { /// reorder can introduce an error the written order avoided. /// /// `b <> 0 AND 1 / b > 2` on data where `b <> 0` holds for 15% of the rows. - /// The written `BinaryExpr` `AND` pre-selects (its own threshold is also - /// 20%), so `1 / b` never sees a zero and the flag-off query succeeds. - /// Once the conjuncts are reordered, `1 / b > 2` runs first — on every row, - /// zeros included — and integer division by zero is an error. + /// The written `BinaryExpr` `AND` pre-selects on `b <> 0` (15 of 100 rows, + /// within its 20% threshold), so `1 / b` never sees a zero and the flag-off + /// query succeeds. In the rebuilt chain `1 / b > 2` is the outermost left + /// operand, so it runs first — on every row, zeros included — and integer + /// division by zero is an error. #[test] fn adopted_reorder_can_introduce_a_divide_by_zero() { let schema = int64_schema(); @@ -1229,7 +1360,7 @@ mod tests { // compaction on `b <> 0` also keeps `1 / b` away from the zeros. adaptive.evaluate(&rb).unwrap(); assert_eq!(adaptive.order, vec![1, 0]); - assert!(adaptive.compact); + assert!(adaptive.reordered); // The next batch runs the adopted reorder, and errors. let err = adaptive.evaluate(&rb).unwrap_err().to_string(); @@ -1237,12 +1368,14 @@ mod tests { } /// The mirror of the case above: an error the written order *does* raise, - /// which the reordered compact-once evaluation avoids. + /// which the adopted order avoids. /// /// `1 / b > 2 AND a < 10`, with `b = 0` on exactly the rows `a < 10` /// discards. The written `AND` evaluates its left side on every row and - /// errors; the adopted order runs `a < 10` first, compacts to its 10 - /// survivors (all with `b = 1`), and never divides by zero. + /// errors. The rebuilt chain is `a < 10 AND (1 / b > 2)`, and `a < 10` + /// keeps 10 of the 100 rows with no nulls — inside `BinaryExpr`'s 20% + /// pre-selection threshold — so the batch is filtered down to those + /// survivors (all with `b = 1`) before `1 / b > 2` is evaluated at all. #[test] fn adopted_reorder_can_avoid_a_divide_by_zero_the_written_order_raises() { let schema = int64_schema(); @@ -1285,8 +1418,8 @@ mod tests { // The written order divides by zero... let err = p.evaluate(&rb).unwrap_err().to_string(); assert!(err.contains("Divide by zero"), "unexpected error: {err}"); - // ...while the adopted order compacts `1 / b > 2` down to the rows - // `a < 10` kept, none of which is zero. + // ...while the rebuilt chain pre-selects on `a < 10` and evaluates + // `1 / b > 2` only on the rows it kept, none of which is zero. let got = adaptive.evaluate(&rb).unwrap(); assert!(passing_rows(&got).is_empty(), "1 / 1 > 2 is false"); } From eeecd9dd70a34096221cd8d5e4d094a223e92f27 Mon Sep 17 00:00:00 2001 From: Adrian Garcia Badaracco <1755071+adriangb@users.noreply.github.com> Date: Mon, 7 Sep 2026 01:35:33 -0500 Subject: [PATCH 15/27] refactor(physical-plan): measure conjuncts through BinaryExpr instead of a private evaluation loop The warm-up used to walk the conjuncts itself: a private `eval_conjuncts` loop that AND-ed the masks, compacted the working batch past its own selectivity threshold, tracked live row indices and scattered the result back to full length. That was a second conjunction-evaluation engine living next to `BinaryExpr`'s, with its own compaction policy to keep in step and its own null and index handling to get right. Delete it. Each conjunct is now wrapped in a small measuring `PhysicalExpr` that times the call and counts the rows it was handed and the rows it kept, and the wrapped conjuncts are assembled into the written order as the same right-nested `AND` chain the settled path uses. `BinaryExpr` evaluates and pre-selects exactly as it would for the plain predicate, so the compaction is its own and every conjunct is measured on precisely the rows it hands over. The wrapper returns the conjunct's result unchanged, nulls included; three-valued logic stays `BinaryExpr`'s business. The module now holds no evaluation logic of its own on any path: stats and the shared pool, the ranking and cost model, `settle` plus the right-nested chain builder, the measuring wrapper, and the per-stream glue. Behaviour is unchanged where it was observable: the pooled counts for the first measured batch are identical, both divide-by-zero side-effect tests still hold (the warm-up's pre-selection keeps `1 / b` away from the zeros exactly as the old loop's compaction did), and the sqllogictests are unaffected. Co-Authored-By: Claude Fable 5.1 --- .../physical-plan/src/adaptive_filter.rs | 465 +++++++++++------- 1 file changed, 275 insertions(+), 190 deletions(-) diff --git a/datafusion/physical-plan/src/adaptive_filter.rs b/datafusion/physical-plan/src/adaptive_filter.rs index 812a98c479327..241282f89ec88 100644 --- a/datafusion/physical-plan/src/adaptive_filter.rs +++ b/datafusion/physical-plan/src/adaptive_filter.rs @@ -74,17 +74,19 @@ //! //! ## How it evaluates //! -//! While the order is being learned, the conjuncts are evaluated one at a time -//! so each can be timed and counted on exactly the rows that reached it. Their -//! boolean results are combined with `AND`, and the working batch is -//! physically compacted to the surviving rows once the accumulated mask -//! becomes selective enough ([`COMPACTION_SELECTIVITY_THRESHOLD`]); every -//! conjunct after that point is evaluated against the compacted batch. So a -//! run of non-selective conjuncts costs only cheap bitwise `AND`s, everything -//! after a selective conjunct decodes just its survivors, and each measurement -//! is taken on the population that conjunct would really see. +//! This module contains no evaluation logic of its own. Every batch, learning +//! or settled, is evaluated by [`BinaryExpr`] as a right-nested `AND` chain. //! -//! Once the order is settled that loop is gone. If the warm-up kept the +//! While the order is being learned the chain is built over the *written* +//! order, with each conjunct wrapped in a measuring expression that times the +//! call and counts the rows it saw and the rows it kept. `BinaryExpr` is +//! therefore what performs the compaction, exactly as it does for the plain +//! predicate: its pre-selection filters the batch before evaluating the rest +//! of the chain, so every conjunct is measured on precisely the rows +//! `BinaryExpr` hands it — the population it would really see in that +//! position. +//! +//! Once the order settles the wrappers are gone. If the warm-up kept the //! written order, the written predicate is evaluated as one expression, //! exactly as it would be with the flag off. If it adopted a reorder, the //! learned order is materialised once as a right-nested `AND` chain, @@ -127,12 +129,12 @@ //! - Reordering never changes query *results* — the value of a conjunction does //! not depend on evaluation order — but it can change the observable *side //! effects* of fallible predicates, in either direction: a conjunct evaluated -//! after a compaction sees only the rows that survived, so an error the +//! after a pre-selection sees only the rows that survived, so an error the //! written order raises can disappear and one it avoided can appear. //! Predicates containing volatile expressions are never reordered. //! - The measurements are *conditional*: each conjunct is measured on the rows //! that survived the conjuncts before it in written order, and after a -//! compaction on small survivor batches whose per-row cost is inflated by +//! pre-selection on small survivor batches whose per-row cost is inflated by //! fixed overheads. Correlated conjuncts can therefore look more selective in //! a late position than they would be up front; the material-win guard makes //! adoption conservative but cannot detect correlation. @@ -143,18 +145,19 @@ //! See for the measurements //! behind this design. +use std::fmt; +use std::fmt::Formatter; use std::sync::Arc; use std::sync::Mutex; +use std::sync::atomic::{AtomicU64, Ordering::Relaxed}; -use arrow::array::{Array, ArrayRef, BooleanArray, BooleanBufferBuilder, UInt32Array}; -use arrow::buffer::BooleanBuffer; -use arrow::compute::kernels::boolean::and; -use arrow::compute::{filter, filter_record_batch, prep_null_mask_filter}; +use arrow::array::ArrayRef; +use arrow::datatypes::{DataType, FieldRef, Schema}; use arrow::record_batch::RecordBatch; +use datafusion_common::Result; use datafusion_common::cast::as_boolean_array; use datafusion_common::instant::Instant; -use datafusion_common::{Result, internal_err}; -use datafusion_expr::Operator; +use datafusion_expr::{ColumnarValue, Operator}; use datafusion_physical_expr::PhysicalExpr; use datafusion_physical_expr::expressions::BinaryExpr; use datafusion_physical_expr::utils::split_conjunction; @@ -169,12 +172,6 @@ const WARMUP_BATCHES: u64 = 8; /// conjuncts never trigger a reorder. const TIE_COST_FRACTION: f64 = 0.05; -/// Physically compact the working batch to the surviving rows only when the -/// accumulated mask keeps at most this fraction of them. Above this, the cost -/// of materializing a barely-smaller batch is not repaid, so we keep evaluating -/// against the full working batch and just `AND` the boolean masks. -const COMPACTION_SELECTIVITY_THRESHOLD: f64 = 0.2; - /// Per-conjunct measurement: marginal pass rate and per-row evaluation cost, /// accumulated over the warm-up window on exactly the rows that reached the /// conjunct. @@ -189,12 +186,6 @@ struct ConjunctStats { } impl ConjunctStats { - fn record(&mut self, matched: u64, rows: u64, nanos: u64) { - self.rows += rows; - self.matched += matched; - self.nanos += nanos; - } - /// Fold another accumulator's counts into this one (they are plain sums, so /// merging is addition). Used to pool measurements across partition streams. fn merge(&mut self, other: &Self) { @@ -255,9 +246,10 @@ struct SharedInner { /// cloning it, so reporting costs nothing on the per-batch path. #[derive(Debug, Clone, Copy, PartialEq, Eq)] enum BatchStrategy<'a> { - /// Still learning: the written order through the measuring loop, each - /// conjunct instrumented and its counts pooled. (Empty batches are - /// evaluated but measure nothing and do not consume the warm-up.) + /// Still learning: the written order as a right-nested `AND` chain over + /// the conjuncts wrapped in [`MeasuredConjunct`], their counts pooled. + /// (Empty batches are evaluated but measure nothing and do not consume the + /// warm-up.) Measure, /// Settled without a reorder: the written predicate evaluated as one /// expression, exactly as if the feature were off. @@ -321,6 +313,120 @@ impl AdaptiveFilterShared { } } +/// A conjunct wrapped so that evaluating it records what it saw. +/// +/// This is the whole of the warm-up's instrumentation. The wrapped conjuncts +/// are assembled into the written order's right-nested `AND` chain and handed +/// to [`BinaryExpr`], which evaluates and pre-selects exactly as it would for +/// the plain predicate; each wrapper therefore records the rows, matches and +/// elapsed time of the population `BinaryExpr` actually handed its conjunct. +/// The wrapper adds nothing but the counters: it returns the conjunct's own +/// result unchanged, nulls included, because three-valued logic is +/// `BinaryExpr`'s business and not this module's. +/// +/// The counters are per stream and uncontended, so `Relaxed` ordering is +/// enough; [`take`](Self::take) drains them. +/// +/// `Display`, [`fmt_sql`](PhysicalExpr::fmt_sql), `data_type`, `nullable` and +/// `return_field` all delegate to the wrapped conjunct, so a rendered warm-up +/// predicate is indistinguishable from the plain one. Equality and hashing +/// likewise consider only the wrapped conjunct: two wrappers around the same +/// expression are the same expression, whatever their counters hold. +#[derive(Debug)] +struct MeasuredConjunct { + inner: Arc, + /// Rows handed to the conjunct since the last [`take`](Self::take). + rows: AtomicU64, + /// Of those, the rows it kept: non-null `true`, matching SQL filter + /// semantics. + matched: AtomicU64, + /// Time spent inside the conjunct over those rows, in nanoseconds. + nanos: AtomicU64, +} + +impl MeasuredConjunct { + fn new(inner: Arc) -> Self { + Self { + inner, + rows: AtomicU64::new(0), + matched: AtomicU64::new(0), + nanos: AtomicU64::new(0), + } + } + + /// Drain the counters, returning what they held. + fn take(&self) -> ConjunctStats { + ConjunctStats { + rows: self.rows.swap(0, Relaxed), + matched: self.matched.swap(0, Relaxed), + nanos: self.nanos.swap(0, Relaxed), + } + } +} + +impl PartialEq for MeasuredConjunct { + fn eq(&self, other: &Self) -> bool { + self.inner.eq(&other.inner) + } +} + +impl Eq for MeasuredConjunct {} + +impl std::hash::Hash for MeasuredConjunct { + fn hash(&self, state: &mut H) { + self.inner.hash(state); + } +} + +impl fmt::Display for MeasuredConjunct { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.inner) + } +} + +impl PhysicalExpr for MeasuredConjunct { + fn data_type(&self, input_schema: &Schema) -> Result { + self.inner.data_type(input_schema) + } + + fn nullable(&self, input_schema: &Schema) -> Result { + self.inner.nullable(input_schema) + } + + fn return_field(&self, input_schema: &Schema) -> Result { + self.inner.return_field(input_schema) + } + + fn evaluate(&self, batch: &RecordBatch) -> Result { + let rows = batch.num_rows(); + let timer = Instant::now(); + let array = self.inner.evaluate(batch)?.into_array(rows)?; + let nanos = timer.elapsed().as_nanos() as u64; + let matched = as_boolean_array(&array)?.true_count() as u64; + + self.rows.fetch_add(rows as u64, Relaxed); + self.matched.fetch_add(matched, Relaxed); + self.nanos.fetch_add(nanos, Relaxed); + + Ok(ColumnarValue::Array(array)) + } + + fn children(&self) -> Vec<&Arc> { + vec![&self.inner] + } + + fn with_new_children( + self: Arc, + children: Vec>, + ) -> Result> { + Ok(Arc::new(Self::new(Arc::clone(&children[0])))) + } + + fn fmt_sql(&self, f: &mut Formatter<'_>) -> fmt::Result { + self.inner.fmt_sql(f) + } +} + /// Adaptive evaluator for a single conjunctive predicate, owned per partition /// stream. Measurements are pooled into the shared [`AdaptiveFilterShared`]; /// the per-stream state is just the current order and how far it has caught up. @@ -334,6 +440,14 @@ pub(crate) struct AdaptiveConjunction { predicate: Arc, /// Measurements and the settled decision, shared by every partition stream. shared: Arc, + /// What the warm-up evaluates: the *written* order as a right-nested `AND` + /// chain over [`measured`](Self::measured), so [`BinaryExpr`] does the + /// evaluating and the pre-selection while the wrappers do the counting. + /// Unused once settled. + warmup_predicate: Arc, + /// The wrappers inside [`warmup_predicate`](Self::warmup_predicate), in + /// written order, so each batch's counts can be drained out of them. + measured: Vec>, /// The expression [`evaluate_settled`](Self::evaluate_settled) evaluates: /// the written predicate, or — once a reorder is adopted — the learned /// order as a right-nested `AND` chain. Unused until `settled`, and equal @@ -377,11 +491,23 @@ impl AdaptiveConjunction { if conjuncts.len() < 2 || conjuncts.iter().any(is_volatile) { return None; } - let order = (0..conjuncts.len()).collect(); + let order: Vec = (0..conjuncts.len()).collect(); + let measured: Vec> = conjuncts + .iter() + .map(|c| Arc::new(MeasuredConjunct::new(Arc::clone(c)))) + .collect(); + let wrapped: Vec> = measured + .iter() + .map(|m| Arc::clone(m) as Arc) + .collect(); + let warmup_predicate = + right_nested_conjunction(&wrapped, &order).expect("at least two conjuncts"); Some(Self { conjuncts, predicate: Arc::clone(predicate), shared, + warmup_predicate, + measured, settled_predicate: Arc::clone(predicate), order, reordered: false, @@ -438,18 +564,37 @@ impl AdaptiveConjunction { // warm-up (a run of empty batches would otherwise settle the written // order on no evidence, permanently). if batch.num_rows() == 0 { - let mask = eval_conjuncts(&self.conjuncts, &self.order, batch, None)?; + let mask = self.evaluate_warmup(batch)?; + // Discard what the wrappers recorded: no rows, but a real call + // cost, which would otherwise inflate the next batch's per-row + // cost. + self.take_measurements(); return Ok((mask, BatchStrategy::Measure)); } - // Measure this batch into a local accumulator, then pool it. - let mut local = vec![ConjunctStats::default(); self.conjuncts.len()]; - let result = - eval_conjuncts(&self.conjuncts, &self.order, batch, Some(&mut local))?; + // Evaluate the written order through the wrappers, then drain and pool + // what they recorded. + let result = self.evaluate_warmup(batch)?; + let local = self.take_measurements(); self.pool_and_maybe_settle(&local); Ok((result, BatchStrategy::Measure)) } + /// Evaluate the warm-up arrangement: the written order as a right-nested + /// `AND` chain over the measuring wrappers, leaving this batch's counts in + /// them. + fn evaluate_warmup(&self, batch: &RecordBatch) -> Result { + self.warmup_predicate + .evaluate(batch)? + .into_array(batch.num_rows()) + } + + /// Drain the wrappers into per-conjunct counts, indexed by written + /// position. + fn take_measurements(&self) -> Vec { + self.measured.iter().map(|m| m.take()).collect() + } + /// Evaluate the settled arrangement with no instrumentation: one /// expression, either the written predicate or the right-nested `AND` /// chain built from the adopted order. @@ -559,125 +704,6 @@ fn right_nested_conjunction( }) } -/// The measuring loop: evaluate `conjuncts` one at a time in `order` against -/// `batch`, returning the boolean mask (over the batch's original rows) of rows -/// that passed every conjunct. With `stats`, each conjunct is additionally -/// timed and counted on exactly the rows it evaluated (its marginal selectivity -/// and cost on the current working population) — which is why the warm-up -/// evaluates conjunct by conjunct rather than handing the predicate to -/// [`BinaryExpr`], as the settled path does. -/// -/// The working batch is physically compacted to the surviving rows only once -/// the accumulated mask becomes selective enough (see -/// [`COMPACTION_SELECTIVITY_THRESHOLD`]); until then masks are combined with a -/// cheap bitwise `AND`, so a run of non-selective conjuncts pays no -/// materialization cost. Once compacted, the survivors stay compacted for the -/// conjuncts that follow. -fn eval_conjuncts( - conjuncts: &[Arc], - order: &[usize], - batch: &RecordBatch, - mut stats: Option<&mut [ConjunctStats]>, -) -> Result { - let num_rows = batch.num_rows(); - if num_rows == 0 { - return Ok(Arc::new(BooleanArray::from(Vec::::new()))); - } - // Live-row indices are tracked as `u32` (arrow's `filter`/`take` index - // space); a larger batch would silently wrap the indices, so refuse it. - if num_rows > u32::MAX as usize { - return internal_err!("adaptive filter: batch exceeds u32::MAX rows"); - } - - // `working` is the batch conjuncts are evaluated against. `acc` is the - // accumulated (`AND`-combined, null-free) result over `working`'s rows since - // the last compaction; `None` means all of them are still live. `live` maps - // `working`'s rows back to original row indices; `None` until a compaction - // first drops rows. - let mut working = batch.clone(); - let mut acc: Option = None; - let mut live: Option = None; - - for &id in order { - let rows_in = working.num_rows(); - - let timer = stats.is_some().then(Instant::now); - let array = conjuncts[id].evaluate(&working)?.into_array(rows_in)?; - let mask = as_boolean_array(&array)?; - // `matched` counts non-null trues (SQL filter semantics). - let matched = mask.true_count() as u64; - - if let (Some(stats), Some(timer)) = (stats.as_deref_mut(), timer) { - let eval_nanos = timer.elapsed().as_nanos() as u64; - stats[id].record(matched, rows_in as u64, eval_nanos); - } - - // An all-true mask leaves the accumulated result untouched. - if matched == rows_in as u64 && mask.null_count() == 0 { - continue; - } - - // Fold this conjunct into the accumulated mask (null -> false). - let mask = if mask.null_count() > 0 { - prep_null_mask_filter(mask) - } else { - mask.clone() - }; - let folded = match &acc { - None => mask, - Some(prev) => and(prev, &mask)?, - }; - - let alive = folded.true_count(); - if alive == 0 { - // Nothing survives; the result is all-false over the original rows. - return Ok(Arc::new(BooleanArray::new( - BooleanBuffer::new_unset(num_rows), - None, - ))); - } - // Compact only when the survivors are a small fraction of the working - // batch — otherwise the copy is not worth it. - if (alive as f64) <= COMPACTION_SELECTIVITY_THRESHOLD * rows_in as f64 { - working = filter_record_batch(&working, &folded)?; - let indices = live.take().unwrap_or_else(|| { - Arc::new(UInt32Array::from_iter_values(0..num_rows as u32)) - }); - live = Some(filter(&indices, &folded)?); - acc = None; - } else { - acc = Some(folded); - } - } - - match live { - // Never compacted: `acc` (or all-true) already covers the original rows. - None => Ok(match acc { - Some(acc) => Arc::new(acc), - None => Arc::new(BooleanArray::new(BooleanBuffer::new_set(num_rows), None)), - }), - // Compacted at least once: scatter the surviving original indices - // (`live`, narrowed by any residual `acc`) into a full-length mask. - Some(indices) => { - let indices = match acc { - Some(acc) => filter(&indices, &acc)?, - None => indices, - }; - let Some(indices) = indices.as_any().downcast_ref::() else { - return internal_err!( - "adaptive filter: live row indices are not a UInt32Array" - ); - }; - let mut builder = BooleanBufferBuilder::new(num_rows); - builder.append_n(num_rows, false); - for &idx in indices.values() { - builder.set_bit(idx as usize, true); - } - Ok(Arc::new(BooleanArray::new(builder.finish(), None))) - } - } -} - /// Rank conjunct ids by effectiveness (discards per nanosecond) descending; /// ids without measurements sort last. Stable, so equal ids keep their order. fn rank_by_effectiveness(stats: &[ConjunctStats]) -> Vec { @@ -715,7 +741,7 @@ fn expected_cost_per_row(stats: &[ConjunctStats], order: &[usize]) -> f64 { mod tests { use super::*; - use arrow::array::{Int32Array, Int64Array}; + use arrow::array::{Array, Int32Array, Int64Array}; use arrow::datatypes::{DataType, Field, Schema}; use datafusion_physical_expr::expressions::{binary, col, lit}; @@ -752,10 +778,6 @@ mod tests { .collect() } - fn conjuncts(schema: &Arc) -> Vec> { - split(&predicate(schema)) - } - fn passing_rows(mask: &ArrayRef) -> Vec { let mask = as_boolean_array(mask).unwrap(); (0..mask.len()) @@ -838,6 +860,57 @@ mod tests { assert_eq!(rank_by_effectiveness(&s), vec![0, 1]); } + /// The measuring wrapper counts the rows it was handed and the rows its + /// conjunct kept, returns the conjunct's own array untouched (nulls and + /// all), and drains to zero. + /// + /// The elapsed time is deliberately not asserted: on a coarse timer a + /// five-row evaluation can legitimately measure zero nanoseconds, which + /// [`ConjunctStats::cost_per_row`] already handles by clamping. + #[test] + fn measured_conjunct_counts_rows_and_matches() { + let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, true)])); + let rb = RecordBatch::try_new( + Arc::clone(&schema), + vec![Arc::new(Int32Array::from(vec![ + Some(1), + None, + Some(5), + Some(7), + None, + ]))], + ) + .unwrap(); + let inner = + binary(col("a", &schema).unwrap(), Operator::Gt, lit(2i32), &schema).unwrap(); + let measured = Arc::new(MeasuredConjunct::new(Arc::clone(&inner))); + + let got = measured + .evaluate(&rb) + .unwrap() + .into_array(rb.num_rows()) + .unwrap(); + let want = inner + .evaluate(&rb) + .unwrap() + .into_array(rb.num_rows()) + .unwrap(); + assert_eq!(&got, &want, "the conjunct's own result, unchanged"); + assert_eq!( + as_boolean_array(&got).unwrap().null_count(), + 2, + "nulls are left for `BinaryExpr` to interpret" + ); + + // Every row was handed to the conjunct; only the non-null trues count + // as matches. + let s = measured.take(); + assert_eq!((s.rows, s.matched), (5, 2)); + // ...and `take` drains. + let s = measured.take(); + assert_eq!((s.rows, s.matched, s.nanos), (0, 0, 0)); + } + /// Empty batches measure nothing, so they must not consume the warm-up: /// a stream fed only empty batches keeps learning instead of settling the /// written order on no evidence. @@ -865,43 +938,56 @@ mod tests { assert!((expected_cost_per_row(&s, &[1, 0]) - 10.5).abs() < 1e-9); } - /// The measuring loop returns exactly the rows the written predicate - /// keeps, in any order and whether or not compaction triggers. + /// Across the warm-up boundary the mask must always equal the written + /// predicate's, before and after the order settles. #[test] - fn eval_conjuncts_matches_predicate_in_any_order() { + fn evaluate_matches_predicate_across_warmup() { let schema = schema(); - let cs = conjuncts(&schema); let p = predicate(&schema); + let mut adaptive = try_new(&p).unwrap(); - // A batch where `b < 5` is rare (forces a compaction) and one where it - // is common (no compaction). - for b in [ - (0..100).map(|x| x % 50).collect::>(), // b<5 rare - (0..100).map(|x| x % 3).collect::>(), // b<5 common - ] { - let a: Vec = (0..100).collect(); + for round in 0..(WARMUP_BATCHES as i32 + 4) { + let base = round * 10; + let a: Vec = (base..base + 10).collect(); + let b: Vec = (base..base + 10).map(|x| x.rem_euclid(9)).collect(); let rb = batch(&schema, a, b); + + let got = adaptive.evaluate(&rb).unwrap(); let want = p.evaluate(&rb).unwrap().into_array(rb.num_rows()).unwrap(); - for order in [vec![0, 1], vec![1, 0]] { - let got = eval_conjuncts(&cs, &order, &rb, None).unwrap(); - assert_eq!(passing_rows(&got), passing_rows(&want), "order {order:?}"); - } + assert_eq!( + passing_rows(&got), + passing_rows(&want), + "mismatch on round {round}" + ); } + assert!(adaptive.settled); } - /// Across the warm-up boundary the mask must always equal the written - /// predicate's, before and after the order settles. + /// A conjunct that produces nulls must come through the warm-up unchanged: + /// the wrapper hands `BinaryExpr` the conjunct's own three-valued result, + /// so the mask matches the plain predicate's on every batch. #[test] - fn evaluate_matches_predicate_across_warmup() { - let schema = schema(); + fn nullable_conjuncts_match_the_plain_predicate_across_warmup() { + let schema = Arc::new(Schema::new(vec![ + Field::new("a", DataType::Int32, true), + Field::new("b", DataType::Int32, true), + ])); let p = predicate(&schema); let mut adaptive = try_new(&p).unwrap(); for round in 0..(WARMUP_BATCHES as i32 + 4) { let base = round * 10; - let a: Vec = (base..base + 10).collect(); - let b: Vec = (base..base + 10).map(|x| x.rem_euclid(9)).collect(); - let rb = batch(&schema, a, b); + let a: Vec> = (base..base + 10) + .map(|x| (x.rem_euclid(3) != 0).then_some(x)) + .collect(); + let b: Vec> = (base..base + 10) + .map(|x| (x.rem_euclid(4) != 0).then_some(x.rem_euclid(9))) + .collect(); + let rb = RecordBatch::try_new( + Arc::clone(&schema), + vec![Arc::new(Int32Array::from(a)), Arc::new(Int32Array::from(b))], + ) + .unwrap(); let got = adaptive.evaluate(&rb).unwrap(); let want = p.evaluate(&rb).unwrap().into_array(rb.num_rows()).unwrap(); @@ -1063,8 +1149,7 @@ mod tests { let mut s2 = AdaptiveConjunction::try_new(&p, Arc::clone(&shared)).unwrap(); // `b < 5` (conjunct 1) is the selective one; drive both streams with - // batches where it keeps 5 rows in 25 (20%, exactly the compaction - // threshold). + // batches where it keeps 5 rows in 25. let mk = |round: i32| { let base = round * 100; let a: Vec = (base..base + 100).collect(); @@ -1271,8 +1356,8 @@ mod tests { let inner = shared.inner.lock().unwrap(); assert_eq!(inner.stats.len(), 2, "sized to the conjunct count"); assert_eq!(inner.measured_batches, 1); - // `a > 2` keeps 7 of 10 rows: too many to compact, so `b < 5` is - // evaluated on all 10 rows too and keeps 5 of them. + // `a > 2` keeps 7 of 10 rows: too many for `BinaryExpr` to pre-select + // on, so `b < 5` is evaluated on all 10 rows too and keeps 5 of them. assert_eq!((inner.stats[0].rows, inner.stats[0].matched), (10, 7)); assert_eq!((inner.stats[1].rows, inner.stats[1].matched), (10, 5)); } @@ -1357,7 +1442,7 @@ mod tests { let mut adaptive = AdaptiveConjunction::try_new(&p, Arc::clone(&shared)).unwrap(); // The settling batch is still measured in the written order, whose - // compaction on `b <> 0` also keeps `1 / b` away from the zeros. + // pre-selection on `b <> 0` also keeps `1 / b` away from the zeros. adaptive.evaluate(&rb).unwrap(); assert_eq!(adaptive.order, vec![1, 0]); assert!(adaptive.reordered); From 8dac5eea4dc6741e122479e9e55c10d64def5a45 Mon Sep 17 00:00:00 2001 From: Adrian Garcia Badaracco <1755071+adriangb@users.noreply.github.com> Date: Mon, 7 Sep 2026 01:54:35 -0500 Subject: [PATCH 16/27] refactor(physical-plan): keep FilterExecMetrics::new signature; add with_adaptive_reorder_metrics Co-Authored-By: Claude Fable 5.1 --- datafusion/physical-plan/src/filter.rs | 39 +++++++++++++++++--------- 1 file changed, 25 insertions(+), 14 deletions(-) diff --git a/datafusion/physical-plan/src/filter.rs b/datafusion/physical-plan/src/filter.rs index 10e9a9e16e42b..cd8b18a7a8573 100644 --- a/datafusion/physical-plan/src/filter.rs +++ b/datafusion/physical-plan/src/filter.rs @@ -687,8 +687,12 @@ impl ExecutionPlan for FilterExec { ) }) .flatten(); - let metrics = - FilterExecMetrics::new(&self.metrics, partition, adaptive.is_some()); + let metrics = FilterExecMetrics::new(&self.metrics, partition); + let metrics = if adaptive.is_some() { + metrics.with_adaptive_reorder_metrics(&self.metrics, partition) + } else { + metrics + }; Ok(Box::pin(FilterExecStream { schema: self.schema(), predicate: Arc::clone(&self.predicate), @@ -1436,26 +1440,33 @@ struct FilterExecMetrics { } impl FilterExecMetrics { - pub fn new( - metrics: &ExecutionPlanMetricsSet, - partition: usize, - adaptive: bool, - ) -> Self { + pub fn new(metrics: &ExecutionPlanMetricsSet, partition: usize) -> Self { Self { baseline_metrics: BaselineMetrics::new(metrics, partition), selectivity: MetricBuilder::new(metrics) .with_type(MetricType::Summary) .ratio_metrics("selectivity", partition), - adaptive_reorders: adaptive.then(|| { - MetricBuilder::new(metrics) - // A deterministic, dimensionless counter: it depends on - // the plan and the data, not on wall-clock timings. - .with_category(MetricCategory::Rows) - .counter("adaptive_reorders", partition) - }), + adaptive_reorders: None, } } + /// Also register the `adaptive_reorders` counter; see + /// [`Self::adaptive_reorders`]. + fn with_adaptive_reorder_metrics( + mut self, + metrics: &ExecutionPlanMetricsSet, + partition: usize, + ) -> Self { + self.adaptive_reorders = Some( + MetricBuilder::new(metrics) + // A deterministic, dimensionless counter: it depends on + // the plan and the data, not on wall-clock timings. + .with_category(MetricCategory::Rows) + .counter("adaptive_reorders", partition), + ); + self + } + /// Record that this stream adopted a reordered evaluation order. fn record_adaptive_reorder(&self) { if let Some(count) = &self.adaptive_reorders { From 19c8120f1a5d3e914ab49eacf8ff8f229e703fb1 Mon Sep 17 00:00:00 2001 From: Adrian Garcia Badaracco <1755071+adriangb@users.noreply.github.com> Date: Mon, 7 Sep 2026 02:04:02 -0500 Subject: [PATCH 17/27] refactor(physical-plan): rank conjuncts by Velox's discards-per-time key Co-Authored-By: Claude Fable 5.1 --- .../physical-plan/src/adaptive_filter.rs | 25 +++++++++++-------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/datafusion/physical-plan/src/adaptive_filter.rs b/datafusion/physical-plan/src/adaptive_filter.rs index 241282f89ec88..28cc7cd4a6322 100644 --- a/datafusion/physical-plan/src/adaptive_filter.rs +++ b/datafusion/physical-plan/src/adaptive_filter.rs @@ -106,7 +106,7 @@ //! Each conjunct is timed and counted on exactly the rows it evaluated, giving //! its marginal selectivity and per-row cost. After a short warm-up the //! conjuncts are ranked by rows discarded per nanosecond -//! (`(1 - pass_rate) / cost_per_row`, the classic optimal ordering key for +//! (`(1 + rows_in - rows_out) / time`, the ordering key Velox uses for //! independent conjuncts). The ranking is adopted only if it is materially //! cheaper than the written order ([`TIE_COST_FRACTION`]); otherwise the //! written predicate is evaluated unchanged, so a conjunction that does not @@ -207,16 +207,21 @@ impl ConjunctStats { (self.rows > 0).then(|| self.nanos.max(1) as f64 / self.rows as f64) } - /// Ranking key: rows discarded per nanosecond of evaluation - /// (`(1 - pass_rate) / cost_per_row`). Maximising this is exactly - /// minimising `cost_per_row / (1 - pass_rate)`, the classic optimal - /// ordering key for independent conjuncts — so a selective-but-expensive - /// predicate correctly sorts ahead of a cheap-but-unselective one. - /// `None` when unmeasured, so such conjuncts sort last. + /// Ranking key: rows discarded per nanosecond of evaluation, + /// `(1 + rows_in - rows_out) / time`. This is the reciprocal of the + /// `time / (1 + n_in - n_out)` score Velox sorts its filters by + /// (Pedreira et al., "Velox: Meta's Unified Execution Engine", VLDB 2022, + /// ): maximising it + /// minimises time per discarded row, the optimal ordering key for + /// independent conjuncts, so a selective-but-expensive predicate sorts + /// ahead of a cheap-but-unselective one. The `1 +` keeps conjuncts that + /// discard nothing ordered cheapest-first instead of tied; the time is + /// clamped to one nanosecond so an evaluation faster than the timer's + /// resolution ranks as very cheap. `None` when unmeasured, so such + /// conjuncts sort last. fn effectiveness(&self) -> Option { - let cost = self.cost_per_row()?; - let pass = self.pass_rate()?; - Some((1.0 - pass) / cost) + (self.rows > 0) + .then(|| (1 + self.rows - self.matched) as f64 / self.nanos.max(1) as f64) } } From 15a5be3bf4f9ba5edb1978cf491ee9a43a0fe4ef Mon Sep 17 00:00:00 2001 From: Adrian Garcia Badaracco <1755071+adriangb@users.noreply.github.com> Date: Mon, 7 Sep 2026 10:22:45 -0500 Subject: [PATCH 18/27] perf(physical-plan): right-nest the kept written order; pool without a pre-evaluation lock When the warm-up keeps the written order, evaluate it as the same right-nested AND chain an adopted order uses instead of the planner's left-nested expression: a left-nested chain pre-selects on the accumulated prefix and pays a whole-batch filter and scatter at every level where that prefix crosses the threshold. Unsettled streams no longer take the shared lock before evaluating a batch; a decision made meanwhile is picked up when the batch's counts are pooled, and those counts are discarded. Co-Authored-By: Claude Fable 5.1 --- .../physical-plan/src/adaptive_filter.rs | 107 +++++++++--------- 1 file changed, 54 insertions(+), 53 deletions(-) diff --git a/datafusion/physical-plan/src/adaptive_filter.rs b/datafusion/physical-plan/src/adaptive_filter.rs index 28cc7cd4a6322..47e78fb8de5f8 100644 --- a/datafusion/physical-plan/src/adaptive_filter.rs +++ b/datafusion/physical-plan/src/adaptive_filter.rs @@ -86,10 +86,9 @@ //! `BinaryExpr` hands it — the population it would really see in that //! position. //! -//! Once the order settles the wrappers are gone. If the warm-up kept the -//! written order, the written predicate is evaluated as one expression, -//! exactly as it would be with the flag off. If it adopted a reorder, the -//! learned order is materialised once as a right-nested `AND` chain, +//! Once the order settles the wrappers are gone: the settled order — the +//! written one if the warm-up found nothing materially better, otherwise the +//! learned one — is materialised once as a right-nested `AND` chain, //! `(c_first AND (c_second AND (... AND c_last)))`, and from then on evaluated //! by [`BinaryExpr`] like any other predicate. //! @@ -109,9 +108,9 @@ //! (`(1 + rows_in - rows_out) / time`, the ordering key Velox uses for //! independent conjuncts). The ranking is adopted only if it is materially //! cheaper than the written order ([`TIE_COST_FRACTION`]); otherwise the -//! written predicate is evaluated unchanged, so a conjunction that does not -//! benefit from reordering carries none of this module's machinery past the -//! warm-up. The decision then stays fixed. +//! written order is kept, so a conjunction that does not benefit from +//! reordering carries none of this module's machinery past the warm-up. The +//! decision then stays fixed. //! //! ## How it shares //! @@ -256,8 +255,8 @@ enum BatchStrategy<'a> { /// (Empty batches are evaluated but measure nothing and do not consume the /// warm-up.) Measure, - /// Settled without a reorder: the written predicate evaluated as one - /// expression, exactly as if the feature were off. + /// Settled without a reorder: the written order, as a right-nested `AND` + /// chain. Fused, /// Settled on an adopted reorder, evaluated as the right-nested `AND` /// chain built from it. The payload is the adopted order: positions in the @@ -269,15 +268,13 @@ enum BatchStrategy<'a> { /// is evaluated with, and the order it evaluates the conjuncts in. #[derive(Debug, Clone)] struct Settled { - /// The predicate to evaluate from now on: the written predicate when the - /// warm-up kept the written order, otherwise `order` materialised as a + /// The predicate to evaluate from now on: `order` materialised as a /// right-nested `AND` chain (see [`settle`]). predicate: Arc, /// Evaluation order: indices into the conjunct list. Carried for reporting /// ([`BatchStrategy::Reordered`]) rather than for evaluation. order: Vec, - /// Whether `order` reorders the written conjuncts — equivalently, whether - /// `predicate` is the rebuilt chain rather than the written predicate. + /// Whether `order` reorders the written conjuncts. reordered: bool, } @@ -287,6 +284,7 @@ impl AdaptiveFilterShared { } /// The settled decision, or `None` if the streams are still learning. + #[cfg(test)] fn settled(&self) -> Option { self.inner.lock().expect("poisoned").settled.clone() } @@ -440,8 +438,7 @@ pub(crate) struct AdaptiveConjunction { /// The split conjuncts. `order` indices refer to positions here. conjuncts: Vec>, /// The written predicate as one expression: what the measured conjuncts - /// are ranked against, and what is evaluated when the settled order does - /// not reorder them. + /// are ranked against. predicate: Arc, /// Measurements and the settled decision, shared by every partition stream. shared: Arc, @@ -454,9 +451,8 @@ pub(crate) struct AdaptiveConjunction { /// written order, so each batch's counts can be drained out of them. measured: Vec>, /// The expression [`evaluate_settled`](Self::evaluate_settled) evaluates: - /// the written predicate, or — once a reorder is adopted — the learned - /// order as a right-nested `AND` chain. Unused until `settled`, and equal - /// to the written predicate until then. + /// the settled order as a right-nested `AND` chain. Unused until + /// `settled`, and equal to the written predicate until then. settled_predicate: Arc, /// Evaluation order: indices into `conjuncts`. The written order until a /// settled order is adopted. @@ -546,15 +542,6 @@ impl AdaptiveConjunction { &mut self, batch: &RecordBatch, ) -> Result<(ArrayRef, BatchStrategy<'_>)> { - // Take up an order another stream settled since our last batch. An - // unsettled stream already locks the shared state once per measured - // batch to pool its counts, so this brief extra lock is in the same - // cost class; a settled stream never touches it again. - if !self.settled - && let Some(decision) = self.shared.settled() - { - self.adopt(decision); - } if self.settled { let mask = self.evaluate_settled(batch)?; let strategy = if self.reordered { @@ -600,9 +587,8 @@ impl AdaptiveConjunction { self.measured.iter().map(|m| m.take()).collect() } - /// Evaluate the settled arrangement with no instrumentation: one - /// expression, either the written predicate or the right-nested `AND` - /// chain built from the adopted order. + /// Evaluate the settled arrangement with no instrumentation: the + /// right-nested `AND` chain built from the settled order. fn evaluate_settled(&self, batch: &RecordBatch) -> Result { self.settled_predicate .evaluate(batch)? @@ -624,6 +610,16 @@ impl AdaptiveConjunction { /// them. fn pool_and_maybe_settle(&mut self, local: &[ConjunctStats]) { let mut inner = self.shared.inner.lock().expect("poisoned"); + // Another stream settled since this batch started: its decision stands + // and this batch's counts are discarded (they can no longer change + // anything). Checking here, after evaluating, rather than before keeps + // the shared lock off the path entirely until a stream has something to + // pool; the price is at most one measured batch per stream. + if let Some(decision) = inner.settled.clone() { + drop(inner); + self.adopt(decision); + return; + } if inner.stats.is_empty() { inner.stats = vec![ConjunctStats::default(); local.len()]; } @@ -636,13 +632,6 @@ impl AdaptiveConjunction { s.merge(l); } inner.measured_batches += 1; - // Another stream settled between our two lock acquisitions: adopt its - // decision rather than measuring on. - if let Some(decision) = inner.settled.clone() { - drop(inner); - self.adopt(decision); - return; - } if inner.measured_batches < WARMUP_BATCHES { return; } @@ -656,10 +645,9 @@ impl AdaptiveConjunction { /// Decide the settled arrangement from the pooled measurements. /// /// Rank the conjuncts by effectiveness and take the ranking only if it is -/// materially cheaper than the written order, materialising it as a -/// right-nested `AND` chain over `conjuncts`; otherwise keep `predicate` and -/// evaluate it as one expression, so a conjunction that does not benefit from -/// reordering pays nothing for the attempt. +/// materially cheaper than the written order; otherwise keep the written +/// order. Either way the result is materialised as a right-nested `AND` chain +/// over `conjuncts` (`predicate` is only the fallback for an empty list). fn settle( stats: &[ConjunctStats], conjuncts: &[Arc], @@ -678,8 +666,16 @@ fn settle( reordered: true, } } else { + // The written order, but still right-nested: `predicate` as the + // planner built it is typically left-nested, and a left-nested chain + // pre-selects on the accumulated prefix, paying a whole-batch filter + // and scatter at every level where that prefix crosses the threshold. + // Right-nested, each `AND` decides on a single conjunct and survivors + // stay compacted for the rest of the chain. + let written = right_nested_conjunction(conjuncts, &identity) + .unwrap_or_else(|| Arc::clone(predicate)); Settled { - predicate: Arc::clone(predicate), + predicate: written, order: identity, reordered: false, } @@ -1012,15 +1008,15 @@ mod tests { let schema = schema(); let p = predicate(&schema); // Two equally cheap, equally selective conjuncts: swapping cannot help, - // so the written order stands and nothing is rebuilt. + // so the written order stands, rebuilt as a right-nested chain. let s = vec![stats(1000, 500, 1000), stats(1000, 500, 1000)]; - let d = settle(&s, &split(&p), &p); + let cs = split(&p); + let d = settle(&s, &cs, &p); assert_eq!(d.order, vec![0, 1]); assert!(!d.reordered); - assert!( - Arc::ptr_eq(&d.predicate, &p), - "the written predicate itself" - ); + let chain = d.predicate.downcast_ref::().expect("an AND"); + assert!(Arc::ptr_eq(chain.left(), &cs[0])); + assert!(Arc::ptr_eq(chain.right(), &cs[1])); } #[test] @@ -1092,7 +1088,7 @@ mod tests { } /// When the order does not change, the settled evaluator runs the written - /// predicate as one expression. + /// order as a right-nested chain over the same conjuncts. /// /// This test measures real timings on purpose and is still deterministic: /// both conjuncts pass the same ~96% of rows, and with a pass rate `p` @@ -1122,9 +1118,15 @@ mod tests { assert!(adaptive.settled); assert!( !adaptive.reordered, - "interchangeable conjuncts stay on the plain predicate" + "interchangeable conjuncts keep the written order" ); - assert!(Arc::ptr_eq(&adaptive.settled_predicate, &p)); + let cs = split(&p); + let chain = adaptive + .settled_predicate + .downcast_ref::() + .expect("an AND"); + assert!(Arc::ptr_eq(chain.left(), &cs[0])); + assert!(Arc::ptr_eq(chain.right(), &cs[1])); } /// Two streams sharing one pool settle the order together: the warm-up is @@ -1304,8 +1306,7 @@ mod tests { } /// Contract scenario for the no-win case: interchangeable conjuncts settle - /// on the written predicate evaluated as one expression (as if the feature - /// were off), never a rebuilt chain. + /// on the written order, never a reorder. #[test] fn scenario_measure_batches_then_settle_on_fused() { let schema = schema(); From 9dbdffe90e547fa0c359dfee89fe3a145a2aeeeb Mon Sep 17 00:00:00 2001 From: Adrian Garcia Badaracco <1755071+adriangb@users.noreply.github.com> Date: Mon, 7 Sep 2026 11:00:20 -0500 Subject: [PATCH 19/27] refactor(physical-plan): trim adaptive filter to its essentials Remove the test-only tracing machinery from the production path, simplify the metric plumbing and cut the documentation down to saying each thing once, where a reader first meets it. No behaviour change. - delete `BatchStrategy` and `evaluate_traced`; `evaluate` is the only entry point, and `order` is gone from `Settled` and `AdaptiveConjunction` (`reordered` is the decision). The tests that used the trace now assert on `settled`/`reordered` and on the shape of `settled_predicate`. - `AdaptiveConjunction::try_new` takes the `adaptive_reorders` `Count` and increments it in `adopt`, so `adopted_reorder`, `take_adopted_reorder` and `FilterExecMetrics::record_adaptive_reorder` are gone and the poll loop is a plain two-arm match. `execute` decides `AdaptiveConjunction::applies` first so the counter exists before the evaluator that increments it, and is still registered exactly when the adaptive path is active. - `right_nested_conjunction` is infallible, which lets `settle` drop its `predicate` fallback and `AdaptiveConjunction` drop the field. - one seeding helper (`seed_one_batch_short_of_warmup`) for both the module tests and the `FilterExec` end-to-end test. - drop `AdaptiveFilterShared::new` and `MeasuredConjunct::return_field` (the trait default derives the same field from `data_type`/`nullable`). Co-Authored-By: Claude Fable 5.1 --- .../physical-plan/src/adaptive_filter.rs | 694 +++++++----------- datafusion/physical-plan/src/filter.rs | 64 +- 2 files changed, 298 insertions(+), 460 deletions(-) diff --git a/datafusion/physical-plan/src/adaptive_filter.rs b/datafusion/physical-plan/src/adaptive_filter.rs index 47e78fb8de5f8..58dceefc8433a 100644 --- a/datafusion/physical-plan/src/adaptive_filter.rs +++ b/datafusion/physical-plan/src/adaptive_filter.rs @@ -31,97 +31,62 @@ //! the *written order* and measures against. //! - [`BinaryExpr`]'s `AND` pre-selects: when the conjuncts evaluated so far //! keep at most 20% of the rows and produce no nulls, it filters the batch -//! down to those rows before evaluating the next conjunct. -//! -//! Pre-selection can only gate a conjunct on the conjuncts written *before* it, -//! never on a more selective one written after it; it does not fire while the -//! accumulated result still has nulls; and in a left-nested `AND` chain each -//! level filters the original batch and scatters its result back to full -//! length, so survivors are not carried forward compacted from one level to -//! the next. +//! down to those rows before evaluating the next conjunct. It can only gate +//! a conjunct on the conjuncts written *before* it, never on a more +//! selective one written after it. //! //! This module measures each conjunct's selectivity and cost at runtime and -//! reorders them accordingly, handing the learned order back to `BinaryExpr` -//! as a right-nested `AND` chain so that pre-selection fires on the conjunct -//! that discards the most rows and its survivors stay compacted for the rest -//! of the chain. Whether it runs at all is controlled by -//! `datafusion.execution.adaptive_filter_reordering`. -//! -//! For example: +//! reorders them accordingly, so that pre-selection fires on the conjunct that +//! discards the most rows. Whether it runs at all is controlled by +//! `datafusion.execution.adaptive_filter_reordering`. For example: //! //! ```sql //! WHERE regexp_like(s,'a') AND regexp_like(s,'b') AND regexp_like(s,'rare') //! ``` //! //! All three conjuncts are equally expensive to the static cost class, so they -//! reach `FilterExec` as written. The first two each keep most rows, so `AND` -//! pre-selection never fires and every conjunct runs on the whole batch: +//! reach `FilterExec` as written, and the first two each keep most rows, so +//! `AND` pre-selection never fires. Once the warm-up has measured the three, +//! the selective one is promoted and the batch is compacted behind it: //! //! ```text -//! regexp_like(s,'a') evaluated on every row -//! regexp_like(s,'b') evaluated on every row -//! regexp_like(s,'rare') evaluated on every row -//! ``` +//! before: regexp_like(s,'a') evaluated on every row +//! regexp_like(s,'b') evaluated on every row +//! regexp_like(s,'rare') evaluated on every row //! -//! Once the warm-up has measured the three, the selective one is promoted and -//! the batch is compacted behind it: -//! -//! ```text -//! regexp_like(s,'rare') evaluated on every row, keeps ~1% -> batch compacted -//! regexp_like(s,'a') evaluated on those survivors only -//! regexp_like(s,'b') evaluated on those survivors only +//! after: regexp_like(s,'rare') every row, keeps ~1% -> batch compacted +//! regexp_like(s,'a') evaluated on those survivors only +//! regexp_like(s,'b') evaluated on those survivors only //! ``` //! -//! ## How it evaluates -//! -//! This module contains no evaluation logic of its own. Every batch, learning -//! or settled, is evaluated by [`BinaryExpr`] as a right-nested `AND` chain. -//! -//! While the order is being learned the chain is built over the *written* -//! order, with each conjunct wrapped in a measuring expression that times the -//! call and counts the rows it saw and the rows it kept. `BinaryExpr` is -//! therefore what performs the compaction, exactly as it does for the plain -//! predicate: its pre-selection filters the batch before evaluating the rest -//! of the chain, so every conjunct is measured on precisely the rows -//! `BinaryExpr` hands it — the population it would really see in that -//! position. +//! This module contains no evaluation logic of its own. While the order is +//! being learned, the written order is handed to [`BinaryExpr`] with every +//! conjunct wrapped in a [`MeasuredConjunct`]; `BinaryExpr` evaluates and +//! pre-selects as it would for the plain predicate, so each conjunct is +//! measured on the population it would really see in that position. //! //! Once the order settles the wrappers are gone: the settled order — the //! written one if the warm-up found nothing materially better, otherwise the //! learned one — is materialised once as a right-nested `AND` chain, -//! `(c_first AND (c_second AND (... AND c_last)))`, and from then on evaluated -//! by [`BinaryExpr`] like any other predicate. -//! -//! Right-nesting is what makes that cheap. Pre-selection filters the batch the -//! `AND` is handed before evaluating its right-hand side, so under right -//! nesting the survivors of the first (most selective) conjunct stay compacted -//! for the entire remainder of the chain. A left-nested chain — what -//! [`conjunction`](datafusion_physical_expr::utils::conjunction) builds — -//! would instead re-filter the original batch at every level and scatter each -//! level's result back to full length. -//! -//! ## How it orders +//! `(c_first AND (c_second AND (... AND c_last)))`. Right-nesting is what makes +//! it pay: pre-selection filters the batch an `AND` is handed before evaluating +//! its right-hand side, so the survivors of the first conjunct stay compacted +//! for the rest of the chain, where a left-nested chain — what +//! [`conjunction`](datafusion_physical_expr::utils::conjunction) builds — would +//! re-filter the original batch at every level. //! -//! Each conjunct is timed and counted on exactly the rows it evaluated, giving -//! its marginal selectivity and per-row cost. After a short warm-up the -//! conjuncts are ranked by rows discarded per nanosecond -//! (`(1 + rows_in - rows_out) / time`, the ordering key Velox uses for -//! independent conjuncts). The ranking is adopted only if it is materially -//! cheaper than the written order ([`TIE_COST_FRACTION`]); otherwise the -//! written order is kept, so a conjunction that does not benefit from -//! reordering carries none of this module's machinery past the warm-up. The -//! decision then stays fixed. -//! -//! ## How it shares +//! The ranking key is rows discarded per nanosecond +//! ([`effectiveness`](ConjunctStats::effectiveness)), and the ranking is +//! adopted only if it is materially cheaper than the written order +//! ([`TIE_COST_FRACTION`]), so a conjunction that does not benefit carries none +//! of this machinery past the warm-up. The decision then stays fixed. //! //! A `FilterExec` is split across many partition streams, each seeing only a -//! slice of the data. Measurements are pooled into a shared -//! [`AdaptiveFilterShared`] so the streams learn as one: the first stream to -//! accumulate enough samples settles the order for all of them, and the rest -//! adopt it on their next batch instead of each re-paying the warm-up — which -//! is what makes the win materialise when each stream is only a handful of -//! batches long. Only unsettled streams take the shared lock; a settled stream -//! never touches it again. +//! slice of the data, so measurements are pooled into a shared +//! [`AdaptiveFilterShared`] and the streams learn as one: the first stream with +//! enough samples settles the order for all of them, and the rest adopt it on +//! their next batch instead of each re-paying the warm-up. Only unsettled +//! streams take the shared lock. //! //! ## Known limitations //! @@ -132,17 +97,16 @@ //! written order raises can disappear and one it avoided can appear. //! Predicates containing volatile expressions are never reordered. //! - The measurements are *conditional*: each conjunct is measured on the rows -//! that survived the conjuncts before it in written order, and after a -//! pre-selection on small survivor batches whose per-row cost is inflated by -//! fixed overheads. Correlated conjuncts can therefore look more selective in -//! a late position than they would be up front; the material-win guard makes -//! adoption conservative but cannot detect correlation. +//! that survived the conjuncts before it, and after a pre-selection on small +//! survivor batches whose per-row cost is inflated by fixed overheads. +//! Correlated conjuncts can therefore look more selective in a late position +//! than they would be up front; the material-win guard makes adoption +//! conservative but cannot detect correlation. //! - The decision is one-shot: once settled, the order is never re-measured, so //! a misjudged reorder — or data whose selectivity drifts — is kept for the //! rest of the query. //! -//! See for the measurements -//! behind this design. +//! See . use std::fmt; use std::fmt::Formatter; @@ -150,8 +114,9 @@ use std::sync::Arc; use std::sync::Mutex; use std::sync::atomic::{AtomicU64, Ordering::Relaxed}; +use crate::metrics::Count; use arrow::array::ArrayRef; -use arrow::datatypes::{DataType, FieldRef, Schema}; +use arrow::datatypes::{DataType, Schema}; use arrow::record_batch::RecordBatch; use datafusion_common::Result; use datafusion_common::cast::as_boolean_array; @@ -206,17 +171,10 @@ impl ConjunctStats { (self.rows > 0).then(|| self.nanos.max(1) as f64 / self.rows as f64) } - /// Ranking key: rows discarded per nanosecond of evaluation, - /// `(1 + rows_in - rows_out) / time`. This is the reciprocal of the - /// `time / (1 + n_in - n_out)` score Velox sorts its filters by - /// (Pedreira et al., "Velox: Meta's Unified Execution Engine", VLDB 2022, - /// ): maximising it - /// minimises time per discarded row, the optimal ordering key for - /// independent conjuncts, so a selective-but-expensive predicate sorts - /// ahead of a cheap-but-unselective one. The `1 +` keeps conjuncts that - /// discard nothing ordered cheapest-first instead of tied; the time is - /// clamped to one nanosecond so an evaluation faster than the timer's - /// resolution ranks as very cheap. `None` when unmeasured, so such + /// Ranking key: rows discarded per nanosecond, `(1 + rows_in - rows_out) / + /// time` — the reciprocal of the score Velox sorts its filters by + /// (), so maximising it + /// minimises time per discarded row. `None` when unmeasured, so such /// conjuncts sort last. fn effectiveness(&self) -> Option { (self.rows > 0) @@ -227,7 +185,7 @@ impl ConjunctStats { /// State shared by every partition stream of one `FilterExec`, so the streams /// learn as one: per-conjunct measurements are pooled across streams and the /// first stream to accumulate enough samples settles the order for all of them -/// (see "How it shares" in the [module docs](self)). +/// (see the [module docs](self)). #[derive(Debug, Default)] pub(crate) struct AdaptiveFilterShared { inner: Mutex, @@ -244,45 +202,17 @@ struct SharedInner { settled: Option, } -/// How one batch was evaluated, reported by -/// [`AdaptiveConjunction::evaluate_traced`] so the `scenario_*` tests can -/// assert the strategy batch by batch. Borrows the adopted order rather than -/// cloning it, so reporting costs nothing on the per-batch path. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum BatchStrategy<'a> { - /// Still learning: the written order as a right-nested `AND` chain over - /// the conjuncts wrapped in [`MeasuredConjunct`], their counts pooled. - /// (Empty batches are evaluated but measure nothing and do not consume the - /// warm-up.) - Measure, - /// Settled without a reorder: the written order, as a right-nested `AND` - /// chain. - Fused, - /// Settled on an adopted reorder, evaluated as the right-nested `AND` - /// chain built from it. The payload is the adopted order: positions in the - /// written conjunct list, first-evaluated first. - Reordered(&'a [usize]), -} - -/// The settled outcome of the warm-up: the expression every subsequent batch -/// is evaluated with, and the order it evaluates the conjuncts in. +/// The settled outcome of the warm-up. #[derive(Debug, Clone)] struct Settled { - /// The predicate to evaluate from now on: `order` materialised as a + /// The predicate to evaluate from now on: the settled order as a /// right-nested `AND` chain (see [`settle`]). predicate: Arc, - /// Evaluation order: indices into the conjunct list. Carried for reporting - /// ([`BatchStrategy::Reordered`]) rather than for evaluation. - order: Vec, - /// Whether `order` reorders the written conjuncts. + /// Whether that order reorders the written conjuncts. reordered: bool, } impl AdaptiveFilterShared { - pub(crate) fn new() -> Self { - Self::default() - } - /// The settled decision, or `None` if the streams are still learning. #[cfg(test)] fn settled(&self) -> Option { @@ -296,8 +226,7 @@ impl AdaptiveFilterShared { /// This stands in for a mocked clock: it pins the settle decision instead /// of leaving it to real timer values, which a scheduling hiccup on a /// shared CI runner can perturb by far more than the evaluation being - /// measured. Used by `FilterExec`'s end-to-end tests; the tests in this - /// module use the equivalent `tests::seed`. + /// measured. #[cfg(test)] pub(crate) fn seed_one_batch_short_of_warmup( &self, @@ -316,25 +245,12 @@ impl AdaptiveFilterShared { } } -/// A conjunct wrapped so that evaluating it records what it saw. -/// -/// This is the whole of the warm-up's instrumentation. The wrapped conjuncts -/// are assembled into the written order's right-nested `AND` chain and handed -/// to [`BinaryExpr`], which evaluates and pre-selects exactly as it would for -/// the plain predicate; each wrapper therefore records the rows, matches and -/// elapsed time of the population `BinaryExpr` actually handed its conjunct. -/// The wrapper adds nothing but the counters: it returns the conjunct's own -/// result unchanged, nulls included, because three-valued logic is -/// `BinaryExpr`'s business and not this module's. -/// -/// The counters are per stream and uncontended, so `Relaxed` ordering is -/// enough; [`take`](Self::take) drains them. -/// -/// `Display`, [`fmt_sql`](PhysicalExpr::fmt_sql), `data_type`, `nullable` and -/// `return_field` all delegate to the wrapped conjunct, so a rendered warm-up -/// predicate is indistinguishable from the plain one. Equality and hashing -/// likewise consider only the wrapped conjunct: two wrappers around the same -/// expression are the same expression, whatever their counters hold. +/// A conjunct wrapped so that evaluating it records the rows it was handed, the +/// rows it kept and the time it took; [`take`](Self::take) drains the counters. +/// It returns the conjunct's own result unchanged, nulls included — three-valued +/// logic is [`BinaryExpr`]'s business, not this module's — and delegates +/// rendering, typing, equality and hashing to it, so a wrapped predicate is +/// indistinguishable from the plain one. #[derive(Debug)] struct MeasuredConjunct { inner: Arc, @@ -357,7 +273,8 @@ impl MeasuredConjunct { } } - /// Drain the counters, returning what they held. + /// Drain the counters, returning what they held. They are per stream and + /// uncontended, so `Relaxed` ordering is enough. fn take(&self) -> ConjunctStats { ConjunctStats { rows: self.rows.swap(0, Relaxed), @@ -396,10 +313,6 @@ impl PhysicalExpr for MeasuredConjunct { self.inner.nullable(input_schema) } - fn return_field(&self, input_schema: &Schema) -> Result { - self.inner.return_field(input_schema) - } - fn evaluate(&self, batch: &RecordBatch) -> Result { let rows = batch.num_rows(); let timer = Instant::now(); @@ -435,63 +348,54 @@ impl PhysicalExpr for MeasuredConjunct { /// the per-stream state is just the current order and how far it has caught up. #[derive(Debug)] pub(crate) struct AdaptiveConjunction { - /// The split conjuncts. `order` indices refer to positions here. + /// The split conjuncts, in written order. conjuncts: Vec>, - /// The written predicate as one expression: what the measured conjuncts - /// are ranked against. - predicate: Arc, /// Measurements and the settled decision, shared by every partition stream. shared: Arc, - /// What the warm-up evaluates: the *written* order as a right-nested `AND` - /// chain over [`measured`](Self::measured), so [`BinaryExpr`] does the - /// evaluating and the pre-selection while the wrappers do the counting. - /// Unused once settled. + /// The written order as a right-nested `AND` chain over the wrappers. warmup_predicate: Arc, - /// The wrappers inside [`warmup_predicate`](Self::warmup_predicate), in - /// written order, so each batch's counts can be drained out of them. + /// The wrappers inside `warmup_predicate`, in written order. measured: Vec>, - /// The expression [`evaluate_settled`](Self::evaluate_settled) evaluates: - /// the settled order as a right-nested `AND` chain. Unused until - /// `settled`, and equal to the written predicate until then. + /// The settled order as a right-nested `AND` chain; the warm-up chain until then. settled_predicate: Arc, - /// Evaluation order: indices into `conjuncts`. The written order until a - /// settled order is adopted. - order: Vec, - /// Whether the settled decision reordered the conjuncts — equivalently, - /// whether [`settled_predicate`](Self::settled_predicate) is the rebuilt - /// chain rather than [`predicate`](Self::predicate). + /// Whether the settled decision reordered the conjuncts. reordered: bool, /// Whether the order is settled: this stream no longer measures. settled: bool, - /// Set when this stream adopts a *reordered* decision, and cleared by - /// [`take_adopted_reorder`](Self::take_adopted_reorder) — a one-shot - /// transition signal so the owner (`FilterExec`'s stream) can report that - /// the reorder happened without this type knowing about metrics. - adopted_reorder: bool, + /// Incremented once, if and when this stream adopts a *reordered* decision. + adaptive_reorders: Option, } impl AdaptiveConjunction { - /// Build an adaptive evaluator for `predicate`, or `None` if adaptive - /// reordering does not structurally apply: + /// Whether `predicate` is structurally a candidate for adaptive + /// reordering: it has at least two `AND` conjuncts (something to reorder) + /// and none of them is volatile (reordering could change side effects). /// - /// - the predicate has fewer than two `AND` conjuncts (nothing to reorder); - /// - any conjunct is volatile (reordering could change side effects). + /// Whether adaptive reordering is *enabled* is the caller's policy: the + /// config flag lives with `FilterExec`. + pub(crate) fn applies(predicate: &Arc) -> bool { + let conjuncts = split_conjunction(predicate); + conjuncts.len() >= 2 && !conjuncts.iter().any(|c| is_volatile(c)) + } + + /// Build an adaptive evaluator for `predicate`, or `None` if adaptive + /// reordering does not [apply](Self::applies) to it. /// - /// Whether adaptive reordering is *enabled* is the caller's policy (the - /// config flag lives with `FilterExec`); this constructor only answers - /// whether the predicate is a reorderable conjunction. `shared` is the - /// state common to all partition streams of the owning `FilterExec`. + /// `shared` is the state common to all partition streams of the owning + /// `FilterExec`; `adaptive_reorders`, when given, is the metric counting + /// this stream's adoption of a reordered evaluation order. pub(crate) fn try_new( predicate: &Arc, shared: Arc, + adaptive_reorders: Option, ) -> Option { + if !Self::applies(predicate) { + return None; + } let conjuncts: Vec> = split_conjunction(predicate) .into_iter() .map(Arc::clone) .collect(); - if conjuncts.len() < 2 || conjuncts.iter().any(is_volatile) { - return None; - } let order: Vec = (0..conjuncts.len()).collect(); let measured: Vec> = conjuncts .iter() @@ -501,55 +405,27 @@ impl AdaptiveConjunction { .iter() .map(|m| Arc::clone(m) as Arc) .collect(); - let warmup_predicate = - right_nested_conjunction(&wrapped, &order).expect("at least two conjuncts"); + let warmup_predicate = right_nested_conjunction(&wrapped, &order); Some(Self { conjuncts, - predicate: Arc::clone(predicate), shared, + settled_predicate: Arc::clone(&warmup_predicate), warmup_predicate, measured, - settled_predicate: Arc::clone(predicate), - order, reordered: false, settled: false, - adopted_reorder: false, + adaptive_reorders, }) } - /// Whether this stream has just adopted a reordered evaluation order, - /// clearing the signal. - /// - /// Fires exactly once per stream, on the batch at which it adopts a - /// reorder — whether it settled the order itself or took up one another - /// stream settled. A stream that settles on the written order never fires. - pub(crate) fn take_adopted_reorder(&mut self) -> bool { - std::mem::take(&mut self.adopted_reorder) - } - /// Evaluate the conjunction against `batch`, returning the boolean mask /// (over the batch's rows) of rows that passed every conjunct. /// /// Until the order settles, each batch is measured and its counts pooled /// into the shared state. pub(crate) fn evaluate(&mut self, batch: &RecordBatch) -> Result { - self.evaluate_traced(batch).map(|(mask, _)| mask) - } - - /// [`evaluate`](Self::evaluate), additionally reporting the - /// [`BatchStrategy`] used for this batch. - fn evaluate_traced( - &mut self, - batch: &RecordBatch, - ) -> Result<(ArrayRef, BatchStrategy<'_>)> { if self.settled { - let mask = self.evaluate_settled(batch)?; - let strategy = if self.reordered { - BatchStrategy::Reordered(&self.order) - } else { - BatchStrategy::Fused - }; - return Ok((mask, strategy)); + return self.evaluate_settled(batch); } // An empty batch measures nothing; evaluating it must not consume the @@ -561,7 +437,7 @@ impl AdaptiveConjunction { // cost, which would otherwise inflate the next batch's per-row // cost. self.take_measurements(); - return Ok((mask, BatchStrategy::Measure)); + return Ok(mask); } // Evaluate the written order through the wrappers, then drain and pool @@ -569,7 +445,7 @@ impl AdaptiveConjunction { let result = self.evaluate_warmup(batch)?; let local = self.take_measurements(); self.pool_and_maybe_settle(&local); - Ok((result, BatchStrategy::Measure)) + Ok(result) } /// Evaluate the warm-up arrangement: the written order as a right-nested @@ -597,12 +473,15 @@ impl AdaptiveConjunction { fn adopt(&mut self, decision: Settled) { self.settled_predicate = decision.predicate; - self.order = decision.order; self.reordered = decision.reordered; self.settled = true; // Only a genuine reorder is worth reporting; settling on the written // order is indistinguishable from the feature being off. - self.adopted_reorder = self.reordered; + if self.reordered + && let Some(count) = &self.adaptive_reorders + { + count.add(1); + } } /// Merge this batch's measurements into the shared pool and, once enough @@ -623,10 +502,7 @@ impl AdaptiveConjunction { if inner.stats.is_empty() { inner.stats = vec![ConjunctStats::default(); local.len()]; } - // One `AdaptiveFilterShared` only ever backs one predicate: the builder, - // predicate rewrites and `reset_state` each allocate a fresh instance, - // and the paths that share one (`Clone`, `with_fetch`, - // `with_batch_size`) keep the same predicate. + // One `AdaptiveFilterShared` only ever backs one predicate. debug_assert_eq!(inner.stats.len(), local.len()); for (s, l) in inner.stats.iter_mut().zip(local) { s.merge(l); @@ -635,7 +511,7 @@ impl AdaptiveConjunction { if inner.measured_batches < WARMUP_BATCHES { return; } - let decision = settle(&inner.stats, &self.conjuncts, &self.predicate); + let decision = settle(&inner.stats, &self.conjuncts); inner.settled = Some(decision.clone()); drop(inner); self.adopt(decision); @@ -647,62 +523,48 @@ impl AdaptiveConjunction { /// Rank the conjuncts by effectiveness and take the ranking only if it is /// materially cheaper than the written order; otherwise keep the written /// order. Either way the result is materialised as a right-nested `AND` chain -/// over `conjuncts` (`predicate` is only the fallback for an empty list). -fn settle( - stats: &[ConjunctStats], - conjuncts: &[Arc], - predicate: &Arc, -) -> Settled { +/// over `conjuncts`, which `stats` indexes. +fn settle(stats: &[ConjunctStats], conjuncts: &[Arc]) -> Settled { let identity: Vec = (0..stats.len()).collect(); let candidate = rank_by_effectiveness(stats); if candidate != identity && expected_cost_per_row(stats, &candidate) < (1.0 - TIE_COST_FRACTION) * expected_cost_per_row(stats, &identity) - && let Some(reordered) = right_nested_conjunction(conjuncts, &candidate) { Settled { - predicate: reordered, - order: candidate, + predicate: right_nested_conjunction(conjuncts, &candidate), reordered: true, } } else { - // The written order, but still right-nested: `predicate` as the - // planner built it is typically left-nested, and a left-nested chain - // pre-selects on the accumulated prefix, paying a whole-batch filter - // and scatter at every level where that prefix crosses the threshold. - // Right-nested, each `AND` decides on a single conjunct and survivors - // stay compacted for the rest of the chain. - let written = right_nested_conjunction(conjuncts, &identity) - .unwrap_or_else(|| Arc::clone(predicate)); Settled { - predicate: written, - order: identity, + predicate: right_nested_conjunction(conjuncts, &identity), reordered: false, } } } /// Build `conjuncts` in `order` into one right-nested `AND` chain, -/// `(c_first AND (c_second AND (... AND c_last)))`, or `None` if `order` is -/// empty. +/// `(c_first AND (c_second AND (... AND c_last)))`. /// -/// The nesting is the point. [`BinaryExpr`]'s `AND` pre-selects by filtering -/// the batch it is given before evaluating its right-hand side, so nesting to -/// the right keeps the survivors of the first conjunct compacted for every -/// conjunct after it. Nesting to the left — what -/// [`conjunction`](datafusion_physical_expr::utils::conjunction) builds — -/// would re-filter the original batch at each level instead. +/// Nesting to the right keeps the survivors of the first conjunct compacted for +/// every conjunct after it, because [`BinaryExpr`]'s `AND` pre-selects on the +/// batch it is given before evaluating its right-hand side. `order` must be +/// non-empty and index into `conjuncts`; both hold by construction, an adaptive +/// conjunction having at least two conjuncts. fn right_nested_conjunction( conjuncts: &[Arc], order: &[usize], -) -> Option> { - order.iter().rev().fold(None, |acc, &id| { - let conjunct = Arc::clone(&conjuncts[id]); - Some(match acc { - None => conjunct, - Some(acc) => Arc::new(BinaryExpr::new(conjunct, Operator::And, acc)) as _, +) -> Arc { + let (&last, rest) = order.split_last().expect("a non-empty order"); + rest.iter() + .rev() + .fold(Arc::clone(&conjuncts[last]), |acc, &id| { + Arc::new(BinaryExpr::new( + Arc::clone(&conjuncts[id]), + Operator::And, + acc, + )) as _ }) - }) } /// Rank conjunct ids by effectiveness (discards per nanosecond) descending; @@ -779,6 +641,30 @@ mod tests { .collect() } + /// Assert that `chain` is the right-nested `AND` chain over `conjuncts` in + /// `order` — `(c_first AND (c_second AND (... AND c_last)))` — with every + /// leaf pointer-equal to the conjunct it names. + fn assert_chain( + chain: &Arc, + conjuncts: &[Arc], + order: &[usize], + ) { + let (&last, rest) = order.split_last().expect("a non-empty order"); + let mut node = Arc::clone(chain); + for (depth, &id) in rest.iter().enumerate() { + let and = node + .downcast_ref::() + .unwrap_or_else(|| panic!("an AND at depth {depth}")); + assert_eq!(*and.op(), Operator::And); + assert!( + Arc::ptr_eq(and.left(), &conjuncts[id]), + "conjunct {id} at depth {depth}" + ); + node = Arc::clone(and.right()); + } + assert!(Arc::ptr_eq(&node, &conjuncts[last]), "last conjunct {last}"); + } + fn passing_rows(mask: &ArrayRef) -> Vec { let mask = as_boolean_array(mask).unwrap(); (0..mask.len()) @@ -794,19 +680,13 @@ mod tests { } } - /// `try_new` with a fresh, unshared registry. + /// `try_new` with a fresh, unshared registry and no metric. fn try_new(predicate: &Arc) -> Option { - AdaptiveConjunction::try_new(predicate, Arc::new(AdaptiveFilterShared::new())) - } - - /// Seed the shared pool as if `batches` instrumented batches had already - /// recorded `stats` — a stand-in for a mocked clock, giving scenario tests - /// deterministic control over each conjunct's measured cost and - /// selectivity. - fn seed(shared: &AdaptiveFilterShared, stats: Vec, batches: u64) { - let mut inner = shared.inner.lock().unwrap(); - inner.stats = stats; - inner.measured_batches = batches; + AdaptiveConjunction::try_new( + predicate, + Arc::new(AdaptiveFilterShared::default()), + None, + ) } #[test] @@ -822,8 +702,8 @@ mod tests { let schema = schema(); let adaptive = try_new(&predicate(&schema)).unwrap(); assert_eq!(adaptive.conjuncts.len(), 2); - assert_eq!(adaptive.order, vec![0, 1]); assert!(!adaptive.settled); + assert!(!adaptive.reordered); } #[test] @@ -1002,21 +882,19 @@ mod tests { } /// A reorder is adopted only when materially cheaper; an already-good order - /// is left untouched and keeps evaluating the written predicate itself. + /// is left untouched, rebuilt as a right-nested chain over the same + /// conjuncts. #[test] fn settle_keeps_order_when_not_materially_better() { let schema = schema(); let p = predicate(&schema); // Two equally cheap, equally selective conjuncts: swapping cannot help, - // so the written order stands, rebuilt as a right-nested chain. + // so the written order stands. let s = vec![stats(1000, 500, 1000), stats(1000, 500, 1000)]; let cs = split(&p); - let d = settle(&s, &cs, &p); - assert_eq!(d.order, vec![0, 1]); + let d = settle(&s, &cs); assert!(!d.reordered); - let chain = d.predicate.downcast_ref::().expect("an AND"); - assert!(Arc::ptr_eq(chain.left(), &cs[0])); - assert!(Arc::ptr_eq(chain.right(), &cs[1])); + assert_chain(&d.predicate, &cs, &[0, 1]); } #[test] @@ -1027,12 +905,9 @@ mod tests { // id 1 is far more selective and equally cheap: it should move first, // as the outermost left operand of the rebuilt chain. let s = vec![stats(1000, 900, 1000), stats(1000, 10, 1000)]; - let d = settle(&s, &cs, &p); - assert_eq!(d.order, vec![1, 0]); + let d = settle(&s, &cs); assert!(d.reordered); - let chain = d.predicate.downcast_ref::().expect("an AND"); - assert!(Arc::ptr_eq(chain.left(), &cs[1])); - assert!(Arc::ptr_eq(chain.right(), &cs[0])); + assert_chain(&d.predicate, &cs, &[1, 0]); } /// The adopted order is materialised as a *right*-nested `AND` chain: @@ -1067,8 +942,7 @@ mod tests { stats(1000, 500, 1000), stats(1000, 10, 1000), ]; - let d = settle(&s, &cs, &p); - assert_eq!(d.order, vec![2, 1, 0]); + let d = settle(&s, &cs); assert!(d.reordered); // `(cs[2] AND (cs[1] AND cs[0]))`. @@ -1120,40 +994,30 @@ mod tests { !adaptive.reordered, "interchangeable conjuncts keep the written order" ); - let cs = split(&p); - let chain = adaptive - .settled_predicate - .downcast_ref::() - .expect("an AND"); - assert!(Arc::ptr_eq(chain.left(), &cs[0])); - assert!(Arc::ptr_eq(chain.right(), &cs[1])); + assert_chain(&adaptive.settled_predicate, &split(&p), &[0, 1]); } /// Two streams sharing one pool settle the order together: the warm-up is /// `WARMUP_BATCHES` batches total across both streams, and once one stream /// settles the order the other adopts it on its next batch. /// - /// The pool is seeded (see [`seed`]) two batches short of the warm-up, so - /// the two real measured batches that complete it cannot move the ranking: - /// their counts are orders of magnitude smaller than the seeded ones. + /// The pool is seeded one batch short of the warm-up, so the real measured + /// batches cannot move the ranking: their counts are orders of magnitude + /// smaller than the seeded ones. #[test] fn streams_pool_measurements_and_share_settled_order() { let schema = schema(); let p = predicate(&schema); // `a > 2 AND b < 5`, written order [0, 1] - let shared = Arc::new(AdaptiveFilterShared::new()); + let cs = split(&p); + let shared = Arc::new(AdaptiveFilterShared::default()); // Conjunct 1 is far more selective, so promoting it is materially - // cheaper. Two batches short of the warm-up: the two streams below - // pool one measured batch each to complete it. - seed( - &shared, - vec![ - stats(70_000_000, 63_000_000, 70_000_000), // pass 0.9, ~1ns/row - stats(70_000_000, 700_000, 350_000_000), // pass 0.01, ~5ns/row - ], - WARMUP_BATCHES - 2, - ); - let mut s1 = AdaptiveConjunction::try_new(&p, Arc::clone(&shared)).unwrap(); - let mut s2 = AdaptiveConjunction::try_new(&p, Arc::clone(&shared)).unwrap(); + // cheaper. + shared.seed_one_batch_short_of_warmup(&[ + (70_000_000, 63_000_000, 70_000_000), // pass 0.9, ~1ns/row + (70_000_000, 700_000, 350_000_000), // pass 0.01, ~5ns/row + ]); + let mut s1 = AdaptiveConjunction::try_new(&p, Arc::clone(&shared), None).unwrap(); + let mut s2 = AdaptiveConjunction::try_new(&p, Arc::clone(&shared), None).unwrap(); // `b < 5` (conjunct 1) is the selective one; drive both streams with // batches where it keeps 5 rows in 25. @@ -1165,7 +1029,7 @@ mod tests { }; // Alternate the two streams for `WARMUP_BATCHES` pooled batches; the - // order settles partway through and both streams must end settled. + // order settles on the first of them and both streams must end settled. for round in 0..(WARMUP_BATCHES as i32) { let rb = mk(round); for s in [&mut s1, &mut s2] { @@ -1176,133 +1040,130 @@ mod tests { } assert!(shared.settled().is_some()); - // One more batch each lets a not-yet-settled stream adopt the decision. - s1.evaluate(&mk(99)).unwrap(); - s2.evaluate(&mk(99)).unwrap(); assert!(s1.settled && s2.settled); // The selective conjunct was promoted to the front for both, and the // reorder is evaluated as the rebuilt chain. - assert_eq!(s1.order, vec![1, 0]); - assert_eq!(s2.order, vec![1, 0]); assert!(s1.reordered && s2.reordered); + assert_chain(&s1.settled_predicate, &cs, &[1, 0]); + assert_chain(&s2.settled_predicate, &cs, &[1, 0]); } - /// The reorder-adoption signal (`take_adopted_reorder`, what `FilterExec` - /// counts into its `adaptive_reorders` metric) fires exactly once per - /// stream: once for the stream that settles the order, and once for a - /// stream that later takes it up. + /// The `adaptive_reorders` counter (what `FilterExec` reports) is + /// incremented exactly once per stream: once for the stream that settles + /// the order, and once for a stream that later takes it up. /// - /// The per-conjunct costs are seeded (see [`seed`]) so the decision is a - /// reorder regardless of real timer values. + /// The per-conjunct costs are seeded so the decision is a reorder + /// regardless of real timer values. #[test] fn adopted_reorder_signals_once_per_stream() { let schema = schema(); let p = predicate(&schema); // `a > 2 AND b < 5`, written order [0, 1] - let shared = Arc::new(AdaptiveFilterShared::new()); + let shared = Arc::new(AdaptiveFilterShared::default()); // Conjunct 1 is far more selective; promoting it is materially cheaper, - // so the warm-up settles on a reorder. One batch short of the warm-up. - seed( - &shared, - vec![ - stats(70_000_000, 63_000_000, 70_000_000), // pass 0.9, ~1ns/row - stats(70_000_000, 700_000, 350_000_000), // pass 0.01, ~5ns/row - ], - WARMUP_BATCHES - 1, - ); - let mut settler = AdaptiveConjunction::try_new(&p, Arc::clone(&shared)).unwrap(); - let mut adopter = AdaptiveConjunction::try_new(&p, Arc::clone(&shared)).unwrap(); + // so the warm-up settles on a reorder. + shared.seed_one_batch_short_of_warmup(&[ + (70_000_000, 63_000_000, 70_000_000), // pass 0.9, ~1ns/row + (70_000_000, 700_000, 350_000_000), // pass 0.01, ~5ns/row + ]); + let settler_count = Count::new(); + let adopter_count = Count::new(); + let mut settler = AdaptiveConjunction::try_new( + &p, + Arc::clone(&shared), + Some(settler_count.clone()), + ) + .unwrap(); + let mut adopter = AdaptiveConjunction::try_new( + &p, + Arc::clone(&shared), + Some(adopter_count.clone()), + ) + .unwrap(); let a: Vec = (0..100).collect(); let b: Vec = a.iter().map(|x| x.rem_euclid(25)).collect(); let rb = batch(&schema, a, b); // Nothing adopted yet. - assert!(!settler.take_adopted_reorder()); + assert_eq!(settler_count.value(), 0); // This batch completes the warm-up: `settler` settles on the reorder - // and signals it, exactly once. + // and counts it, exactly once. settler.evaluate(&rb).unwrap(); assert!(settler.reordered); - assert!(settler.take_adopted_reorder()); + assert_eq!(settler_count.value(), 1); settler.evaluate(&rb).unwrap(); - assert!(!settler.take_adopted_reorder()); + assert_eq!(settler_count.value(), 1); // `adopter` never measured its way to a decision: it takes up the - // settled one on its next batch, and signals that once too. + // settled one on its next batch, and counts that once too. adopter.evaluate(&rb).unwrap(); assert!(adopter.reordered); - assert!(adopter.take_adopted_reorder()); + assert_eq!(adopter_count.value(), 1); adopter.evaluate(&rb).unwrap(); - assert!(!adopter.take_adopted_reorder()); + assert_eq!(adopter_count.value(), 1); } /// Settling on the written order is indistinguishable from the feature - /// being off, so it must not signal a reorder. + /// being off, so it must not count a reorder. #[test] fn settling_without_reorder_signals_nothing() { let schema = schema(); let p = predicate(&schema); - let shared = Arc::new(AdaptiveFilterShared::new()); + let shared = Arc::new(AdaptiveFilterShared::default()); // Identical cost and selectivity: no order can be materially cheaper. - seed( - &shared, - vec![ - stats(70_000_000, 35_000_000, 70_000_000), - stats(70_000_000, 35_000_000, 70_000_000), - ], - WARMUP_BATCHES - 1, - ); - let mut adaptive = AdaptiveConjunction::try_new(&p, Arc::clone(&shared)).unwrap(); + shared.seed_one_batch_short_of_warmup(&[ + (70_000_000, 35_000_000, 70_000_000), + (70_000_000, 35_000_000, 70_000_000), + ]); + let count = Count::new(); + let mut adaptive = + AdaptiveConjunction::try_new(&p, Arc::clone(&shared), Some(count.clone())) + .unwrap(); let a: Vec = (0..100).collect(); let rb = batch(&schema, a.clone(), a); adaptive.evaluate(&rb).unwrap(); assert!(adaptive.settled && !adaptive.reordered); - assert!(!adaptive.take_adopted_reorder()); + assert_eq!(count.value(), 0); } - /// End-to-end contract scenario: feed batches, observe the strategy used - /// for each one alongside the masks. + /// End-to-end contract scenario: feed batches and watch the stream settle + /// on a reorder, then stay there. /// - /// The seeded costs (see [`seed`]) make conjunct 0 cheap but unselective - /// and conjunct 1 expensive but very selective, so the warm-up must settle - /// on promoting conjunct 1 and evaluate the rebuilt chain, regardless of - /// real timer values. + /// The seeded costs make conjunct 0 cheap but unselective and conjunct 1 + /// expensive but very selective, so the warm-up must settle on promoting + /// conjunct 1, regardless of real timer values. #[test] fn scenario_measure_batches_then_settle_on_reorder() { let schema = schema(); let p = predicate(&schema); // `a > 2 AND b < 5`, written order [0, 1] - let shared = Arc::new(AdaptiveFilterShared::new()); + let cs = split(&p); + let shared = Arc::new(AdaptiveFilterShared::default()); // One batch short of the warm-up: the next measured batch settles. - seed( - &shared, - vec![ - stats(70_000_000, 63_000_000, 70_000_000), // pass 0.9, ~1ns/row - stats(70_000_000, 700_000, 350_000_000), // pass 0.01, ~5ns/row - ], - WARMUP_BATCHES - 1, - ); - let mut adaptive = AdaptiveConjunction::try_new(&p, Arc::clone(&shared)).unwrap(); + shared.seed_one_batch_short_of_warmup(&[ + (70_000_000, 63_000_000, 70_000_000), // pass 0.9, ~1ns/row + (70_000_000, 700_000, 350_000_000), // pass 0.01, ~5ns/row + ]); + let mut adaptive = + AdaptiveConjunction::try_new(&p, Arc::clone(&shared), None).unwrap(); + assert!(!adaptive.settled, "the first batch is still measured"); - let mut trace = vec![]; for round in 0..3 { let base = round * 100; let a: Vec = (base..base + 100).collect(); let b: Vec = (base..base + 100).map(|x| x.rem_euclid(25)).collect(); let rb = batch(&schema, a, b); - let (got, strategy) = adaptive.evaluate_traced(&rb).unwrap(); - trace.push(format!("{strategy:?}")); + let got = adaptive.evaluate(&rb).unwrap(); let want = p.evaluate(&rb).unwrap().into_array(rb.num_rows()).unwrap(); assert_eq!(passing_rows(&got), passing_rows(&want), "round {round}"); + // Round 0 completes the warm-up and settles on the reorder; every + // round after it evaluates the adopted chain, which promotes the + // selective conjunct to the front. + assert!(adaptive.settled, "settled after round {round}"); + assert!(adaptive.reordered, "reordered after round {round}"); + assert_chain(&adaptive.settled_predicate, &cs, &[1, 0]); } - - // Batch 1 completes the warm-up and settles; batches 2+ evaluate the - // adopted reorder (selective conjunct promoted to the front) as a - // right-nested `AND` chain. - assert_eq!( - trace, - vec!["Measure", "Reordered([1, 0])", "Reordered([1, 0])"] - ); } /// Contract scenario for the no-win case: interchangeable conjuncts settle @@ -1311,34 +1172,31 @@ mod tests { fn scenario_measure_batches_then_settle_on_fused() { let schema = schema(); let p = predicate(&schema); - let shared = Arc::new(AdaptiveFilterShared::new()); + let cs = split(&p); + let shared = Arc::new(AdaptiveFilterShared::default()); // Identical cost and selectivity: no order can be materially cheaper. // The seeded magnitudes dominate the one real measured batch, so even // if real timings nudge the ranking, the 5% material-win guard holds. - seed( - &shared, - vec![ - stats(70_000_000, 35_000_000, 70_000_000), - stats(70_000_000, 35_000_000, 70_000_000), - ], - WARMUP_BATCHES - 1, - ); - let mut adaptive = AdaptiveConjunction::try_new(&p, Arc::clone(&shared)).unwrap(); + shared.seed_one_batch_short_of_warmup(&[ + (70_000_000, 35_000_000, 70_000_000), + (70_000_000, 35_000_000, 70_000_000), + ]); + let mut adaptive = + AdaptiveConjunction::try_new(&p, Arc::clone(&shared), None).unwrap(); + assert!(!adaptive.settled, "the first batch is still measured"); - let mut trace = vec![]; for round in 0..3 { let base = round * 100; let a: Vec = (base..base + 100).collect(); let b: Vec = (base..base + 100).collect(); let rb = batch(&schema, a, b); - let (got, strategy) = adaptive.evaluate_traced(&rb).unwrap(); - trace.push(format!("{strategy:?}")); + let got = adaptive.evaluate(&rb).unwrap(); let want = p.evaluate(&rb).unwrap().into_array(rb.num_rows()).unwrap(); assert_eq!(passing_rows(&got), passing_rows(&want), "round {round}"); + assert!(adaptive.settled, "settled after round {round}"); + assert!(!adaptive.reordered, "not reordered after round {round}"); + assert_chain(&adaptive.settled_predicate, &cs, &[0, 1]); } - - assert_eq!(trace, vec!["Measure", "Fused", "Fused"]); - assert_eq!(adaptive.order, vec![0, 1]); } /// The pooled registry is sized lazily by the first measured batch (the @@ -1349,9 +1207,10 @@ mod tests { fn first_measured_batch_initialises_the_shared_pool() { let schema = schema(); let p = predicate(&schema); // `a > 2 AND b < 5` - let shared = Arc::new(AdaptiveFilterShared::new()); + let shared = Arc::new(AdaptiveFilterShared::default()); assert!(shared.inner.lock().unwrap().stats.is_empty()); - let mut adaptive = AdaptiveConjunction::try_new(&p, Arc::clone(&shared)).unwrap(); + let mut adaptive = + AdaptiveConjunction::try_new(&p, Arc::clone(&shared), None).unwrap(); // Empty batches measure nothing, so the pool is still unsized after one. adaptive.evaluate(&batch(&schema, vec![], vec![])).unwrap(); assert!(shared.inner.lock().unwrap().stats.is_empty()); @@ -1426,6 +1285,7 @@ mod tests { let (non_zero, divide) = divide_by_zero_conjuncts(&schema); // Written order [0, 1] = [`b <> 0`, `1 / b > 2`]. let p = binary(non_zero, Operator::And, divide, &schema).unwrap(); + let cs = split(&p); let a: Vec = (0..100).collect(); let b: Vec = (0..100).map(|i| i64::from(i < 15)).collect(); let rb = int64_batch(&schema, a, b); @@ -1434,24 +1294,21 @@ mod tests { // rows, within its 20% threshold) and succeeds. assert!(p.evaluate(&rb).is_ok(), "flag-off evaluation must succeed"); - let shared = Arc::new(AdaptiveFilterShared::new()); + let shared = Arc::new(AdaptiveFilterShared::default()); // `1 / b > 2` (conjunct 1) seeded as cheap and very selective, so the - // warm-up settles on promoting it. One batch short of the warm-up. - seed( - &shared, - vec![ - stats(70_000_000, 63_000_000, 70_000_000), // pass 0.9, ~1ns/row - stats(70_000_000, 700_000, 70_000_000), // pass 0.01, ~1ns/row - ], - WARMUP_BATCHES - 1, - ); - let mut adaptive = AdaptiveConjunction::try_new(&p, Arc::clone(&shared)).unwrap(); + // warm-up settles on promoting it. + shared.seed_one_batch_short_of_warmup(&[ + (70_000_000, 63_000_000, 70_000_000), // pass 0.9, ~1ns/row + (70_000_000, 700_000, 70_000_000), // pass 0.01, ~1ns/row + ]); + let mut adaptive = + AdaptiveConjunction::try_new(&p, Arc::clone(&shared), None).unwrap(); // The settling batch is still measured in the written order, whose // pre-selection on `b <> 0` also keeps `1 / b` away from the zeros. adaptive.evaluate(&rb).unwrap(); - assert_eq!(adaptive.order, vec![1, 0]); assert!(adaptive.reordered); + assert_chain(&adaptive.settled_predicate, &cs, &[1, 0]); // The next batch runs the adopted reorder, and errors. let err = adaptive.evaluate(&rb).unwrap_err().to_string(); @@ -1480,19 +1337,17 @@ mod tests { .unwrap(); // Written order [0, 1] = [`1 / b > 2`, `a < 10`]. let p = binary(divide, Operator::And, selective, &schema).unwrap(); + let cs = split(&p); - let shared = Arc::new(AdaptiveFilterShared::new()); + let shared = Arc::new(AdaptiveFilterShared::default()); // Conjunct 1 (`a < 10`) seeded as cheap and very selective, conjunct 0 // as expensive and unselective, so the warm-up promotes conjunct 1. - seed( - &shared, - vec![ - stats(70_000_000, 63_000_000, 350_000_000), // pass 0.9, ~5ns/row - stats(70_000_000, 700_000, 70_000_000), // pass 0.01, ~1ns/row - ], - WARMUP_BATCHES - 1, - ); - let mut adaptive = AdaptiveConjunction::try_new(&p, Arc::clone(&shared)).unwrap(); + shared.seed_one_batch_short_of_warmup(&[ + (70_000_000, 63_000_000, 350_000_000), // pass 0.9, ~5ns/row + (70_000_000, 700_000, 70_000_000), // pass 0.01, ~1ns/row + ]); + let mut adaptive = + AdaptiveConjunction::try_new(&p, Arc::clone(&shared), None).unwrap(); // Settle on a batch with no zeros at all, so the warm-up itself (which // evaluates in the written order) cannot hit the error. @@ -1500,7 +1355,8 @@ mod tests { adaptive .evaluate(&int64_batch(&schema, a.clone(), vec![1; 100])) .unwrap(); - assert_eq!(adaptive.order, vec![1, 0]); + assert!(adaptive.reordered); + assert_chain(&adaptive.settled_predicate, &cs, &[1, 0]); // Now a batch whose `b` is zero on every row `a < 10` discards. let b: Vec = (0..100).map(|i| i64::from(i < 10)).collect(); diff --git a/datafusion/physical-plan/src/filter.rs b/datafusion/physical-plan/src/filter.rs index cd8b18a7a8573..5db08351ce48e 100644 --- a/datafusion/physical-plan/src/filter.rs +++ b/datafusion/physical-plan/src/filter.rs @@ -230,7 +230,7 @@ impl FilterExecBuilder { projection: self.projection, batch_size: self.batch_size, fetch: self.fetch, - adaptive_stats: Arc::new(AdaptiveFilterShared::new()), + adaptive_stats: Arc::new(AdaptiveFilterShared::default()), }) } } @@ -647,18 +647,13 @@ impl ExecutionPlan for FilterExec { } /// Reset per-execution state so an independent re-execution (e.g. a - /// recursive query) does not inherit runtime state from a prior run. - /// - /// The per-execution state is the pooled adaptive-conjunct measurements - /// (`AdaptiveFilterShared`) and the execution metrics; both are replaced - /// with fresh instances. The predicate, input, and cached plan properties - /// remain valid, so they are preserved (unlike - /// [`with_new_children`](Self::with_new_children), this does not recompute - /// them). Any dynamic filters *inside* the predicate are owned and reset by - /// the operator that created them, not by `FilterExec`. + /// recursive query) does not inherit runtime state from a prior run: the + /// pooled adaptive-conjunct measurements and the execution metrics are + /// replaced with fresh instances, while the predicate, input and cached + /// plan properties remain valid and are preserved. fn reset_state(self: Arc) -> Result> { let mut new = (*self).clone(); - new.adaptive_stats = Arc::new(AdaptiveFilterShared::new()); + new.adaptive_stats = Arc::new(AdaptiveFilterShared::default()); new.metrics = ExecutionPlanMetricsSet::new(); Ok(Arc::new(new)) } @@ -679,20 +674,22 @@ impl ExecutionPlan for FilterExec { .options() .execution .adaptive_filter_reordering; - let adaptive = enabled + // The metric must exist before the evaluator that increments it, and + // must be registered exactly when the adaptive path is active. + let adaptive_applies = enabled && AdaptiveConjunction::applies(&self.predicate); + let mut metrics = FilterExecMetrics::new(&self.metrics, partition); + if adaptive_applies { + metrics = metrics.with_adaptive_reorder_metrics(&self.metrics, partition); + } + let adaptive = adaptive_applies .then(|| { AdaptiveConjunction::try_new( &self.predicate, Arc::clone(&self.adaptive_stats), + metrics.adaptive_reorders.clone(), ) }) .flatten(); - let metrics = FilterExecMetrics::new(&self.metrics, partition); - let metrics = if adaptive.is_some() { - metrics.with_adaptive_reorder_metrics(&self.metrics, partition) - } else { - metrics - }; Ok(Box::pin(FilterExecStream { schema: self.schema(), predicate: Arc::clone(&self.predicate), @@ -905,7 +902,7 @@ impl ExecutionPlan for FilterExec { fetch: self.fetch, // The predicate changed; pooled per-conjunct stats no longer // describe it. - adaptive_stats: Arc::new(AdaptiveFilterShared::new()), + adaptive_stats: Arc::new(AdaptiveFilterShared::default()), }; Some(Arc::new(new) as _) }; @@ -1466,13 +1463,6 @@ impl FilterExecMetrics { ); self } - - /// Record that this stream adopted a reordered evaluation order. - fn record_adaptive_reorder(&self) { - if let Some(count) = &self.adaptive_reorders { - count.add(1); - } - } } pub fn batch_filter( @@ -1539,22 +1529,14 @@ impl Stream for FilterExecStream { } Some(Ok(batch)) => { let timer = elapsed_compute.timer(); - let (array, adopted_reorder) = match self.adaptive.as_mut() { - Some(adaptive) => { - let array = adaptive.evaluate(&batch); - (array, adaptive.take_adopted_reorder()) - } - None => ( - self.predicate - .as_ref() - .evaluate(&batch) - .and_then(|v| v.into_array(batch.num_rows())), - false, - ), + let array = match self.adaptive.as_mut() { + Some(adaptive) => adaptive.evaluate(&batch), + None => self + .predicate + .as_ref() + .evaluate(&batch) + .and_then(|v| v.into_array(batch.num_rows())), }; - if adopted_reorder { - self.metrics.record_adaptive_reorder(); - } let status = array .and_then(|array| { Ok(match self.projection.as_ref() { From 784e0851204416b97cab8ae8d5d629bf86b009fe Mon Sep 17 00:00:00 2001 From: Adrian Garcia Badaracco <1755071+adriangb@users.noreply.github.com> Date: Mon, 7 Sep 2026 11:07:02 -0500 Subject: [PATCH 20/27] docs(physical-plan): drop stale order wording and fused test name Co-Authored-By: Claude Fable 5.1 --- datafusion/physical-plan/src/adaptive_filter.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/datafusion/physical-plan/src/adaptive_filter.rs b/datafusion/physical-plan/src/adaptive_filter.rs index 58dceefc8433a..b0e83ad813dfc 100644 --- a/datafusion/physical-plan/src/adaptive_filter.rs +++ b/datafusion/physical-plan/src/adaptive_filter.rs @@ -345,7 +345,7 @@ impl PhysicalExpr for MeasuredConjunct { /// Adaptive evaluator for a single conjunctive predicate, owned per partition /// stream. Measurements are pooled into the shared [`AdaptiveFilterShared`]; -/// the per-stream state is just the current order and how far it has caught up. +/// the per-stream state is just the chain this stream currently evaluates. #[derive(Debug)] pub(crate) struct AdaptiveConjunction { /// The split conjuncts, in written order. @@ -1169,7 +1169,7 @@ mod tests { /// Contract scenario for the no-win case: interchangeable conjuncts settle /// on the written order, never a reorder. #[test] - fn scenario_measure_batches_then_settle_on_fused() { + fn scenario_measure_batches_then_settle_on_written_order() { let schema = schema(); let p = predicate(&schema); let cs = split(&p); From 1e4c62c1dc4e45d86a044edf585f238b41c7071b Mon Sep 17 00:00:00 2001 From: Adrian Garcia Badaracco <1755071+adriangb@users.noreply.github.com> Date: Mon, 7 Sep 2026 11:17:21 -0500 Subject: [PATCH 21/27] docs(physical-plan): cut comment bloat in adaptive filter Co-Authored-By: Claude Fable 5.1 --- .../physical-plan/src/adaptive_filter.rs | 335 +++++------------- datafusion/physical-plan/src/filter.rs | 80 ++--- .../test_files/adaptive_filter.slt | 23 +- docs/source/user-guide/metrics.md | 8 +- 4 files changed, 122 insertions(+), 324 deletions(-) diff --git a/datafusion/physical-plan/src/adaptive_filter.rs b/datafusion/physical-plan/src/adaptive_filter.rs index b0e83ad813dfc..359c65b8a764e 100644 --- a/datafusion/physical-plan/src/adaptive_filter.rs +++ b/datafusion/physical-plan/src/adaptive_filter.rs @@ -90,21 +90,17 @@ //! //! ## Known limitations //! -//! - Reordering never changes query *results* — the value of a conjunction does -//! not depend on evaluation order — but it can change the observable *side -//! effects* of fallible predicates, in either direction: a conjunct evaluated -//! after a pre-selection sees only the rows that survived, so an error the -//! written order raises can disappear and one it avoided can appear. -//! Predicates containing volatile expressions are never reordered. -//! - The measurements are *conditional*: each conjunct is measured on the rows -//! that survived the conjuncts before it, and after a pre-selection on small -//! survivor batches whose per-row cost is inflated by fixed overheads. -//! Correlated conjuncts can therefore look more selective in a late position -//! than they would be up front; the material-win guard makes adoption -//! conservative but cannot detect correlation. -//! - The decision is one-shot: once settled, the order is never re-measured, so -//! a misjudged reorder — or data whose selectivity drifts — is kept for the -//! rest of the query. +//! - Results never change (a conjunction's value does not depend on evaluation +//! order), but the side effects of fallible predicates can, in either +//! direction: a conjunct evaluated after a pre-selection sees only the rows +//! that survived, so an error the written order raises can disappear and one +//! it avoided can appear. Volatile predicates are never reordered. +//! - Measurements are conditional on the written order and, after a +//! pre-selection, taken on small batches whose per-row cost is inflated by +//! fixed overheads. Correlated conjuncts can be misjudged; the material-win +//! guard only makes adoption conservative. +//! - The decision is one-shot: a misjudged reorder, or drifting data, is kept +//! for the rest of the query. //! //! See . @@ -130,15 +126,11 @@ use datafusion_physical_expr_common::physical_expr::is_volatile; /// Batches measured before the order is settled. const WARMUP_BATCHES: u64 = 8; -/// Fraction of the conjunction's expected per-row cost below which a reorder is -/// immaterial. A candidate order is adopted only if it is expected to cost less -/// than `(1 - TIE_COST_FRACTION)` of the written order, so interchangeable -/// conjuncts never trigger a reorder. +/// A candidate order is adopted only if its expected cost is below +/// `(1 - TIE_COST_FRACTION)` of the written order's. const TIE_COST_FRACTION: f64 = 0.05; -/// Per-conjunct measurement: marginal pass rate and per-row evaluation cost, -/// accumulated over the warm-up window on exactly the rows that reached the -/// conjunct. +/// Per-conjunct counts over the warm-up, on exactly the rows that reached it. #[derive(Debug, Default, Clone)] struct ConjunctStats { /// Total rows the conjunct was evaluated on. @@ -150,8 +142,7 @@ struct ConjunctStats { } impl ConjunctStats { - /// Fold another accumulator's counts into this one (they are plain sums, so - /// merging is addition). Used to pool measurements across partition streams. + /// Pool another stream's counts into this one. fn merge(&mut self, other: &Self) { self.rows += other.rows; self.matched += other.matched; @@ -163,10 +154,8 @@ impl ConjunctStats { (self.rows > 0).then(|| self.matched as f64 / self.rows as f64) } - /// Per-row evaluation cost in nanoseconds, or `None` if the conjunct was - /// never evaluated on any row. An evaluation faster than the timer's - /// resolution is clamped to one nanosecond total: "too cheap to measure" - /// must rank as very cheap, not drop out of the ranking as unmeasured. + /// Per-row cost in nanoseconds, or `None` if never evaluated. Time is + /// clamped to 1ns so "too cheap to measure" ranks as very cheap. fn cost_per_row(&self) -> Option { (self.rows > 0).then(|| self.nanos.max(1) as f64 / self.rows as f64) } @@ -182,10 +171,8 @@ impl ConjunctStats { } } -/// State shared by every partition stream of one `FilterExec`, so the streams -/// learn as one: per-conjunct measurements are pooled across streams and the -/// first stream to accumulate enough samples settles the order for all of them -/// (see the [module docs](self)). +/// Measurements pooled across the partition streams of one `FilterExec`, and +/// the decision the first stream to fill the warm-up makes for all of them. #[derive(Debug, Default)] pub(crate) struct AdaptiveFilterShared { inner: Mutex, @@ -193,8 +180,7 @@ pub(crate) struct AdaptiveFilterShared { #[derive(Debug, Default)] struct SharedInner { - /// Per-conjunct counts pooled across all streams (indexed by conjunct - /// position). Empty until the first measured batch sizes it. + /// Pooled per-conjunct counts, sized by the first measured batch. stats: Vec, /// Measured batches contributed by all streams so far. measured_batches: u64, @@ -205,8 +191,7 @@ struct SharedInner { /// The settled outcome of the warm-up. #[derive(Debug, Clone)] struct Settled { - /// The predicate to evaluate from now on: the settled order as a - /// right-nested `AND` chain (see [`settle`]). + /// The settled order as a right-nested `AND` chain. predicate: Arc, /// Whether that order reorders the written conjuncts. reordered: bool, @@ -219,14 +204,9 @@ impl AdaptiveFilterShared { self.inner.lock().expect("poisoned").settled.clone() } - /// Test-only: seed the pooled measurements with `(rows, matched, nanos)` - /// per conjunct and leave the pool exactly one batch short of the warm-up, - /// so the next measured batch settles on the seeded decision. - /// - /// This stands in for a mocked clock: it pins the settle decision instead - /// of leaving it to real timer values, which a scheduling hiccup on a - /// shared CI runner can perturb by far more than the evaluation being - /// measured. + /// Seed `(rows, matched, nanos)` per conjunct one batch short of the + /// warm-up, so the next measured batch settles on the seeded decision + /// regardless of real timings. #[cfg(test)] pub(crate) fn seed_one_batch_short_of_warmup( &self, @@ -245,19 +225,15 @@ impl AdaptiveFilterShared { } } -/// A conjunct wrapped so that evaluating it records the rows it was handed, the -/// rows it kept and the time it took; [`take`](Self::take) drains the counters. -/// It returns the conjunct's own result unchanged, nulls included — three-valued -/// logic is [`BinaryExpr`]'s business, not this module's — and delegates -/// rendering, typing, equality and hashing to it, so a wrapped predicate is -/// indistinguishable from the plain one. +/// A conjunct that records the rows it was handed, the rows it kept and the +/// time it took, returning its result unchanged (nulls included). Everything +/// else delegates to the wrapped conjunct. #[derive(Debug)] struct MeasuredConjunct { inner: Arc, /// Rows handed to the conjunct since the last [`take`](Self::take). rows: AtomicU64, - /// Of those, the rows it kept: non-null `true`, matching SQL filter - /// semantics. + /// Of those, the non-null `true`s. matched: AtomicU64, /// Time spent inside the conjunct over those rows, in nanoseconds. nanos: AtomicU64, @@ -273,8 +249,7 @@ impl MeasuredConjunct { } } - /// Drain the counters, returning what they held. They are per stream and - /// uncontended, so `Relaxed` ordering is enough. + /// Drain the counters (per stream and uncontended, hence `Relaxed`). fn take(&self) -> ConjunctStats { ConjunctStats { rows: self.rows.swap(0, Relaxed), @@ -367,23 +342,15 @@ pub(crate) struct AdaptiveConjunction { } impl AdaptiveConjunction { - /// Whether `predicate` is structurally a candidate for adaptive - /// reordering: it has at least two `AND` conjuncts (something to reorder) - /// and none of them is volatile (reordering could change side effects). - /// - /// Whether adaptive reordering is *enabled* is the caller's policy: the - /// config flag lives with `FilterExec`. + /// Whether `predicate` has at least two `AND` conjuncts, none volatile. + /// (Whether the feature is enabled is the caller's business.) pub(crate) fn applies(predicate: &Arc) -> bool { let conjuncts = split_conjunction(predicate); conjuncts.len() >= 2 && !conjuncts.iter().any(|c| is_volatile(c)) } - /// Build an adaptive evaluator for `predicate`, or `None` if adaptive - /// reordering does not [apply](Self::applies) to it. - /// - /// `shared` is the state common to all partition streams of the owning - /// `FilterExec`; `adaptive_reorders`, when given, is the metric counting - /// this stream's adoption of a reordered evaluation order. + /// `None` if adaptive reordering does not [apply](Self::applies). + /// `adaptive_reorders` is bumped if this stream adopts a reorder. pub(crate) fn try_new( predicate: &Arc, shared: Arc, @@ -418,53 +385,39 @@ impl AdaptiveConjunction { }) } - /// Evaluate the conjunction against `batch`, returning the boolean mask - /// (over the batch's rows) of rows that passed every conjunct. - /// - /// Until the order settles, each batch is measured and its counts pooled - /// into the shared state. + /// The boolean mask of rows passing every conjunct. Until the order + /// settles, each batch is measured and its counts pooled. pub(crate) fn evaluate(&mut self, batch: &RecordBatch) -> Result { if self.settled { return self.evaluate_settled(batch); } - // An empty batch measures nothing; evaluating it must not consume the - // warm-up (a run of empty batches would otherwise settle the written - // order on no evidence, permanently). + // Empty batches measure nothing and must not consume the warm-up. if batch.num_rows() == 0 { let mask = self.evaluate_warmup(batch)?; - // Discard what the wrappers recorded: no rows, but a real call - // cost, which would otherwise inflate the next batch's per-row - // cost. self.take_measurements(); return Ok(mask); } - // Evaluate the written order through the wrappers, then drain and pool - // what they recorded. let result = self.evaluate_warmup(batch)?; let local = self.take_measurements(); self.pool_and_maybe_settle(&local); Ok(result) } - /// Evaluate the warm-up arrangement: the written order as a right-nested - /// `AND` chain over the measuring wrappers, leaving this batch's counts in - /// them. + /// Evaluate the written order through the wrappers. fn evaluate_warmup(&self, batch: &RecordBatch) -> Result { self.warmup_predicate .evaluate(batch)? .into_array(batch.num_rows()) } - /// Drain the wrappers into per-conjunct counts, indexed by written - /// position. + /// Drain the wrappers, indexed by written position. fn take_measurements(&self) -> Vec { self.measured.iter().map(|m| m.take()).collect() } - /// Evaluate the settled arrangement with no instrumentation: the - /// right-nested `AND` chain built from the settled order. + /// Evaluate the settled chain, uninstrumented. fn evaluate_settled(&self, batch: &RecordBatch) -> Result { self.settled_predicate .evaluate(batch)? @@ -475,8 +428,6 @@ impl AdaptiveConjunction { self.settled_predicate = decision.predicate; self.reordered = decision.reordered; self.settled = true; - // Only a genuine reorder is worth reporting; settling on the written - // order is indistinguishable from the feature being off. if self.reordered && let Some(count) = &self.adaptive_reorders { @@ -484,16 +435,12 @@ impl AdaptiveConjunction { } } - /// Merge this batch's measurements into the shared pool and, once enough - /// batches have accrued across all streams, settle the order for all of - /// them. + /// Pool this batch's counts and settle once the warm-up is full. fn pool_and_maybe_settle(&mut self, local: &[ConjunctStats]) { let mut inner = self.shared.inner.lock().expect("poisoned"); - // Another stream settled since this batch started: its decision stands - // and this batch's counts are discarded (they can no longer change - // anything). Checking here, after evaluating, rather than before keeps - // the shared lock off the path entirely until a stream has something to - // pool; the price is at most one measured batch per stream. + // Another stream settled meanwhile: take its decision and drop this + // batch's counts. Checking here rather than before evaluating keeps the + // lock off the path until there is something to pool. if let Some(decision) = inner.settled.clone() { drop(inner); self.adopt(decision); @@ -518,12 +465,9 @@ impl AdaptiveConjunction { } } -/// Decide the settled arrangement from the pooled measurements. -/// -/// Rank the conjuncts by effectiveness and take the ranking only if it is -/// materially cheaper than the written order; otherwise keep the written -/// order. Either way the result is materialised as a right-nested `AND` chain -/// over `conjuncts`, which `stats` indexes. +/// Rank by effectiveness and adopt the ranking only if it is materially +/// cheaper than the written order; either way, build the result as a +/// right-nested `AND` chain. fn settle(stats: &[ConjunctStats], conjuncts: &[Arc]) -> Settled { let identity: Vec = (0..stats.len()).collect(); let candidate = rank_by_effectiveness(stats); @@ -543,14 +487,9 @@ fn settle(stats: &[ConjunctStats], conjuncts: &[Arc]) -> Settl } } -/// Build `conjuncts` in `order` into one right-nested `AND` chain, -/// `(c_first AND (c_second AND (... AND c_last)))`. -/// -/// Nesting to the right keeps the survivors of the first conjunct compacted for -/// every conjunct after it, because [`BinaryExpr`]'s `AND` pre-selects on the -/// batch it is given before evaluating its right-hand side. `order` must be -/// non-empty and index into `conjuncts`; both hold by construction, an adaptive -/// conjunction having at least two conjuncts. +/// `conjuncts` in `order` as `(c_first AND (c_second AND (... AND c_last)))`. +/// Right-nesting lets [`BinaryExpr`]'s pre-selection keep the first conjunct's +/// survivors compacted for the rest of the chain. `order` must be non-empty. fn right_nested_conjunction( conjuncts: &[Arc], order: &[usize], @@ -582,10 +521,9 @@ fn rank_by_effectiveness(stats: &[ConjunctStats]) -> Vec { ids } -/// Expected cost of evaluating the conjuncts in `order`, in nanoseconds per -/// input row: each conjunct's measured per-row cost weighted by the fraction of -/// rows expected to reach it (the product of the pass rates of the conjuncts -/// before it, treated as independent). Unmeasured conjuncts contribute nothing. +/// Expected nanoseconds per input row for `order`: each conjunct's per-row +/// cost weighted by the product of the pass rates before it (assumed +/// independent). Unmeasured conjuncts contribute nothing. fn expected_cost_per_row(stats: &[ConjunctStats], order: &[usize]) -> f64 { let mut weight = 1.0_f64; let mut total = 0.0_f64; @@ -632,8 +570,7 @@ mod tests { binary(left, Operator::And, right, schema).unwrap() } - /// The conjuncts of `predicate`, exactly as `AdaptiveConjunction` splits - /// them (so the `Arc`s are pointer-equal to the ones inside `predicate`). + /// The conjuncts of `predicate`, pointer-equal to the ones inside it. fn split(predicate: &Arc) -> Vec> { split_conjunction(predicate) .into_iter() @@ -641,9 +578,7 @@ mod tests { .collect() } - /// Assert that `chain` is the right-nested `AND` chain over `conjuncts` in - /// `order` — `(c_first AND (c_second AND (... AND c_last)))` — with every - /// leaf pointer-equal to the conjunct it names. + /// Assert `chain` is the right-nested `AND` of `conjuncts` in `order`. fn assert_chain( chain: &Arc, conjuncts: &[Arc], @@ -741,13 +676,9 @@ mod tests { assert_eq!(rank_by_effectiveness(&s), vec![0, 1]); } - /// The measuring wrapper counts the rows it was handed and the rows its - /// conjunct kept, returns the conjunct's own array untouched (nulls and - /// all), and drains to zero. - /// - /// The elapsed time is deliberately not asserted: on a coarse timer a - /// five-row evaluation can legitimately measure zero nanoseconds, which - /// [`ConjunctStats::cost_per_row`] already handles by clamping. + /// The wrapper counts rows in and non-null trues, returns the array + /// untouched, and drains to zero. Elapsed time is not asserted: five rows + /// can legitimately measure 0ns on a coarse timer. #[test] fn measured_conjunct_counts_rows_and_matches() { let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, true)])); @@ -783,18 +714,13 @@ mod tests { "nulls are left for `BinaryExpr` to interpret" ); - // Every row was handed to the conjunct; only the non-null trues count - // as matches. let s = measured.take(); assert_eq!((s.rows, s.matched), (5, 2)); - // ...and `take` drains. let s = measured.take(); assert_eq!((s.rows, s.matched, s.nanos), (0, 0, 0)); } - /// Empty batches measure nothing, so they must not consume the warm-up: - /// a stream fed only empty batches keeps learning instead of settling the - /// written order on no evidence. + /// Empty batches must not consume the warm-up. #[test] fn empty_batches_do_not_consume_warmup() { let schema = schema(); @@ -819,8 +745,7 @@ mod tests { assert!((expected_cost_per_row(&s, &[1, 0]) - 10.5).abs() < 1e-9); } - /// Across the warm-up boundary the mask must always equal the written - /// predicate's, before and after the order settles. + /// The mask equals the written predicate's before and after settling. #[test] fn evaluate_matches_predicate_across_warmup() { let schema = schema(); @@ -844,9 +769,7 @@ mod tests { assert!(adaptive.settled); } - /// A conjunct that produces nulls must come through the warm-up unchanged: - /// the wrapper hands `BinaryExpr` the conjunct's own three-valued result, - /// so the mask matches the plain predicate's on every batch. + /// Null-producing conjuncts match the written predicate on every batch. #[test] fn nullable_conjuncts_match_the_plain_predicate_across_warmup() { let schema = Arc::new(Schema::new(vec![ @@ -881,15 +804,12 @@ mod tests { assert!(adaptive.settled); } - /// A reorder is adopted only when materially cheaper; an already-good order - /// is left untouched, rebuilt as a right-nested chain over the same - /// conjuncts. + /// An already-good order is kept, as a right-nested chain. #[test] fn settle_keeps_order_when_not_materially_better() { let schema = schema(); let p = predicate(&schema); - // Two equally cheap, equally selective conjuncts: swapping cannot help, - // so the written order stands. + // Equally cheap and selective: swapping cannot help. let s = vec![stats(1000, 500, 1000), stats(1000, 500, 1000)]; let cs = split(&p); let d = settle(&s, &cs); @@ -902,19 +822,14 @@ mod tests { let schema = schema(); let p = predicate(&schema); let cs = split(&p); - // id 1 is far more selective and equally cheap: it should move first, - // as the outermost left operand of the rebuilt chain. + // id 1 is far more selective at equal cost: it moves first. let s = vec![stats(1000, 900, 1000), stats(1000, 10, 1000)]; let d = settle(&s, &cs); assert!(d.reordered); assert_chain(&d.predicate, &cs, &[1, 0]); } - /// The adopted order is materialised as a *right*-nested `AND` chain: - /// `(c_first AND (c_second AND c_last))`. That is what lets `BinaryExpr`'s - /// pre-selection keep the survivors of the first conjunct compacted for the - /// whole remainder of the chain; a left-nested chain would re-filter the - /// original batch at every level. + /// The adopted order is a right-nested chain. #[test] fn adopted_order_is_a_right_nested_and_chain() { let schema = schema(); @@ -935,8 +850,7 @@ mod tests { let cs = split(&p); assert_eq!(cs.len(), 3); - // Equal cost, decreasing pass rate: the written order is exactly - // reversed, and reversing it is materially cheaper. + // Equal cost, decreasing pass rate: the ranking reverses the order. let s = vec![ stats(1000, 900, 1000), stats(1000, 500, 1000), @@ -961,18 +875,12 @@ mod tests { assert!(Arc::ptr_eq(inner.right(), &cs[0])); } - /// When the order does not change, the settled evaluator runs the written - /// order as a right-nested chain over the same conjuncts. - /// - /// This test measures real timings on purpose and is still deterministic: - /// both conjuncts pass the same ~96% of rows, and with a pass rate `p` - /// above `1 - TIE_COST_FRACTION` the material-win guard - /// (`c1 + p*c0 < 0.95 * (c0 + p*c1)`) cannot hold for any positive costs, - /// so no timing can produce a reorder here. + /// A kept written order runs as a right-nested chain. Real timings are + /// used on purpose: with both pass rates above `1 - TIE_COST_FRACTION` the + /// guard `c1 + p*c0 < 0.95 * (c0 + p*c1)` cannot hold for any costs. #[test] fn no_reorder_evaluates_plain_predicate() { let schema = schema(); - // Both conjuncts equally cheap and selective: nothing to reorder. let left = binary(col("a", &schema).unwrap(), Operator::Gt, lit(2i32), &schema).unwrap(); let right = @@ -997,21 +905,14 @@ mod tests { assert_chain(&adaptive.settled_predicate, &split(&p), &[0, 1]); } - /// Two streams sharing one pool settle the order together: the warm-up is - /// `WARMUP_BATCHES` batches total across both streams, and once one stream - /// settles the order the other adopts it on its next batch. - /// - /// The pool is seeded one batch short of the warm-up, so the real measured - /// batches cannot move the ranking: their counts are orders of magnitude - /// smaller than the seeded ones. + /// Two streams share one pool: the warm-up is pooled across both, and the + /// second adopts the first's decision on its next batch. #[test] fn streams_pool_measurements_and_share_settled_order() { let schema = schema(); let p = predicate(&schema); // `a > 2 AND b < 5`, written order [0, 1] let cs = split(&p); let shared = Arc::new(AdaptiveFilterShared::default()); - // Conjunct 1 is far more selective, so promoting it is materially - // cheaper. shared.seed_one_batch_short_of_warmup(&[ (70_000_000, 63_000_000, 70_000_000), // pass 0.9, ~1ns/row (70_000_000, 700_000, 350_000_000), // pass 0.01, ~5ns/row @@ -1019,8 +920,6 @@ mod tests { let mut s1 = AdaptiveConjunction::try_new(&p, Arc::clone(&shared), None).unwrap(); let mut s2 = AdaptiveConjunction::try_new(&p, Arc::clone(&shared), None).unwrap(); - // `b < 5` (conjunct 1) is the selective one; drive both streams with - // batches where it keeps 5 rows in 25. let mk = |round: i32| { let base = round * 100; let a: Vec = (base..base + 100).collect(); @@ -1028,8 +927,6 @@ mod tests { batch(&schema, a, b) }; - // Alternate the two streams for `WARMUP_BATCHES` pooled batches; the - // order settles on the first of them and both streams must end settled. for round in 0..(WARMUP_BATCHES as i32) { let rb = mk(round); for s in [&mut s1, &mut s2] { @@ -1041,26 +938,18 @@ mod tests { assert!(shared.settled().is_some()); assert!(s1.settled && s2.settled); - // The selective conjunct was promoted to the front for both, and the - // reorder is evaluated as the rebuilt chain. assert!(s1.reordered && s2.reordered); assert_chain(&s1.settled_predicate, &cs, &[1, 0]); assert_chain(&s2.settled_predicate, &cs, &[1, 0]); } - /// The `adaptive_reorders` counter (what `FilterExec` reports) is - /// incremented exactly once per stream: once for the stream that settles - /// the order, and once for a stream that later takes it up. - /// - /// The per-conjunct costs are seeded so the decision is a reorder - /// regardless of real timer values. + /// `adaptive_reorders` is bumped once per stream: by the settler and by a + /// stream that later takes up its decision. #[test] fn adopted_reorder_signals_once_per_stream() { let schema = schema(); let p = predicate(&schema); // `a > 2 AND b < 5`, written order [0, 1] let shared = Arc::new(AdaptiveFilterShared::default()); - // Conjunct 1 is far more selective; promoting it is materially cheaper, - // so the warm-up settles on a reorder. shared.seed_one_batch_short_of_warmup(&[ (70_000_000, 63_000_000, 70_000_000), // pass 0.9, ~1ns/row (70_000_000, 700_000, 350_000_000), // pass 0.01, ~5ns/row @@ -1084,19 +973,15 @@ mod tests { let b: Vec = a.iter().map(|x| x.rem_euclid(25)).collect(); let rb = batch(&schema, a, b); - // Nothing adopted yet. assert_eq!(settler_count.value(), 0); - // This batch completes the warm-up: `settler` settles on the reorder - // and counts it, exactly once. + // This batch completes the warm-up. settler.evaluate(&rb).unwrap(); assert!(settler.reordered); assert_eq!(settler_count.value(), 1); settler.evaluate(&rb).unwrap(); assert_eq!(settler_count.value(), 1); - // `adopter` never measured its way to a decision: it takes up the - // settled one on its next batch, and counts that once too. adopter.evaluate(&rb).unwrap(); assert!(adopter.reordered); assert_eq!(adopter_count.value(), 1); @@ -1104,8 +989,7 @@ mod tests { assert_eq!(adopter_count.value(), 1); } - /// Settling on the written order is indistinguishable from the feature - /// being off, so it must not count a reorder. + /// Keeping the written order is not a reorder. #[test] fn settling_without_reorder_signals_nothing() { let schema = schema(); @@ -1128,19 +1012,13 @@ mod tests { assert_eq!(count.value(), 0); } - /// End-to-end contract scenario: feed batches and watch the stream settle - /// on a reorder, then stay there. - /// - /// The seeded costs make conjunct 0 cheap but unselective and conjunct 1 - /// expensive but very selective, so the warm-up must settle on promoting - /// conjunct 1, regardless of real timer values. + /// Feed batches, settle on a reorder, stay there. #[test] fn scenario_measure_batches_then_settle_on_reorder() { let schema = schema(); let p = predicate(&schema); // `a > 2 AND b < 5`, written order [0, 1] let cs = split(&p); let shared = Arc::new(AdaptiveFilterShared::default()); - // One batch short of the warm-up: the next measured batch settles. shared.seed_one_batch_short_of_warmup(&[ (70_000_000, 63_000_000, 70_000_000), // pass 0.9, ~1ns/row (70_000_000, 700_000, 350_000_000), // pass 0.01, ~5ns/row @@ -1157,17 +1035,13 @@ mod tests { let got = adaptive.evaluate(&rb).unwrap(); let want = p.evaluate(&rb).unwrap().into_array(rb.num_rows()).unwrap(); assert_eq!(passing_rows(&got), passing_rows(&want), "round {round}"); - // Round 0 completes the warm-up and settles on the reorder; every - // round after it evaluates the adopted chain, which promotes the - // selective conjunct to the front. assert!(adaptive.settled, "settled after round {round}"); assert!(adaptive.reordered, "reordered after round {round}"); assert_chain(&adaptive.settled_predicate, &cs, &[1, 0]); } } - /// Contract scenario for the no-win case: interchangeable conjuncts settle - /// on the written order, never a reorder. + /// Interchangeable conjuncts settle on the written order. #[test] fn scenario_measure_batches_then_settle_on_written_order() { let schema = schema(); @@ -1175,8 +1049,6 @@ mod tests { let cs = split(&p); let shared = Arc::new(AdaptiveFilterShared::default()); // Identical cost and selectivity: no order can be materially cheaper. - // The seeded magnitudes dominate the one real measured batch, so even - // if real timings nudge the ranking, the 5% material-win guard holds. shared.seed_one_batch_short_of_warmup(&[ (70_000_000, 35_000_000, 70_000_000), (70_000_000, 35_000_000, 70_000_000), @@ -1199,10 +1071,7 @@ mod tests { } } - /// The pooled registry is sized lazily by the first measured batch (the - /// conjunct count is not known to `AdaptiveFilterShared`, which is built - /// before the predicate is split), and the counts of that first batch land - /// in it. + /// The pool is sized by the first measured batch and receives its counts. #[test] fn first_measured_batch_initialises_the_shared_pool() { let schema = schema(); @@ -1211,7 +1080,6 @@ mod tests { assert!(shared.inner.lock().unwrap().stats.is_empty()); let mut adaptive = AdaptiveConjunction::try_new(&p, Arc::clone(&shared), None).unwrap(); - // Empty batches measure nothing, so the pool is still unsized after one. adaptive.evaluate(&batch(&schema, vec![], vec![])).unwrap(); assert!(shared.inner.lock().unwrap().stats.is_empty()); @@ -1221,8 +1089,7 @@ mod tests { let inner = shared.inner.lock().unwrap(); assert_eq!(inner.stats.len(), 2, "sized to the conjunct count"); assert_eq!(inner.measured_batches, 1); - // `a > 2` keeps 7 of 10 rows: too many for `BinaryExpr` to pre-select - // on, so `b < 5` is evaluated on all 10 rows too and keeps 5 of them. + // `a > 2` keeps 7 of 10 (no pre-selection), so `b < 5` sees all 10. assert_eq!((inner.stats[0].rows, inner.stats[0].matched), (10, 7)); assert_eq!((inner.stats[1].rows, inner.stats[1].matched), (10, 5)); } @@ -1270,33 +1137,24 @@ mod tests { (non_zero, divide) } - /// This is the behaviour the config option's doc warns about: adopting a - /// reorder can introduce an error the written order avoided. - /// - /// `b <> 0 AND 1 / b > 2` on data where `b <> 0` holds for 15% of the rows. - /// The written `BinaryExpr` `AND` pre-selects on `b <> 0` (15 of 100 rows, - /// within its 20% threshold), so `1 / b` never sees a zero and the flag-off - /// query succeeds. In the rebuilt chain `1 / b > 2` is the outermost left - /// operand, so it runs first — on every row, zeros included — and integer - /// division by zero is an error. + /// The side effect the config doc warns about: a reorder can introduce an + /// error the written order avoided. `b <> 0` holds on 15% of rows, so the + /// written `AND` pre-selects and `1 / b` never sees a zero; reordered, + /// `1 / b > 2` runs first on every row. #[test] fn adopted_reorder_can_introduce_a_divide_by_zero() { let schema = int64_schema(); let (non_zero, divide) = divide_by_zero_conjuncts(&schema); - // Written order [0, 1] = [`b <> 0`, `1 / b > 2`]. let p = binary(non_zero, Operator::And, divide, &schema).unwrap(); let cs = split(&p); let a: Vec = (0..100).collect(); let b: Vec = (0..100).map(|i| i64::from(i < 15)).collect(); let rb = int64_batch(&schema, a, b); - // Flag off: the written predicate pre-selects on `b <> 0` (15 of 100 - // rows, within its 20% threshold) and succeeds. + // Flag off: succeeds. assert!(p.evaluate(&rb).is_ok(), "flag-off evaluation must succeed"); let shared = Arc::new(AdaptiveFilterShared::default()); - // `1 / b > 2` (conjunct 1) seeded as cheap and very selective, so the - // warm-up settles on promoting it. shared.seed_one_batch_short_of_warmup(&[ (70_000_000, 63_000_000, 70_000_000), // pass 0.9, ~1ns/row (70_000_000, 700_000, 70_000_000), // pass 0.01, ~1ns/row @@ -1304,26 +1162,19 @@ mod tests { let mut adaptive = AdaptiveConjunction::try_new(&p, Arc::clone(&shared), None).unwrap(); - // The settling batch is still measured in the written order, whose - // pre-selection on `b <> 0` also keeps `1 / b` away from the zeros. + // The settling batch still runs in the written order. adaptive.evaluate(&rb).unwrap(); assert!(adaptive.reordered); assert_chain(&adaptive.settled_predicate, &cs, &[1, 0]); - // The next batch runs the adopted reorder, and errors. + // The next batch runs the reorder. let err = adaptive.evaluate(&rb).unwrap_err().to_string(); assert!(err.contains("Divide by zero"), "unexpected error: {err}"); } - /// The mirror of the case above: an error the written order *does* raise, - /// which the adopted order avoids. - /// - /// `1 / b > 2 AND a < 10`, with `b = 0` on exactly the rows `a < 10` - /// discards. The written `AND` evaluates its left side on every row and - /// errors. The rebuilt chain is `a < 10 AND (1 / b > 2)`, and `a < 10` - /// keeps 10 of the 100 rows with no nulls — inside `BinaryExpr`'s 20% - /// pre-selection threshold — so the batch is filtered down to those - /// survivors (all with `b = 1`) before `1 / b > 2` is evaluated at all. + /// The mirror: `1 / b > 2 AND a < 10` with `b = 0` exactly where `a < 10` + /// discards. The written order divides by zero; reordered, `a < 10` keeps + /// 10% of rows, so pre-selection keeps `1 / b` away from the zeros. #[test] fn adopted_reorder_can_avoid_a_divide_by_zero_the_written_order_raises() { let schema = int64_schema(); @@ -1335,13 +1186,10 @@ mod tests { &schema, ) .unwrap(); - // Written order [0, 1] = [`1 / b > 2`, `a < 10`]. let p = binary(divide, Operator::And, selective, &schema).unwrap(); let cs = split(&p); let shared = Arc::new(AdaptiveFilterShared::default()); - // Conjunct 1 (`a < 10`) seeded as cheap and very selective, conjunct 0 - // as expensive and unselective, so the warm-up promotes conjunct 1. shared.seed_one_batch_short_of_warmup(&[ (70_000_000, 63_000_000, 350_000_000), // pass 0.9, ~5ns/row (70_000_000, 700_000, 70_000_000), // pass 0.01, ~1ns/row @@ -1349,8 +1197,7 @@ mod tests { let mut adaptive = AdaptiveConjunction::try_new(&p, Arc::clone(&shared), None).unwrap(); - // Settle on a batch with no zeros at all, so the warm-up itself (which - // evaluates in the written order) cannot hit the error. + // Settle on a batch with no zeros, so the warm-up cannot error. let a: Vec = (0..100).collect(); adaptive .evaluate(&int64_batch(&schema, a.clone(), vec![1; 100])) @@ -1358,15 +1205,11 @@ mod tests { assert!(adaptive.reordered); assert_chain(&adaptive.settled_predicate, &cs, &[1, 0]); - // Now a batch whose `b` is zero on every row `a < 10` discards. let b: Vec = (0..100).map(|i| i64::from(i < 10)).collect(); let rb = int64_batch(&schema, a, b); - // The written order divides by zero... let err = p.evaluate(&rb).unwrap_err().to_string(); assert!(err.contains("Divide by zero"), "unexpected error: {err}"); - // ...while the rebuilt chain pre-selects on `a < 10` and evaluates - // `1 / b > 2` only on the rows it kept, none of which is zero. let got = adaptive.evaluate(&rb).unwrap(); assert!(passing_rows(&got).is_empty(), "1 / 1 > 2 is false"); } diff --git a/datafusion/physical-plan/src/filter.rs b/datafusion/physical-plan/src/filter.rs index 5db08351ce48e..54ae1126b0039 100644 --- a/datafusion/physical-plan/src/filter.rs +++ b/datafusion/physical-plan/src/filter.rs @@ -101,11 +101,9 @@ pub struct FilterExec { batch_size: usize, /// Number of rows to fetch fetch: Option, - /// Per-execution measurements pooled across this node's partition streams - /// by adaptive conjunct reordering (see [`AdaptiveConjunction`]). Not part - /// of the plan shape. `Clone` shares it, since a clone filters the same - /// predicate; [`reset_state`](ExecutionPlan::reset_state) and predicate - /// rewrites replace it with a fresh instance. + /// Adaptive conjunct reordering state (see [`AdaptiveConjunction`]), + /// pooled across partition streams. Shared by `Clone`; replaced by + /// [`reset_state`](ExecutionPlan::reset_state) and predicate rewrites. adaptive_stats: Arc, } @@ -608,9 +606,7 @@ impl ExecutionPlan for FilterExec { ) -> Result> { validate_child_count!(self, children); match options.children_properties { - // `adaptive_stats` is carried over by the struct update: the - // predicate is unchanged, so any pooled measurements still - // describe it. + // `adaptive_stats` is kept: the predicate is unchanged. ChildrenPropertiesMode::Keep => Ok(Arc::new(Self { input: children.swap_remove(0), metrics: ExecutionPlanMetricsSet::new(), @@ -646,11 +642,8 @@ impl ExecutionPlan for FilterExec { ) } - /// Reset per-execution state so an independent re-execution (e.g. a - /// recursive query) does not inherit runtime state from a prior run: the - /// pooled adaptive-conjunct measurements and the execution metrics are - /// replaced with fresh instances, while the predicate, input and cached - /// plan properties remain valid and are preserved. + /// Fresh adaptive-reordering state and metrics for a re-execution; the + /// predicate, input and cached properties are still valid and kept. fn reset_state(self: Arc) -> Result> { let mut new = (*self).clone(); new.adaptive_stats = Arc::new(AdaptiveFilterShared::default()); @@ -674,8 +667,7 @@ impl ExecutionPlan for FilterExec { .options() .execution .adaptive_filter_reordering; - // The metric must exist before the evaluator that increments it, and - // must be registered exactly when the adaptive path is active. + // Register the counter exactly when the adaptive path is active. let adaptive_applies = enabled && AdaptiveConjunction::applies(&self.predicate); let mut metrics = FilterExecMetrics::new(&self.metrics, partition); if adaptive_applies { @@ -900,8 +892,7 @@ impl ExecutionPlan for FilterExec { projection: self.projection.clone(), batch_size: self.batch_size, fetch: self.fetch, - // The predicate changed; pooled per-conjunct stats no longer - // describe it. + // The predicate changed. adaptive_stats: Arc::new(AdaptiveFilterShared::default()), }; Some(Arc::new(new) as _) @@ -1407,8 +1398,7 @@ struct FilterExecStream { schema: SchemaRef, /// The expression to filter on. This expression must evaluate to a boolean value. predicate: Arc, - /// When set, the predicate is a reorderable conjunction evaluated - /// adaptively instead of via `predicate`. + /// Evaluates `predicate` adaptively when set. adaptive: Option, /// The input partition to filter. input: SendableRecordBatchStream, @@ -1426,11 +1416,8 @@ struct FilterExecMetrics { baseline_metrics: BaselineMetrics, /// Selectivity of the filter, calculated as output_rows / input_rows selectivity: RatioMetrics, - /// Number of partition streams that adopted an adaptively reordered - /// evaluation order for the predicate's conjuncts (at most one per - /// stream). Registered only when adaptive conjunct reordering is enabled - /// and the predicate is a reorderable conjunction, so the flag-off path's - /// metrics are unchanged. + /// Partition streams that adopted an adaptively reordered conjunct order. + /// Registered only when adaptive reordering applies. adaptive_reorders: Option, // Remember to update `docs/source/user-guide/metrics.md` when adding new metrics, // or modifying metrics comments @@ -1456,8 +1443,6 @@ impl FilterExecMetrics { ) -> Self { self.adaptive_reorders = Some( MetricBuilder::new(metrics) - // A deterministic, dimensionless counter: it depends on - // the plan and the data, not on wall-clock timings. .with_category(MetricCategory::Rows) .counter("adaptive_reorders", partition), ); @@ -2549,30 +2534,20 @@ mod tests { .downcast_ref::() .expect("reset_state returns a FilterExec"); - // The per-execution adaptive state is a fresh instance, so learning - // cannot leak across independent executions... assert!(!Arc::ptr_eq(&filter.adaptive_stats, &reset.adaptive_stats)); - // ...while the predicate (which the reset does not touch) is preserved. assert!(Arc::ptr_eq(&filter.predicate, &reset.predicate)); Ok(()) } - /// End-to-end `FilterExec` run with `adaptive_filter_reordering` enabled: - /// four partitions of sixteen small batches, a nullable column, and a - /// conjunction written selective-*last*. - /// - /// Both conjuncts are cheap arithmetic, so real timings cannot separate - /// them reliably; the shared pool is therefore seeded one batch short of - /// the warm-up so the reorder is adopted deterministically. Everything - /// else — the streams, the metric, the coalescer, the projection-free - /// output path — is the real thing. + /// `FilterExec` with the flag on: four partitions, a nullable column, a + /// conjunction written selective-last. The pool is seeded so the reorder + /// is adopted regardless of timings; everything else is the real path. #[tokio::test] async fn adaptive_filter_reordering_end_to_end() -> Result<()> { const PARTITIONS: usize = 4; const BATCHES: usize = 16; const ROWS: i64 = 64; - // `b` is NULL on every 37th row, including inside the range the - // predicate selects, so the run exercises SQL filter semantics. + // `b` is NULL on every 37th row, including inside the selected range. const NULL_EVERY: i64 = 37; let schema = Arc::new(Schema::new(vec![ @@ -2601,11 +2576,8 @@ mod tests { let input: Arc = TestMemoryExec::try_new_exec(&partitions, Arc::clone(&schema), None)?; - // `(a % 97 + a % 89 >= 0) AND (b > 3990)`: the left conjunct is always - // true and the more expensive of the two, the right one keeps a - // hundred-odd rows out of four thousand — the selective conjunct is - // written last, and both are cheap arithmetic, so the logical - // heuristics would not have reordered this. + // `(a % 97 + a % 89 >= 0) AND (b > 3990)`: the always-true conjunct is + // written first; both are cheap, so the static heuristic leaves them. let threshold = total_rows - 106; // 3990 let cheap = binary( binary( @@ -2623,8 +2595,7 @@ mod tests { let predicate = binary(cheap, Operator::And, selective, &schema)?; let filter = Arc::new(FilterExec::try_new(predicate, input)?); - // Execute every partition with the flag set as given and return the - // output rows, sorted so partition interleaving cannot matter. + // Output rows across all partitions, sorted. async fn run( filter: &Arc, adaptive: bool, @@ -2647,7 +2618,6 @@ mod tests { .downcast_ref::() .unwrap(); for i in 0..batch.num_rows() { - // SQL filter semantics: `NULL > 3990` is not `true`. assert!(!b.is_null(i), "a NULL row survived the filter"); rows.push((a.value(i), b.value(i))); } @@ -2657,8 +2627,7 @@ mod tests { Ok(rows) } - // The rows the predicate selects, computed independently: `b > 3990` - // spans 105 values, three of which have a NULL `b` and must be dropped. + // Expected rows: 105 values of `b > 3990`, three of them NULL. let candidates: Vec = (threshold + 1..total_rows).collect(); assert_eq!(candidates.len(), 105); let expected: Vec<(i64, i64)> = candidates @@ -2679,9 +2648,7 @@ mod tests { "the flag-off path must not register the adaptive metric" ); - // Seed the pooled measurements so the warm-up settles on promoting the - // selective conjunct: conjunct 0 keeps every row and is ~5x the cost of - // conjunct 1, which keeps 1%. + // Conjunct 0 keeps every row at ~5x the cost of conjunct 1 (keeps 1%). filter.adaptive_stats.seed_one_batch_short_of_warmup(&[ (70_000_000, 70_000_000, 350_000_000), (70_000_000, 700_000, 70_000_000), @@ -2697,13 +2664,10 @@ mod tests { .unwrap_or(0); assert!(reorders >= 1, "expected an adopted reorder, got {reorders}"); - // Re-executing the same node keeps the learned state (the streams - // adopt the settled order on their first batch) but cannot change the - // rows. + // Re-executing the same node keeps the learned state; same rows. assert_eq!(run(&filter, true).await?, flag_off, "state persists"); - // `reset_state` drops the pooled measurements, so the fresh node learns - // from scratch — and still produces the same rows. + // A reset node learns from scratch; same rows. let reset = Arc::clone(&filter).reset_state()?; let reset: Arc = Arc::new( reset diff --git a/datafusion/sqllogictest/test_files/adaptive_filter.slt b/datafusion/sqllogictest/test_files/adaptive_filter.slt index 4678c5ce4e25b..e3f0876eace32 100644 --- a/datafusion/sqllogictest/test_files/adaptive_filter.slt +++ b/datafusion/sqllogictest/test_files/adaptive_filter.slt @@ -18,10 +18,7 @@ # Tests for execution.adaptive_filter_reordering. Runtime reordering of a # conjunction must never change query results, only evaluation order. -# Store the table as many small batches so that, with the flag on, the -# adaptive evaluator's warm-up (8 measured batches) completes and the settled -# (possibly reordered) path is exercised end-to-end — not just the measuring -# path a single-batch table would reach. +# Small batches, so the 8-batch warm-up completes and the settled path runs. statement ok SET datafusion.execution.batch_size = 64; @@ -41,8 +38,7 @@ SELECT count(*) FROM t WHERE b = 3 AND s LIKE '1%'; ---- 160 -# Baseline plan (flag off), so the "EXPLAIN is identical on and off" claim -# below is asserted against something rather than merely stated. +# Baseline plan (flag off); the flag-on plan below must be identical. query TT EXPLAIN SELECT count(*) FROM t WHERE b = 3 AND s LIKE '1%'; ---- @@ -108,16 +104,11 @@ physical_plan 05)--------FilterExec: b@0 = 3 AND s@1 LIKE 1%, projection=[] 06)----------DataSourceExec: partitions=4, partition_sizes=[16, 16, 16, 15] -# EXPLAIN ANALYZE: the `adaptive_reorders` metric makes a reorder that actually -# happened at runtime visible. The predicate is written selective-conjunct-last: -# the arithmetic chain is the more expensive conjunct and matches every row, -# while `a > 3990` is a single comparison keeping 10 rows in 4000 — so the -# learned order flips the two. (Both conjuncts are "cheap" to the static -# cheap-first `reorder_predicates` rule, which is cost-only and blind to -# selectivity, so the written order reaches `FilterExec` untouched.) Each of the -# 4 partition streams adopts the reorder exactly once — the first to finish the -# pooled warm-up settles the order and the others take it up on their next -# batch — so the aggregated counter is 4. +# EXPLAIN ANALYZE shows the reorder via `adaptive_reorders`. The predicate is +# written selective-last: the arithmetic chain matches every row and costs +# more than `a > 3990`, which keeps 10 rows in 4000. Both are cheap to the +# static `reorder_predicates` rule, so the written order reaches `FilterExec`. +# Each of the 4 partition streams adopts the reorder once, hence 4. query I SELECT count(*) FROM t WHERE a % 97 + a % 89 + a % 83 + b % 13 >= 0 AND a > 3990; ---- diff --git a/docs/source/user-guide/metrics.md b/docs/source/user-guide/metrics.md index 46aad02fa806d..a4c89e6db1d8f 100644 --- a/docs/source/user-guide/metrics.md +++ b/docs/source/user-guide/metrics.md @@ -38,10 +38,10 @@ DataFusion operators expose runtime metrics so you can understand where time is ### FilterExec -| Metric | Description | -| ----------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| selectivity | Selectivity of the filter, calculated as output_rows / input_rows | -| adaptive_reorders | Number of partition streams that adopted an adaptively reordered evaluation order for the predicate's conjuncts (at most one per stream). Only present when `datafusion.execution.adaptive_filter_reordering` is enabled and the predicate is a reorderable conjunction; `0` means the written order was kept. | +| Metric | Description | +| ----------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| selectivity | Selectivity of the filter, calculated as output_rows / input_rows | +| adaptive_reorders | Partition streams that adopted an adaptively reordered conjunct order. Present only when `datafusion.execution.adaptive_filter_reordering` applies; `0` means the written order was kept. | ### HashJoinExec From 571a388155033077267f6fae9114d793faefe0c9 Mon Sep 17 00:00:00 2001 From: Adrian Garcia Badaracco <1755071+adriangb@users.noreply.github.com> Date: Wed, 16 Sep 2026 14:29:29 +0000 Subject: [PATCH 22/27] refactor(physical-expr): share the AND pre-selection rule with its callers `check_short_circuit` decides what an `AND` does with its right-hand side from the left-hand side's true count, null count and length: skip it when the left side is all false, pre-select when the left side has no nulls and keeps at most `PRE_SELECTION_THRESHOLD` of the rows, otherwise evaluate it on the whole batch. Anything that models the cost of a conjunction has to reproduce that rule, and a private threshold behind an inline condition makes it easy to drift from. Lift the decision into `and_rhs_evaluation`, returning the new `AndRhsEvaluation`, and have `check_short_circuit` decide by it, so the rule has one definition. Export it, and the threshold, alongside `BinaryExpr`. Behaviour is unchanged: the null and empty-batch cases return exactly what they returned before. The `true` count is now computed before the null check rather than after it, which costs a popcount over the values buffer on the nulls-present path that previously bailed out first. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01SZtdyL1QXmSiQdMRYmqTkV --- .../physical-expr/src/expressions/binary.rs | 96 +++++++++++++++---- .../physical-expr/src/expressions/mod.rs | 5 +- 2 files changed, 79 insertions(+), 22 deletions(-) diff --git a/datafusion/physical-expr/src/expressions/binary.rs b/datafusion/physical-expr/src/expressions/binary.rs index dfb1d136d0ff0..1c316b62c08a6 100644 --- a/datafusion/physical-expr/src/expressions/binary.rs +++ b/datafusion/physical-expr/src/expressions/binary.rs @@ -1164,7 +1164,55 @@ enum ShortCircuitStrategy { /// the side that cannot short-circuit the operator is rare: /// - for `AND`, when the proportion of `true` is less than or equal to 0.2 /// - for `OR`, when the proportion of `false` is less than or equal to 0.2 -const PRE_SELECTION_THRESHOLD: f32 = 0.2; +pub const PRE_SELECTION_THRESHOLD: f32 = 0.2; + +/// How much of the batch an `AND`'s right-hand side is evaluated on, given the +/// shape of its left-hand side's result. +/// +/// This is the observable consequence of [`check_short_circuit`] for `AND`, +/// exposed so that consumers modelling the cost of a conjunction share one +/// definition with the code that implements it. See [`and_rhs_evaluation`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum AndRhsEvaluation { + /// The left-hand side is `false` on every row, so the right-hand side is + /// not evaluated at all. + Skipped, + /// The left-hand side has no nulls and is `true` on few enough rows + /// ([`PRE_SELECTION_THRESHOLD`]), so the right-hand side is evaluated only + /// on the rows where it is `true`. + PreSelected, + /// The right-hand side is evaluated on the whole batch. This is the case + /// whenever the left-hand side produces a null, however selective it looks. + FullBatch, +} + +/// What an `AND` does with its right-hand side, given its left-hand side's +/// `true` count, null count and length. +/// +/// `true_count` counts non-null `true`s; it is only consulted when +/// `null_count` is zero, where the two conventions coincide. +/// +/// [`check_short_circuit`] decides by this function, so a caller that models +/// conjunction cost cannot drift away from what evaluation actually does. +pub fn and_rhs_evaluation( + true_count: usize, + null_count: usize, + len: usize, +) -> AndRhsEvaluation { + // A null makes `AND` fall through to a plain full-batch evaluation: it can + // neither skip the right-hand side nor build a pre-selection mask from a + // left-hand side it cannot interpret row by row. + if null_count > 0 || len == 0 { + return AndRhsEvaluation::FullBatch; + } + if true_count == 0 { + return AndRhsEvaluation::Skipped; + } + if true_count < len && true_count as f32 / len as f32 <= PRE_SELECTION_THRESHOLD { + return AndRhsEvaluation::PreSelected; + } + AndRhsEvaluation::FullBatch +} /// Checks if a logical operator (`AND`/`OR`) can short-circuit evaluation based on the left-hand side (lhs) result. /// @@ -1202,35 +1250,41 @@ fn check_short_circuit(lhs: &ColumnarValue, op: &Operator) -> ShortCircuitStrate ColumnarValue::Array(array) => { // Fast path for arrays - try to downcast to boolean array if let Ok(bool_array) = as_boolean_array(array) { - // Arrays with nulls can't be short-circuited - if bool_array.null_count() > 0 { - return ShortCircuitStrategy::None; - } - let len = bool_array.len(); if len == 0 { return ShortCircuitStrategy::None; } + let null_count = bool_array.null_count(); let true_count = bool_array.values().count_set_bits(); if is_and { - if true_count == 0 { - return ShortCircuitStrategy::ReturnLeft; - } - - if true_count == len { - return ShortCircuitStrategy::ReturnRight; - } - - if true_count as f32 / len as f32 <= PRE_SELECTION_THRESHOLD { - // Select rows where the LHS is true; rows where the LHS - // is false are false regardless of the RHS. - return ShortCircuitStrategy::PreSelection { - mask: bool_array.clone(), - fill_value: false, - }; + // Decided by `and_rhs_evaluation` so that consumers + // modelling this behaviour cannot drift away from it. + match and_rhs_evaluation(true_count, null_count, len) { + AndRhsEvaluation::Skipped => { + return ShortCircuitStrategy::ReturnLeft; + } + AndRhsEvaluation::PreSelected => { + // Select rows where the LHS is true; rows where the + // LHS is false are false regardless of the RHS. + return ShortCircuitStrategy::PreSelection { + mask: bool_array.clone(), + fill_value: false, + }; + } + // All true: the RHS alone decides. Otherwise fall + // through to a plain full-batch evaluation. + AndRhsEvaluation::FullBatch => { + if null_count == 0 && true_count == len { + return ShortCircuitStrategy::ReturnRight; + } + } } } else { + // Arrays with nulls can't be short-circuited + if null_count > 0 { + return ShortCircuitStrategy::None; + } if true_count == len { return ShortCircuitStrategy::ReturnLeft; } diff --git a/datafusion/physical-expr/src/expressions/mod.rs b/datafusion/physical-expr/src/expressions/mod.rs index f2f9285de560a..2c2816304faf1 100644 --- a/datafusion/physical-expr/src/expressions/mod.rs +++ b/datafusion/physical-expr/src/expressions/mod.rs @@ -41,7 +41,10 @@ pub use crate::PhysicalSortExpr; /// Module with some convenient methods used in expression building pub use crate::aggregate::stats::StatsType; -pub use binary::{BinaryExpr, binary, similar_to}; +pub use binary::{ + AndRhsEvaluation, BinaryExpr, PRE_SELECTION_THRESHOLD, and_rhs_evaluation, binary, + similar_to, +}; pub use case::{CaseExpr, case}; pub use cast::{CastExpr, cast}; pub use column::{Column, col, with_new_schema}; From c54d917df751ba8519395200ed7af75b8cbb69fc Mon Sep 17 00:00:00 2001 From: Adrian Garcia Badaracco <1755071+adriangb@users.noreply.github.com> Date: Wed, 16 Sep 2026 14:29:44 +0000 Subject: [PATCH 23/27] fix(physical-plan): keep the written AND tree, and cost it as AND runs it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two review findings on the adaptive filter, both of which made the flag claim more than it did. Keep the written tree. The warm-up rebuilt the predicate as a right-nested chain over the wrappers, and a warm-up that found nothing better returned a right-nested rebuild of the written order too. Reassociating an `AND` changes where pre-selection fires even when the conjunct sequence is unchanged, so enabling the flag could change what a fallible conjunct sees while `adaptive_reorders` stayed 0. For `((a < 50 AND b < 3) AND 1/z > 2)`, the written tree gates `1/z` on the 15% that `a < 50 AND b < 3` keeps, while the right-nested form gates it on `b < 3` alone, at 30%, which does not pre-select at all — so it divides by zero. The warm-up now wraps the conjunct leaves in place, leaving the tree exactly as written, and a warm-up that adopts no reorder hands back the written `Arc` itself. An adopted reorder is still materialised right-nested, which is what makes it pay. Cost what `AND` really hands on. `expected_cost_per_row` multiplied by each conjunct's pass rate, as if every conjunct narrowed the batch for the ones after it. `BinaryExpr` only narrows it when it pre-selects: with no nulls and at most `PRE_SELECTION_THRESHOLD` of the rows kept. Conjuncts keeping 30% and 90% both leave the next one facing the whole batch, yet both were credited with a discount; a conjunct that looked selective only because it produced nulls was credited most of all, though nulls disable pre-selection outright. `MeasuredConjunct` now classifies every batch through `and_rhs_evaluation` — the function `BinaryExpr` itself decides by, so the two cannot drift — and `ConjunctStats::downstream_weight` reports the rows the conjuncts after it were actually handed. Measuring this per batch, rather than inferring it from pooled rates, also keeps a conjunct whose selectivity straddles the threshold honestly weighted. Tests cover the reassociation case end to end, the wrapped tree's shape, the weight just below, exactly at and just above the threshold, a nullable conjunct measured against the same conjunct without nulls, and a reorder that ranks better but cannot pre-select and so is rejected. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01SZtdyL1QXmSiQdMRYmqTkV --- datafusion/common/src/config.rs | 7 +- .../physical-plan/src/adaptive_filter.rs | 522 +++++++++++++++--- .../test_files/information_schema.slt | 2 +- docs/source/user-guide/configs.md | 2 +- 4 files changed, 462 insertions(+), 71 deletions(-) diff --git a/datafusion/common/src/config.rs b/datafusion/common/src/config.rs index 97a2d61f8e5d5..5c68051ecc4cd 100644 --- a/datafusion/common/src/config.rs +++ b/datafusion/common/src/config.rs @@ -1145,8 +1145,11 @@ config_namespace! { /// the observable side effects of a fallible predicate can, in either /// direction: reordering `b <> 0 AND 1/b > 2` can make a /// divide-by-zero error appear or disappear, since each conjunct is - /// evaluated only on the rows the conjuncts before it kept. Predicates - /// containing volatile expressions are never reordered. + /// evaluated only on the rows the conjuncts before it kept. A + /// predicate whose written order is kept is evaluated exactly as + /// written, so enabling this has no observable effect until a reorder + /// is actually adopted (reported by the `adaptive_reorders` metric). + /// Predicates containing volatile expressions are never reordered. pub adaptive_filter_reordering: bool, default = false /// Size (bytes) of data buffer DataFusion uses when writing output files. diff --git a/datafusion/physical-plan/src/adaptive_filter.rs b/datafusion/physical-plan/src/adaptive_filter.rs index 359c65b8a764e..603e7fa6f4b9e 100644 --- a/datafusion/physical-plan/src/adaptive_filter.rs +++ b/datafusion/physical-plan/src/adaptive_filter.rs @@ -60,14 +60,20 @@ //! ``` //! //! This module contains no evaluation logic of its own. While the order is -//! being learned, the written order is handed to [`BinaryExpr`] with every -//! conjunct wrapped in a [`MeasuredConjunct`]; `BinaryExpr` evaluates and -//! pre-selects as it would for the plain predicate, so each conjunct is -//! measured on the population it would really see in that position. +//! being learned, the written predicate is handed to [`BinaryExpr`] with every +//! conjunct *leaf* wrapped in a [`MeasuredConjunct`] and the `AND` tree left +//! exactly as written; `BinaryExpr` evaluates and pre-selects as it would for +//! the plain predicate, so each conjunct is measured on the population it +//! would really see in that position. //! -//! Once the order settles the wrappers are gone: the settled order — the -//! written one if the warm-up found nothing materially better, otherwise the -//! learned one — is materialised once as a right-nested `AND` chain, +//! If the warm-up finds nothing materially better, the written predicate is +//! handed back as written — the same `Arc`, not an equivalent rebuilt from its +//! conjuncts. Reassociating an `AND` tree changes where pre-selection fires +//! even when the conjunct sequence is unchanged, so rebuilding it would let +//! the flag alter evaluation, and the side effects of fallible conjuncts with +//! it, without any reorder having been adopted. +//! +//! An adopted reorder *is* materialised, as a right-nested `AND` chain, //! `(c_first AND (c_second AND (... AND c_last)))`. Right-nesting is what makes //! it pay: pre-selection filters the batch an `AND` is handed before evaluating //! its right-hand side, so the survivors of the first conjunct stay compacted @@ -81,6 +87,18 @@ //! ([`TIE_COST_FRACTION`]), so a conjunction that does not benefit carries none //! of this machinery past the warm-up. The decision then stays fixed. //! +//! Cost is counted the way `AND` actually behaves rather than by pass rate +//! alone: a conjunct shortens the work after it only on a batch where +//! `BinaryExpr` pre-selects, which it does only when the conjunct produced no +//! nulls and kept at most +//! [`PRE_SELECTION_THRESHOLD`](datafusion_physical_expr::expressions::PRE_SELECTION_THRESHOLD) +//! of the rows. Conjuncts keeping 30% and 90% therefore both leave the next +//! one facing the whole batch and are credited alike, and a conjunct that +//! looks selective only because it produced nulls is credited with nothing. +//! Which of these happened is recorded per batch through +//! [`and_rhs_evaluation`], the same function `BinaryExpr` decides by, so the +//! model cannot drift away from the behaviour it models. +//! //! A `FilterExec` is split across many partition streams, each seeing only a //! slice of the data, so measurements are pooled into a shared //! [`AdaptiveFilterShared`] and the streams learn as one: the first stream with @@ -99,6 +117,9 @@ //! pre-selection, taken on small batches whose per-row cost is inflated by //! fixed overheads. Correlated conjuncts can be misjudged; the material-win //! guard only makes adoption conservative. +//! - A conjunct is only ever observed where it was written, so whether it +//! would pre-select somewhere else in the order is projected from what it +//! did in its own position, not measured. //! - The decision is one-shot: a misjudged reorder, or drifting data, is kept //! for the rest of the query. //! @@ -111,7 +132,7 @@ use std::sync::Mutex; use std::sync::atomic::{AtomicU64, Ordering::Relaxed}; use crate::metrics::Count; -use arrow::array::ArrayRef; +use arrow::array::{Array, ArrayRef}; use arrow::datatypes::{DataType, Schema}; use arrow::record_batch::RecordBatch; use datafusion_common::Result; @@ -119,7 +140,9 @@ use datafusion_common::cast::as_boolean_array; use datafusion_common::instant::Instant; use datafusion_expr::{ColumnarValue, Operator}; use datafusion_physical_expr::PhysicalExpr; -use datafusion_physical_expr::expressions::BinaryExpr; +use datafusion_physical_expr::expressions::{ + AndRhsEvaluation, BinaryExpr, and_rhs_evaluation, +}; use datafusion_physical_expr::utils::split_conjunction; use datafusion_physical_expr_common::physical_expr::is_volatile; @@ -131,6 +154,12 @@ const WARMUP_BATCHES: u64 = 8; const TIE_COST_FRACTION: f64 = 0.05; /// Per-conjunct counts over the warm-up, on exactly the rows that reached it. +/// +/// Besides the totals, the counts are split by what [`BinaryExpr`]'s `AND` +/// actually did with the conjuncts *after* this one on each batch +/// ([`and_rhs_evaluation`]), because that — not the raw pass rate — is what +/// decides how much work this conjunct saves them. See +/// [`downstream_weight`](Self::downstream_weight). #[derive(Debug, Default, Clone)] struct ConjunctStats { /// Total rows the conjunct was evaluated on. @@ -139,19 +168,49 @@ struct ConjunctStats { matched: u64, /// Total evaluation time, nanoseconds. nanos: u64, + /// Of `rows`, those in batches where the result let `AND` pre-select: the + /// conjuncts after this one saw only the matching rows. + gated_rows: u64, + /// Of `gated_rows`, the rows that passed — what the conjuncts after this + /// one were actually handed. + gated_matched: u64, + /// Of `rows`, those in batches where the result was all `false`: the + /// conjuncts after this one were not evaluated at all. + skipped_rows: u64, } impl ConjunctStats { + /// Counts for a conjunct that saw `rows` rows across null-free batches + /// that all had the same shape, keeping `matched` of them in `nanos` + /// nanoseconds. Used by tests and by seeding; real measurements come from + /// [`MeasuredConjunct`] batch by batch. + #[cfg(test)] + fn from_null_free_batches(rows: u64, matched: u64, nanos: u64) -> Self { + let mut stats = Self { + rows, + matched, + nanos, + ..Default::default() + }; + match and_rhs_evaluation(matched as usize, 0, rows as usize) { + AndRhsEvaluation::Skipped => stats.skipped_rows = rows, + AndRhsEvaluation::PreSelected => { + stats.gated_rows = rows; + stats.gated_matched = matched; + } + AndRhsEvaluation::FullBatch => {} + } + stats + } + /// Pool another stream's counts into this one. fn merge(&mut self, other: &Self) { self.rows += other.rows; self.matched += other.matched; self.nanos += other.nanos; - } - - /// Fraction of rows that pass, or `None` if never evaluated on any row. - fn pass_rate(&self) -> Option { - (self.rows > 0).then(|| self.matched as f64 / self.rows as f64) + self.gated_rows += other.gated_rows; + self.gated_matched += other.gated_matched; + self.skipped_rows += other.skipped_rows; } /// Per-row cost in nanoseconds, or `None` if never evaluated. Time is @@ -160,6 +219,32 @@ impl ConjunctStats { (self.rows > 0).then(|| self.nanos.max(1) as f64 / self.rows as f64) } + /// Rows the conjuncts after this one were handed, per row this one saw. + /// + /// This is *not* the pass rate. `AND` only narrows what follows when it + /// pre-selects, which it does on a null-free batch keeping at most + /// [`PRE_SELECTION_THRESHOLD`] of the rows; otherwise the conjuncts after + /// it see the whole batch however many rows this one rejected. So a + /// conjunct keeping 30% and one keeping 90% both leave a weight of 1, and + /// a conjunct that looks selective only because it produced nulls also + /// leaves 1, since nulls disable pre-selection entirely. + /// + /// Unmeasured conjuncts weigh 1: they are assumed to narrow nothing. + /// + /// [`PRE_SELECTION_THRESHOLD`]: datafusion_physical_expr::expressions::PRE_SELECTION_THRESHOLD + fn downstream_weight(&self) -> f64 { + if self.rows == 0 { + return 1.0; + } + // Batches that pre-selected pass on their matching rows; batches that + // skipped pass on nothing; the rest pass on everything they saw. + let full_batch_rows = self + .rows + .saturating_sub(self.gated_rows) + .saturating_sub(self.skipped_rows); + (self.gated_matched + full_batch_rows) as f64 / self.rows as f64 + } + /// Ranking key: rows discarded per nanosecond, `(1 + rows_in - rows_out) / /// time` — the reciprocal of the score Velox sorts its filters by /// (), so maximising it @@ -215,19 +300,18 @@ impl AdaptiveFilterShared { let mut inner = self.inner.lock().expect("poisoned"); inner.stats = per_conjunct .iter() - .map(|&(rows, matched, nanos)| ConjunctStats { - rows, - matched, - nanos, + .map(|&(rows, matched, nanos)| { + ConjunctStats::from_null_free_batches(rows, matched, nanos) }) .collect(); inner.measured_batches = WARMUP_BATCHES - 1; } } -/// A conjunct that records the rows it was handed, the rows it kept and the -/// time it took, returning its result unchanged (nulls included). Everything -/// else delegates to the wrapped conjunct. +/// A conjunct that records the rows it was handed, the rows it kept, the time +/// it took and what `AND` then did with the conjuncts after it, returning its +/// result unchanged (nulls included). Everything else delegates to the wrapped +/// conjunct. #[derive(Debug)] struct MeasuredConjunct { inner: Arc, @@ -237,6 +321,12 @@ struct MeasuredConjunct { matched: AtomicU64, /// Time spent inside the conjunct over those rows, in nanoseconds. nanos: AtomicU64, + /// Rows in batches whose result let `AND` pre-select. + gated_rows: AtomicU64, + /// Of `gated_rows`, the rows that passed. + gated_matched: AtomicU64, + /// Rows in batches whose result was all `false`. + skipped_rows: AtomicU64, } impl MeasuredConjunct { @@ -246,6 +336,9 @@ impl MeasuredConjunct { rows: AtomicU64::new(0), matched: AtomicU64::new(0), nanos: AtomicU64::new(0), + gated_rows: AtomicU64::new(0), + gated_matched: AtomicU64::new(0), + skipped_rows: AtomicU64::new(0), } } @@ -255,6 +348,9 @@ impl MeasuredConjunct { rows: self.rows.swap(0, Relaxed), matched: self.matched.swap(0, Relaxed), nanos: self.nanos.swap(0, Relaxed), + gated_rows: self.gated_rows.swap(0, Relaxed), + gated_matched: self.gated_matched.swap(0, Relaxed), + skipped_rows: self.skipped_rows.swap(0, Relaxed), } } } @@ -293,12 +389,27 @@ impl PhysicalExpr for MeasuredConjunct { let timer = Instant::now(); let array = self.inner.evaluate(batch)?.into_array(rows)?; let nanos = timer.elapsed().as_nanos() as u64; - let matched = as_boolean_array(&array)?.true_count() as u64; + let bools = as_boolean_array(&array)?; + let matched = bools.true_count() as u64; self.rows.fetch_add(rows as u64, Relaxed); self.matched.fetch_add(matched, Relaxed); self.nanos.fetch_add(nanos, Relaxed); + // Record what this result lets `AND` do with the conjuncts after it, + // by the same rule evaluation uses, rather than inferring it from the + // pass rate afterwards. + match and_rhs_evaluation(matched as usize, bools.null_count(), rows) { + AndRhsEvaluation::Skipped => { + self.skipped_rows.fetch_add(rows as u64, Relaxed); + } + AndRhsEvaluation::PreSelected => { + self.gated_rows.fetch_add(rows as u64, Relaxed); + self.gated_matched.fetch_add(matched, Relaxed); + } + AndRhsEvaluation::FullBatch => {} + } + Ok(ColumnarValue::Array(array)) } @@ -323,11 +434,15 @@ impl PhysicalExpr for MeasuredConjunct { /// the per-stream state is just the chain this stream currently evaluates. #[derive(Debug)] pub(crate) struct AdaptiveConjunction { + /// The predicate exactly as written, kept so that a warm-up that finds + /// nothing better hands back the very tree `FilterExec` would have run. + written: Arc, /// The split conjuncts, in written order. conjuncts: Vec>, /// Measurements and the settled decision, shared by every partition stream. shared: Arc, - /// The written order as a right-nested `AND` chain over the wrappers. + /// The written tree with every conjunct leaf wrapped in a + /// [`MeasuredConjunct`] — same shape, same evaluation, plus counters. warmup_predicate: Arc, /// The wrappers inside `warmup_predicate`, in written order. measured: Vec>, @@ -363,17 +478,11 @@ impl AdaptiveConjunction { .into_iter() .map(Arc::clone) .collect(); - let order: Vec = (0..conjuncts.len()).collect(); - let measured: Vec> = conjuncts - .iter() - .map(|c| Arc::new(MeasuredConjunct::new(Arc::clone(c)))) - .collect(); - let wrapped: Vec> = measured - .iter() - .map(|m| Arc::clone(m) as Arc) - .collect(); - let warmup_predicate = right_nested_conjunction(&wrapped, &order); + let mut measured = Vec::with_capacity(conjuncts.len()); + let warmup_predicate = wrap_conjuncts_in_place(predicate, &mut measured); + debug_assert_eq!(measured.len(), conjuncts.len()); Some(Self { + written: Arc::clone(predicate), conjuncts, shared, settled_predicate: Arc::clone(&warmup_predicate), @@ -458,17 +567,23 @@ impl AdaptiveConjunction { if inner.measured_batches < WARMUP_BATCHES { return; } - let decision = settle(&inner.stats, &self.conjuncts); + let decision = settle(&inner.stats, &self.conjuncts, &self.written); inner.settled = Some(decision.clone()); drop(inner); self.adopt(decision); } } -/// Rank by effectiveness and adopt the ranking only if it is materially -/// cheaper than the written order; either way, build the result as a -/// right-nested `AND` chain. -fn settle(stats: &[ConjunctStats], conjuncts: &[Arc]) -> Settled { +/// Rank by effectiveness and adopt the ranking as a right-nested `AND` chain, +/// but only if it is materially cheaper than the written order. Otherwise hand +/// back `written` untouched: rebuilding it would reassociate the `AND` tree, +/// which changes where pre-selection fires and so what a fallible conjunct +/// sees, even though the conjunct sequence is unchanged. +fn settle( + stats: &[ConjunctStats], + conjuncts: &[Arc], + written: &Arc, +) -> Settled { let identity: Vec = (0..stats.len()).collect(); let candidate = rank_by_effectiveness(stats); if candidate != identity @@ -481,12 +596,36 @@ fn settle(stats: &[ConjunctStats], conjuncts: &[Arc]) -> Settl } } else { Settled { - predicate: right_nested_conjunction(conjuncts, &identity), + predicate: Arc::clone(written), reordered: false, } } } +/// The `AND` tree of `predicate` with every conjunct leaf replaced by a +/// [`MeasuredConjunct`] around it, collected into `measured`. +/// +/// The tree keeps its shape, so the warm-up evaluates exactly what the written +/// predicate would — same nesting, same pre-selection points — and only adds +/// the counters. Leaves are collected left to right, the order +/// [`split_conjunction`] yields them in, so `measured[i]` is the wrapper for +/// conjunct `i`. +fn wrap_conjuncts_in_place( + predicate: &Arc, + measured: &mut Vec>, +) -> Arc { + if let Some(binary) = predicate.downcast_ref::() + && *binary.op() == Operator::And + { + let left = wrap_conjuncts_in_place(binary.left(), measured); + let right = wrap_conjuncts_in_place(binary.right(), measured); + return Arc::new(BinaryExpr::new(left, Operator::And, right)) as _; + } + let wrapper = Arc::new(MeasuredConjunct::new(Arc::clone(predicate))); + measured.push(Arc::clone(&wrapper)); + wrapper as _ +} + /// `conjuncts` in `order` as `(c_first AND (c_second AND (... AND c_last)))`. /// Right-nesting lets [`BinaryExpr`]'s pre-selection keep the first conjunct's /// survivors compacted for the rest of the chain. `order` must be non-empty. @@ -522,18 +661,23 @@ fn rank_by_effectiveness(stats: &[ConjunctStats]) -> Vec { } /// Expected nanoseconds per input row for `order`: each conjunct's per-row -/// cost weighted by the product of the pass rates before it (assumed -/// independent). Unmeasured conjuncts contribute nothing. +/// cost weighted by the product of the +/// [`downstream_weight`](ConjunctStats::downstream_weight)s of the conjuncts +/// before it (assumed independent). Unmeasured conjuncts contribute no cost +/// and narrow nothing. +/// +/// The weights are what `AND` really hands on — a conjunct only shrinks the +/// work after it when it pre-selects — so an order is credited for a discard +/// only where evaluation would act on it. fn expected_cost_per_row(stats: &[ConjunctStats], order: &[usize]) -> f64 { let mut weight = 1.0_f64; let mut total = 0.0_f64; for &id in order { - let (Some(cost), Some(pass)) = (stats[id].cost_per_row(), stats[id].pass_rate()) - else { + let Some(cost) = stats[id].cost_per_row() else { continue; }; total += weight * cost; - weight *= pass; + weight *= stats[id].downstream_weight(); } total } @@ -607,12 +751,9 @@ mod tests { .collect() } + /// Counts as if measured over null-free batches of a uniform shape. fn stats(rows: u64, matched: u64, nanos: u64) -> ConjunctStats { - ConjunctStats { - rows, - matched, - nanos, - } + ConjunctStats::from_null_free_batches(rows, matched, nanos) } /// `try_new` with a fresh, unshared registry and no metric. @@ -735,14 +876,112 @@ mod tests { assert_eq!(adaptive.shared.inner.lock().unwrap().measured_batches, 0); } + fn close(got: f64, want: f64) -> bool { + (got - want).abs() < 1e-9 + } + + /// A conjunct narrows the work after it only where `AND` pre-selects. + /// Below the threshold the weight is the pass rate; above it, and for an + /// all-`true` conjunct, the conjuncts after it still see every row. #[test] - fn expected_cost_weights_by_upstream_pass_rate() { - // a: cost 1, pass 0.5 ; b: cost 10, pass 0.5 + fn downstream_weight_follows_the_pre_selection_threshold() { + // Just below the 20% threshold: pre-selects, so the weight is the + // pass rate. + assert!(close(stats(1000, 190, 1000).downstream_weight(), 0.19)); + // Exactly at it: `check_short_circuit` uses `<=`, so it pre-selects. + assert!(close(stats(1000, 200, 1000).downstream_weight(), 0.20)); + // Just above it: no pre-selection, so the full batch carries on. + assert!(close(stats(1000, 210, 1000).downstream_weight(), 1.0)); + // Well above it, and all rows passing: likewise the full batch. + assert!(close(stats(1000, 900, 1000).downstream_weight(), 1.0)); + assert!(close(stats(1000, 1000, 1000).downstream_weight(), 1.0)); + // All rows rejected: nothing after it is evaluated at all. + assert!(close(stats(1000, 0, 1000).downstream_weight(), 0.0)); + // Never evaluated: assumed to narrow nothing. + assert!(close(stats(0, 0, 0).downstream_weight(), 1.0)); + } + + /// The cost model charges a conjunct's followers for the rows `AND` really + /// hands them, not for its pass rate. + #[test] + fn expected_cost_weights_by_what_and_hands_on() { + // Both keep half the rows, so neither pre-selects and the second + // conjunct is charged for the whole batch either way. let s = vec![stats(1000, 500, 1000), stats(1000, 500, 10_000)]; - // order [0,1]: 1 + 0.5*10 = 6 - assert!((expected_cost_per_row(&s, &[0, 1]) - 6.0).abs() < 1e-9); - // order [1,0]: 10 + 0.5*1 = 10.5 - assert!((expected_cost_per_row(&s, &[1, 0]) - 10.5).abs() < 1e-9); + assert!(close(expected_cost_per_row(&s, &[0, 1]), 11.0)); + assert!(close(expected_cost_per_row(&s, &[1, 0]), 11.0)); + + // Below the threshold conjunct 0 does narrow the batch, and running it + // first pays: 1 + 0.1 * 10, against 10 + 1 the other way round. + let s = vec![stats(1000, 100, 1000), stats(1000, 500, 10_000)]; + assert!(close(expected_cost_per_row(&s, &[0, 1]), 2.0)); + assert!(close(expected_cost_per_row(&s, &[1, 0]), 11.0)); + } + + /// The reviewed case: a cheap conjunct keeping 30% ranks ahead of an + /// expensive one keeping 90%, but neither can pre-select, so promoting it + /// saves nothing and the reorder must not be adopted. + #[test] + fn no_reorder_when_the_better_ranked_conjunct_cannot_pre_select() { + let schema = schema(); + let p = predicate(&schema); + let cs = split(&p); + // id 0: 10ns/row, keeps 90% ; id 1: 1ns/row, keeps 30%. + let s = vec![stats(1000, 900, 10_000), stats(1000, 300, 1000)]; + assert_eq!(rank_by_effectiveness(&s), vec![1, 0], "id 1 ranks first"); + // Both orders cost 10 + 1: neither conjunct narrows the other. + assert!(close(expected_cost_per_row(&s, &[0, 1]), 11.0)); + assert!(close(expected_cost_per_row(&s, &[1, 0]), 11.0)); + + let d = settle(&s, &cs, &p); + assert!(!d.reordered, "a reorder that cannot pay must be rejected"); + assert!(Arc::ptr_eq(&d.predicate, &p)); + } + + /// Nulls disable pre-selection entirely, so a conjunct that looks + /// selective only because it produced them narrows nothing. Measured + /// through the wrapper, against the same conjunct on a null-free batch. + #[test] + fn nulls_disable_the_downstream_discount() { + let nullable = + Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, true)])); + let inner = binary( + col("a", &nullable).unwrap(), + Operator::Gt, + lit(8i32), + &nullable, + ) + .unwrap(); + + // 1 true, 1 null, 8 false: a 10% pass rate that cannot pre-select. + let measured = Arc::new(MeasuredConjunct::new(Arc::clone(&inner))); + let mut with_null: Vec> = (0..10).map(Some).collect(); + with_null[0] = None; + let rb = RecordBatch::try_new( + Arc::clone(&nullable), + vec![Arc::new(Int32Array::from(with_null))], + ) + .unwrap(); + measured.evaluate(&rb).unwrap(); + let s = measured.take(); + assert_eq!((s.rows, s.matched), (10, 1)); + assert_eq!(s.gated_rows, 0, "a null batch never pre-selects"); + assert!(close(s.downstream_weight(), 1.0)); + + // The same conjunct and the same pass rate without the null does. + let measured = Arc::new(MeasuredConjunct::new(inner)); + let rb = RecordBatch::try_new( + Arc::clone(&nullable), + vec![Arc::new(Int32Array::from((0..10).collect::>()))], + ) + .unwrap(); + measured.evaluate(&rb).unwrap(); + let s = measured.take(); + assert_eq!( + (s.rows, s.matched, s.gated_rows, s.gated_matched), + (10, 1, 10, 1) + ); + assert!(close(s.downstream_weight(), 0.1)); } /// The mask equals the written predicate's before and after settling. @@ -804,17 +1043,21 @@ mod tests { assert!(adaptive.settled); } - /// An already-good order is kept, as a right-nested chain. + /// An already-good order is kept as the written expression itself, not + /// rebuilt: reassociating it would change where pre-selection fires. #[test] - fn settle_keeps_order_when_not_materially_better() { + fn settle_keeps_the_written_expression_when_not_materially_better() { let schema = schema(); let p = predicate(&schema); // Equally cheap and selective: swapping cannot help. let s = vec![stats(1000, 500, 1000), stats(1000, 500, 1000)]; let cs = split(&p); - let d = settle(&s, &cs); + let d = settle(&s, &cs, &p); assert!(!d.reordered); - assert_chain(&d.predicate, &cs, &[0, 1]); + assert!( + Arc::ptr_eq(&d.predicate, &p), + "the written predicate is handed back untouched" + ); } #[test] @@ -824,7 +1067,7 @@ mod tests { let cs = split(&p); // id 1 is far more selective at equal cost: it moves first. let s = vec![stats(1000, 900, 1000), stats(1000, 10, 1000)]; - let d = settle(&s, &cs); + let d = settle(&s, &cs, &p); assert!(d.reordered); assert_chain(&d.predicate, &cs, &[1, 0]); } @@ -856,7 +1099,7 @@ mod tests { stats(1000, 500, 1000), stats(1000, 10, 1000), ]; - let d = settle(&s, &cs); + let d = settle(&s, &cs, &p); assert!(d.reordered); // `(cs[2] AND (cs[1] AND cs[0]))`. @@ -875,9 +1118,9 @@ mod tests { assert!(Arc::ptr_eq(inner.right(), &cs[0])); } - /// A kept written order runs as a right-nested chain. Real timings are - /// used on purpose: with both pass rates above `1 - TIE_COST_FRACTION` the - /// guard `c1 + p*c0 < 0.95 * (c0 + p*c1)` cannot hold for any costs. + /// A kept written order runs the written expression itself. Real timings + /// are used on purpose: neither conjunct pre-selects, so both downstream + /// weights are 1 and no order can be cheaper than another. #[test] fn no_reorder_evaluates_plain_predicate() { let schema = schema(); @@ -902,7 +1145,153 @@ mod tests { !adaptive.reordered, "interchangeable conjuncts keep the written order" ); - assert_chain(&adaptive.settled_predicate, &split(&p), &[0, 1]); + assert!( + Arc::ptr_eq(&adaptive.settled_predicate, &p), + "and run the written expression itself" + ); + } + + /// A left-nested three-conjunct predicate, `((a > 2 AND b < 5) AND a < 90)`. + fn left_nested_predicate(schema: &Arc) -> Arc { + let third = + binary(col("a", schema).unwrap(), Operator::Lt, lit(90i32), schema).unwrap(); + binary(predicate(schema), Operator::And, third, schema).unwrap() + } + + /// The warm-up wraps the conjunct leaves without reshaping the `AND` tree, + /// so it evaluates exactly what the written predicate would — same + /// nesting, so the same pre-selection points. + #[test] + fn warmup_preserves_the_written_tree_shape() { + let schema = schema(); + let p = left_nested_predicate(&schema); + let cs = split(&p); + assert_eq!(cs.len(), 3); + + let adaptive = try_new(&p).unwrap(); + + // `((M(cs[0]) AND M(cs[1])) AND M(cs[2]))`: left-nested, as written. + let outer = adaptive + .warmup_predicate + .downcast_ref::() + .expect("an AND"); + assert_eq!(*outer.op(), Operator::And); + let inner = outer + .left() + .downcast_ref::() + .expect("the written tree nests to the left"); + assert_eq!(*inner.op(), Operator::And); + + // Every leaf is the wrapper for the conjunct written in that position. + for (leaf, id) in [(inner.left(), 0), (inner.right(), 1), (outer.right(), 2)] { + let wrapper = leaf + .downcast_ref::() + .unwrap_or_else(|| panic!("conjunct {id} is wrapped")); + assert!(Arc::ptr_eq(&wrapper.inner, &cs[id]), "conjunct {id}"); + assert!( + Arc::ptr_eq(&adaptive.measured[id].inner, &cs[id]), + "measured[{id}] is the wrapper in written position {id}" + ); + } + } + + /// Until a reorder is adopted the flag must be inert, side effects + /// included. + /// + /// `((a < 50 AND b < 3) AND 1 / z > 2)` over 100 rows: `a < 50` keeps 50% + /// and `b < 3` keeps 30%, so neither pre-selects on its own, but their + /// conjunction keeps 15% and the outer `AND` does — which is the only + /// reason `1 / z` never meets the zeros. Rebuilding the same conjuncts + /// right-nested would gate `1 / z` on `b < 3` alone, at 30%, and divide by + /// zero. + #[test] + fn keeping_the_written_order_keeps_its_side_effects() { + let schema = Arc::new(Schema::new(vec![ + Field::new("a", DataType::Int64, false), + Field::new("b", DataType::Int64, false), + Field::new("z", DataType::Int64, false), + ])); + let lt = |name: &str, v: i64| { + binary(col(name, &schema).unwrap(), Operator::Lt, lit(v), &schema).unwrap() + }; + let divide = || { + binary( + binary( + lit(1i64), + Operator::Divide, + col("z", &schema).unwrap(), + &schema, + ) + .unwrap(), + Operator::Gt, + lit(2i64), + &schema, + ) + .unwrap() + }; + // Written left-nested, as `conjunction` and the parser build it. + let p = binary( + binary(lt("a", 50), Operator::And, lt("b", 3), &schema).unwrap(), + Operator::And, + divide(), + &schema, + ) + .unwrap(); + + // `a < 50` on 50 rows, `b < 3` on 30 spread across them: 15 together. + let a: Vec = (0..100).collect(); + let b: Vec = (0..100).map(|i| i % 10).collect(); + let z: Vec = (0..100).map(|i| i64::from(i < 50 && i % 10 < 3)).collect(); + let rb = RecordBatch::try_new( + Arc::clone(&schema), + vec![ + Arc::new(Int64Array::from(a)), + Arc::new(Int64Array::from(b)), + Arc::new(Int64Array::from(z)), + ], + ) + .unwrap(); + + // Flag off: pre-selection keeps `1 / z` away from the zeros. + let want = p.evaluate(&rb).unwrap().into_array(rb.num_rows()).unwrap(); + assert_eq!(passing_rows(&want).len(), 0, "1 / 1 > 2 is false"); + + // The same conjuncts in the same order, but right-nested, gate + // `1 / z` on `b < 3` alone and raise. This is what settling on the + // written order must not do. + let right_nested = binary( + lt("a", 50), + Operator::And, + binary(lt("b", 3), Operator::And, divide(), &schema).unwrap(), + &schema, + ) + .unwrap(); + let err = right_nested.evaluate(&rb).unwrap_err().to_string(); + assert!(err.contains("Divide by zero"), "unexpected error: {err}"); + + // Flag on, settling on the written order: identical counts rank as a + // tie, so no candidate can be materially cheaper. + let shared = Arc::new(AdaptiveFilterShared::default()); + shared.seed_one_batch_short_of_warmup(&[ + (70_000_000, 35_000_000, 70_000_000), + (70_000_000, 35_000_000, 70_000_000), + (70_000_000, 35_000_000, 70_000_000), + ]); + let mut adaptive = + AdaptiveConjunction::try_new(&p, Arc::clone(&shared), None).unwrap(); + + // The settling batch runs through the wrappers ... + let got = adaptive.evaluate(&rb).unwrap(); + assert_eq!(passing_rows(&got), passing_rows(&want)); + assert!(adaptive.settled && !adaptive.reordered); + assert!( + Arc::ptr_eq(&adaptive.settled_predicate, &p), + "the written expression is run, not a rebuilt one" + ); + + // ... and every batch after it runs the written expression. + let got = adaptive.evaluate(&rb).unwrap(); + assert_eq!(passing_rows(&got), passing_rows(&want)); } /// Two streams share one pool: the warm-up is pooled across both, and the @@ -1046,7 +1435,6 @@ mod tests { fn scenario_measure_batches_then_settle_on_written_order() { let schema = schema(); let p = predicate(&schema); - let cs = split(&p); let shared = Arc::new(AdaptiveFilterShared::default()); // Identical cost and selectivity: no order can be materially cheaper. shared.seed_one_batch_short_of_warmup(&[ @@ -1067,7 +1455,7 @@ mod tests { assert_eq!(passing_rows(&got), passing_rows(&want), "round {round}"); assert!(adaptive.settled, "settled after round {round}"); assert!(!adaptive.reordered, "not reordered after round {round}"); - assert_chain(&adaptive.settled_predicate, &cs, &[0, 1]); + assert!(Arc::ptr_eq(&adaptive.settled_predicate, &p)); } } diff --git a/datafusion/sqllogictest/test_files/information_schema.slt b/datafusion/sqllogictest/test_files/information_schema.slt index 81004c6176c81..d7de27f024535 100644 --- a/datafusion/sqllogictest/test_files/information_schema.slt +++ b/datafusion/sqllogictest/test_files/information_schema.slt @@ -375,7 +375,7 @@ datafusion.catalog.has_header true Default value for `format.has_header` for `CR datafusion.catalog.information_schema true Should DataFusion provide access to `information_schema` virtual tables for displaying schema information datafusion.catalog.location NULL Location scanned to load tables for `default` schema datafusion.catalog.newlines_in_values false Specifies whether newlines in (quoted) CSV values are supported. This is the default value for `format.newlines_in_values` for `CREATE EXTERNAL TABLE` if not specified explicitly in the statement. Parsing newlines in quoted values may be affected by execution behaviour such as parallel file scanning. Setting this to `true` ensures that newlines in values are parsed successfully, which may reduce performance. -datafusion.execution.adaptive_filter_reordering false (experimental) When enabled, `FilterExec` measures the selectivity and evaluation cost of each conjunct of an `AND` predicate at runtime and reorders them to run the ones that discard the most rows per unit of CPU time first. Query results never change, but the observable side effects of a fallible predicate can, in either direction: reordering `b <> 0 AND 1/b > 2` can make a divide-by-zero error appear or disappear, since each conjunct is evaluated only on the rows the conjuncts before it kept. Predicates containing volatile expressions are never reordered. +datafusion.execution.adaptive_filter_reordering false (experimental) When enabled, `FilterExec` measures the selectivity and evaluation cost of each conjunct of an `AND` predicate at runtime and reorders them to run the ones that discard the most rows per unit of CPU time first. Query results never change, but the observable side effects of a fallible predicate can, in either direction: reordering `b <> 0 AND 1/b > 2` can make a divide-by-zero error appear or disappear, since each conjunct is evaluated only on the rows the conjuncts before it kept. A predicate whose written order is kept is evaluated exactly as written, so enabling this has no observable effect until a reorder is actually adopted (reported by the `adaptive_reorders` metric). Predicates containing volatile expressions are never reordered. datafusion.execution.batch_size 8192 Default batch size while creating new batches, it's especially useful for buffer-in-memory batches since creating tiny batches would result in too much metadata memory consumption datafusion.execution.coalesce_batches true When set to true, record batches will be examined between each operator and small batches will be coalesced into larger batches. This is helpful when there are highly selective filters or joins that could produce tiny output batches. The target batch size is determined by the configuration setting datafusion.execution.collect_statistics true Should DataFusion collect statistics when first creating a table. Has no effect after the table is created. Defaults to true. diff --git a/docs/source/user-guide/configs.md b/docs/source/user-guide/configs.md index e85caacc1ff9f..4d048e8c77ae5 100644 --- a/docs/source/user-guide/configs.md +++ b/docs/source/user-guide/configs.md @@ -142,7 +142,7 @@ The following configuration settings are available: | datafusion.execution.skip_partial_aggregation_probe_rows_threshold | 100000 | Number of input rows partial aggregation partition should process, before aggregation ratio check and trying to switch to skipping aggregation mode | | datafusion.execution.use_row_number_estimates_to_optimize_partitioning | false | Should DataFusion use row number estimates at the input to decide whether increasing parallelism is beneficial or not. By default, only exact row numbers (not estimates) are used for this decision. Setting this flag to `true` will likely produce better plans. if the source of statistics is accurate. We plan to make this the default in the future. | | datafusion.execution.enforce_batch_size_in_joins | false | Should DataFusion enforce batch size in joins or not. By default, DataFusion will not enforce batch size in joins. Enforcing batch size in joins can reduce memory usage when joining large tables with a highly-selective join filter, but is also slightly slower. Note: this option currently only applies to the symmetric hash join. | -| datafusion.execution.adaptive_filter_reordering | false | (experimental) When enabled, `FilterExec` measures the selectivity and evaluation cost of each conjunct of an `AND` predicate at runtime and reorders them to run the ones that discard the most rows per unit of CPU time first. Query results never change, but the observable side effects of a fallible predicate can, in either direction: reordering `b <> 0 AND 1/b > 2` can make a divide-by-zero error appear or disappear, since each conjunct is evaluated only on the rows the conjuncts before it kept. Predicates containing volatile expressions are never reordered. | +| datafusion.execution.adaptive_filter_reordering | false | (experimental) When enabled, `FilterExec` measures the selectivity and evaluation cost of each conjunct of an `AND` predicate at runtime and reorders them to run the ones that discard the most rows per unit of CPU time first. Query results never change, but the observable side effects of a fallible predicate can, in either direction: reordering `b <> 0 AND 1/b > 2` can make a divide-by-zero error appear or disappear, since each conjunct is evaluated only on the rows the conjuncts before it kept. A predicate whose written order is kept is evaluated exactly as written, so enabling this has no observable effect until a reorder is actually adopted (reported by the `adaptive_reorders` metric). Predicates containing volatile expressions are never reordered. | | datafusion.execution.objectstore_writer_buffer_size | 10485760 | Size (bytes) of data buffer DataFusion uses when writing output files. This affects the size of the data chunks that are uploaded to remote object stores (e.g. AWS S3). If very large (>= 100 GiB) output files are being written, it may be necessary to increase this size to avoid errors from the remote end point. | | datafusion.execution.enable_ansi_mode | false | Whether to enable ANSI SQL mode. The flag is experimental and relevant only for DataFusion Spark built-in functions When `enable_ansi_mode` is set to `true`, the query engine follows ANSI SQL semantics for expressions, casting, and error handling. This means: - **Strict type coercion rules:** implicit casts between incompatible types are disallowed. - **Standard SQL arithmetic behavior:** operations such as division by zero, numeric overflow, or invalid casts raise runtime errors rather than returning `NULL` or adjusted values. - **Consistent ANSI behavior** for string concatenation, comparisons, and `NULL` handling. When `enable_ansi_mode` is `false` (the default), the engine uses a more permissive, non-ANSI mode designed for user convenience and backward compatibility. In this mode: - Implicit casts between types are allowed (e.g., string to integer when possible). - Arithmetic operations are more lenient — for example, `abs()` on the minimum representable integer value returns the input value instead of raising overflow. - Division by zero or invalid casts may return `NULL` instead of failing. # Default `false` — ANSI SQL mode is disabled by default. | | datafusion.execution.hash_join_buffering_capacity | 0 | How many bytes to buffer in the probe side of hash joins while the build side is concurrently being built. Without this, hash joins will wait until the full materialization of the build side before polling the probe side. This is useful in scenarios where the query is not completely CPU bounded, allowing to do some early work concurrently and reducing the latency of the query. Note that when hash join buffering is enabled, the probe side will start eagerly polling data, not giving time for the producer side of dynamic filters to produce any meaningful predicate. Queries with dynamic filters might see performance degradation. Disabled by default, set to a number greater than 0 for enabling it. | From df919f9d29524ca86ce38d9b8c312c76a852e7ed Mon Sep 17 00:00:00 2001 From: Adrian Garcia Badaracco <1755071+adriangb@users.noreply.github.com> Date: Wed, 16 Sep 2026 14:39:55 +0000 Subject: [PATCH 24/27] test(physical-plan): pin the 30% downstream weight directly The reviewed example pairs a conjunct keeping 30% with one keeping 90%; the 30% half was only covered through a cost-model assertion. Assert its weight alongside the others. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01SZtdyL1QXmSiQdMRYmqTkV --- datafusion/physical-plan/src/adaptive_filter.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/datafusion/physical-plan/src/adaptive_filter.rs b/datafusion/physical-plan/src/adaptive_filter.rs index 603e7fa6f4b9e..60b4f29d943a6 100644 --- a/datafusion/physical-plan/src/adaptive_filter.rs +++ b/datafusion/physical-plan/src/adaptive_filter.rs @@ -892,8 +892,11 @@ mod tests { assert!(close(stats(1000, 200, 1000).downstream_weight(), 0.20)); // Just above it: no pre-selection, so the full batch carries on. assert!(close(stats(1000, 210, 1000).downstream_weight(), 1.0)); - // Well above it, and all rows passing: likewise the full batch. + // Well above it: keeping 30% and keeping 90% both leave the conjuncts + // after them facing every row, so both weigh the same. + assert!(close(stats(1000, 300, 1000).downstream_weight(), 1.0)); assert!(close(stats(1000, 900, 1000).downstream_weight(), 1.0)); + // All rows passing: likewise the full batch. assert!(close(stats(1000, 1000, 1000).downstream_weight(), 1.0)); // All rows rejected: nothing after it is evaluated at all. assert!(close(stats(1000, 0, 1000).downstream_weight(), 0.0)); From 0c0642ec67c22933bd552fc5aff8055bc932b334 Mon Sep 17 00:00:00 2001 From: Adrian Garcia Badaracco <1755071+adriangb@users.noreply.github.com> Date: Wed, 16 Sep 2026 19:33:37 +0000 Subject: [PATCH 25/27] fix(physical-expr): do not link private check_short_circuit from public docs `AndRhsEvaluation` and `and_rhs_evaluation` are public, and their docs linked `[`check_short_circuit`]`, which is private. The link resolves only under `--document-private-items`, so the workspace doc build fails with `rustdoc::private_intra_doc_links` under `RUSTDOCFLAGS=-D warnings`. Name the function in plain backticks instead. Verified with the CI command, `ci/scripts/rust_docs.sh`. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01SZtdyL1QXmSiQdMRYmqTkV --- datafusion/physical-expr/src/expressions/binary.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/datafusion/physical-expr/src/expressions/binary.rs b/datafusion/physical-expr/src/expressions/binary.rs index 1c316b62c08a6..667c9e057f758 100644 --- a/datafusion/physical-expr/src/expressions/binary.rs +++ b/datafusion/physical-expr/src/expressions/binary.rs @@ -1169,7 +1169,7 @@ pub const PRE_SELECTION_THRESHOLD: f32 = 0.2; /// How much of the batch an `AND`'s right-hand side is evaluated on, given the /// shape of its left-hand side's result. /// -/// This is the observable consequence of [`check_short_circuit`] for `AND`, +/// This is the observable consequence of `check_short_circuit` for `AND`, /// exposed so that consumers modelling the cost of a conjunction share one /// definition with the code that implements it. See [`and_rhs_evaluation`]. #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -1192,7 +1192,7 @@ pub enum AndRhsEvaluation { /// `true_count` counts non-null `true`s; it is only consulted when /// `null_count` is zero, where the two conventions coincide. /// -/// [`check_short_circuit`] decides by this function, so a caller that models +/// `check_short_circuit` decides by this function, so a caller that models /// conjunction cost cannot drift away from what evaluation actually does. pub fn and_rhs_evaluation( true_count: usize, From 8385e54c6db342948a51ec8506f25079b2340d57 Mon Sep 17 00:00:00 2001 From: Adrian Garcia Badaracco <1755071+adriangb@users.noreply.github.com> Date: Wed, 16 Sep 2026 19:47:49 +0000 Subject: [PATCH 26/27] refactor(physical-expr): hide the shared AND pre-selection rule from public docs `PRE_SELECTION_THRESHOLD`, `AndRhsEvaluation` and `and_rhs_evaluation` are `pub` only so that `physical-plan` can model `AND` pre-selection from the same definition evaluation uses. They are not API worth advertising, so mark them and their re-export `#[doc(hidden)]` and say why in a line of doc comment. `AndRhsEvaluation` is hidden alongside the function that returns it; left visible it would document a type no visible signature mentions. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01SZtdyL1QXmSiQdMRYmqTkV --- datafusion/physical-expr/src/expressions/binary.rs | 12 ++++++++++++ datafusion/physical-expr/src/expressions/mod.rs | 8 ++++---- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/datafusion/physical-expr/src/expressions/binary.rs b/datafusion/physical-expr/src/expressions/binary.rs index 667c9e057f758..03a624e2980d9 100644 --- a/datafusion/physical-expr/src/expressions/binary.rs +++ b/datafusion/physical-expr/src/expressions/binary.rs @@ -1164,6 +1164,10 @@ enum ShortCircuitStrategy { /// the side that cannot short-circuit the operator is rare: /// - for `AND`, when the proportion of `true` is less than or equal to 0.2 /// - for `OR`, when the proportion of `false` is less than or equal to 0.2 +/// +/// Public only so that crates modelling this behaviour can share the +/// definition; not part of the API surface DataFusion advertises. +#[doc(hidden)] pub const PRE_SELECTION_THRESHOLD: f32 = 0.2; /// How much of the batch an `AND`'s right-hand side is evaluated on, given the @@ -1172,6 +1176,10 @@ pub const PRE_SELECTION_THRESHOLD: f32 = 0.2; /// This is the observable consequence of `check_short_circuit` for `AND`, /// exposed so that consumers modelling the cost of a conjunction share one /// definition with the code that implements it. See [`and_rhs_evaluation`]. +/// +/// Public only so that crates modelling this behaviour can share the +/// definition; not part of the API surface DataFusion advertises. +#[doc(hidden)] #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum AndRhsEvaluation { /// The left-hand side is `false` on every row, so the right-hand side is @@ -1194,6 +1202,10 @@ pub enum AndRhsEvaluation { /// /// `check_short_circuit` decides by this function, so a caller that models /// conjunction cost cannot drift away from what evaluation actually does. +/// +/// Public only so that crates modelling this behaviour can share the +/// definition; not part of the API surface DataFusion advertises. +#[doc(hidden)] pub fn and_rhs_evaluation( true_count: usize, null_count: usize, diff --git a/datafusion/physical-expr/src/expressions/mod.rs b/datafusion/physical-expr/src/expressions/mod.rs index 2c2816304faf1..24a2879af6e44 100644 --- a/datafusion/physical-expr/src/expressions/mod.rs +++ b/datafusion/physical-expr/src/expressions/mod.rs @@ -41,10 +41,10 @@ pub use crate::PhysicalSortExpr; /// Module with some convenient methods used in expression building pub use crate::aggregate::stats::StatsType; -pub use binary::{ - AndRhsEvaluation, BinaryExpr, PRE_SELECTION_THRESHOLD, and_rhs_evaluation, binary, - similar_to, -}; +/// Shared with crates that model `AND` pre-selection; not advertised API. +#[doc(hidden)] +pub use binary::{AndRhsEvaluation, PRE_SELECTION_THRESHOLD, and_rhs_evaluation}; +pub use binary::{BinaryExpr, binary, similar_to}; pub use case::{CaseExpr, case}; pub use cast::{CastExpr, cast}; pub use column::{Column, col, with_new_schema}; From 8f5dcd18e45a2913a1574690116d31d93d6432da Mon Sep 17 00:00:00 2001 From: Adrian Garcia Badaracco <1755071+adriangb@users.noreply.github.com> Date: Wed, 16 Sep 2026 21:04:50 +0000 Subject: [PATCH 27/27] docs(physical-plan): describe the adaptive pool as node state, not per-execution The pooled measurements and the settled decision live on the `FilterExec` node. `execute()` clones them, and the `ExecutionPlan` contract does not promise `reset_state()` before every execution, so a second or concurrent run of the same plan reuses what the first one learned. Two comments implied otherwise: one called the measurements "per-execution", and `reset_state`'s doc read as though re-executing were enough to reset them. Say what actually holds, and record the lifetime among the module's known limitations, where the other caveats are. Results are unaffected either way; speed, `adaptive_reorders` and the side effects of a fallible conjunct are not. `reset_state()` discarding what was learned is now asserted on the state itself rather than on `Arc` identity, and the end-to-end test pins the other half: a second `execute()` keeps the settled decision. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01SZtdyL1QXmSiQdMRYmqTkV --- .../physical-plan/src/adaptive_filter.rs | 14 +++++++++ datafusion/physical-plan/src/filter.rs | 31 ++++++++++++++++--- 2 files changed, 41 insertions(+), 4 deletions(-) diff --git a/datafusion/physical-plan/src/adaptive_filter.rs b/datafusion/physical-plan/src/adaptive_filter.rs index 60b4f29d943a6..f0cd48d2aeb5a 100644 --- a/datafusion/physical-plan/src/adaptive_filter.rs +++ b/datafusion/physical-plan/src/adaptive_filter.rs @@ -122,6 +122,13 @@ //! did in its own position, not measured. //! - The decision is one-shot: a misjudged reorder, or drifting data, is kept //! for the rest of the query. +//! - The pooled state lives on the `FilterExec` node, not on one execution of +//! it. Executing the same plan again, or concurrently, reuses the earlier +//! measurements and the settled decision; only +//! [`reset_state`](crate::ExecutionPlan::reset_state) — which the execution +//! API does not promise to call — starts over. Results are unaffected, but a +//! second run can differ from the first in speed, in `adaptive_reorders`, +//! and in the side effects of a fallible conjunct. //! //! See . @@ -289,6 +296,13 @@ impl AdaptiveFilterShared { self.inner.lock().expect("poisoned").settled.clone() } + /// Whether nothing has been measured or settled yet. + #[cfg(test)] + pub(crate) fn is_pristine(&self) -> bool { + let inner = self.inner.lock().expect("poisoned"); + inner.stats.is_empty() && inner.measured_batches == 0 && inner.settled.is_none() + } + /// Seed `(rows, matched, nanos)` per conjunct one batch short of the /// warm-up, so the next measured batch settles on the seeded decision /// regardless of real timings. diff --git a/datafusion/physical-plan/src/filter.rs b/datafusion/physical-plan/src/filter.rs index 54ae1126b0039..ab9e34e99bc5f 100644 --- a/datafusion/physical-plan/src/filter.rs +++ b/datafusion/physical-plan/src/filter.rs @@ -642,8 +642,12 @@ impl ExecutionPlan for FilterExec { ) } - /// Fresh adaptive-reordering state and metrics for a re-execution; the - /// predicate, input and cached properties are still valid and kept. + /// Fresh adaptive-reordering state and metrics; the predicate, input and + /// cached properties are still valid and kept. + /// + /// Callers wanting a query to learn from scratch must call this: plain + /// [`execute`](ExecutionPlan::execute) reuses whatever the node has + /// already measured. fn reset_state(self: Arc) -> Result> { let mut new = (*self).clone(); new.adaptive_stats = Arc::new(AdaptiveFilterShared::default()); @@ -955,7 +959,7 @@ impl ExecutionPlan for FilterExec { projection, batch_size, fetch, - // Per-execution adaptive measurements, not part of the plan shape. + // Adaptive measurements: node state, not part of the plan shape. adaptive_stats: _, } = self; let input_node = ctx.encode_child(input)?; @@ -2514,8 +2518,9 @@ mod tests { Ok(()) } + /// `reset_state()` discards what the node learned; nothing else does. #[tokio::test] - async fn test_reset_state_gives_fresh_adaptive_stats() -> Result<()> { + async fn test_reset_state_resets_adaptive_learning() -> Result<()> { let schema = Schema::new(vec![Field::new("a", DataType::Int32, false)]); let input = Arc::new(StatisticsExec::new( Statistics::new_unknown(&schema), @@ -2528,6 +2533,12 @@ mod tests { )); let filter = Arc::new(FilterExec::try_new(predicate, input)?); + assert!(filter.adaptive_stats.is_pristine()); + filter + .adaptive_stats + .seed_one_batch_short_of_warmup(&[(100, 50, 100)]); + assert!(!filter.adaptive_stats.is_pristine(), "the node has learned"); + let reset = Arc::clone(&filter).reset_state()?; let reset = reset .as_ref() @@ -2535,6 +2546,14 @@ mod tests { .expect("reset_state returns a FilterExec"); assert!(!Arc::ptr_eq(&filter.adaptive_stats, &reset.adaptive_stats)); + assert!( + reset.adaptive_stats.is_pristine(), + "the reset node learns from scratch" + ); + assert!( + !filter.adaptive_stats.is_pristine(), + "and the node it came from is left as it was" + ); assert!(Arc::ptr_eq(&filter.predicate, &reset.predicate)); Ok(()) } @@ -2666,6 +2685,10 @@ mod tests { // Re-executing the same node keeps the learned state; same rows. assert_eq!(run(&filter, true).await?, flag_off, "state persists"); + assert!( + !filter.adaptive_stats.is_pristine(), + "a second execute() reuses the settled decision, it does not reset it" + ); // A reset node learns from scratch; same rows. let reset = Arc::clone(&filter).reset_state()?;