bench: SQL benchmark suite for null-aware (NOT IN) joins

benchmarks/README.md

@@ -966,6 +966,36 @@ Several queries are included to test hash joins under various workloads.
 ./bench.sh run hj
 ```
 
+## Null-Aware Join
+
+This benchmark focuses on `NOT IN` subqueries, which plan as null-aware joins: an
+outer row that finds no match is TRUE only if neither side has a NULL in scope,
+and UNKNOWN otherwise.
+
+Deciding which outer rows are UNKNOWN is cheap when the `NOT IN` is uncorrelated
+(Q01-Q03) and is linear in the table sizes. When the subquery is correlated, the
+correlation predicate stays behind as a join filter, and the join has to evaluate
+that filter per candidate (build row x probe row) pair to decide which rows the
+NULLs actually reach. A non-equality correlation leaves no equality key to narrow
+those pairs, so Q04-Q07 measure how that cost grows with the NULL fraction, while
+Q08 adds an equality correlation that turns the candidate pairs into a hash
+lookup.
+
+Both table sizes are knobs: `NAJ_ROWS` (default `10000`) sizes the correlated
+queries, whose cost grows with its square, and `NAJ_LARGE_ROWS` (default
+`1000000`) sizes the uncorrelated ones.
+
+### Example Run
+
+```bash
+# No need to generate data: this benchmark uses table function `range()` as the data source
+
+./bench.sh run null_aware_join
+
+# Or, with more rows for the correlated queries (~4x the per-pair filter work)
+NAJ_ROWS=20000 ./bench.sh run null_aware_join
+```
+
 ## Sort Merge Join
 
 This benchmark focuses on the performance of queries with sort merge joins, minimizing other overheads such as scanning data sources or evaluating predicates.

benchmarks/bench.sh

@@ -112,6 +112,9 @@ parquet_row_filter_skip: Per-RG fully-matched RowFilter skip on Parquet (apache/
                           range filter + pushdown, so most row groups are fully matched and the per-row RowFilter is skipped on them
                           (subgroups via BENCH_SUBGROUP: skip = clustered key so the skip fires, control = scrambled key so it never fires)
                           (data generated inline by the suite's load SQL; knobs: PRED_ROWS, RG_SIZE)
+null_aware_join:        Null-aware (NOT IN) hash join micro-benchmarks: uncorrelated, non-equality-correlated and equality-correlated
+                          NOT IN across NULL fractions, to measure the per-pair join-filter work the correlated cases do
+                          (data generated inline by the suite's load SQL from range(); knobs: NAJ_ROWS, NAJ_LARGE_ROWS)
 
 # ClickBench Benchmarks
 clickbench_1:           ClickBench queries against a single parquet file
@@ -271,6 +274,10 @@ main() {
                     # Data is generated inline by the suite's load SQL (COPY).
                     echo "parquet_row_filter_skip: no external data to generate"
                     ;;
+                null_aware_join)
+                    # Data is generated inline by the suite's load SQL from range().
+                    echo "null_aware_join: no external data to generate"
+                    ;;
                 asof_join)
                     data_asof_join
                     ;;
@@ -518,6 +525,9 @@ main() {
                 parquet_row_filter_skip)
                     run_parquet_row_filter_skip
                     ;;
+                null_aware_join)
+                    run_null_aware_join
+                    ;;
                 asof_join)
                     run_asof_join
                     ;;
@@ -933,6 +943,27 @@ run_parquet_row_filter_skip() {
       bash -c "$SQL_CARGO_COMMAND"
 }
 
+# Runs the null_aware_join suite: NOT IN (null-aware) hash joins. The load SQL
+# builds every table inline from range(), so there is no data step.
+#
+# Q01-Q03 are uncorrelated NOT IN and are linear in the table size; they are the
+# regression guard for the plain null-aware path. Q04-Q08 are correlated, where
+# the correlation predicate stays behind as a join filter that the join applies
+# per candidate (build row x probe row) pair while deciding which rows are
+# UNKNOWN; with no equality correlation there are no scope keys to narrow those
+# pairs, so Q05-Q07 scale with the NULL count times the opposite table's size.
+#
+#   NAJ_ROWS         rows per table for the correlated queries (default 10_000)
+#   NAJ_LARGE_ROWS   rows per table for the uncorrelated queries (default 1_000_000)
+run_null_aware_join() {
+    echo "Running null_aware_join benchmark (rows=${NAJ_ROWS:-10000}, large_rows=${NAJ_LARGE_ROWS:-1000000})..."
+    debug_run env BENCH_NAME=null_aware_join \
+      NAJ_ROWS="${NAJ_ROWS:-10000}" \
+      NAJ_LARGE_ROWS="${NAJ_LARGE_ROWS:-1000000}" \
+      ${QUERY:+BENCH_QUERY="${QUERY}"}  \
+      bash -c "$SQL_CARGO_COMMAND"
+}
+
 # Runs the tpch in memory (needs tpch parquet data)
 run_tpch_mem() {
     SCALE_FACTOR=$1

benchmarks/sql_benchmarks/README.md

@@ -37,6 +37,7 @@ in the community:
 | `hj`                  | Hash join benchmark                                                |
 | `imdb`                | IMDb benchmark                                                     |
 | `nlj`                 | Nested‑loop join benchmark                                         |
+| `null_aware_join`     | Null-aware (`NOT IN`) hash join micro-benchmarks. Q01-Q03 are uncorrelated `NOT IN` across NULL fractions and are linear in the table size (`NAJ_LARGE_ROWS`, default `1000000`). Q04-Q08 are correlated, so the correlation predicate stays behind as a join filter that the join applies per candidate (build row × probe row) pair while deciding which outer rows are UNKNOWN; without an equality correlation there are no scope keys to narrow those pairs, so their cost grows with the square of `NAJ_ROWS` (default `10000`). Q08 adds an equality correlation, which turns those pairs into a hash lookup. All tables are built inline from `range()`, so there is no data step. |
 | `push_down_topk`      | `ORDER BY ... LIMIT` over outer joins (TPC-H data); exercises pushing a TopK through a join |
 | `smj`                 | Sort‑merge join benchmark                                          |
 | `sort tpch`           | Sorting benchmarks against the TPC-H lineitem table                |

benchmarks/sql_benchmarks/null_aware_join/benchmarks/q01.benchmark

@@ -0,0 +1,16 @@
+name Q01
+group null_aware_join
+
+load sql_benchmarks/null_aware_join/init/load.sql
+
+expect_plan HashJoinExec
+
+run
+-- Q1: uncorrelated NOT IN, non-nullable key on both sides.
+-- No null-aware handling is needed at all, so this is the floor for the shape
+-- and a guard against the anti-join path getting slower for everyone else.
+SELECT count(*)
+FROM large_outer o
+WHERE o.id NOT IN (SELECT i.id FROM large_inner i);
+
+cleanup sql_benchmarks/null_aware_join/init/cleanup.sql

benchmarks/sql_benchmarks/null_aware_join/benchmarks/q02.benchmark

@@ -0,0 +1,16 @@
+name Q02
+group null_aware_join
+
+load sql_benchmarks/null_aware_join/init/load.sql
+
+expect_plan HashJoinExec
+
+run
+-- Q2: uncorrelated NOT IN, 1% NULL on the subquery side.
+-- A single NULL anywhere in the subquery makes every unmatched outer row
+-- UNKNOWN, which is the cheap bulk path. Linear in the table sizes.
+SELECT count(*)
+FROM large_outer o
+WHERE o.id NOT IN (SELECT i.id_n1 FROM large_inner i);
+
+cleanup sql_benchmarks/null_aware_join/init/cleanup.sql

benchmarks/sql_benchmarks/null_aware_join/benchmarks/q03.benchmark

@@ -0,0 +1,15 @@
+name Q03
+group null_aware_join
+
+load sql_benchmarks/null_aware_join/init/load.sql
+
+expect_plan HashJoinExec
+
+run
+-- Q3: uncorrelated NOT IN, 50% NULL on the outer side.
+-- Every NULL outer row is UNKNOWN on its own. Linear in the table sizes.
+SELECT count(*)
+FROM large_outer o
+WHERE o.id_n50 NOT IN (SELECT i.id FROM large_inner i);
+
+cleanup sql_benchmarks/null_aware_join/init/cleanup.sql

benchmarks/sql_benchmarks/null_aware_join/benchmarks/q04.benchmark

@@ -0,0 +1,18 @@
+name Q04
+group null_aware_join
+
+load sql_benchmarks/null_aware_join/init/load.sql
+
+expect_plan HashJoinExec
+
+run
+-- Q4: non-equality-correlated NOT IN, nullable keys that hold no NULL.
+-- The correlation `i.z < o.z` has no equality, so it stays behind as a join
+-- filter and the join has no correlation scope keys. With no NULL on either
+-- side there is nothing to mark, so this is the zero-NULL baseline: it shows
+-- what the correlated NOT IN shape costs before any per-pair filter work.
+SELECT count(*)
+FROM small_outer o
+WHERE o.id_n0 NOT IN (SELECT i.id_n0 FROM small_inner i WHERE i.z < o.z);
+
+cleanup sql_benchmarks/null_aware_join/init/cleanup.sql

benchmarks/sql_benchmarks/null_aware_join/benchmarks/q05.benchmark

@@ -0,0 +1,18 @@
+name Q05
+group null_aware_join
+
+load sql_benchmarks/null_aware_join/init/load.sql
+
+expect_plan HashJoinExec
+
+run
+-- Q5: non-equality-correlated NOT IN, 1% NULL on the outer side.
+-- With no correlation scope keys the join filter is evaluated over every
+-- (NULL outer row x probe row) pair, so the work scales with the NULL count
+-- times the subquery size. At 1% NULL that product is still small; Q06 is the
+-- same query at 50%.
+SELECT count(*)
+FROM small_outer o
+WHERE o.id_n1 NOT IN (SELECT i.id_n0 FROM small_inner i WHERE i.z < o.z);
+
+cleanup sql_benchmarks/null_aware_join/init/cleanup.sql

benchmarks/sql_benchmarks/null_aware_join/benchmarks/q06.benchmark

@@ -0,0 +1,19 @@
+name Q06
+group null_aware_join
+
+load sql_benchmarks/null_aware_join/init/load.sql
+
+expect_plan HashJoinExec
+
+run
+-- Q6: non-equality-correlated NOT IN, 50% NULL on the outer side.
+-- Worst case for the pairing that has no correlation scope keys: half the
+-- outer rows are NULL, and the join filter is evaluated for each of them
+-- against every subquery row, including for outer rows already known to be
+-- UNKNOWN. Compare against Q05 (same query, 1% NULL) to see how the cost
+-- tracks the NULL fraction.
+SELECT count(*)
+FROM small_outer o
+WHERE o.id_n50 NOT IN (SELECT i.id_n0 FROM small_inner i WHERE i.z < o.z);
+
+cleanup sql_benchmarks/null_aware_join/init/cleanup.sql

benchmarks/sql_benchmarks/null_aware_join/benchmarks/q07.benchmark

@@ -0,0 +1,17 @@
+name Q07
+group null_aware_join
+
+load sql_benchmarks/null_aware_join/init/load.sql
+
+expect_plan HashJoinExec
+
+run
+-- Q7: non-equality-correlated NOT IN, 50% NULL on the subquery side.
+-- The mirror of Q06: a NULL subquery row makes every outer row in its
+-- correlation scope UNKNOWN, so the filter is evaluated over every
+-- (outer row x NULL probe row) pair.
+SELECT count(*)
+FROM small_outer o
+WHERE o.id_n0 NOT IN (SELECT i.id_n50 FROM small_inner i WHERE i.z < o.z);
+
+cleanup sql_benchmarks/null_aware_join/init/cleanup.sql

benchmarks/sql_benchmarks/null_aware_join/benchmarks/q08.benchmark

@@ -0,0 +1,26 @@
+name Q08
+group null_aware_join
+
+load sql_benchmarks/null_aware_join/init/load.sql
+
+expect_plan HashJoinExec
+
+run
+-- Q8: NOT IN correlated by both an equality and a non-equality, 50% NULL on
+-- the outer side. The equality correlation becomes a correlation scope key, so
+-- the candidate pairs come from a hash lookup on `k` (16 groups here) instead
+-- of the full cross product, and only those pairs reach the join filter. Same
+-- NULL fraction as Q06, so the gap between the two is what the scope key buys.
+--
+-- The `OR` keeps this a mark join: a bare `WHERE ... NOT IN` with an equality
+-- correlation is the shape that does not plan at all before this suite's
+-- motivating fix, and the point here is to measure the scope-key path, not to
+-- pick a shape that only runs on one branch.
+SELECT count(*)
+FROM small_outer o
+WHERE o.z > 900
+   OR o.id_n50 NOT IN (
+        SELECT i.id_n0 FROM small_inner i WHERE i.k = o.k AND i.z < o.z
+      );
+
+cleanup sql_benchmarks/null_aware_join/init/cleanup.sql

benchmarks/sql_benchmarks/null_aware_join/init/cleanup.sql

@@ -0,0 +1,4 @@
+DROP TABLE small_outer;
+DROP TABLE small_inner;
+DROP TABLE large_outer;
+DROP TABLE large_inner;

benchmarks/sql_benchmarks/null_aware_join/init/load.sql

@@ -0,0 +1,55 @@
+-- Synthetic tables for the null-aware (NOT IN) hash join benchmarks. Built from
+-- `range()`, so there is no data generation step.
+--
+-- Columns, on both sides:
+--   id       Int64 NOT NULL         -- NOT IN over this never needs null-aware handling
+--   id_n0    Int64 NULL, 0% NULL    -- nullable, but no NULL is ever present
+--   id_n1    Int64 NULL, 1% NULL
+--   id_n50   Int64 NULL, 50% NULL
+--   z        correlation value for a non-equality correlation (`i.z < o.z`)
+--   k        correlation key for an equality correlation (`i.k = o.k`), 16 groups
+--
+-- `id_n0` is nullable so the planner still builds a null-aware join, but holds no
+-- NULL, which is what makes it the zero-NULL baseline for the correlated queries.
+-- `id` is only ~half covered by the subquery side, so the anti join returns rows
+-- rather than degenerating to an empty or full result.
+
+-- Small tables: used by the correlated queries (Q04-Q08), whose cost grows with
+-- the product of the two table sizes.
+CREATE TABLE small_outer AS
+SELECT
+  value AS id,
+  CASE WHEN value < 0 THEN NULL ELSE value END AS id_n0,
+  CASE WHEN value % 100 = 0 THEN NULL ELSE value END AS id_n1,
+  CASE WHEN value % 2 = 0 THEN NULL ELSE value END AS id_n50,
+  value % 1000 AS z,
+  value % 16 AS k
+FROM range(0, ${NAJ_ROWS:-10000});
+
+CREATE TABLE small_inner AS
+SELECT
+  value * 2 AS id,
+  CASE WHEN value < 0 THEN NULL ELSE value * 2 END AS id_n0,
+  CASE WHEN value % 100 = 0 THEN NULL ELSE value * 2 END AS id_n1,
+  CASE WHEN value % 2 = 0 THEN NULL ELSE value * 2 END AS id_n50,
+  value % 1000 AS z,
+  value % 16 AS k
+FROM range(0, ${NAJ_ROWS:-10000});
+
+-- Large tables: used by the uncorrelated queries (Q01-Q03), whose cost is linear
+-- in the table size.
+CREATE TABLE large_outer AS
+SELECT
+  value AS id,
+  CASE WHEN value < 0 THEN NULL ELSE value END AS id_n0,
+  CASE WHEN value % 100 = 0 THEN NULL ELSE value END AS id_n1,
+  CASE WHEN value % 2 = 0 THEN NULL ELSE value END AS id_n50
+FROM range(0, ${NAJ_LARGE_ROWS:-1000000});
+
+CREATE TABLE large_inner AS
+SELECT
+  value * 2 AS id,
+  CASE WHEN value < 0 THEN NULL ELSE value * 2 END AS id_n0,
+  CASE WHEN value % 100 = 0 THEN NULL ELSE value * 2 END AS id_n1,
+  CASE WHEN value % 2 = 0 THEN NULL ELSE value * 2 END AS id_n50
+FROM range(0, ${NAJ_LARGE_ROWS:-1000000});

benchmarks/sql_benchmarks/null_aware_join/null_aware_join.suite

@@ -0,0 +1,31 @@
+description = "Null-aware (NOT IN) hash join benchmarks: uncorrelated, non-equality-correlated, and equality-correlated NOT IN across NULL fractions"
+
+query_pattern = "q{QUERY_ID_PADDED}.benchmark"
+
+[[options]]
+name = "rows"
+short = "r"
+env = "NAJ_ROWS"
+default = "10000"
+values = ["10000", "..."]
+help = "Rows per table for the correlated NOT IN queries (Q04-Q08). These evaluate the join filter over candidate build x probe pairs, so their cost grows with the square of this value."
+
+[[options]]
+name = "large-rows"
+short = "l"
+env = "NAJ_LARGE_ROWS"
+default = "1000000"
+values = ["1000000", "..."]
+help = "Rows per table for the uncorrelated NOT IN queries (Q01-Q03), whose cost is linear in the table size."
+
+[[examples]]
+command = "cargo run --release --bin benchmark_runner -- null_aware_join"
+description = "Run all null-aware join queries at the default sizes."
+
+[[examples]]
+command = "cargo run --release --bin benchmark_runner -- null_aware_join --query 6"
+description = "Run only Q06, the worst case for filter evaluation over NULL build rows."
+
+[[examples]]
+command = "cargo run --release --bin benchmark_runner -- null_aware_join -r 20000"
+description = "Run with 20,000 rows per table for the correlated queries (~4x the filter work of the default)."